webrecipe 0.1.0

package/dist/src/benchmark/discovery.js

@@ -0,0 +1,355 @@
+import { mkdtemp, mkdir, writeFile, appendFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { load } from 'cheerio';
+import { teach } from '../authoring/teach.js';
+import { HttpHtmlStrategy } from '../executor/strategies/http-html.js';
+import { HttpJsonStrategy } from '../executor/strategies/http-json.js';
+import { PolitenessLayer } from '../net/politeness.js';
+import { StaticSiteResolver, WILD_ORIGINS } from '../sites.js';
+import { verifyReadable, UserError } from '../local.js';
+import { PLANS } from '../../benchmark/plans.js';
+/**
+ * What the frozen verifier says about real sites, beside what an independent
+ * account says the answer was.
+ *
+ * The verifier is not modified while this runs. A failure found on the first
+ * site and fixed before the last is a batch whose halves came from two
+ * different systems, and the first case is the one a fix overfits to.
+ */
+export const BASELINE = '7f51ca0';
+/** At most four, plus every task that expects an empty result: those carry a
+ *  deterministic oracle and are worth more than another ordinary query. */
+const UNSEEN_PER_CASE = 4;
+export function buildCases(tasks, domains) {
+    const groups = new Map();
+    for (const task of tasks) {
+        if (!domains.includes(task.site))
+            continue;
+        const key = `${task.site}\u0000${task.intent}`;
+        groups.set(key, [...(groups.get(key) ?? []), task]);
+    }
+    return [...groups.values()].map((all) => {
+        const [taught, ...rest] = all;
+        const empty = rest.filter((t) => t.expectEmpty === true);
+        const ordinary = rest.filter((t) => t.expectEmpty !== true).slice(0, UNSEEN_PER_CASE - empty.length);
+        return { site: taught.site, intent: taught.intent, taught: taught, unseen: [...ordinary, ...empty] };
+    });
+}
+/** Reads the signals each verifier already records; no verifier was changed. */
+export function classifyNotTested(evidence) {
+    if (evidence === undefined)
+        return 'insufficient_evidence';
+    const declined = (evidence.probes ?? []).find((p) => p.unavailable !== undefined)?.unavailable ?? '';
+    if (/429|retry later/i.test(declined))
+        return 'rate_limited';
+    if (declined !== '')
+        return 'transport_failure';
+    const s = evidence.signals ?? {};
+    if (s.stable === false)
+        return 'volatile';
+    if (s.applicable === false)
+        return 'no_visible_lexical_match';
+    if (s.consistent === false)
+        return 'lexical_inconsistent';
+    if (s.agreed === false)
+        return 'browser_disagreement';
+    if (s.moved === false)
+        return 'page_control_ambiguous';
+    if ((evidence.probes ?? []).some((p) => p.role === 'alternate' && p.items === 0))
+        return 'empty_alternate_page';
+    if (!(evidence.probes ?? []).some((p) => p.role === 'contrast' || p.role === 'alternate'))
+        return 'insufficient_evidence';
+    return 'unclassified';
+}
+const WORD = /^[a-z][a-z0-9-]{3,}$/;
+const tokens = (text) => [...new Set(text.toLowerCase().split(/[^a-z0-9-]+/i).filter((t) => WORD.test(t)))];
+const carriesId = (items, id) => {
+    const wanted = id.toLowerCase();
+    return items.some((item) => Object.values(item).some((v) => String(v ?? '').toLowerCase().includes(wanted)));
+};
+/**
+ * Whether the id rule can be applied to this plan at all.
+ *
+ * Decided from the taught observation, which is the browser's reading of the
+ * url the task names and so is the right entity by the task's own definition.
+ * If the id cannot be seen even there, this plan does not extract anything that
+ * carries one and the rule tells us nothing about any other input.
+ *
+ * This answers only "can this oracle be used here". It is never the answer to
+ * "was this result right", which is what keeps it from grading itself.
+ */
+export function entityIdApplicable(taught, taughtItems) {
+    const type = 'entity_id';
+    if (taught.intent !== 'detail')
+        return { type, applicable: false, reason: 'not a detail lookup' };
+    const id = taught.input.id;
+    if (id === undefined || String(id) === '')
+        return { type, applicable: false, reason: 'the task names no entity id' };
+    if (taughtItems === null || taughtItems.length === 0)
+        return { type, applicable: false, reason: 'the taught observation could not be read' };
+    return carriesId(taughtItems, String(id))
+        ? { type, applicable: true, reason: `the taught output carries the entity id ${id}` }
+        : { type, applicable: false, reason: 'requested entity id is not observable in the taught output' };
+}
+/**
+ * T1. A deterministic reading of the task itself, owing nothing to the probes.
+ */
+export function judgeByTask(task, items, role, entityId) {
+    if (task.expectEmpty === true) {
+        const oracle = { type: 'expect_empty', applicable: true, reason: 'the task declares this query matches nothing' };
+        return items.length === 0
+            ? { judgement: 'correct', source: 'expect_empty', reason: 'a query declared to match nothing returned nothing', oracle }
+            : { judgement: 'wrong', source: 'expect_empty', reason: `a query declared to match nothing returned ${items.length} rows`, oracle };
+    }
+    if (!entityId.applicable)
+        return { judgement: 'indeterminate', source: 'none', reason: entityId.reason, oracle: entityId };
+    // The input the gate was calibrated on cannot also be judged by it.
+    if (role === 'taught') {
+        return {
+            judgement: 'indeterminate', source: 'none',
+            reason: 'this is the observation the id oracle was calibrated on, so it cannot also answer for it',
+            oracle: { ...entityId, applicable: false, reason: 'calibration input' },
+        };
+    }
+    const id = String(task.input.id);
+    if (items.length === 0) {
+        return { judgement: 'indeterminate', source: 'none', reason: 'nothing returned to check the id against', oracle: entityId };
+    }
+    return carriesId(items, id)
+        ? { judgement: 'correct', source: 'entity_id', reason: `the requested id ${id} appears in the answer`, oracle: entityId }
+        : { judgement: 'wrong', source: 'entity_id', reason: `the requested id ${id} appears nowhere in the answer`, oracle: entityId };
+}
+/**
+ * T2. One-sided on purpose.
+ *
+ * A row that does not appear on the page it links to is a row describing
+ * something else, which is evidence the answer is wrong. A row that does appear
+ * there proves only that the row and its own link agree: move a whole row to
+ * the wrong entity — title, url and all — and this check still passes it. So a
+ * match settles nothing and goes to the next tier.
+ */
+export async function detailMismatch(items, net, origin, sample = 2) {
+    let requests = 0;
+    for (const item of items.slice(0, sample)) {
+        const href = String(item.url ?? '');
+        const title = String(item.title ?? '');
+        if (href === '' || title === '')
+            continue;
+        const wanted = tokens(title);
+        if (wanted.length === 0)
+            continue;
+        let url;
+        try {
+            url = new URL(href, origin).toString();
+        }
+        catch {
+            continue;
+        }
+        if (new URL(url).origin !== origin)
+            continue;
+        if (!(await net.isAllowed(url)))
+            continue;
+        let text;
+        try {
+            requests += 1;
+            const res = await net.fetch(url);
+            if (res.status !== 200)
+                continue;
+            text = load(res.body).text().toLowerCase();
+        }
+        catch {
+            continue;
+        }
+        // A shell that rendered nothing cannot disagree with anything.
+        if (text.length < 500)
+            continue;
+        if (!wanted.some((t) => text.includes(t))) {
+            return { wrong: true, requests, reason: `the row "${title}" shares no word with the page at ${href} that it links to` };
+        }
+    }
+    return { wrong: false, requests, reason: '' };
+}
+export const DISCOVERY_DOMAINS = [
+    'hex.pm', 'docs.rs', 'jsr.io', 'bandcamp.com',
+    'meta.discourse.org', 'lemmy.world', 'openlibrary.org', 'dev.to',
+];
+/** Untouched by this batch, and run once when a changed verifier is first evaluated. */
+export const HELD_OUT_DOMAINS = ['npmjs.com', 'musicbrainz.org', 'mastodon.social', 'itch.io'];
+export async function runDiscovery(cases, opts) {
+    await mkdir(join(opts.outDir, 'evidence'), { recursive: true });
+    const records = [];
+    const skips = [];
+    const net = new PolitenessLayer({ minIntervalMs: opts.minIntervalMs });
+    const skip = async (s) => {
+        skips.push(s);
+        opts.onSkip?.(s);
+        await appendFile(join(opts.outDir, 'skips.jsonl'), `${JSON.stringify(s)}\n`);
+    };
+    const keep = async (r) => {
+        records.push(r);
+        opts.onRecord?.(r);
+        await appendFile(join(opts.outDir, 'runs.jsonl'), `${JSON.stringify(r)}\n`);
+    };
+    for (const one of cases) {
+        const origin = WILD_ORIGINS[one.site];
+        const plan = PLANS[one.site]?.[one.intent];
+        if (origin === undefined || plan === undefined) {
+            await skip({ baseline: BASELINE, site: one.site, taskId: one.taught.id, reason: 'no_plan', detail: 'no hand-written plan for this site and intent' });
+            continue;
+        }
+        const taughtUrl = plan.url(origin, one.taught);
+        if (!(await net.isAllowed(taughtUrl))) {
+            await skip({ baseline: BASELINE, site: one.site, taskId: one.taught.id, reason: 'skipped_by_robots', detail: taughtUrl });
+            continue;
+        }
+        const planDir = await mkdtemp(join(tmpdir(), 'disc-plans-'));
+        const recipeDir = await mkdtemp(join(tmpdir(), 'disc-recipes-'));
+        const startedTeach = performance.now();
+        let taught;
+        try {
+            taught = await teach({
+                site: one.site, intent: one.intent, url: taughtUrl, input: one.taught.input,
+                itemSelector: plan.itemSelector, fields: plan.fields,
+                planDir, recipeDir, minIntervalMs: opts.minIntervalMs,
+            });
+        }
+        catch (error) {
+            await skip({ baseline: BASELINE, site: one.site, taskId: one.taught.id, reason: 'teach_failed', detail: error instanceof Error ? error.message : String(error) });
+            continue;
+        }
+        const teachMs = Math.round(performance.now() - startedTeach);
+        const verification = taught.plan.verification;
+        const contract = verification?.contract.required ?? [];
+        const notTestedReasons = {};
+        for (const check of contract) {
+            const evidence = verification?.evidence[check];
+            if (check === 'non_empty' || check === 'required_fields')
+                continue;
+            if (evidence?.status === 'passed' || evidence?.status === 'failed')
+                continue;
+            notTestedReasons[check] = classifyNotTested(evidence);
+        }
+        await writeFile(join(opts.outDir, 'evidence', `${one.site}-${one.intent}.json`), `${JSON.stringify({ baseline: BASELINE, site: one.site, intent: one.intent, taughtUrl, plan: taught.plan, refused: taught.refused, sample: taught.sample }, null, 2)}\n`);
+        const sites = new StaticSiteResolver({ [one.site]: origin });
+        const fields = Object.keys(plan.fields);
+        // Calibrated once per plan, from the browser's reading of the url the task
+        // names. It decides whether the id rule applies here, never what the answer was.
+        const entityId = entityIdApplicable(one.taught, taught.sample);
+        for (const [role, task] of [['taught', one.taught], ...one.unseen.map((t) => ['unseen', t])]) {
+            const url = plan.url(origin, task);
+            if (!(await net.isAllowed(url))) {
+                await skip({ baseline: BASELINE, site: one.site, taskId: task.id, reason: 'skipped_by_robots', detail: url });
+                continue;
+            }
+            const started = performance.now();
+            let items = null;
+            let error;
+            let requests = 0;
+            if (taught.recipe !== null) {
+                const strategy = taught.recipe.output.type === 'json' ? new HttpJsonStrategy(net, sites) : new HttpHtmlStrategy(net, sites);
+                try {
+                    requests += 1;
+                    const result = await strategy.execute(taught.recipe, { id: task.id, site: one.site, intent: one.intent, input: task.input });
+                    items = result.status !== undefined && result.status !== taught.recipe.validation.status ? null : result.items;
+                    if (items === null)
+                        error = `status ${result.status}`;
+                }
+                catch (err) {
+                    error = err instanceof Error ? err.message : String(err);
+                }
+            }
+            else {
+                error = `no recipe: ${taught.refused ?? 'unknown'}`;
+            }
+            let finalStatus = 'error';
+            let checks = {};
+            if (items !== null) {
+                try {
+                    const read = verifyReadable(items, fields, verification);
+                    finalStatus = read.status;
+                    checks = read.checks;
+                }
+                catch (err) {
+                    finalStatus = 'unverified';
+                    if (err instanceof UserError)
+                        error = err.message;
+                }
+            }
+            let { judgement, source, reason, oracle } = items === null
+                ? {
+                    judgement: 'indeterminate', source: 'none',
+                    reason: error ?? 'no answer',
+                    oracle: { type: 'none', applicable: false, reason: 'no answer to judge' },
+                }
+                : judgeByTask(task, items, role, entityId);
+            if (judgement === 'indeterminate' && items !== null && items.length > 0 && one.intent !== 'detail') {
+                const probe = await detailMismatch(items, net, origin);
+                requests += probe.requests;
+                if (probe.wrong) {
+                    judgement = 'wrong';
+                    source = 'detail_mismatch';
+                    reason = probe.reason;
+                    oracle = { type: 'detail_mismatch', applicable: true, reason: 'a returned row could be checked against the page it links to' };
+                }
+            }
+            // A person settles what reached verified, and a sample of what did not.
+            const needsManual = judgement === 'indeterminate' && (finalStatus === 'verified' || finalStatus === 'partially_verified');
+            await keep({
+                baseline: BASELINE, cached: false, site: one.site, taskId: task.id, intent: one.intent,
+                inputRole: role, input: task.input, contract, checks, finalStatus, notTestedReasons,
+                judgement, judgementSource: source, judgementReason: reason, oracle, needsManual,
+                items: items ?? [], itemCount: items?.length ?? 0,
+                cost: { httpRequests: requests, browserRuns: role === 'taught' ? 3 : 0, elapsedMs: Math.round(performance.now() - started) + (role === 'taught' ? teachMs : 0) },
+                ...(error === undefined ? {} : { error }),
+            });
+        }
+    }
+    return { records, skips };
+}
+export function formatDiscovery(records, skips, eligible) {
+    const lines = [];
+    const judged = (j) => (r) => r.judgement === j;
+    const status = (s) => (r) => r.finalStatus === s;
+    const count = (f) => records.filter(f).length;
+    const both = (a, b) => count((r) => a(r) && b(r));
+    const domains = new Set(records.map((r) => r.site));
+    lines.push(`baseline: ${BASELINE}   cached: false (every request hit the origin)`);
+    lines.push(`domains: ${domains.size}   learned tasks: ${eligible}   evaluated inputs: ${records.length}`);
+    const byReason = new Map();
+    for (const s of skips)
+        byReason.set(s.reason, (byReason.get(s.reason) ?? 0) + 1);
+    lines.push(`skipped: ${skips.length}${skips.length === 0 ? '' : ` (${[...byReason].map(([k, v]) => `${k} ${v}`).join(', ')})`}`);
+    lines.push('');
+    lines.push('                  correct   wrong   indeterminate');
+    for (const s of ['verified', 'partially_verified', 'structural', 'unverified', 'error']) {
+        const row = records.filter(status(s));
+        if (row.length === 0)
+            continue;
+        lines.push(`  ${s.padEnd(20)}${String(both(status(s), judged('correct'))).padStart(5)}${String(both(status(s), judged('wrong'))).padStart(8)}${String(both(status(s), judged('indeterminate'))).padStart(14)}`);
+    }
+    lines.push('');
+    const falseSuccess = records.filter((r) => r.finalStatus === 'verified' && r.judgement === 'wrong');
+    lines.push(`false success (verified + wrong): ${falseSuccess.length}`);
+    for (const r of falseSuccess)
+        lines.push(`  ${r.site} ${r.taskId} ${JSON.stringify(r.input)} — ${r.judgementReason}`);
+    lines.push('');
+    lines.push('correct but not verified, by reason:');
+    const reasons = new Map();
+    for (const r of records) {
+        if (r.judgement !== 'correct' || r.finalStatus === 'verified')
+            continue;
+        for (const reason of Object.values(r.notTestedReasons))
+            reasons.set(reason, (reasons.get(reason) ?? 0) + 1);
+        if (Object.keys(r.notTestedReasons).length === 0)
+            reasons.set('structural_only', (reasons.get('structural_only') ?? 0) + 1);
+    }
+    for (const [reason, n] of [...reasons].sort((a, b) => b[1] - a[1]))
+        lines.push(`  ${reason.padEnd(26)} ${n}`);
+    lines.push('');
+    const queue = records.filter((r) => r.needsManual);
+    lines.push(`awaiting manual judgement: ${queue.length}`);
+    for (const r of queue.slice(0, 40))
+        lines.push(`  [${r.finalStatus}] ${r.site} ${r.taskId} ${JSON.stringify(r.input)} items=${r.itemCount}`);
+    return lines.join('\n');
+}

package/dist/src/benchmark/golden.js

@@ -0,0 +1,95 @@
+export function diffGolden(golden, actual, volatile = []) {
+    const reasons = [];
+    if (actual.length !== golden.items.length) {
+        reasons.push(`item count ${actual.length} !== golden ${golden.items.length}`);
+        return { match: false, reasons };
+    }
+    const volatileSet = new Set(volatile);
+    golden.items.forEach((expected, index) => {
+        const got = actual[index];
+        if (!got) {
+            reasons.push(`item ${index} missing`);
+            return;
+        }
+        for (const [field, expectedValue] of Object.entries(expected)) {
+            const gotValue = got[field];
+            if (volatileSet.has(field)) {
+                if (!(field in got) || gotValue === null || gotValue === undefined) {
+                    reasons.push(`item ${index}: volatile field "${field}" absent`);
+                }
+                else if (typeof gotValue !== typeof expectedValue) {
+                    reasons.push(`item ${index}: volatile field "${field}" changed type`);
+                }
+                continue;
+            }
+            if (gotValue !== expectedValue) {
+                reasons.push(`item ${index}: "${field}" ${JSON.stringify(gotValue)} !== ${JSON.stringify(expectedValue)}`);
+            }
+        }
+    });
+    return { match: reasons.length === 0, reasons };
+}
+const key = (item, volatile, fields) => JSON.stringify((fields ?? Object.keys(item)).filter((k) => !volatile.has(k)).sort().map((k) => [k, item[k]]));
+/**
+ * Three separate questions, because collapsing them mixes engine failure with
+ * website nondeterminism. flathub and pkg.go.dev reorder equally-ranked results
+ * between one request and the next, which failed even the browser baseline
+ * against its own golden — that is the site being unstable, not the recipe
+ * being wrong, and it should not be reported as the same thing.
+ *
+ * `fields`, when given, restricts both schema and semantic comparison to that
+ * set — the same restriction `oracle.compare.fields` applies under
+ * `paired-live`, so a column the oracle does not score cannot fail a golden
+ * task either.
+ */
+export function gradeGolden(golden, actual, volatile = [], fields) {
+    const volatileSet = new Set(volatile);
+    const reasons = [];
+    // Schema: does every declared field still exist, with the right type?
+    let schema = true;
+    const declared = new Set(fields ?? golden.items.flatMap((i) => Object.keys(i)));
+    for (const field of declared) {
+        const expected = golden.items.find((i) => i[field] !== null && i[field] !== undefined)?.[field];
+        const missing = actual.filter((i) => !(field in i) || i[field] === null || i[field] === undefined);
+        if (actual.length > 0 && missing.length === actual.length) {
+            schema = false;
+            reasons.push(`field "${field}" absent from every item`);
+            continue;
+        }
+        const mistyped = actual.filter((i) => i[field] != null && expected != null && typeof i[field] !== typeof expected);
+        if (mistyped.length > 0) {
+            schema = false;
+            reasons.push(`field "${field}" changed type in ${mistyped.length} items`);
+        }
+    }
+    // Semantic: the same items, as a set.
+    const wanted = golden.items.map((i) => key(i, volatileSet, fields));
+    const got = actual.map((i) => key(i, volatileSet, fields));
+    const pool = [...got];
+    const absent = [];
+    for (const k of wanted) {
+        const at = pool.indexOf(k);
+        if (at === -1)
+            absent.push(k);
+        else
+            pool.splice(at, 1);
+    }
+    const semantic = absent.length === 0 && pool.length === 0;
+    if (absent.length > 0)
+        reasons.push(`${absent.length} golden item(s) missing from the result`);
+    if (pool.length > 0)
+        reasons.push(`${pool.length} unexpected item(s) in the result`);
+    // Volatile fields must still be present and of the right type.
+    for (const field of volatileSet) {
+        const bad = actual.filter((i) => !(field in i) || i[field] === null || i[field] === undefined);
+        if (bad.length > 0) {
+            schema = false;
+            reasons.push(`volatile field "${field}" absent in ${bad.length} items`);
+        }
+    }
+    // Ordering: only answerable when the sets agree.
+    const ordering = semantic ? wanted.every((k, i) => got[i] === k) : null;
+    if (ordering === false)
+        reasons.push('same items, different order');
+    return { schema, semantic, ordering, reasons };
+}

package/dist/src/benchmark/grade.js

@@ -0,0 +1,146 @@
+/** The fields the oracle compares; all of them when it names none. */
+function comparedFields(items, oracle) {
+    return oracle.compare.fields ?? [...new Set(items.flatMap((i) => Object.keys(i)))];
+}
+const present = (value) => value !== null && value !== undefined && value !== '';
+/**
+ * Layer 1. Decides whether the adjacent browser run is usable at all, before
+ * the engine is judged against it. Checks exactly the fields Layer 2 compares:
+ * disqualifying a baseline over a column nobody scores would throw away a
+ * usable measurement.
+ */
+export function baselineValidity(items, oracle, opts) {
+    if (opts.threw)
+        return { usable: false, reason: 'baseline threw' };
+    if (items.length === 0) {
+        return opts.expectEmpty
+            ? { usable: true, reason: null }
+            : { usable: false, reason: 'baseline returned no items' };
+    }
+    for (const field of comparedFields(items, oracle)) {
+        const missing = items.filter((i) => !present(i[field])).length;
+        if (missing > items.length / 2) {
+            return { usable: false, reason: `baseline field "${field}" missing in ${missing}/${items.length} items` };
+        }
+    }
+    return { usable: true, reason: null };
+}
+function keyOf(item, fields) {
+    return JSON.stringify(fields.map((f) => [f, item[f] ?? null]));
+}
+/** Layer 2a. The same items, as a set, on the fields the oracle compares. */
+export function entityEquivalence(reference, actual, oracle) {
+    const fields = comparedFields([...reference, ...actual], oracle);
+    const pool = actual.map((i) => keyOf(i, fields));
+    const reasons = [];
+    let missing = 0;
+    for (const key of reference.map((i) => keyOf(i, fields))) {
+        const at = pool.indexOf(key);
+        if (at === -1)
+            missing += 1;
+        else
+            pool.splice(at, 1);
+    }
+    if (missing > 0)
+        reasons.push(`${missing} baseline item(s) missing from the result`);
+    if (pool.length > 0)
+        reasons.push(`${pool.length} unexpected item(s) in the result`);
+    return { match: reasons.length === 0, reasons };
+}
+/**
+ * Layer 2b. The result answers the question that was asked.
+ *
+ * arbeitnow.com discards its search parameter on redirect and serves the same
+ * unfiltered front page for every query — a result that is internally
+ * consistent, matches its own baseline, and answers nothing. Entity equivalence
+ * alone cannot see that.
+ *
+ * Judged only where an oracle declares a rule, and returns null otherwise.
+ * Applying a generic "the query must appear in the results" heuristic
+ * everywhere would fail honest engines: a search for `senior rust engineer` may
+ * legitimately return `Systems Engineer`, a search for `cars` may return
+ * `automobile`, and a match may live in a description the oracle does not
+ * compare. A rule that fires on sites it was never designed for is an oracle
+ * bug wearing a correctness check's clothes.
+ */
+export function querySemantics(actual, input, oracle) {
+    const rule = oracle.semantics;
+    if (!rule)
+        return { match: null, reasons: [] };
+    if (actual.length === 0)
+        return { match: null, reasons: [] };
+    const raw = input[rule.input];
+    if (raw === undefined || String(raw).trim() === '')
+        return { match: null, reasons: [] };
+    // Token-wise, so a multi-word query is not required to appear verbatim.
+    const tokens = String(raw).toLowerCase().split(/\s+/).filter((t) => t.length >= 2);
+    if (tokens.length === 0)
+        return { match: null, reasons: [] };
+    const hits = actual.filter((item) => rule.fields.some((f) => {
+        const value = String(item[f] ?? '').toLowerCase();
+        return tokens.some((t) => value.includes(t));
+    })).length;
+    return hits >= actual.length * rule.minShare
+        ? { match: true, reasons: [] }
+        : { match: false, reasons: [`no token of "${String(raw)}" in ${actual.length - hits}/${actual.length} items`] };
+}
+/** Layer 2c. Null when the oracle ignores order, or when the sets differ. */
+export function orderingAgreement(reference, actual, oracle) {
+    if (oracle.compare.ordering === 'ignore')
+        return null;
+    if (!entityEquivalence(reference, actual, oracle).match)
+        return null;
+    const fields = comparedFields([...reference, ...actual], oracle);
+    return reference.every((item, i) => actual[i] !== undefined && keyOf(actual[i], fields) === keyOf(item, fields));
+}
+/** A result this much smaller than its golden suggests the plan, not the site. */
+const COLLAPSE_RATIO = 0.5;
+/**
+ * Compares a fresh browser result against its stored golden structurally rather
+ * than by content.
+ *
+ * Under `paired-live` the golden no longer scores correctness, but it still
+ * answers one question nothing else does: does the browser plan still see the
+ * page the way it used to? Items may legitimately all differ; the shape should
+ * not. Returns a description of the decay, or null.
+ */
+export function planAging(golden, baseline, oracle) {
+    if (golden === undefined || golden.length === 0)
+        return null;
+    // A count change is a hint, not a verdict: a search that legitimately returns
+    // forty results today where it returned a hundred last week has not aged its
+    // plan. Field disappearance below is the far stronger signal.
+    if (baseline.length <= golden.length * COLLAPSE_RATIO) {
+        return `possible plan aging: item count changed ${golden.length} -> ${baseline.length}`;
+    }
+    for (const field of comparedFields(golden, oracle)) {
+        const hadIt = golden.filter((i) => present(i[field])).length > golden.length / 2;
+        const hasIt = baseline.filter((i) => present(i[field])).length > baseline.length / 2;
+        if (hadIt && !hasIt)
+            return `plan aging: field "${field}" no longer resolves`;
+    }
+    return null;
+}
+/** Engine equivalence, defined once so a report and a printed line cannot diverge. */
+export const isEquivalent = (grade) => grade.entities === true && grade.query !== false;
+/** An unusable baseline leaves layer two unjudged rather than failed. */
+export function gradePair(reference, actual, oracle, opts) {
+    const validity = baselineValidity(reference, oracle, opts);
+    if (!validity.usable) {
+        return {
+            usable: false, validityReason: validity.reason, excludeReason: 'baseline',
+            entities: null, query: null, ordering: null, reasons: [validity.reason ?? 'baseline unusable'],
+        };
+    }
+    const entities = entityEquivalence(reference, actual, oracle);
+    const query = querySemantics(actual, opts.input, oracle);
+    return {
+        usable: true,
+        validityReason: null,
+        excludeReason: null,
+        entities: entities.match,
+        query: query.match,
+        ordering: orderingAgreement(reference, actual, oracle),
+        reasons: [...entities.reasons, ...query.reasons],
+    };
+}

package/dist/src/benchmark/ground-truth.js

@@ -0,0 +1,35 @@
+import { DATASET, PAGE_SIZE, search } from '../../fixtures/data.js';
+const identity = (item) => String(item.url ?? '');
+const KNOWN = new Set(DATASET.map((r) => `/item/${r.id}`));
+const PAGES = Math.ceil(DATASET.length / PAGE_SIZE);
+/** Every match, not just page one: a different ranking is not a wrong answer. */
+export const datasetMatches = (query) => new Set(Array.from({ length: PAGES }, (_, i) => search(query, i + 1))
+    .flat()
+    .map((r) => `/item/${r.id}`));
+/** Any page's worth of matches: a different ranking is not a wrong answer. */
+export const DATASET_TRUTH = { expected: (input) => datasetMatches(String(input.query ?? '')) };
+/** The one window this page should hold, which is what a page asks about. */
+export const pageWindow = (input) => new Set(search(String(input.query ?? ''), Number(input.page ?? 1)).map((r) => `/item/${r.id}`));
+export const PAGE_TRUTH = { expected: pageWindow };
+/**
+ * Judges precision, not recall: every row returned must be one this query
+ * should have returned. Whether it returned all of them is a question about
+ * ranking and paging, which this benchmark does not ask and which a listing
+ * that inserts a fresh row would fail for no good reason.
+ */
+export function judgeByGroundTruth(items, input, truth) {
+    if (items.length === 0)
+        return { match: null, reason: 'no items to judge' };
+    const expected = truth.expected(input);
+    for (const item of items) {
+        const id = identity(item);
+        if (expected.has(id))
+            continue;
+        if (truth.ephemeral?.test(id) === true)
+            continue;
+        return KNOWN.has(id)
+            ? { match: false, reason: `returned ${id}, a record this query does not match` }
+            : { match: false, reason: `returned ${id}, which is neither a known record nor a declared ephemeral row` };
+    }
+    return { match: true, reason: '' };
+}