webrecipe 0.1.0

package/dist/src/compiler/derive.js

@@ -0,0 +1,144 @@
+/**
+ * Learns the derivation a page performs on top of its API.
+ *
+ * An API is not a machine-readable version of a page; it is the input the page
+ * renders from. The page composes: crates.io builds `/crates/serde` from the
+ * crate id, and `serde v1.0.229` from a name and a version. A recipe that can
+ * only read fields verbatim cannot reproduce those, so it is refused as a
+ * narrower replacement for the browser. Synthesising the template recovers them.
+ */
+/** Shorter values match by coincidence far more often than they mean anything. */
+const MIN_VALUE_LENGTH = 2;
+function scalarEntries(row) {
+    return Object.entries(row)
+        .filter(([, v]) => typeof v === 'string' || typeof v === 'number')
+        .map(([k, v]) => [k, String(v)])
+        .filter(([, v]) => v.length >= MIN_VALUE_LENGTH);
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/**
+ * Rebuilds `target` from the row's own values, longest value first so that a
+ * version like `1.0.229` is not broken up by the `1` sitting in another field.
+ * Returns null when no value contributes.
+ */
+export function synthesizeTemplate(target, row) {
+    const candidates = scalarEntries(row).sort((a, b) => b[1].length - a[1].length);
+    let template = target;
+    let used = false;
+    for (const [key, value] of candidates) {
+        if (!template.includes(value))
+            continue;
+        template = template.replace(new RegExp(escapeRegExp(value), 'g'), `{{${key}}}`);
+        used = true;
+    }
+    return used ? template : null;
+}
+/** Renders a synthesized template back against a row, for verification. */
+export function renderRowTemplate(template, row) {
+    // `{{k1 ?? k2}}` renders the first key that has a value, so a page rule
+    // conditional on a null field is expressible.
+    return template.replace(/\{\{\s*([a-zA-Z0-9_]+(?:\s*\?\?\s*[a-zA-Z0-9_]+)*)\s*\}\}/g, (_match, keys) => {
+        for (const key of keys.split('??')) {
+            const value = row[key.trim()];
+            if (value !== null && value !== undefined && value !== '')
+                return String(value);
+        }
+        return '';
+    });
+}
+function placeholderCount(template) {
+    return [...template.matchAll(/\{\{/g)].length;
+}
+/**
+ * Accepts a template only if it reproduces the browser's value for **every**
+ * row. One row cannot tell `id` from `name`, or `objectID` from `story_id`;
+ * ten rows can. Where several templates still survive, the choice is made
+ * deterministically so that the same trace always compiles to the same recipe.
+ */
+export function synthesizeAcrossRows(targets, rows) {
+    if (rows.length === 0 || targets.length !== rows.length)
+        return null;
+    if (targets.some((t) => t === null || t === undefined))
+        return null;
+    const strings = targets.map((t) => String(t));
+    // Every template that explains the first row is a candidate; the rest filter.
+    const seeds = new Set();
+    const first = synthesizeTemplate(strings[0], rows[0]);
+    if (first === null)
+        return null;
+    seeds.add(first);
+    // Single-field alternatives, so an ambiguous pair is not collapsed too early.
+    for (const [key, value] of scalarEntries(rows[0])) {
+        if (!strings[0].includes(value))
+            continue;
+        seeds.add(strings[0].replace(new RegExp(escapeRegExp(value), 'g'), `{{${key}}}`));
+    }
+    const survives = (template) => strings.every((expected, i) => renderRowTemplate(template, rows[i]) === expected);
+    const survivors = [...seeds].filter(survives);
+    // Only when nothing plain fits: the page's rule may be conditional on a null
+    // field, as bandcamp renders `title by (album_artist ?? band_name)`.
+    if (survivors.length === 0) {
+        // Across every row, not just the first: the field a page falls back to is
+        // null on the rows that made the fallback visible.
+        const keys = [...new Set(rows.flatMap((row) => scalarEntries(row).map(([k]) => k)))].sort();
+        for (const seed of seeds) {
+            for (const slot of seed.matchAll(/\{\{([a-zA-Z0-9_]+)\}\}/g)) {
+                const key = slot[1];
+                for (const other of keys) {
+                    if (other === key)
+                        continue;
+                    for (const pair of [`${key} ?? ${other}`, `${other} ?? ${key}`]) {
+                        const candidate = seed.slice(0, slot.index) + `{{${pair}}}`
+                            + seed.slice(slot.index + slot[0].length);
+                        if (survives(candidate))
+                            survivors.push(candidate);
+                    }
+                }
+            }
+        }
+    }
+    if (survivors.length === 0)
+        return null;
+    // Prefer the most parameterised survivor. A literal left in the template is a
+    // value baked in from the recording: `{{id}} v1.0.229` reproduces the crate it
+    // was learned from and reports that same version for every other crate. An
+    // over-parameterised template at least varies with the data, and the
+    // equivalence check and golden diff still judge it.
+    survivors.sort((a, b) => placeholderCount(b) - placeholderCount(a) || a.localeCompare(b));
+    return survivors[0];
+}
+/**
+ * Decides what a field's spec should be, given what the browser produced for it.
+ *
+ * A spec can be absent, or present and wrong: hn.algolia's payload has a `url`,
+ * so `url` maps onto it by name, but the page's link is the discussion thread
+ * while the payload's is the article. Mapping by name is a guess; the browser's
+ * own values are the evidence. Returns null when neither the existing spec nor
+ * any synthesis reproduces them.
+ */
+export function reconcileField(spec, browserValues, rows, readSpec = defaultRead) {
+    if (spec !== undefined) {
+        const reproduces = rows.every((row, i) => {
+            const expected = browserValues[i];
+            if (expected === null || expected === undefined)
+                return false;
+            const actual = readSpec(spec, row);
+            return actual !== undefined && actual !== null && String(actual) === String(expected);
+        });
+        if (reproduces)
+            return spec;
+    }
+    return synthesizeAcrossRows(browserValues, rows);
+}
+function defaultRead(spec, row) {
+    if (!spec.startsWith('$'))
+        return renderRowTemplate(spec, row);
+    if (spec === '$')
+        return row;
+    return spec
+        .slice(2)
+        .split('.')
+        .reduce((node, key) => (node === null || typeof node !== 'object' ? undefined : node[key]), row);
+}

package/dist/src/compiler/heuristic.js

@@ -0,0 +1,398 @@
+import { scoreRequests } from '../analyzer/score.js';
+import { computeFingerprint } from '../recipes/fingerprint.js';
+import { jsonSignature } from '../executor/extract.js';
+import { resolvePath } from '../recipes/paths.js';
+import { RecipeSchema } from '../recipes/schema.js';
+import { extractJsonItems } from '../executor/extract.js';
+import { verifyAgainstBrowser, browserItemsOf } from './verify.js';
+import { reconcileField } from './derive.js';
+import { readBody } from '../recorder/body.js';
+import { equivalenceRefusal, isRefused } from './types.js';
+/** Values worth substituting; anything shorter matches by coincidence. */
+const MIN_INPUT_LENGTH = 2;
+/** No response got far enough to fail for a reason of its own. */
+export const NO_CANDIDATE = "no candidate response carries the browser's items";
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+const TITLE_KEYS = ['title', 'name', 'headline', 'label'];
+const ID_KEYS = ['id', 'slug', 'key', 'uuid'];
+const URL_KEYS = ['url', 'href', 'link', 'permalink'];
+function isObjectArray(value) {
+    return (Array.isArray(value) &&
+        value.length > 0 &&
+        value.every((v) => v !== null && typeof v === 'object' && !Array.isArray(v)));
+}
+/** The longest array of objects at depth 1 or 2 is the result set. */
+export function findItemsPath(payload) {
+    if (payload === null || typeof payload !== 'object')
+        return null;
+    let bestPath = null;
+    let bestLength = 0;
+    const consider = (path, value) => {
+        if (!isObjectArray(value))
+            return;
+        if (bestPath === null || value.length > bestLength) {
+            bestPath = path;
+            bestLength = value.length;
+        }
+    };
+    for (const [key, value] of Object.entries(payload)) {
+        consider(`$.${key}`, value);
+        if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
+            for (const [k2, v2] of Object.entries(value))
+                consider(`$.${key}.${k2}`, v2);
+        }
+    }
+    return bestPath;
+}
+const RECORD_KEYS = ['result', 'item', 'record', 'data', 'crate', 'entry'];
+const MIN_RECORD_FIELDS = 2;
+function scalarFieldCount(value) {
+    if (value === null || typeof value !== 'object' || Array.isArray(value))
+        return 0;
+    return Object.values(value).filter((v) => v === null || typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean').length;
+}
+/**
+ * A detail response carries one record, not a list, so there is no array for
+ * findItemsPath to latch onto. Picks the wrapped object that looks most like a
+ * record, falling back to the payload root when nothing wraps it.
+ */
+export function findRecordPath(payload) {
+    if (payload === null || typeof payload !== 'object' || Array.isArray(payload))
+        return null;
+    const entries = Object.entries(payload);
+    for (const key of RECORD_KEYS) {
+        const value = payload[key];
+        if (scalarFieldCount(value) >= MIN_RECORD_FIELDS)
+            return `$.${key}`;
+    }
+    let bestKey = null;
+    let bestCount = 0;
+    for (const [key, value] of entries) {
+        const count = scalarFieldCount(value);
+        if (count >= MIN_RECORD_FIELDS && count > bestCount) {
+            bestKey = key;
+            bestCount = count;
+        }
+    }
+    if (bestKey !== null)
+        return `$.${bestKey}`;
+    return scalarFieldCount(payload) >= MIN_RECORD_FIELDS ? '$' : null;
+}
+function pick(sample, candidates) {
+    return candidates.find((k) => k in sample && sample[k] !== null) ?? null;
+}
+const SYNONYMS = {
+    id: ID_KEYS,
+    title: TITLE_KEYS,
+    url: URL_KEYS,
+    author: ['author', 'by', 'owner', 'user', 'creator'],
+};
+/**
+ * When `expected` is given the recipe must be a drop-in replacement for the
+ * browser plan, so it maps those exact output names onto whatever the payload
+ * calls them. Without it, falls back to the conventional id/title/url guess.
+ */
+export function inferFields(sample, expected) {
+    const fields = {};
+    if (expected && expected.length > 0) {
+        for (const name of expected) {
+            const key = pick(sample, SYNONYMS[name] ?? [name]);
+            if (key)
+                fields[name] = `$.${key}`;
+        }
+        return fields;
+    }
+    const id = pick(sample, ID_KEYS) ?? Object.entries(sample).find(([, v]) => typeof v === 'string')?.[0];
+    if (id)
+        fields.id = `$.${id}`;
+    const title = pick(sample, TITLE_KEYS);
+    if (title)
+        fields.title = `$.${title}`;
+    const url = pick(sample, URL_KEYS);
+    if (url)
+        fields.url = `$.${url}`;
+    return fields;
+}
+/**
+ * Substitutes input values anywhere in a path, not only as whole segments.
+ *
+ * A filter is routinely welded into a segment rather than given a query
+ * parameter — `/games/genre-puzzle`, `/remote-python-jobs`. Matching whole
+ * segments only left those literal, and a recipe carrying no placeholder for
+ * its input is refused as a snapshot, so two working sites compiled to nothing.
+ *
+ * A match must be bounded by a non-word character on both sides, so a value is
+ * recognised inside a segment but not inside a longer number or word. Longest
+ * value first, so one value is not broken up by a shorter one sitting inside
+ * it. Anything that still slips through is caught downstream: a path templated
+ * too eagerly returns the wrong thing for a different input and fails the
+ * equivalence check.
+ */
+export function templatePath(pathname, input) {
+    const values = Object.entries(input)
+        .map(([name, v]) => [name, String(v)])
+        .filter(([, v]) => v.length >= MIN_INPUT_LENGTH)
+        .sort((a, b) => b[1].length - a[1].length);
+    let out = pathname;
+    for (const [name, value] of values) {
+        for (const candidate of [value, encodeURIComponent(value)]) {
+            // Bounded by something that is not a word character, so `100` is found in
+            // `/item/100` and in `/genre-100` but not inside `/1000`, where
+            // substituting it would render `/2000` for an id of 200.
+            const bounded = new RegExp(`(?<![A-Za-z0-9])${escapeRegExp(candidate)}(?![A-Za-z0-9])`, 'g');
+            out = out.replace(bounded, `{{${name}}}`);
+        }
+    }
+    // A value below the floor would match by coincidence almost anywhere, so it
+    // is templated only where it fills an entire path segment: `2` is the page in
+    // `/page/2`, but not the tail of `/1000`, the version in `/v2` or part of
+    // `/item-2`.
+    const short = Object.entries(input)
+        .map(([name, v]) => [name, String(v)])
+        .filter(([, v]) => v.length > 0 && v.length < MIN_INPUT_LENGTH);
+    for (const [name, value] of short) {
+        const candidates = new Set([value, encodeURIComponent(value)]);
+        out = out
+            .split('/')
+            .map((segment) => (candidates.has(segment) ? `{{${name}}}` : segment))
+            .join('/');
+    }
+    return out;
+}
+/**
+ * Finds which query parameter carried an input value and replaces it with a
+ * placeholder. A parameter is only templated on an exact value match, so a
+ * coincidental substring cannot turn a constant into a variable.
+ */
+function templateQuery(url, input) {
+    const query = {};
+    for (const [key, value] of url.searchParams.entries()) {
+        const match = Object.entries(input).find(([, v]) => String(v) === value);
+        query[key] = match ? `{{${match[0]}}}` : value;
+    }
+    return query;
+}
+/**
+ * Templates input values inside a POST body. Algolia and friends put the search
+ * term in the body rather than the URL, so a recipe that does not template it
+ * returns the results it was recorded with for every query it is ever asked.
+ *
+ * Only exact whole-value matches are replaced, so prose that merely mentions
+ * the query is left alone. Returns null when the body carries no input value.
+ */
+export function templateBody(postData, input) {
+    const wanted = Object.entries(input).filter(([, v]) => String(v) !== '');
+    if (wanted.length === 0)
+        return null;
+    const placeholderFor = (value) => {
+        const match = wanted.find(([, v]) => String(v) === value);
+        return match ? `{{${match[0]}}}` : null;
+    };
+    try {
+        const parsed = JSON.parse(postData);
+        let replaced = false;
+        const walk = (node) => {
+            if (typeof node === 'string') {
+                const placeholder = placeholderFor(node);
+                if (placeholder === null)
+                    return node;
+                replaced = true;
+                return placeholder;
+            }
+            if (Array.isArray(node))
+                return node.map(walk);
+            if (node !== null && typeof node === 'object') {
+                return Object.fromEntries(Object.entries(node).map(([k, v]) => [k, walk(v)]));
+            }
+            return node;
+        };
+        const out = walk(parsed);
+        return replaced ? JSON.stringify(out) : null;
+    }
+    catch {
+        // Not JSON — try form encoding.
+    }
+    const params = new URLSearchParams(postData);
+    let replaced = false;
+    for (const [key, value] of [...params.entries()]) {
+        const placeholder = placeholderFor(value);
+        if (placeholder === null)
+            continue;
+        params.set(key, placeholder);
+        replaced = true;
+    }
+    return replaced ? params.toString() : null;
+}
+/** Input names that must appear as a placeholder for the recipe to be parameterised. */
+export function templatedInputs(parts) {
+    const found = new Set();
+    for (const part of parts) {
+        for (const match of part.matchAll(/\{\{\s*([a-zA-Z0-9_]+)\s*\}\}/g))
+            found.add(match[1]);
+    }
+    return found;
+}
+export class HeuristicCompiler {
+    expectedFields;
+    /**
+     * `plan` is the browser behaviour the recipe must reproduce. Given one, every
+     * candidate is checked against what the browser actually extracted before it
+     * is accepted.
+     */
+    constructor(expected) {
+        if (Array.isArray(expected))
+            this.expectedFields = expected;
+        else if (expected) {
+            this.plan = expected;
+            this.expectedFields = Object.keys(expected.fields);
+        }
+    }
+    plan;
+    /**
+     * Walks candidates in score order and returns the first that yields a usable
+     * recipe. Taking only the top scorer makes the compiler give up whenever the
+     * highest-ranked request happens not to be parseable — an autocomplete
+     * endpoint returning an array of bare strings, say — even though the real
+     * result set sits one place below it.
+     */
+    async compile(trace) {
+        // The first substantive refusal, not the last: candidates are walked in
+        // score order, so the highest-ranked response that got far enough to fail
+        // for a nameable reason is the one worth reporting.
+        let refused = null;
+        for (const candidate of scoreRequests(trace)) {
+            const outcome = this.compileCandidate(trace, candidate);
+            if (outcome === null)
+                continue;
+            if (isRefused(outcome)) {
+                refused ??= outcome.refused;
+                continue;
+            }
+            if (!this.plan)
+                return outcome;
+            const payload = JSON.parse(readBody(candidate.request));
+            const browserItems = browserItemsOf(trace, this.plan);
+            const { recipe: complete, unreconciled } = this.reconcileFields(outcome, payload, browserItems);
+            const items = extractJsonItems(complete, payload);
+            if (verifyAgainstBrowser(trace, this.plan, items).equivalent)
+                return complete;
+            refused ??= unreconciled ?? equivalenceRefusal(items.length, browserItems.length);
+        }
+        return { refused: refused ?? NO_CANDIDATE };
+    }
+    /**
+     * Settles every expected field against the values the browser produced.
+     *
+     * Inferring a field by name is a guess, and it can be absent or simply wrong:
+     * hn.algolia's payload carries a `url`, so `url` maps onto it, but the page
+     * links to the discussion while the payload holds the article. The browser's
+     * own values are the evidence, so each field keeps its inferred spec only if
+     * that spec reproduces them, and is otherwise re-derived.
+     *
+     * Verification still has the last word over the result.
+     */
+    reconcileFields(recipe, payload, browserItems) {
+        if (!this.expectedFields || recipe.output.type !== 'json')
+            return { recipe, unreconciled: null };
+        if (browserItems.length === 0)
+            return { recipe, unreconciled: null };
+        const located = resolvePath(payload, recipe.output.items.path);
+        const rows = (Array.isArray(located) ? located : [located]);
+        if (rows.length !== browserItems.length)
+            return { recipe, unreconciled: null };
+        const fields = {};
+        let unreconciled = null;
+        for (const name of this.expectedFields) {
+            const settled = reconcileField(recipe.output.items.fields[name], browserItems.map((item) => item[name] ?? null), rows);
+            if (settled !== null)
+                fields[name] = settled;
+            // Verification still has the last word, but when it goes on to reject the
+            // recipe this names the field that could not be made to agree.
+            else
+                unreconciled ??= `field "${name}" could not be reconciled across ${rows.length} rows`;
+        }
+        return {
+            recipe: { ...recipe, output: { ...recipe.output, items: { ...recipe.output.items, fields } } },
+            unreconciled,
+        };
+    }
+    /** null means this response is not a json candidate, which diagnoses nothing. */
+    compileCandidate(trace, picked) {
+        const raw = readBody(picked.request);
+        if (raw === null)
+            return null;
+        let payload;
+        try {
+            payload = JSON.parse(raw);
+        }
+        catch {
+            return null;
+        }
+        // A detail response describes one thing; an array in it is a sidecar
+        // (a crate's versions, a package's imports) rather than the answer.
+        const itemsPath = trace.intent === 'detail'
+            ? findRecordPath(payload) ?? findItemsPath(payload)
+            : findItemsPath(payload) ?? findRecordPath(payload);
+        if (itemsPath === null)
+            return { refused: 'items path not found' };
+        const located = resolvePath(payload, itemsPath);
+        const rows = (Array.isArray(located) ? located : [located]);
+        const sample = rows[0];
+        if (!sample || typeof sample !== 'object')
+            return { refused: 'no object row at the items path' };
+        const fields = inferFields(sample, this.expectedFields);
+        if (Object.keys(fields).length === 0)
+            return { refused: 'no fields could be inferred from the payload' };
+        // With a plan, a gap here is not yet fatal: compile() tries to derive the
+        // missing field and verification judges the result. Without one there is no
+        // safety net, so the field count is all the contract we can enforce.
+        if (!this.plan && this.expectedFields && Object.keys(fields).length < this.expectedFields.length) {
+            return { refused: `inferred ${Object.keys(fields).length} of ${this.expectedFields.length} expected fields` };
+        }
+        const url = new URL(picked.request.url);
+        const path = templatePath(url.pathname, trace.input);
+        const query = templateQuery(url, trace.input);
+        const body = picked.request.postData
+            ? templateBody(picked.request.postData, trace.input) ?? undefined
+            : undefined;
+        // A recipe that does not carry every input as a placeholder is a snapshot,
+        // not a recipe: it returns what it was recorded with whatever it is asked.
+        const templated = templatedInputs([path, ...Object.values(query), body ?? '']);
+        const required = Object.entries(trace.input).filter(([, v]) => String(v) !== '').map(([k]) => k);
+        const untemplated = required.find((name) => !templated.has(name));
+        if (untemplated !== undefined)
+            return { refused: `required input "${untemplated}" not templated` };
+        const contentType = picked.request.requestHeaders['content-type'];
+        const outputSpec = { type: 'json', items: { path: itemsPath, fields } };
+        const signature = jsonSignature({ output: outputSpec }, payload);
+        return RecipeSchema.parse({
+            site: trace.site,
+            intent: trace.intent,
+            inputs: Object.fromEntries(Object.entries(trace.input).map(([k, v]) => [k, { type: typeof v === 'number' ? 'number' : 'string' }])),
+            strategy: { type: 'http-json' },
+            request: {
+                method: picked.request.method === 'POST' ? 'POST' : 'GET',
+                // A search vendor's API is a different host than the site itself.
+                ...(url.origin === trace.origin ? {} : { origin: url.origin }),
+                path,
+                query,
+                ...(body === undefined ? {} : { body }),
+                ...(contentType === undefined ? {} : { headers: { 'content-type': contentType } }),
+            },
+            output: outputSpec,
+            validation: {
+                status: 200,
+                required: Object.keys(fields).filter((f) => f === 'id' || f === 'title'),
+                minItems: 1,
+            },
+            fingerprint: {
+                endpoint: path,
+                hash: computeFingerprint(path, signature),
+                responseFields: signature.fields,
+            },
+            fallback: { type: 'browser' },
+        });
+    }
+}

package/dist/src/compiler/html.js

@@ -0,0 +1,117 @@
+import { classify } from '../analyzer/classify.js';
+import { scoreRequests } from '../analyzer/score.js';
+import { extractHtmlItems, htmlSignature, parseHtmlFragment } from '../executor/extract.js';
+import { NO_CANDIDATE, templatePath } from './heuristic.js';
+import { browserItemsOf, verifyAgainstBrowser } from './verify.js';
+import { computeFingerprint } from '../recipes/fingerprint.js';
+import { RecipeSchema } from '../recipes/schema.js';
+import { readBody } from '../recorder/body.js';
+import { equivalenceRefusal, isRefused } from './types.js';
+const PLACEHOLDER = /\{\{\s*([a-zA-Z0-9_]+)\s*\}\}/g;
+function isHtml(request) {
+    return request.contentType !== null && /text\/html/.test(request.contentType);
+}
+/**
+ * Requests that could carry the items, best first.
+ *
+ * The navigation is only one candidate. A site may answer a search with an XHR
+ * that returns HTML rather than JSON — remoteok.com replies to
+ * `?action=get_jobs` with a bare run of table rows — and such a response is
+ * reachable by neither compiler: the json one cannot parse it and this one used
+ * to look at the navigation alone.
+ */
+function htmlCandidates(trace) {
+    const navigation = trace.requests.filter((r) => classify(r) === 'navigation' && r.status === 200);
+    const xhr = scoreRequests(trace)
+        .map((s) => s.request)
+        .filter(isHtml);
+    return [...navigation, ...xhr];
+}
+/**
+ * Compiles a recipe that fetches HTML and reads the items out of it.
+ *
+ * Validity means the *raw* response already contains them. The rendered DOM is
+ * not evidence: on a SPA it looks identical while the response is empty.
+ */
+export function compileHtmlRecipe(trace, plan) {
+    // Candidates are the navigation first, then scored XHR, so the first
+    // nameable refusal is the one about the page the task actually asked for.
+    let refused = null;
+    for (const candidate of htmlCandidates(trace)) {
+        const outcome = compileFrom(trace, plan, candidate);
+        if (outcome === null)
+            continue;
+        if (isRefused(outcome)) {
+            refused ??= outcome.refused;
+            continue;
+        }
+        return outcome;
+    }
+    return { refused: refused ?? NO_CANDIDATE };
+}
+/** null means this response is not an html candidate, which diagnoses nothing. */
+function compileFrom(trace, plan, source) {
+    const body = readBody(source);
+    if (body === null)
+        return null;
+    if (parseHtmlFragment(body)(plan.itemSelector).length === 0) {
+        return { refused: "items are not in the raw html; only the rendered dom has them" };
+    }
+    // For a navigation, replay the URL the task asked for rather than the one the
+    // server bounced it to: itch.io rewrites /games/tag-puzzle to
+    // /games/genre-puzzle, and templating the target gives a recipe that is right
+    // for the recorded term and a 404 for every tag that is not also a genre.
+    // An XHR was issued by the page itself, so its own URL is the one to replay.
+    const isNavigation = classify(source) === 'navigation';
+    const requested = isNavigation
+        ? trace.actions.find((a) => a.type === 'navigate')?.value ?? source.url
+        : source.url;
+    const url = new URL(requested);
+    const path = templatePath(url.pathname, trace.input);
+    const query = {};
+    for (const [key, value] of url.searchParams.entries()) {
+        const match = Object.entries(trace.input).find(([, v]) => String(v) === value);
+        query[key] = match ? `{{${match[0]}}}` : templatePath(value, trace.input);
+    }
+    // An empty spec means "this element's own text" and is a real field, not a hole.
+    const fields = { ...plan.fields };
+    const draft = {
+        site: trace.site,
+        intent: trace.intent,
+        inputs: Object.fromEntries(Object.entries(trace.input).map(([k, v]) => [k, { type: typeof v === 'number' ? 'number' : 'string' }])),
+        strategy: { type: 'http-html' },
+        request: {
+            method: 'GET',
+            ...(url.origin === trace.origin ? {} : { origin: url.origin }),
+            path,
+            query,
+        },
+        output: { type: 'html', items: { selector: plan.itemSelector, fields } },
+        validation: { status: 200, required: Object.keys(fields).slice(0, 2), minItems: 1 },
+        fingerprint: { endpoint: path, hash: '', responseFields: [] },
+        fallback: { type: 'browser' },
+    };
+    // Same rule as the json compiler: an untemplated input makes this a snapshot.
+    const templated = new Set();
+    for (const part of [path, ...Object.values(query)]) {
+        for (const m of part.matchAll(PLACEHOLDER))
+            templated.add(m[1]);
+    }
+    const required = Object.entries(trace.input).filter(([, v]) => String(v) !== '').map(([k]) => k);
+    const untemplated = required.find((name) => !templated.has(name));
+    if (untemplated !== undefined)
+        return { refused: `required input "${untemplated}" not templated` };
+    const items = extractHtmlItems(draft, body);
+    if (!verifyAgainstBrowser(trace, plan, items).equivalent) {
+        return { refused: equivalenceRefusal(items.length, browserItemsOf(trace, plan).length) };
+    }
+    // Fingerprint what the executor will actually see: which declared fields the
+    // selectors resolve against this very response.
+    const signature = htmlSignature(draft, items);
+    draft.fingerprint = {
+        endpoint: path,
+        hash: computeFingerprint(path, signature),
+        responseFields: signature.fields,
+    };
+    return RecipeSchema.parse(draft);
+}

package/dist/src/compiler/types.js

@@ -0,0 +1,12 @@
+export function isRefused(result) {
+    return 'refused' in result;
+}
+/**
+ * Equal counts that still do not match are a different diagnosis from a short
+ * read, and saying "8 items, browser saw 8" for the first reads as agreement.
+ */
+export function equivalenceRefusal(recipeItems, browserItems) {
+    return recipeItems === browserItems
+        ? `equivalence: recipe and browser both yield ${recipeItems} items, but they differ`
+        : `equivalence: recipe yields ${recipeItems} items, browser saw ${browserItems}`;
+}