webrecipe 0.1.0

package/dist/fixtures/data.js

@@ -0,0 +1,23 @@
+const TOPICS = ['engineer', 'designer', 'analyst', 'manager'];
+const STACKS = ['rust', 'typescript', 'go', 'python', 'elixir'];
+export const DATASET = Array.from({ length: 40 }, (_, i) => {
+    const topic = TOPICS[i % TOPICS.length];
+    const stack = STACKS[i % STACKS.length];
+    return {
+        id: String(100 + i),
+        title: `Senior ${stack} ${topic} #${i}`,
+        author: `author-${i % 7}`,
+        tags: [stack, topic],
+        body: `Full description for record ${100 + i}. Stack is ${stack}.`,
+    };
+});
+export const PAGE_SIZE = 10;
+export function search(query, page = 1) {
+    const q = query.toLowerCase().trim();
+    const hits = q === '' ? DATASET : DATASET.filter((r) => r.title.toLowerCase().includes(q) || r.tags.some((t) => t.includes(q)));
+    const start = (page - 1) * PAGE_SIZE;
+    return hits.slice(start, start + PAGE_SIZE);
+}
+export function byId(id) {
+    return DATASET.find((r) => r.id === id);
+}

package/dist/fixtures/harness.js

@@ -0,0 +1,34 @@
+import { createServer } from 'node:http';
+export async function startFixture(name, handler, port = 0) {
+    const requestLog = [];
+    let version = 'v1';
+    let url = '';
+    const http = createServer((req, res) => {
+        requestLog.push(req.url ?? '');
+        handler(req, res, server);
+    });
+    const server = {
+        name,
+        get url() { return url; },
+        get version() { return version; },
+        set version(v) { version = v; },
+        setVersion(v) { version = v; },
+        requestLog,
+        close: () => new Promise((resolve) => { http.close(() => resolve()); }),
+    };
+    await new Promise((resolve, reject) => { http.once('error', reject); http.listen(port, '127.0.0.1', resolve); });
+    url = `http://127.0.0.1:${http.address().port}`;
+    return server;
+}
+export function sendHtml(res, html, status = 200) {
+    res.writeHead(status, { 'content-type': 'text/html; charset=utf-8' });
+    res.end(html);
+}
+export function sendJson(res, body, status = 200) {
+    res.writeHead(status, { 'content-type': 'application/json; charset=utf-8' });
+    res.end(JSON.stringify(body));
+}
+export function sendJs(res, js) {
+    res.writeHead(200, { 'content-type': 'application/javascript; charset=utf-8' });
+    res.end(js);
+}

package/dist/fixtures/ignoring.js

@@ -0,0 +1,27 @@
+import { startFixture, sendHtml } from './harness.js';
+import { page } from './ssr.js';
+import { DATASET, PAGE_SIZE } from './data.js';
+/**
+ * Accepts a search parameter and discards it, the way arbeitnow.com serves its
+ * unfiltered front page for every query.
+ *
+ * Nothing about the response says so: the status is 200, the markup is the
+ * markup the plan was taught, and every field the recipe extracts is populated.
+ * Only asking it a second, different question reveals the answer never moved.
+ */
+export function startIgnoringFixture() {
+    return startFixture('siteIgnoring', (req, res) => {
+        const url = new URL(req.url ?? '/', 'http://localhost');
+        if (url.pathname !== '/search') {
+            sendHtml(res, page('Home', '<a href="/search?q=rust">search</a>'));
+            return;
+        }
+        const rows = DATASET.slice(0, PAGE_SIZE)
+            .map((r) => `<li class="result" data-id="${r.id}">
+        <a class="title" href="/item/${r.id}">${r.title}</a>
+        <span class="author">${r.author}</span>
+      </li>`)
+            .join('');
+        sendHtml(res, page('Search', `<ul id="results">${rows}</ul>`));
+    });
+}

package/dist/fixtures/limiting.js

@@ -0,0 +1,38 @@
+import { startFixture, sendHtml } from './harness.js';
+import { page } from './ssr.js';
+import { search } from './data.js';
+/**
+ * Answers one query and turns away every other, the way a site that serves a
+ * warmed query and rate-limits the rest does.
+ *
+ * Keyed on the query value rather than on how many requests have arrived: the
+ * probes must be turned away because of what they ask, not because of when
+ * they ask it. Nothing here depends on the caller's probe order, retries or
+ * concurrency.
+ *
+ * A site saying "not so fast" is not a site saying the query means nothing, and
+ * the verdict has to keep them apart.
+ */
+export function startLimitingFixture(status = 429, answers = 'rust') {
+    return startFixture('siteLimiting', (req, res) => {
+        const url = new URL(req.url ?? '/', 'http://localhost');
+        if (url.pathname !== '/search') {
+            sendHtml(res, page('Home', '<a href="/search?q=rust">search</a>'));
+            return;
+        }
+        if ((url.searchParams.get('q') ?? '') !== answers) {
+            // Retry-After only on the rate limit, which is where a site states one.
+            res.writeHead(status, status === 429
+                ? { 'content-type': 'text/plain', 'retry-after': '3600' }
+                : { 'content-type': 'text/plain' });
+            res.end(status === 429 ? 'slow down' : 'temporarily unavailable');
+            return;
+        }
+        const rows = search(answers, 1)
+            .map((r) => `<li class="result" data-id="${r.id}">
+        <a class="title" href="/item/${r.id}">${r.title}</a>
+      </li>`)
+            .join('');
+        sendHtml(res, page('Search', `<ul id="results">${rows}</ul>`));
+    });
+}

package/dist/fixtures/paging.js

@@ -0,0 +1,72 @@
+import { startFixture, sendHtml } from './harness.js';
+import { isBrowser } from './refusing.js';
+import { page as html } from './ssr.js';
+import { search } from './data.js';
+const rows = (records) => records.map((r) => `<li class="result" data-id="${r.id}">
+        <a class="title" href="/item/${r.id}">${r.title}</a>
+      </li>`).join('');
+const pageOf = (url) => Number(url.searchParams.get('page') ?? '1');
+const queryOf = (url) => url.searchParams.get('q') ?? '';
+/**
+ * Serves page one whatever page is asked for.
+ *
+ * With `browserPaginates`, only the engine's HTTP client is pinned and a
+ * browser still moves between pages — a recipe whose page control was never
+ * wired up, which is the one thing a verifier can prove from outside. Without
+ * it, nobody moves, and from outside that is indistinguishable from a site with
+ * a single page: the verdict there has to be that we do not know.
+ */
+export function startPageIgnoringFixture(opts) {
+    return startFixture('sitePageIgnoring', (req, res) => {
+        const url = new URL(req.url ?? '/', 'http://localhost');
+        if (url.pathname !== '/search') {
+            sendHtml(res, html('Home', '<a href="/search?q=rust">search</a>'));
+            return;
+        }
+        const asked = opts.browserPaginates && isBrowser(req) ? pageOf(url) : 1;
+        sendHtml(res, html('Search', `<ul id="results">${rows(search(queryOf(url), asked))}</ul>`));
+    });
+}
+/**
+ * Paginates correctly and repeats one pinned row on every page.
+ *
+ * Overlap between pages is ordinary — a sponsored insert, a live posting, a
+ * ranking that shifted — so a verifier that required the pages to be disjoint
+ * would call this broken. It is here to keep that rule out.
+ */
+export function startPinnedFixture() {
+    return startFixture('sitePinned', (req, res) => {
+        const url = new URL(req.url ?? '/', 'http://localhost');
+        if (url.pathname !== '/search') {
+            sendHtml(res, html('Home', '<a href="/search?q=rust">search</a>'));
+            return;
+        }
+        const query = queryOf(url);
+        const pinned = search(query, 1)[0];
+        const here = search(query, pageOf(url));
+        const shown = pinned === undefined || here.some((r) => r.id === pinned.id) ? here : [pinned, ...here];
+        sendHtml(res, html('Search', `<ul id="results">${rows(shown)}</ul>`));
+    });
+}
+/**
+ * Answers one page and turns away the rest.
+ *
+ * Keyed on the page asked for, not on how many requests have arrived, so the
+ * recording gets its page and the probe for any other is declined however the
+ * probes are ordered.
+ */
+export function startPageLimitedFixture(status = 503, answers = 1) {
+    return startFixture('sitePageLimited', (req, res) => {
+        const url = new URL(req.url ?? '/', 'http://localhost');
+        if (url.pathname !== '/search') {
+            sendHtml(res, html('Home', '<a href="/search?q=rust">search</a>'));
+            return;
+        }
+        if (pageOf(url) !== answers) {
+            res.writeHead(status, { 'content-type': 'text/plain' });
+            res.end('not now');
+            return;
+        }
+        sendHtml(res, html('Search', `<ul id="results">${rows(search(queryOf(url), answers))}</ul>`));
+    });
+}

package/dist/fixtures/refusing.js

@@ -0,0 +1,57 @@
+import { startFixture, sendHtml } from './harness.js';
+import { page, ssrHandler } from './ssr.js';
+/**
+ * Tells the engine's HTTP client from the engine's browser.
+ *
+ * Not the user agent: `src/browser/session.ts` gives the headless browser the
+ * same `webrecipe/` string the fetch client sends, so refusing on that
+ * would refuse both halves and prove nothing. `sec-ch-ua` is a client hint only
+ * a browser emits, which is also what a real challenge wall keys on.
+ */
+/** Client hints are sent by a browser and by nothing else; `fetch` omits them. */
+export const isBrowser = (req) => req.headers['sec-ch-ua'] !== undefined;
+/**
+ * bandcamp's shape: a 200 whose body is an interstitial rather than the page,
+ * padded to the ~3 KB a real one weighs so the size is not what gives it away.
+ */
+const CHALLENGE = page('Client Challenge', `<div id="challenge">${'<p>Checking your browser before you continue.</p>'.repeat(64)}</div>`);
+/**
+ * The two ways a site refuses a compiled recipe while still serving a browser.
+ *
+ * At `v1` it is loc.gov: a non-browser client gets a 403 and a browser gets
+ * siteA's page. At `v2` it is bandcamp: everyone gets a 200 carrying the
+ * challenge page instead of results.
+ */
+export function startRefusingFixture() {
+    return startFixture('siteRefusing', (req, res, server) => {
+        if (server.version === 'v2') {
+            sendHtml(res, CHALLENGE);
+            return;
+        }
+        if (!isBrowser(req)) {
+            sendHtml(res, '<html><body>Forbidden</body></html>', 403);
+            return;
+        }
+        ssrHandler(req, res, server);
+    });
+}
+/**
+ * musicbrainz's shape, and the one rule (b) cannot see: a 200 carrying a
+ * challenge page, served only to the client without `sec-ch-ua`. Short on
+ * purpose — the real one weighs 1.4 KB — because nothing here may key on the
+ * body, only on the browser agreeing with the recipe it already had.
+ */
+const VERIFYING = '<html><head><title>Verifying your browser</title></head><body>Verifying your browser</body></html>';
+/**
+ * Serves the engine's client a 200 challenge page and the browser siteA's real
+ * page, so every re-record compiles the very same recipe again.
+ */
+export function startStubFixture() {
+    return startFixture('siteStub', (req, res, server) => {
+        if (!isBrowser(req)) {
+            sendHtml(res, VERIFYING);
+            return;
+        }
+        ssrHandler(req, res, server);
+    });
+}

package/dist/fixtures/shifted.js

@@ -0,0 +1,32 @@
+import { startFixture, sendHtml } from './harness.js';
+import { page } from './ssr.js';
+import { DATASET, search } from './data.js';
+/**
+ * Filters on the query honestly and then attributes each hit to the record
+ * after it — the shape of a wrong join or an off-by-one in a template.
+ *
+ * This is the case the query-honored probes cannot catch, and it is here to
+ * keep that boundary visible. The parameter is genuinely honoured: the result
+ * set moves with the query, a query nothing matches comes back empty, and the
+ * browser sees exactly what the recipe sees. Every signal the verifier reads is
+ * true and the answer is still about the wrong records, which only something
+ * holding an independent account of the right ones can tell.
+ */
+export function startShiftedFixture() {
+    return startFixture('siteShifted', (req, res) => {
+        const url = new URL(req.url ?? '/', 'http://localhost');
+        if (url.pathname !== '/search') {
+            sendHtml(res, page('Home', '<a href="/search?q=rust">search</a>'));
+            return;
+        }
+        const hits = search(url.searchParams.get('q') ?? '', Number(url.searchParams.get('page') ?? '1'));
+        const rows = hits
+            .map((hit) => DATASET[(DATASET.indexOf(hit) + 1) % DATASET.length])
+            .map((r) => `<li class="result" data-id="${r.id}">
+        <a class="title" href="/item/${r.id}">${r.title}</a>
+        <span class="author">${r.author}</span>
+      </li>`)
+            .join('');
+        sendHtml(res, page('Search', `<ul id="results">${rows}</ul>`));
+    });
+}

package/dist/fixtures/spa.js

@@ -0,0 +1,71 @@
+import { startFixture, sendHtml, sendJson, sendJs } from './harness.js';
+import { search, byId } from './data.js';
+const SHELL = `<!doctype html><html><head><title>Site C</title>
+<script src="/bundle.js" defer></script></head><body><div id="root"></div></body></html>`;
+const BUNDLE = `
+async function render() {
+  const root = document.getElementById('root')
+  root.innerHTML = ''
+  const path = location.pathname
+  if (path.startsWith('/item/')) {
+    const res = await fetch('/api/detail?id=' + path.slice(6))
+    const data = await res.json()
+    const el = document.createElement('article')
+    el.id = 'detail'
+    el.dataset.id = data.result.id
+    el.textContent = data.result.title
+    root.appendChild(el)
+    return
+  }
+  const params = new URLSearchParams(location.search)
+  const q = params.get('q') || ''
+  const page = params.get('page') || '1'
+  const res = await fetch('/api/query?q=' + encodeURIComponent(q) + '&page=' + encodeURIComponent(page))
+  const data = await res.json()
+  const rows = data.results || data.items || []
+  const ul = document.createElement('ul')
+  ul.id = 'results'
+  for (const r of rows) {
+    const li = document.createElement('li')
+    li.className = 'result'
+    li.dataset.id = r.id
+    li.textContent = r.title || r.name
+    ul.appendChild(li)
+  }
+  root.appendChild(ul)
+}
+window.addEventListener('popstate', render)
+render()
+`;
+export function startSpaFixture() {
+    return startFixture('siteC-spa', (req, res, server) => {
+        const url = new URL(req.url ?? '/', 'http://localhost');
+        if (url.pathname === '/bundle.js') {
+            sendJs(res, BUNDLE);
+            return;
+        }
+        if (url.pathname === '/api/query') {
+            const q = url.searchParams.get('q') ?? '';
+            const page = Number(url.searchParams.get('page') ?? '1');
+            const rows = search(q, page);
+            if (server.version === 'v2') {
+                sendJson(res, { items: rows.map((r) => ({ id: r.id, name: r.title, by: r.author })) });
+            }
+            else {
+                sendJson(res, { results: rows.map((r) => ({ id: r.id, title: r.title, author: r.author })) });
+            }
+            return;
+        }
+        if (url.pathname === '/api/detail') {
+            const record = byId(url.searchParams.get('id') ?? '');
+            if (!record) {
+                sendJson(res, { error: 'not found' }, 404);
+                return;
+            }
+            sendJson(res, { result: record });
+            return;
+        }
+        // Client-side routing: every other route gets the same shell.
+        sendHtml(res, SHELL);
+    });
+}

package/dist/fixtures/ssr.js

@@ -0,0 +1,46 @@
+import { startFixture, sendHtml } from './harness.js';
+import { search, byId } from './data.js';
+export function page(title, inner) {
+    return `<!doctype html><html><head><title>${title}</title></head><body>${inner}</body></html>`;
+}
+function renderResults(records, version) {
+    const cls = version === 'v1' ? 'result' : 'hit';
+    const titleCls = version === 'v1' ? 'title' : 'name';
+    const rows = records
+        .map((r) => `<li class="${cls}" data-id="${r.id}">
+        <a class="${titleCls}" href="/item/${r.id}">${r.title}</a>
+        <span class="author">${r.author}</span>
+      </li>`)
+        .join('');
+    return `<ul id="results">${rows}</ul>`;
+}
+/**
+ * Exported so a fixture that only changes who gets served — `refusing.ts` —
+ * wraps this page rather than growing a second copy of it that can drift.
+ */
+export const ssrHandler = (req, res, server) => {
+    const url = new URL(req.url ?? '/', 'http://localhost');
+    if (url.pathname === '/search') {
+        const q = url.searchParams.get('q') ?? '';
+        const p = Number(url.searchParams.get('page') ?? '1');
+        sendHtml(res, page('Search', renderResults(search(q, p), server.version)));
+        return;
+    }
+    if (url.pathname.startsWith('/item/')) {
+        const record = byId(url.pathname.slice('/item/'.length));
+        if (!record) {
+            sendHtml(res, page('Not found', '<p>not found</p>'), 404);
+            return;
+        }
+        sendHtml(res, page(record.title, `<article id="detail" data-id="${record.id}">
+      <h1 class="title">${record.title}</h1>
+      <span class="author">${record.author}</span>
+      <p class="body">${record.body}</p>
+    </article>`));
+        return;
+    }
+    sendHtml(res, page('Home', '<a href="/search?q=rust">search</a>'));
+};
+export function startSsrFixture() {
+    return startFixture('siteA-ssr', ssrHandler);
+}

package/dist/fixtures/volatile.js

@@ -0,0 +1,40 @@
+import { startFixture, sendHtml } from './harness.js';
+import { page } from './ssr.js';
+import { search } from './data.js';
+/**
+ * Filters honestly and alternates one row between consecutive calls for the
+ * same query, the way a listing that mixes in a fresh or sponsored item does.
+ *
+ * The alternation is counted per query, so asking twice for the same thing
+ * always disagrees no matter what else has been asked, in what order, or how
+ * many times. A fixture that instead counted requests would be describing the
+ * caller's probe order rather than the site's behaviour, and would start
+ * failing the moment that order changed.
+ *
+ * A site like this cannot be judged by comparing result sets: two identical
+ * requests already disagree, so a difference between two different queries
+ * proves nothing either.
+ */
+export function startVolatileFixture() {
+    const turn = new Map();
+    return startFixture('siteVolatile', (req, res) => {
+        const url = new URL(req.url ?? '/', 'http://localhost');
+        if (url.pathname !== '/search') {
+            sendHtml(res, page('Home', '<a href="/search?q=rust">search</a>'));
+            return;
+        }
+        const q = url.searchParams.get('q') ?? '';
+        const nth = (turn.get(q) ?? 0) + 1;
+        turn.set(q, nth);
+        const rows = search(q, Number(url.searchParams.get('page') ?? '1'))
+            .map((r, i) => i === 0
+            ? `<li class="result" data-id="live-${nth}">
+        <a class="title" href="/item/live-${nth}">Just posted ${nth}</a>
+      </li>`
+            : `<li class="result" data-id="${r.id}">
+        <a class="title" href="/item/${r.id}">${r.title}</a>
+      </li>`)
+            .join('');
+        sendHtml(res, page('Search', `<ul id="results">${rows}</ul>`));
+    });
+}

package/dist/fixtures/xhr.js

@@ -0,0 +1,120 @@
+import { startFixture, sendHtml, sendJson, sendJs } from './harness.js';
+import { search, byId, DATASET } from './data.js';
+/**
+ * The browser is made to issue these in order on a search:
+ *   /analytics/collect, /api/feature-flags, /api/autocomplete,
+ *   /api/search  <- the only one that matters,
+ *   /api/recommendations, /tracking/event
+ */
+const BOOT_SCRIPT = `
+const params = new URLSearchParams(location.search)
+const q = params.get('q') || ''
+const page = params.get('page') || '1'
+const dataPath = window.__DATA_PATH__
+async function boot() {
+  await fetch('/analytics/collect')
+  await fetch('/api/feature-flags')
+  await fetch('/api/autocomplete?prefix=' + encodeURIComponent(q.slice(0, 2)))
+  const res = await fetch(dataPath + '?q=' + encodeURIComponent(q) + '&page=' + encodeURIComponent(page))
+  const data = await res.json()
+  const ul = document.createElement('ul')
+  ul.id = 'results'
+  for (const r of data.results) {
+    const li = document.createElement('li')
+    li.className = 'result'
+    li.dataset.id = r.id
+    // The title is the link, as in any real listing: the href is composed
+    // client-side from the id, so the API never carries it.
+    const a = document.createElement('a')
+    a.className = 'permalink'
+    a.href = '/item/' + r.id
+    a.textContent = r.title
+    li.appendChild(a)
+    ul.appendChild(li)
+  }
+  document.getElementById('app').appendChild(ul)
+  await fetch('/api/recommendations', { method: 'POST', body: '{}' })
+  await fetch('/tracking/event', { method: 'POST', body: '{}' })
+}
+boot()
+`;
+const DETAIL_SCRIPT = `
+const id = location.pathname.slice('/item/'.length)
+fetch('/api/item?id=' + encodeURIComponent(id))
+  .then((r) => r.json())
+  .then((data) => {
+    if (!data.result) return
+    const el = document.createElement('article')
+    el.id = 'detail'
+    el.dataset.id = data.result.id
+    el.innerHTML = '<h1 class="title"></h1><span class="author"></span>'
+    el.querySelector('.title').textContent = data.result.title
+    el.querySelector('.author').textContent = data.result.author
+    document.getElementById('app').appendChild(el)
+  })
+`;
+export function startXhrFixture() {
+    return startFixture('siteB-xhr', (req, res, server) => {
+        const url = new URL(req.url ?? '/', 'http://localhost');
+        const v2 = server.version === 'v2';
+        const dataPath = v2 ? '/api/v2/search' : '/api/search';
+        switch (true) {
+            case url.pathname === '/search':
+                sendHtml(res, `<!doctype html><html><head><title>Search</title>
+          <script>window.__DATA_PATH__=${JSON.stringify(dataPath)}</script>
+          <script src="/app.js" defer></script></head>
+          <body><header>Site B</header><div id="app"></div></body></html>`);
+                return;
+            case url.pathname === '/app.js':
+                sendJs(res, BOOT_SCRIPT);
+                return;
+            case url.pathname === '/detail.js':
+                sendJs(res, DETAIL_SCRIPT);
+                return;
+            // Detail is a shell too: an XHR site renders it from /api/item, not from HTML.
+            case url.pathname.startsWith('/item/'):
+                sendHtml(res, `<!doctype html><html><head><title>Item</title>
+          <script src="/detail.js" defer></script></head>
+          <body><header>Site B</header><div id="app"></div></body></html>`);
+                return;
+            case url.pathname === dataPath: {
+                const q = url.searchParams.get('q') ?? '';
+                const page = Number(url.searchParams.get('page') ?? '1');
+                sendJson(res, {
+                    query: q,
+                    page,
+                    total: DATASET.length,
+                    // No url: a real API often returns the id and lets the page build the link.
+                    results: search(q, page).map((r) => ({ id: r.id, title: r.title, author: r.author })),
+                });
+                return;
+            }
+            case url.pathname === '/api/item': {
+                const record = byId(url.searchParams.get('id') ?? '');
+                if (!record) {
+                    sendJson(res, { error: 'not found' }, 404);
+                    return;
+                }
+                sendJson(res, { result: record });
+                return;
+            }
+            case url.pathname === '/api/feature-flags':
+                sendJson(res, { flags: { newSearch: true, darkMode: false } });
+                return;
+            case url.pathname === '/api/autocomplete': {
+                const prefix = (url.searchParams.get('prefix') ?? '').toLowerCase();
+                sendJson(res, { suggestions: ['rust', 'ruby', 'react'].filter((s) => s.startsWith(prefix)).slice(0, 3) });
+                return;
+            }
+            case url.pathname === '/api/recommendations':
+                sendJson(res, { recommendations: DATASET.slice(0, 3).map((r) => r.id) });
+                return;
+            case url.pathname === '/analytics/collect':
+            case url.pathname === '/tracking/event':
+                res.writeHead(204).end();
+                return;
+            default:
+                sendJson(res, { error: 'not found' }, 404);
+        }
+    });
+}

package/dist/src/analyzer/classify.js

@@ -0,0 +1,16 @@
+const ASSET_TYPES = new Set(['image', 'font', 'stylesheet', 'script', 'media', 'texttrack', 'manifest']);
+const ANALYTICS = /analytics|googletagmanager|google-analytics|doubleclick|segment\.io|mixpanel|amplitude|hotjar|adservice|\/ads?\//i;
+const TRACKING = /tracking|telemetry|beacon|\/collect\b|pixel|\/event\b/i;
+export function classify(req) {
+    if (ANALYTICS.test(req.url))
+        return 'analytics';
+    if (TRACKING.test(req.url))
+        return 'tracking';
+    if (ASSET_TYPES.has(req.resourceType))
+        return 'asset';
+    if (req.resourceType === 'document')
+        return 'navigation';
+    if (req.resourceType === 'xhr' || req.resourceType === 'fetch')
+        return 'data_candidate';
+    return 'unknown';
+}

package/dist/src/analyzer/score.js

@@ -0,0 +1,52 @@
+import { classify } from './classify.js';
+const LARGE_PAYLOAD_BYTES = 1024;
+function isJson(req) {
+    return req.contentType !== null && /json/.test(req.contentType);
+}
+function containsInput(req, input) {
+    const values = Object.values(input).map((v) => String(v)).filter((v) => v.length >= 2);
+    if (values.length === 0)
+        return false;
+    const haystack = decodeURIComponent(req.url) + (req.postData ?? '');
+    return values.some((v) => haystack.toLowerCase().includes(v.toLowerCase()));
+}
+/**
+ * The scoring table is the whole model. It is deliberately small and legible so
+ * that a wrong pick can be explained by reading the signals, not by guessing.
+ */
+export function scoreRequests(trace) {
+    const scored = [];
+    for (const request of trace.requests) {
+        if (classify(request) !== 'data_candidate')
+            continue;
+        if (request.status !== 200)
+            continue;
+        const signals = [];
+        let score = 0;
+        if (isJson(request)) {
+            score += 3;
+            signals.push('json-response');
+        }
+        if (containsInput(request, trace.input)) {
+            score += 3;
+            signals.push('contains-input');
+        }
+        if (request.afterAction !== null) {
+            score += 2;
+            signals.push('after-action');
+        }
+        if (request.domChanged) {
+            score += 2;
+            signals.push('dom-changed');
+        }
+        if (request.bodySize > LARGE_PAYLOAD_BYTES) {
+            score += 1;
+            signals.push('large-payload');
+        }
+        scored.push({ request, score, signals });
+    }
+    return scored.sort((a, b) => b.score - a.score);
+}
+export function pickDataRequest(trace) {
+    return scoreRequests(trace)[0] ?? null;
+}