website-to-markdown 1.1.1 → 2.0.0

package/dist/fetch-page.js

@@ -1,4 +1,4 @@
-import { parseRetryAfter } from 'hanzio';
+import { exponentialBackoff, fileExtension, getUniqueValues, invariant, isOneOf, isRecord, must, parseRetryAfter, startTimer, timeoutSignal, typedSwitch, typedSwitchGroup } from 'hanzio';
 import { CloudflareRateLimitedError, createCloudflareClient } from './client.js';
 import { classifyContentType, convertibleDocumentType } from './content-type.js';
 import { currentCrawlSnapshot, throughCrawlSnapshot, withCrawlSnapshot } from './crawl-snapshot.js';
@@ -7,40 +7,50 @@ import { sha256Hex } from './hash.js';
 import { extractHtmlMetadata } from './html-metadata.js';
 import { createHostLimits } from './limit.js';
 import { errorLogAttributes, silentLogger } from './logger.js';
-import { htmlImages, htmlLinks, markdownRefs, stripDocumentPreamble, stripFrontMatter, titleFromMarkdown } from './markdown-refs.js';
+import { cleanConvertedMarkdown, htmlImages, htmlLinks, markdownRefs, stripDocumentPreamble, stripFrontMatter, titleFromMarkdown } from './markdown-refs.js';
 import { createMemoryConcurrencyLimit } from './memory-limit.js';
-import { isHostAllowed, isLocalhostHostname } from './net-guard.js';
+import { followRedirects, guardedFetch, HostNotAllowedError, isHostAllowed, isLocalhostHostname } from './net-guard.js';
 import { currentOperationSignal } from './operation-signal.js';
+import { decodeText, readBody } from './read-body.js';
+import { asUnchanged, isUnchangedFor } from './unchanged-page.js';
 import { BROWSER_USER_AGENT } from './user-agent.js';
 import { preserveXlsxZeros } from './xlsx-zeros.js';
 const FETCH_TIMEOUT_MS = 30_000;
 /** Browser Rendering once returned a 6 MB markdown page from a 6 MB upload. */
 const MAX_HTML_BYTES = 2 * 1024 * 1024;
+/**
+ * Read cap for a textual body. Above `MAX_HTML_BYTES` a page is not converted
+ * but still yields its metadata and links; above this it is not downloaded.
+ */
+const MAX_TEXT_READ_BYTES = 2 * MAX_HTML_BYTES;
 const MAX_DOCUMENT_BYTES = 2 * 1024 * 1024;
 const MAX_PDF_BYTES = 12 * 1024 * 1024;
 const MAX_PDF_PAGES = 60;
 /** Stored markdown ceiling; a page over this is a listing or an upload, not knowledge. */
 export const MAX_MARKDOWN_CHARS = 300_000;
+/**
+ * A NUL byte is never text, and Postgres refuses a string that holds one: a
+ * single page with one (a Wix site's conversion) failed a whole import.
+ * Composed Unicode, because string comparisons downstream assume it: a site
+ * that wrote "Gård" as "a" plus a combining ring read as "Ga" to anything
+ * matching names in the text.
+ */
+function cleanText(text) {
+    return text.replaceAll('\u0000', '').normalize('NFC');
+}
 /**
  * The one place markdown, front-matter and HTML metadata are reconciled: the
  * HTML `<title>` wins, the Workers AI front-matter is the fallback for pages we
  * only ever saw as markdown, and links/images are resolved against `finalUrl`.
  */
 function buildPage(base, rawBody, logger) {
-    // A NUL byte is never text, and Postgres refuses a string that holds one:
-    // a single page with one (a Wix site's conversion) failed the whole import.
-    // Composed Unicode, because every comparison downstream assumes it: a site
-    // that wrote "Gård" as "a" plus a combining ring gave the fact guard the
-    // name "Broges Ga", which no draft could contain, and the document shipped
-    // as its verbatim source.
-    const clean = (text) => text?.replaceAll('\u0000', '').normalize('NFC');
     const body = {
         ...rawBody,
-        markdown: clean(rawBody.markdown),
-        html: clean(rawBody.html)
+        markdown: rawBody.markdown ? cleanText(rawBody.markdown) : null,
+        html: rawBody.html ? cleanText(rawBody.html) : null
     };
     const frontMatter = stripFrontMatter(body.markdown ?? '');
-    let markdown = frontMatter.markdown;
+    let markdown = cleanConvertedMarkdown(frontMatter.markdown);
     if (markdown.length > MAX_MARKDOWN_CHARS) {
         logger.warn('url-to-markdown.truncated', {
             url: base.finalUrl,
@@ -56,7 +66,7 @@ function buildPage(base, rawBody, logger) {
     // navigation (image-wrapped anchors, folded menus), so when we hold the
     // HTML its anchors decide what discovery can still reach.
     const links = body.html
-        ? [...new Set([...refs.links, ...htmlLinks(body.html, base.finalUrl)])]
+        ? getUniqueValues([...refs.links, ...htmlLinks(body.html, base.finalUrl)])
         : refs.links;
     return {
         url: base.url,
@@ -76,25 +86,25 @@ function buildPage(base, rawBody, logger) {
         metaDescription: metadata?.metaDescription ?? frontMatter.description ?? null,
         jsonLd: metadata?.jsonLd ?? [],
         links,
-        images: [
-            ...new Set([
-                ...refs.images,
-                ...(body.html ? htmlImages(body.html, base.finalUrl) : [])
-            ])
-        ],
+        images: getUniqueValues([
+            ...refs.images,
+            ...(body.html ? htmlImages(body.html, base.finalUrl) : [])
+        ]),
         captureIssue: frontMatter.markdown.length > MAX_MARKDOWN_CHARS
             ? 'markdown_limit'
             : (body.captureIssue ?? null),
-        durationMs: Math.round(performance.now() - base.startedAt),
+        durationMs: Math.round(base.elapsedMs()),
+        // Over the cleaned HTML, as `convertTextual` compares it.
         htmlHash: body.html ? htmlHash(body.html) : null
     };
 }
 /**
  * `http://` is upgraded rather than rejected: two carpenter sites in the probe
  * link internally with `http://` and the https-only guard threw away most of
- * their pages. Localhost http stays reachable outside production.
+ * their pages. Localhost http stays http, and is reachable only with
+ * `allowLocalhost`.
  */
-function prepareRequestUrl(raw) {
+function prepareRequestUrl(raw, allowLocalhost) {
     let parsed;
     try {
         parsed = new URL(raw);
@@ -107,7 +117,7 @@ function prepareRequestUrl(raw) {
     if (parsed.protocol !== 'http:')
         return null;
     if (isLocalhostHostname(parsed.hostname)) {
-        return process.env.NODE_ENV === 'production' ? null : parsed;
+        return allowLocalhost ? parsed : null;
     }
     parsed.protocol = 'https:';
     if (parsed.port === '80')
@@ -130,7 +140,7 @@ function wwwFallbackUrl(url) {
 }
 /**
  * A 429 is an origin asking us to slow down, not a bot wall. Shopify answered
- * 117 of Good Habit's 244 reads with it; sent on to Browser Rendering they
+ * 117 of one store's 244 reads with it; sent on to Browser Rendering they
  * tripped Cloudflare's own limit (79 times, 25-30 s each) and 7 came back.
  * So the host is narrowed and paused (`coolDownHost`), the URL is asked again,
  * and a page still refused after that is `transient`: its stored copy stands
@@ -141,7 +151,12 @@ const MAX_RATE_LIMIT_WAIT_MS = 30_000;
 /** The origin's `Retry-After` when it sent one, else 1 s, 2 s, 4 s. */
 function rateLimitWaitMs(response, retry) {
     const askedMs = parseRetryAfter(response.headers.get('retry-after'));
-    return Math.min(MAX_RATE_LIMIT_WAIT_MS, askedMs ?? 1_000 * 2 ** (retry - 1));
+    return Math.min(MAX_RATE_LIMIT_WAIT_MS, askedMs ??
+        exponentialBackoff(retry, {
+            baseMs: 1_000,
+            maxMs: MAX_RATE_LIMIT_WAIT_MS,
+            jitter: false
+        }));
 }
 /**
  * Bump whenever conversion changes what the same HTML becomes (`withoutHiddenUi`,
@@ -149,7 +164,7 @@ function rateLimitWaitMs(response, retry) {
  * matches, and every page is converted again once instead of keeping the
  * markdown the old conversion made.
  */
-const CONVERSION_VERSION = 1;
+const CONVERSION_VERSION = 2;
 function htmlHash(html) {
     return sha256Hex(`${CONVERSION_VERSION}\n${html}`);
 }
@@ -173,11 +188,10 @@ function parseFinalUrl(responseUrl, requestUrl) {
     }
 }
 function isAbortError(error) {
-    const name = typeof error === 'object' && error && 'name' in error
-        ? String(error.name)
-        : '';
-    return name === 'AbortError' || name === 'TimeoutError';
+    return isRecord(error) && isOneOf(['AbortError', 'TimeoutError'], error.name);
 }
+/** The content kinds whose body is read as text; the rest are documents or assets. */
+const TEXTUAL_KINDS = ['markdown', 'html', 'text'];
 /** Reading the body is optional; an unsupported type is dropped at the headers. */
 function discardBody(response) {
     void response.body?.cancel().catch(() => { });
@@ -195,75 +209,67 @@ async function readOriginBody(response, finalUrl, logger) {
     }
     const contentType = response.headers.get('content-type');
     const kind = classifyContentType(contentType);
-    if (kind === 'asset') {
-        discardBody(response);
-        return { kind: 'none' };
-    }
-    if (kind === 'document') {
-        const type = convertibleDocumentType(contentType);
-        if (!type) {
+    return typedSwitch(kind, {
+        asset: () => {
             discardBody(response);
             return { kind: 'none' };
-        }
-        const limit = type.extension === 'pdf' ? MAX_PDF_BYTES : MAX_DOCUMENT_BYTES;
-        const declared = Number(response.headers.get('content-length'));
-        if (Number.isFinite(declared) && declared > limit) {
-            discardBody(response);
-            logger.info('url-to-markdown.document_oversized', {
-                url: finalUrl.href,
-                bytes: declared
-            });
-            return { kind: 'none', captureIssue: 'document_byte_limit' };
-        }
-        const bytes = await readBoundedBody(response, limit);
-        return bytes
-            ? { kind: 'bytes', bytes }
-            : { kind: 'none', captureIssue: 'document_byte_limit' };
-    }
-    return { kind: 'text', text: await response.text() };
+        },
+        document: async () => {
+            const type = convertibleDocumentType(contentType);
+            if (!type) {
+                discardBody(response);
+                return { kind: 'none' };
+            }
+            const limit = type.extension === 'pdf' ? MAX_PDF_BYTES : MAX_DOCUMENT_BYTES;
+            const declared = Number(response.headers.get('content-length'));
+            if (Number.isFinite(declared) && declared > limit) {
+                discardBody(response);
+                logger.info('url-to-markdown.document_oversized', {
+                    url: finalUrl.href,
+                    bytes: declared
+                });
+                return { kind: 'none', captureIssue: 'document_byte_limit' };
+            }
+            const bytes = await readBoundedBody(response, limit);
+            return bytes
+                ? { kind: 'bytes', bytes }
+                : { kind: 'none', captureIssue: 'document_byte_limit' };
+        },
+        ...typedSwitchGroup(kind, TEXTUAL_KINDS, async () => {
+            const declared = Number(response.headers.get('content-length'));
+            const read = Number.isFinite(declared) && declared > MAX_TEXT_READ_BYTES
+                ? null
+                : await readBody(response, MAX_TEXT_READ_BYTES);
+            if (!read || read.truncated) {
+                if (!read)
+                    discardBody(response);
+                logger.info('url-to-markdown.html_oversized', {
+                    url: finalUrl.href,
+                    bytes: read ? MAX_TEXT_READ_BYTES : declared
+                });
+                return { kind: 'none', captureIssue: 'html_byte_limit' };
+            }
+            return { kind: 'text', text: decodeText(read.bytes) };
+        })
+    });
 }
 function documentFilename(finalUrl, extension) {
     const segment = finalUrl.split(/[?#]/)[0]?.split('/').pop() || 'document';
     const name = segment.slice(0, 80);
-    return name.toLowerCase().endsWith(`.${extension}`)
-        ? name
-        : `${name}.${extension}`;
+    return fileExtension(name) === extension ? name : `${name}.${extension}`;
 }
-/** Streaming limits bound chunked bodies as well as declared Content-Length. */
 async function readBoundedBody(response, maxBytes) {
-    const reader = response.body?.getReader();
-    if (!reader)
-        return new ArrayBuffer(0);
-    const chunks = [];
-    let size = 0;
-    while (true) {
-        const next = await reader.read();
-        if (next.done)
-            break;
-        size += next.value.byteLength;
-        if (size > maxBytes) {
-            await reader.cancel();
-            return null;
-        }
-        chunks.push(next.value);
-    }
-    const bytes = new Uint8Array(size);
-    let offset = 0;
-    for (const chunk of chunks) {
-        bytes.set(chunk, offset);
-        offset += chunk.byteLength;
-    }
-    return bytes.buffer;
+    const read = await readBody(response, maxBytes);
+    return read.truncated ? null : read.bytes.buffer;
 }
 /**
  * Markup a visitor never sees until they open it: `<template>` content and
  * closed `<dialog>`s. A Shopify theme ships its cart drawer ("Your cart is
  * empty") and search overlay (a product list with prices) in every page this
- * way; converted, they reached 70 of one store's 89 pages and a price table
- * in seven of its documents (Good Habit audit §3.4). An open dialog is shown
- * and stays. A contact or newsletter form (Shopify posts both to `/contact`)
- * is labels and a button, and "Telefon Kommentar Send us a note" read as the
- * name `Kommentar Send` the fact guard then demanded of the document.
+ * way; converted, they reached 70 of one store's 89 pages. An open dialog is
+ * shown and stays. A contact or newsletter form (Shopify posts both to
+ * `/contact`) is labels and a button, and "Telefon Kommentar Send us a note"
+ * read as if `Kommentar Send` were a name on the page.
  *
  * Plus the form fields a visitor cannot see: an anti-spam honeypot inside a
  * `display:none` block renamed its label ("Navn" / "Url") on every request,
@@ -288,12 +294,7 @@ function withoutHiddenUi(html) {
         .transform(html);
 }
 /** A real browser passes these; every other 4xx has nothing for it to find. */
-const BOT_WALL_STATUSES = new Set([401, 403]);
-/** `AbortSignal.timeout` and the caller's signal, whichever fires first. */
-function timeoutSignal(ms, signal) {
-    const timeout = AbortSignal.timeout(ms);
-    return signal ? AbortSignal.any([timeout, signal]) : timeout;
-}
+const BOT_WALL_STATUSES = [401, 403];
 /**
  * One converter: its Cloudflare client, pools and per-host politeness limits.
  * Create it once per process; every call shares the pools.
@@ -309,14 +310,13 @@ export function createWebsiteToMarkdown(options) {
         logger
     });
     const hostLimits = createHostLimits(createLimit);
+    const allowLocalhost = options.allowLocalhost === true;
     const allowHost = options.isHostAllowed ??
-        ((hostname) => isHostAllowed(hostname, logger));
+        ((hostname) => isHostAllowed(hostname, { logger, allowLocalhost }));
+    const guard = { isHostAllowed: allowHost, fetch: options.fetch };
     async function requestOrigin(url, headers, signal) {
         if (signal?.aborted)
             return { kind: 'failed', retryable: false };
-        if (!(await allowHost(url.hostname))) {
-            return { kind: 'refused' };
-        }
         try {
             // The politeness slot spans the connection *and* the body: freed at the
             // response headers it would bound connections only, and the whole 80-wide
@@ -326,12 +326,9 @@ export function createWebsiteToMarkdown(options) {
             // our upstreams with their own pools, and holding a customer's slot
             // across a 60s render would stall the crawl for no politeness gain.
             const read = await hostLimits.withHostLimit(url.hostname, async () => {
-                const response = await fetch(url, {
-                    redirect: 'follow',
-                    signal: timeoutSignal(FETCH_TIMEOUT_MS, signal),
-                    headers
-                });
-                const finalUrl = parseFinalUrl(response.url, url);
+                const hop = await followRedirects(url, { signal: timeoutSignal(FETCH_TIMEOUT_MS, signal), headers }, guard);
+                const response = hop.response;
+                const finalUrl = parseFinalUrl(response.url, hop.url);
                 return {
                     response,
                     finalUrl,
@@ -341,6 +338,9 @@ export function createWebsiteToMarkdown(options) {
             return { kind: 'read', read };
         }
         catch (error) {
+            // The guard refused the URL or a redirect: policy, not a network blip.
+            if (error instanceof HostNotAllowedError)
+                return { kind: 'refused' };
             logger.warn('url-to-markdown.origin_failed', {
                 url: url.href,
                 ...errorLogAttributes(error)
@@ -395,9 +395,8 @@ export function createWebsiteToMarkdown(options) {
         if (input.signal?.aborted)
             return { outcome: 'transient', source: null };
         try {
-            if (!options.parsePdf)
-                throw new Error('No PDF parser configured');
-            const parsed = await options.parsePdf({
+            const parsePdf = must(options.parsePdf, 'No PDF parser configured');
+            const parsed = await parsePdf({
                 bytes,
                 filename: input.filename,
                 mimeType: input.mimeType,
@@ -435,7 +434,11 @@ export function createWebsiteToMarkdown(options) {
             return {
                 outcome: 'unsupported',
                 source: null,
-                captureIssue: input.body.kind === 'none' ? input.body.captureIssue : null
+                captureIssue: typedSwitch(input.body, 'kind', {
+                    none: (body) => body.captureIssue,
+                    text: () => null,
+                    bytes: () => null
+                })
             };
         }
         return convertDocumentBytes({
@@ -451,15 +454,20 @@ export function createWebsiteToMarkdown(options) {
         // The `Content-Type` header is authoritative - sniffing for a leading
         // `<html` misses every page that opens with a comment or a CDN banner - but
         // a `text/plain` body that is plainly HTML still needs the HTML tier.
-        const isHtml = input.kind === 'html' ||
-            (input.kind !== 'markdown' && looksLikeHtml(body));
+        const isHtml = typedSwitch(input.kind, {
+            html: () => true,
+            markdown: () => false,
+            text: () => looksLikeHtml(body)
+        });
         if (!isHtml && !isBlankMarkdown(body)) {
             return { outcome: 'ok', source: 'content-negotiation', markdown: body };
         }
         const html = isHtml ? body : null;
         // Most origins send no validators, so without this every page of a quiet
         // site went through Workers AI again every night to learn nothing moved.
-        if (html && input.htmlHash && htmlHash(html) === input.htmlHash) {
+        if (html &&
+            input.htmlHash &&
+            htmlHash(cleanText(html)) === input.htmlHash) {
             return { outcome: 'unchanged', source: 'not-modified' };
         }
         if (html && Buffer.byteLength(html) > MAX_HTML_BYTES) {
@@ -470,10 +478,16 @@ export function createWebsiteToMarkdown(options) {
                 url: input.finalUrl.href,
                 bytes: Buffer.byteLength(html)
             });
-            return { outcome: 'unsupported', source: null, html };
+            return {
+                outcome: 'unsupported',
+                source: null,
+                html,
+                captureIssue: 'html_byte_limit'
+            };
         }
         let shell = false;
         let workersAiRateLimited = false;
+        let renderRateLimited = false;
         if (html) {
             const visible = withoutHiddenUi(html);
             let converted = null;
@@ -511,7 +525,7 @@ export function createWebsiteToMarkdown(options) {
             catch (error) {
                 if (!(error instanceof CloudflareRateLimitedError))
                     throw error;
-                workersAiRateLimited = true;
+                renderRateLimited = true;
             }
             if (rendered &&
                 !isBlankMarkdown(rendered) &&
@@ -532,7 +546,9 @@ export function createWebsiteToMarkdown(options) {
                 ? 'render_incomplete'
                 : workersAiRateLimited
                     ? 'workers_ai_rate_limited'
-                    : null
+                    : renderRateLimited
+                        ? 'browser_rendering_rate_limited'
+                        : null
         };
     }
     /** Runs after the origin's slot is released: every tier it reaches is ours. */
@@ -542,7 +558,7 @@ export function createWebsiteToMarkdown(options) {
         const contentType = response.headers.get('content-type');
         if (status === 304)
             return { outcome: 'unchanged', source: 'not-modified' };
-        if (status === 404 || status === 410) {
+        if (isOneOf([404, 410], status)) {
             return { outcome: 'gone', source: null };
         }
         // Still refused after the host's pauses: rendering would only be refused
@@ -550,10 +566,23 @@ export function createWebsiteToMarkdown(options) {
         if (status >= 500 || status === 429)
             return { outcome: 'transient', source: null };
         if (status >= 400) {
-            if (!input.render || !BOT_WALL_STATUSES.has(status)) {
+            if (!input.render || !isOneOf(BOT_WALL_STATUSES, status)) {
                 return { outcome: 'blocked', source: null };
             }
-            const rendered = await client.renderUrlToMarkdown(input.finalUrl.href, input.signal);
+            let rendered;
+            try {
+                rendered = await client.renderUrlToMarkdown(input.finalUrl.href, input.signal);
+            }
+            catch (error) {
+                if (!(error instanceof CloudflareRateLimitedError))
+                    throw error;
+                // The wall may well let a browser through on the next run.
+                return {
+                    outcome: 'transient',
+                    source: null,
+                    captureIssue: 'browser_rendering_rate_limited'
+                };
+            }
             return rendered
                 ? { outcome: 'ok', source: 'browser-rendering', markdown: rendered }
                 : { outcome: 'blocked', source: null };
@@ -562,25 +591,28 @@ export function createWebsiteToMarkdown(options) {
             return { outcome: 'transient', source: null };
         }
         const kind = classifyContentType(contentType);
-        if (kind === 'asset')
-            return { outcome: 'unsupported', source: null };
-        if (kind === 'document') {
-            return convertDocument({
+        return typedSwitch(kind, {
+            asset: () => ({ outcome: 'unsupported', source: null }),
+            document: () => convertDocument({
                 body,
                 contentType,
                 finalUrl: input.finalUrl,
                 signal: input.signal
-            });
-        }
-        if (body.kind !== 'text')
-            return { outcome: 'unsupported', source: null };
-        return convertTextual({
-            text: body.text,
-            kind,
-            finalUrl: input.finalUrl,
-            render: input.render,
-            htmlHash: input.htmlHash,
-            signal: input.signal
+            }),
+            ...typedSwitchGroup(kind, TEXTUAL_KINDS, async (textualKind) => body.kind === 'text'
+                ? convertTextual({
+                    text: body.text,
+                    kind: textualKind,
+                    finalUrl: input.finalUrl,
+                    render: input.render,
+                    htmlHash: input.htmlHash,
+                    signal: input.signal
+                })
+                : {
+                    outcome: 'unsupported',
+                    source: null,
+                    captureIssue: body.kind === 'none' ? body.captureIssue : null
+                })
         });
     }
     /**
@@ -590,10 +622,33 @@ export function createWebsiteToMarkdown(options) {
      * target, validators, HTML metadata and absolute links.
      *
      * Never throws: a page that cannot be read is an outcome, not an exception,
-     * because the caller is crawling hundreds of URLs at a time. A knowledge bench
-     * run reads through its crawl snapshot (`crawl-snapshot.ts`).
+     * because the caller is crawling hundreds of URLs at a time. Anything
+     * unexpected (a failing shared limiter, a corrupt snapshot) is logged and
+     * comes back `transient`. Inside `withCrawlSnapshot` it reads through the
+     * snapshot.
      */
     async function fetchPageAsMarkdown(input) {
+        const elapsedMs = startTimer();
+        try {
+            return await readPage(input);
+        }
+        catch (error) {
+            logger.warn('url-to-markdown.failed', {
+                url: input.url,
+                ...errorLogAttributes(error)
+            });
+            return buildPage({
+                url: input.url,
+                finalUrl: input.url,
+                status: 0,
+                contentType: null,
+                etag: null,
+                lastModified: null,
+                elapsedMs
+            }, { outcome: 'transient', source: null }, logger);
+        }
+    }
+    async function readPage(input) {
         if (!currentCrawlSnapshot())
             return fetchPageLive(input);
         // The snapshot holds the whole page, read without validators; whether it is
@@ -604,34 +659,13 @@ export function createWebsiteToMarkdown(options) {
             render: input.render,
             signal: input.signal
         }));
-        const unchanged = page.outcome === 'ok' &&
-            ((input.htmlHash && page.htmlHash === input.htmlHash) ||
-                (input.etag && page.etag === input.etag));
-        if (!unchanged)
-            return page;
-        return {
-            ...page,
-            outcome: 'unchanged',
-            source: 'not-modified',
-            markdown: '',
-            title: null,
-            lang: null,
-            canonical: null,
-            alternates: [],
-            alternateLanguages: {},
-            metaDescription: null,
-            jsonLd: [],
-            links: [],
-            images: [],
-            captureIssue: null,
-            htmlHash: null
-        };
+        return isUnchangedFor(page, input) ? asUnchanged(page) : page;
     }
     async function fetchPageLive(input) {
         const signal = input.signal ?? currentOperationSignal();
-        const startedAt = performance.now();
+        const elapsedMs = startTimer();
         const render = input.render ?? true;
-        const prepared = prepareRequestUrl(input.url);
+        const prepared = prepareRequestUrl(input.url, allowLocalhost);
         if (!prepared) {
             return buildPage({
                 url: input.url,
@@ -640,7 +674,7 @@ export function createWebsiteToMarkdown(options) {
                 contentType: null,
                 etag: null,
                 lastModified: null,
-                startedAt
+                elapsedMs
             }, { outcome: 'unsupported', source: null }, logger);
         }
         const headers = {
@@ -686,11 +720,13 @@ export function createWebsiteToMarkdown(options) {
                 contentType: null,
                 etag: null,
                 lastModified: null,
-                startedAt
-            }, 
-            // A refused host is a policy decision, not a retryable network blip.
-            {
-                outcome: attempt.kind === 'refused' ? 'blocked' : 'transient',
+                elapsedMs
+            }, {
+                outcome: typedSwitch(attempt, 'kind', {
+                    // A refused host is a policy decision, not a retryable network blip.
+                    refused: () => 'blocked',
+                    failed: () => 'transient'
+                }),
                 source: null
             }, logger);
         }
@@ -713,7 +749,7 @@ export function createWebsiteToMarkdown(options) {
             etag: response.headers.get('etag') ?? notModified(input.etag),
             lastModified: response.headers.get('last-modified') ??
                 notModified(input.lastModified),
-            startedAt
+            elapsedMs
         }, body, logger);
         logger.info('url-to-markdown.completed', {
             url: page.finalUrl,
@@ -742,13 +778,19 @@ export function createWebsiteToMarkdown(options) {
         renderUrlToMarkdown: client.renderUrlToMarkdown,
         /** The SSRF guard the converter runs, bound to its `logger`. */
         isHostAllowed: allowHost,
+        /**
+         * `fetch` behind the converter's guard, run on the URL and on every
+         * redirect. Throws `HostNotAllowedError` when one is refused. Use it for
+         * any other request to a customer-supplied URL.
+         */
+        guardedFetch(input, init, fetchOptions) {
+            return guardedFetch(input, init, { ...guard, ...fetchOptions });
+        },
         /** `withCrawlSnapshot` bound to the `crawlSnapshot` option. */
         withCrawlSnapshot(input, run) {
             if (!input)
                 return run();
-            if (!options.crawlSnapshot) {
-                throw new Error('Set crawlSnapshot.root to use crawl snapshots');
-            }
+            invariant(options.crawlSnapshot, 'Set crawlSnapshot.root to use crawl snapshots');
             return withCrawlSnapshot(input, run, {
                 root: options.crawlSnapshot.root,
                 logger: options.crawlSnapshot.logger ?? logger