website-to-markdown 0.0.0-stage → 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hans Kristoffer Hansen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,79 @@
-# Temporary Holding Version
+# website-to-markdown
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Turn URLs, HTML and files into markdown through Cloudflare's tiers (content
+negotiation, Workers AI `toMarkdown`, Browser Rendering), with an SSRF guard,
+per-host politeness limits, `Retry-After` handling, crawl snapshots for
+benchmarks, and robots.txt + sitemap discovery. Runs on Bun (it uses
+`HTMLRewriter`).
+
+```sh
+bun add website-to-markdown hanzio zod
+```
+
+## Convert
+
+```ts
+import { createWebsiteToMarkdown } from 'website-to-markdown'
+
+const converter = createWebsiteToMarkdown({
+	// Read on first use, so getters over a secret store work.
+	cloudflare: { apiToken: '…', accountId: '…' }
+})
+
+const page = await converter.fetchPageAsMarkdown({ url: 'https://example.com' })
+if (page.outcome === 'ok') console.log(page.title, page.markdown, page.links)
+```
+
+`fetchPageAsMarkdown` never throws: a page that cannot be read comes back as
+an `outcome` (`unchanged`, `gone`, `blocked`, `transient`, `unsupported`).
+Pass the last read's `etag`, `lastModified` or `htmlHash` to get `unchanged`
+without converting again. `convertDocumentBytes` converts an uploaded file;
+`urlToMarkdown` is the one-shot form that throws.
+
+Create one converter per process: it owns the Cloudflare pools and the
+per-host limits.
+
+### Options
+
+| Option | Default | |
+|---|---|---|
+| `cloudflare` | required | `{ apiToken, accountId }` |
+| `createLimit` | `createMemoryConcurrencyLimit` | Builds every pool. Pass a shared (Redis) limiter when several processes share the Cloudflare quota. |
+| `onApiError` | none | Workers AI client errors (hanzio `ApiError`) other than 429s. |
+| `logger` | silent | `{ info, warn }` |
+| `parsePdf` | none | OCR tier for a PDF Workers AI returned nothing for. |
+| `isHostAllowed` | public addresses only | SSRF guard run before every origin request. |
+
+A limiter is a function `run => Promise` with `tripCooldown(ms)` and
+`dispose()`:
+
+```ts
+import type { CreateConcurrencyLimit } from 'website-to-markdown'
+```
+
+### Cancellation
+
+Pass `signal`, or run inside `withOperationSignal(signal, run)` so every read
+in a job inherits it.
+
+### Crawl snapshots
+
+`withCrawlSnapshot({ mode: 'record' | 'replay', name: 'label/host' }, run, { root })`
+records every page and sitemap read under `root` and replays them, so two
+benchmark runs read the same site. Never in production.
+
+## Sitemaps
+
+```ts
+import { discoverSitemapUrls, fetchRobots, robotsAllows } from 'website-to-markdown/sitemap'
+
+const robots = await fetchRobots('https://example.com')
+const pages = await discoverSitemapUrls('https://example.com', robots.sitemaps)
+const readable = pages.filter((page) => robotsAllows(robots.rules, page.url))
+```
+
+## CLI
+
+```sh
+CLOUDFLARE_API_TOKEN=… CLOUDFLARE_ACCOUNT_ID=… bunx website-to-markdown https://example.com [--no-render]
+```

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env bun
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,66 @@
+#!/usr/bin/env bun
+/**
+ * Fetch a public URL as markdown and print the result. Page metadata goes to
+ * stderr; markdown goes to stdout.
+ */
+import { createWebsiteToMarkdown } from './fetch-page.js';
+const HELP = `
+Fetch a public URL as markdown.
+
+Usage:
+  website-to-markdown <url> [--no-render]
+
+Options:
+  --no-render   Skip the Browser Rendering tier (HTML tiers only).
+
+Environment:
+  CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID
+
+Example:
+  website-to-markdown https://example.com/about
+`.trim();
+const args = Bun.argv.slice(2);
+const url = args.find((arg) => !arg.startsWith('-'));
+if (!url || args.includes('-h') || args.includes('--help')) {
+    console.log(HELP);
+    process.exit(url ? 0 : 1);
+}
+const { CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID } = process.env;
+if (!CLOUDFLARE_API_TOKEN || !CLOUDFLARE_ACCOUNT_ID) {
+    console.error('Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.');
+    process.exit(1);
+}
+const { fetchPageAsMarkdown } = createWebsiteToMarkdown({
+    cloudflare: {
+        apiToken: CLOUDFLARE_API_TOKEN,
+        accountId: CLOUDFLARE_ACCOUNT_ID
+    }
+});
+console.error(`Fetching ${url}…`);
+const page = await fetchPageAsMarkdown({
+    url,
+    render: !args.includes('--no-render')
+});
+console.error(JSON.stringify({
+    outcome: page.outcome,
+    status: page.status,
+    source: page.source,
+    url: page.url,
+    finalUrl: page.finalUrl,
+    contentType: page.contentType,
+    etag: page.etag,
+    lastModified: page.lastModified,
+    title: page.title,
+    lang: page.lang,
+    canonical: page.canonical,
+    metaDescription: page.metaDescription,
+    jsonLdBlocks: page.jsonLd.length,
+    links: page.links.length,
+    images: page.images.length,
+    markdownChars: page.markdown.length,
+    durationMs: page.durationMs
+}, null, 2));
+console.log(page.markdown);
+if (page.outcome !== 'ok')
+    process.exit(1);
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AAEzD,MAAM,IAAI,GAAG;;;;;;;;;;;;;;CAcZ,CAAC,IAAI,EAAE,CAAA;AAER,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;AAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAA;AACpD,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACjB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AAC1B,CAAC;AAED,MAAM,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,GAAG,CAAA;AACnE,IAAI,CAAC,oBAAoB,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACrD,OAAO,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,MAAM,EAAE,mBAAmB,EAAE,GAAG,uBAAuB,CAAC;IACvD,UAAU,EAAE;QACX,QAAQ,EAAE,oBAAoB;QAC9B,SAAS,EAAE,qBAAqB;KAChC;CACD,CAAC,CAAA;AAEF,OAAO,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC,CAAA;AAEjC,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC;IACtC,GAAG;IACH,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;CACrC,CAAC,CAAA;AAEF,OAAO,CAAC,KAAK,CACZ,IAAI,CAAC,SAAS,CACb;IACC,OAAO,EAAE,IAAI,CAAC,OAAO;IACrB,MAAM,EAAE,IAAI,CAAC,MAAM;IACnB,MAAM,EAAE,IAAI,CAAC,MAAM;IACnB,GAAG,EAAE,IAAI,CAAC,GAAG;IACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;IACvB,WAAW,EAAE,IAAI,CAAC,WAAW;IAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;IACf,YAAY,EAAE,IAAI,CAAC,YAAY;IAC/B,KAAK,EAAE,IAAI,CAAC,KAAK;IACjB,IAAI,EAAE,IAAI,CAAC,IAAI;IACf,SAAS,EAAE,IAAI,CAAC,SAAS;IACzB,eAAe,EAAE,IAAI,CAAC,eAAe;IACrC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;IAChC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;IACxB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;IAC1B,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;IACnC,UAAU,EAAE,IAAI,CAAC,UAAU;CAC3B,EACD,IAAI,EACJ,CAAC,CACD,CACD,CAAA;AAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;AAC1B,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;IAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA","sourcesContent":["#!/usr/bin/env bun\n/**\n * Fetch a public URL as markdown and print the result. Page metadata goes to\n * stderr; markdown goes to stdout.\n */\nimport { createWebsiteToMarkdown } from './fetch-page.js'\n\nconst HELP = `\nFetch a public URL as markdown.\n\nUsage:\n  website-to-markdown <url> [--no-render]\n\nOptions:\n  --no-render   Skip the Browser Rendering tier (HTML tiers only).\n\nEnvironment:\n  CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID\n\nExample:\n  website-to-markdown https://example.com/about\n`.trim()\n\nconst args = Bun.argv.slice(2)\nconst url = args.find((arg) => !arg.startsWith('-'))\nif (!url || args.includes('-h') || args.includes('--help')) {\n\tconsole.log(HELP)\n\tprocess.exit(url ? 0 : 1)\n}\n\nconst { CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID } = process.env\nif (!CLOUDFLARE_API_TOKEN || !CLOUDFLARE_ACCOUNT_ID) {\n\tconsole.error('Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.')\n\tprocess.exit(1)\n}\n\nconst { fetchPageAsMarkdown } = createWebsiteToMarkdown({\n\tcloudflare: {\n\t\tapiToken: CLOUDFLARE_API_TOKEN,\n\t\taccountId: CLOUDFLARE_ACCOUNT_ID\n\t}\n})\n\nconsole.error(`Fetching ${url}…`)\n\nconst page = await fetchPageAsMarkdown({\n\turl,\n\trender: !args.includes('--no-render')\n})\n\nconsole.error(\n\tJSON.stringify(\n\t\t{\n\t\t\toutcome: page.outcome,\n\t\t\tstatus: page.status,\n\t\t\tsource: page.source,\n\t\t\turl: page.url,\n\t\t\tfinalUrl: page.finalUrl,\n\t\t\tcontentType: page.contentType,\n\t\t\tetag: page.etag,\n\t\t\tlastModified: page.lastModified,\n\t\t\ttitle: page.title,\n\t\t\tlang: page.lang,\n\t\t\tcanonical: page.canonical,\n\t\t\tmetaDescription: page.metaDescription,\n\t\t\tjsonLdBlocks: page.jsonLd.length,\n\t\t\tlinks: page.links.length,\n\t\t\timages: page.images.length,\n\t\t\tmarkdownChars: page.markdown.length,\n\t\t\tdurationMs: page.durationMs\n\t\t},\n\t\tnull,\n\t\t2\n\t)\n)\n\nconsole.log(page.markdown)\nif (page.outcome !== 'ok') process.exit(1)\n"]}

package/dist/client.d.ts

@@ -0,0 +1,40 @@
+import { type ApiError } from 'hanzio/api-wrapper';
+import { type CreateConcurrencyLimit } from './limit.js';
+import { type Logger } from './logger.js';
+import { CloudflareRateLimitedError } from './rate-limit-retry.js';
+export { CloudflareRateLimitedError };
+/** Read on first use, so getters over a secret store work. */
+export type CloudflareCredentials = {
+    apiToken: string;
+    accountId: string;
+};
+export type ConvertFileInput = {
+    signal?: AbortSignal | undefined;
+    body: string | ArrayBuffer;
+    /** Cloudflare picks its converter from the extension. */
+    filename: string;
+    mimeType: string;
+    /** HTML only: base host for relative links. */
+    hostname?: string | undefined;
+};
+/**
+ * The two Cloudflare tiers behind their pools. Rate-limit retries live in
+ * `withCloudflareRateLimitRetry` so a 429 releases the slot and trips the
+ * shared cooldown before the next try.
+ */
+export declare function createCloudflareClient(options: {
+    credentials: CloudflareCredentials;
+    createLimit: CreateConcurrencyLimit;
+    /** Every Workers AI client error except a 429, which is retried instead. */
+    onApiError?: ((error: ApiError) => void) | undefined;
+    logger: Logger;
+}): {
+    convertFileWithWorkersAi: (input: ConvertFileInput) => Promise<string | null>;
+    convertHtmlWithWorkersAi: (input: {
+        html: string;
+        hostname: string;
+        signal?: AbortSignal | undefined;
+    }) => Promise<string | null>;
+    renderUrlToMarkdown: (url: string, signal?: AbortSignal) => Promise<string | null>;
+};
+//# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AACA,OAAO,EACN,KAAK,QAAQ,EAGb,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAEN,KAAK,sBAAsB,EAE3B,MAAM,YAAY,CAAA;AACnB,OAAO,EAAsB,KAAK,MAAM,EAAE,MAAM,aAAa,CAAA;AAC7D,OAAO,EACN,0BAA0B,EAE1B,MAAM,uBAAuB,CAAA;AAK9B,OAAO,EAAE,0BAA0B,EAAE,CAAA;AAErC,8DAA8D;AAC9D,MAAM,MAAM,qBAAqB,GAAG;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;CACjB,CAAA;AASD,MAAM,MAAM,gBAAgB,GAAG;IAC9B,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;IAChC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAA;IAC1B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAC7B,CAAA;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE;IAC/C,WAAW,EAAE,qBAAqB,CAAA;IAClC,WAAW,EAAE,sBAAsB,CAAA;IACnC,4EAA4E;IAC5E,UAAU,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC,GAAG,SAAS,CAAA;IACpD,MAAM,EAAE,MAAM,CAAA;CACd;sCAkF+C,gBAAgB;sCAmCtB;QACxC,IAAI,EAAE,MAAM,CAAA;QACZ,QAAQ,EAAE,MAAM,CAAA;QAChB,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;KAChC;+BAUuC,MAAM,WAAW,WAAW;EA+BpE"}

package/dist/client.js

@@ -0,0 +1,164 @@
+import Cloudflare from 'cloudflare';
+import { createApiClient, HttpResponseError } from 'hanzio/api-wrapper';
+import { createCloudflareLimits } from './limit.js';
+import { errorLogAttributes } from './logger.js';
+import { CloudflareRateLimitedError, withCloudflareRateLimitRetry } from './rate-limit-retry.js';
+import { rateLimitFromError } from './retry-after.js';
+import { toMarkdown } from './to-markdown.js';
+import { BROWSER_USER_AGENT } from './user-agent.js';
+export { CloudflareRateLimitedError };
+const BROWSER_RENDERING_TIMEOUT_MS = 60_000;
+/**
+ * `networkidle0` never settles on Squarespace commerce pages and burned 30s
+ * timeouts in the probe; `load` plus a short settle gets the same markdown.
+ */
+const BROWSER_RENDERING_SETTLE_MS = 1_500;
+/**
+ * The two Cloudflare tiers behind their pools. Rate-limit retries live in
+ * `withCloudflareRateLimitRetry` so a 429 releases the slot and trips the
+ * shared cooldown before the next try.
+ */
+export function createCloudflareClient(options) {
+    const { credentials, logger } = options;
+    const limits = createCloudflareLimits(options.createLimit);
+    function reportUnlessRateLimited(error) {
+        // Expected transient throttles: we retry and trip the shared cooldown.
+        // Reporting every 429 floods the error tracker without adding signal.
+        if (error instanceof HttpResponseError && error.status === 429)
+            return;
+        options.onApiError?.(error);
+    }
+    let cloudflareSdk;
+    let toMarkdownApi;
+    function createToMarkdownApi() {
+        return createApiClient({
+            name: 'cloudflareToMarkdownApi',
+            baseApiUrls: {
+                default: 'https://api.cloudflare.com/client/v4'
+            },
+            defaultHeaders: {
+                Authorization: `Bearer ${credentials.apiToken}`
+            },
+            timeoutMs: 30_000,
+            retries: 0,
+            onError: reportUnlessRateLimited,
+            endpoints: {
+                toMarkdown
+            }
+        });
+    }
+    function getSdk() {
+        cloudflareSdk ??= new Cloudflare({ apiToken: credentials.apiToken });
+        return cloudflareSdk;
+    }
+    async function transform(file) {
+        toMarkdownApi ??= createToMarkdownApi();
+        const res = await toMarkdownApi.toMarkdown({
+            ...(file.signal && { signal: file.signal }),
+            reqParams: { accountId: credentials.accountId },
+            reqBody: {
+                files: new File([file.body], file.filename, {
+                    type: file.mimeType
+                }),
+                // Only the HTML converter takes options; it uses the hostname to
+                // resolve relative links while it walks the document.
+                conversionOptions: file.hostname
+                    ? JSON.stringify({ html: { hostname: file.hostname } })
+                    : undefined
+            }
+        });
+        return res.data;
+    }
+    async function runWithCloudflareRateLimit(input) {
+        return withCloudflareRateLimitRetry({
+            event: input.event,
+            signal: input.signal,
+            rateLimitFromError,
+            tripCooldown: (waitMs) => input.limit.tripCooldown(waitMs),
+            onRateLimit: ({ attempt, waitMs }) => {
+                logger.warn(`${input.event}.rate_limited`, {
+                    attempt,
+                    wait_ms: waitMs
+                });
+            },
+            run: () => input.limit(input.run)
+        });
+    }
+    /**
+     * Workers AI `toMarkdown` over any file it accepts - HTML, PDF, docx, xlsx,
+     * pptx. Returns null for a soft tier miss. Throws `CloudflareRateLimitedError`
+     * when 429 retries are exhausted so callers can surface a clear capture issue.
+     */
+    async function convertFileWithWorkersAi(input) {
+        try {
+            const envelope = await runWithCloudflareRateLimit({
+                event: 'workers-ai.to-markdown',
+                limit: limits.workersAi,
+                signal: input.signal,
+                run: () => {
+                    input.signal?.throwIfAborted();
+                    return transform(input);
+                }
+            });
+            input.signal?.throwIfAborted();
+            const first = envelope.result[0];
+            if (!envelope.success ||
+                !first ||
+                first.format === 'error' ||
+                !first.data?.trim()) {
+                logger.warn('workers-ai.to-markdown.unusable', {
+                    success: envelope.success,
+                    format: first?.format ?? '',
+                    mime_type: input.mimeType,
+                    has_error: Boolean(first?.error)
+                });
+                return null;
+            }
+            return first.data;
+        }
+        catch (error) {
+            if (error instanceof CloudflareRateLimitedError)
+                throw error;
+            logger.warn('workers-ai.to-markdown.failed', errorLogAttributes(error));
+            return null;
+        }
+    }
+    function convertHtmlWithWorkersAi(input) {
+        return convertFileWithWorkersAi({
+            body: input.html,
+            filename: 'page.html',
+            mimeType: 'text/html',
+            hostname: input.hostname,
+            signal: input.signal
+        });
+    }
+    async function renderUrlToMarkdown(url, signal) {
+        try {
+            const markdown = await runWithCloudflareRateLimit({
+                event: 'browser-rendering',
+                limit: limits.browserRendering,
+                signal,
+                run: () => getSdk().browserRendering.markdown.create({
+                    account_id: credentials.accountId,
+                    url,
+                    userAgent: BROWSER_USER_AGENT,
+                    gotoOptions: { waitUntil: 'load' },
+                    waitForTimeout: BROWSER_RENDERING_SETTLE_MS
+                }, { timeout: BROWSER_RENDERING_TIMEOUT_MS, signal })
+            });
+            return markdown.trim() || null;
+        }
+        catch (error) {
+            if (error instanceof CloudflareRateLimitedError)
+                throw error;
+            logger.warn('browser-rendering.failed', errorLogAttributes(error));
+            return null;
+        }
+    }
+    return {
+        convertFileWithWorkersAi,
+        convertHtmlWithWorkersAi,
+        renderUrlToMarkdown
+    };
+}
+//# sourceMappingURL=client.js.map

package/dist/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,YAAY,CAAA;AACnC,OAAO,EAEN,eAAe,EACf,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAGN,sBAAsB,EACtB,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAA;AAC7D,OAAO,EACN,0BAA0B,EAC1B,4BAA4B,EAC5B,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAEpD,OAAO,EAAE,0BAA0B,EAAE,CAAA;AAQrC,MAAM,4BAA4B,GAAG,MAAM,CAAA;AAC3C;;;GAGG;AACH,MAAM,2BAA2B,GAAG,KAAK,CAAA;AAYzC;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAMtC;IACA,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACvC,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAE1D,SAAS,uBAAuB,CAAC,KAAe;QAC/C,uEAAuE;QACvE,sEAAsE;QACtE,IAAI,KAAK,YAAY,iBAAiB,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG;YAAE,OAAM;QACtE,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAA;IAC5B,CAAC;IAED,IAAI,aAAqC,CAAA;IACzC,IAAI,aAAiE,CAAA;IAErE,SAAS,mBAAmB;QAC3B,OAAO,eAAe,CAAC;YACtB,IAAI,EAAE,yBAAyB;YAC/B,WAAW,EAAE;gBACZ,OAAO,EAAE,sCAAsC;aAC/C;YACD,cAAc,EAAE;gBACf,aAAa,EAAE,UAAU,WAAW,CAAC,QAAQ,EAAE;aAC/C;YACD,SAAS,EAAE,MAAM;YACjB,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,uBAAuB;YAChC,SAAS,EAAE;gBACV,UAAU;aACV;SACD,CAAC,CAAA;IACH,CAAC;IAED,SAAS,MAAM;QACd,aAAa,KAAK,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAA;QACpE,OAAO,aAAa,CAAA;IACrB,CAAC;IAED,KAAK,UAAU,SAAS,CAAC,IAAsB;QAC9C,aAAa,KAAK,mBAAmB,EAAE,CAAA;QACvC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,UAAU,CAAC;YAC1C,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3C,SAAS,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE;YAC/C,OAAO,EAAE;gBACR,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC3C,IAAI,EAAE,IAAI,CAAC,QAAQ;iBACnB,CAAC;gBACF,iEAAiE;gBACjE,sDAAsD;gBACtD,iBAAiB,EAAE,IAAI,CAAC,QAAQ;oBAC/B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;oBACvD,CAAC,CAAC,SAAS;aACZ;SACD,CAAC,CAAA;QACF,OAAO,GAAG,CAAC,IAAI,CAAA;IAChB,CAAC;IAED,KAAK,UAAU,0BAA0B,CAAI,KAK5C;QACA,OAAO,4BAA4B,CAAC;YACnC,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,kBAAkB;YAClB,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC;YAC1D,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE;gBACpC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,eAAe,EAAE;oBAC1C,OAAO;oBACP,OAAO,EAAE,MAAM;iBACf,CAAC,CAAA;YACH,CAAC;YACD,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC;SACjC,CAAC,CAAA;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,UAAU,wBAAwB,CAAC,KAAuB;QAC9D,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBACjD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,MAAM,CAAC,SAAS;gBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,GAAG,EAAE;oBACT,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;oBAC9B,OAAO,SAAS,CAAC,KAAK,CAAC,CAAA;gBACxB,CAAC;aACD,CAAC,CAAA;YACF,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;YAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YAChC,IACC,CAAC,QAAQ,CAAC,OAAO;gBACjB,CAAC,KAAK;gBACN,KAAK,CAAC,MAAM,KAAK,OAAO;gBACxB,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,EAClB,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;oBAC9C,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,EAAE;oBAC3B,SAAS,EAAE,KAAK,CAAC,QAAQ;oBACzB,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;iBAChC,CAAC,CAAA;gBACF,OAAO,IAAI,CAAA;YACZ,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,0BAA0B;gBAAE,MAAM,KAAK,CAAA;YAC5D,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YACvE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,SAAS,wBAAwB,CAAC,KAIjC;QACA,OAAO,wBAAwB,CAAC;YAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,UAAU,mBAAmB,CAAC,GAAW,EAAE,MAAoB;QACnE,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBACjD,KAAK,EAAE,mBAAmB;gBAC1B,KAAK,EAAE,MAAM,CAAC,gBAAgB;gBAC9B,MAAM;gBACN,GAAG,EAAE,GAAG,EAAE,CACT,MAAM,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CACxC;oBACC,UAAU,EAAE,WAAW,CAAC,SAAS;oBACjC,GAAG;oBACH,SAAS,EAAE,kBAAkB;oBAC7B,WAAW,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE;oBAClC,cAAc,EAAE,2BAA2B;iBAC3C,EACD,EAAE,OAAO,EAAE,4BAA4B,EAAE,MAAM,EAAE,CACjD;aACF,CAAC,CAAA;YACF,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,IAAI,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,0BAA0B;gBAAE,MAAM,KAAK,CAAA;YAC5D,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YAClE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO;QACN,wBAAwB;QACxB,wBAAwB;QACxB,mBAAmB;KACnB,CAAA;AACF,CAAC","sourcesContent":["import Cloudflare from 'cloudflare'\nimport {\n\ttype ApiError,\n\tcreateApiClient,\n\tHttpResponseError\n} from 'hanzio/api-wrapper'\nimport {\n\ttype ConcurrencyLimit,\n\ttype CreateConcurrencyLimit,\n\tcreateCloudflareLimits\n} from './limit.js'\nimport { errorLogAttributes, type Logger } from './logger.js'\nimport {\n\tCloudflareRateLimitedError,\n\twithCloudflareRateLimitRetry\n} from './rate-limit-retry.js'\nimport { rateLimitFromError } from './retry-after.js'\nimport { toMarkdown } from './to-markdown.js'\nimport { BROWSER_USER_AGENT } from './user-agent.js'\n\nexport { CloudflareRateLimitedError }\n\n/** Read on first use, so getters over a secret store work. */\nexport type CloudflareCredentials = {\n\tapiToken: string\n\taccountId: string\n}\n\nconst BROWSER_RENDERING_TIMEOUT_MS = 60_000\n/**\n * `networkidle0` never settles on Squarespace commerce pages and burned 30s\n * timeouts in the probe; `load` plus a short settle gets the same markdown.\n */\nconst BROWSER_RENDERING_SETTLE_MS = 1_500\n\nexport type ConvertFileInput = {\n\tsignal?: AbortSignal | undefined\n\tbody: string | ArrayBuffer\n\t/** Cloudflare picks its converter from the extension. */\n\tfilename: string\n\tmimeType: string\n\t/** HTML only: base host for relative links. */\n\thostname?: string | undefined\n}\n\n/**\n * The two Cloudflare tiers behind their pools. Rate-limit retries live in\n * `withCloudflareRateLimitRetry` so a 429 releases the slot and trips the\n * shared cooldown before the next try.\n */\nexport function createCloudflareClient(options: {\n\tcredentials: CloudflareCredentials\n\tcreateLimit: CreateConcurrencyLimit\n\t/** Every Workers AI client error except a 429, which is retried instead. */\n\tonApiError?: ((error: ApiError) => void) | undefined\n\tlogger: Logger\n}) {\n\tconst { credentials, logger } = options\n\tconst limits = createCloudflareLimits(options.createLimit)\n\n\tfunction reportUnlessRateLimited(error: ApiError) {\n\t\t// Expected transient throttles: we retry and trip the shared cooldown.\n\t\t// Reporting every 429 floods the error tracker without adding signal.\n\t\tif (error instanceof HttpResponseError && error.status === 429) return\n\t\toptions.onApiError?.(error)\n\t}\n\n\tlet cloudflareSdk: Cloudflare | undefined\n\tlet toMarkdownApi: ReturnType<typeof createToMarkdownApi> | undefined\n\n\tfunction createToMarkdownApi() {\n\t\treturn createApiClient({\n\t\t\tname: 'cloudflareToMarkdownApi',\n\t\t\tbaseApiUrls: {\n\t\t\t\tdefault: 'https://api.cloudflare.com/client/v4'\n\t\t\t},\n\t\t\tdefaultHeaders: {\n\t\t\t\tAuthorization: `Bearer ${credentials.apiToken}`\n\t\t\t},\n\t\t\ttimeoutMs: 30_000,\n\t\t\tretries: 0,\n\t\t\tonError: reportUnlessRateLimited,\n\t\t\tendpoints: {\n\t\t\t\ttoMarkdown\n\t\t\t}\n\t\t})\n\t}\n\n\tfunction getSdk() {\n\t\tcloudflareSdk ??= new Cloudflare({ apiToken: credentials.apiToken })\n\t\treturn cloudflareSdk\n\t}\n\n\tasync function transform(file: ConvertFileInput) {\n\t\ttoMarkdownApi ??= createToMarkdownApi()\n\t\tconst res = await toMarkdownApi.toMarkdown({\n\t\t\t...(file.signal && { signal: file.signal }),\n\t\t\treqParams: { accountId: credentials.accountId },\n\t\t\treqBody: {\n\t\t\t\tfiles: new File([file.body], file.filename, {\n\t\t\t\t\ttype: file.mimeType\n\t\t\t\t}),\n\t\t\t\t// Only the HTML converter takes options; it uses the hostname to\n\t\t\t\t// resolve relative links while it walks the document.\n\t\t\t\tconversionOptions: file.hostname\n\t\t\t\t\t? JSON.stringify({ html: { hostname: file.hostname } })\n\t\t\t\t\t: undefined\n\t\t\t}\n\t\t})\n\t\treturn res.data\n\t}\n\n\tasync function runWithCloudflareRateLimit<T>(input: {\n\t\tevent: string\n\t\tlimit: ConcurrencyLimit\n\t\tsignal?: AbortSignal | undefined\n\t\trun: () => Promise<T>\n\t}) {\n\t\treturn withCloudflareRateLimitRetry({\n\t\t\tevent: input.event,\n\t\t\tsignal: input.signal,\n\t\t\trateLimitFromError,\n\t\t\ttripCooldown: (waitMs) => input.limit.tripCooldown(waitMs),\n\t\t\tonRateLimit: ({ attempt, waitMs }) => {\n\t\t\t\tlogger.warn(`${input.event}.rate_limited`, {\n\t\t\t\t\tattempt,\n\t\t\t\t\twait_ms: waitMs\n\t\t\t\t})\n\t\t\t},\n\t\t\trun: () => input.limit(input.run)\n\t\t})\n\t}\n\n\t/**\n\t * Workers AI `toMarkdown` over any file it accepts - HTML, PDF, docx, xlsx,\n\t * pptx. Returns null for a soft tier miss. Throws `CloudflareRateLimitedError`\n\t * when 429 retries are exhausted so callers can surface a clear capture issue.\n\t */\n\tasync function convertFileWithWorkersAi(input: ConvertFileInput) {\n\t\ttry {\n\t\t\tconst envelope = await runWithCloudflareRateLimit({\n\t\t\t\tevent: 'workers-ai.to-markdown',\n\t\t\t\tlimit: limits.workersAi,\n\t\t\t\tsignal: input.signal,\n\t\t\t\trun: () => {\n\t\t\t\t\tinput.signal?.throwIfAborted()\n\t\t\t\t\treturn transform(input)\n\t\t\t\t}\n\t\t\t})\n\t\t\tinput.signal?.throwIfAborted()\n\t\t\tconst first = envelope.result[0]\n\t\t\tif (\n\t\t\t\t!envelope.success ||\n\t\t\t\t!first ||\n\t\t\t\tfirst.format === 'error' ||\n\t\t\t\t!first.data?.trim()\n\t\t\t) {\n\t\t\t\tlogger.warn('workers-ai.to-markdown.unusable', {\n\t\t\t\t\tsuccess: envelope.success,\n\t\t\t\t\tformat: first?.format ?? '',\n\t\t\t\t\tmime_type: input.mimeType,\n\t\t\t\t\thas_error: Boolean(first?.error)\n\t\t\t\t})\n\t\t\t\treturn null\n\t\t\t}\n\t\t\treturn first.data\n\t\t} catch (error) {\n\t\t\tif (error instanceof CloudflareRateLimitedError) throw error\n\t\t\tlogger.warn('workers-ai.to-markdown.failed', errorLogAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\tfunction convertHtmlWithWorkersAi(input: {\n\t\thtml: string\n\t\thostname: string\n\t\tsignal?: AbortSignal | undefined\n\t}) {\n\t\treturn convertFileWithWorkersAi({\n\t\t\tbody: input.html,\n\t\t\tfilename: 'page.html',\n\t\t\tmimeType: 'text/html',\n\t\t\thostname: input.hostname,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\tasync function renderUrlToMarkdown(url: string, signal?: AbortSignal) {\n\t\ttry {\n\t\t\tconst markdown = await runWithCloudflareRateLimit({\n\t\t\t\tevent: 'browser-rendering',\n\t\t\t\tlimit: limits.browserRendering,\n\t\t\t\tsignal,\n\t\t\t\trun: () =>\n\t\t\t\t\tgetSdk().browserRendering.markdown.create(\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\taccount_id: credentials.accountId,\n\t\t\t\t\t\t\turl,\n\t\t\t\t\t\t\tuserAgent: BROWSER_USER_AGENT,\n\t\t\t\t\t\t\tgotoOptions: { waitUntil: 'load' },\n\t\t\t\t\t\t\twaitForTimeout: BROWSER_RENDERING_SETTLE_MS\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{ timeout: BROWSER_RENDERING_TIMEOUT_MS, signal }\n\t\t\t\t\t)\n\t\t\t})\n\t\t\treturn markdown.trim() || null\n\t\t} catch (error) {\n\t\t\tif (error instanceof CloudflareRateLimitedError) throw error\n\t\t\tlogger.warn('browser-rendering.failed', errorLogAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\treturn {\n\t\tconvertFileWithWorkersAi,\n\t\tconvertHtmlWithWorkersAi,\n\t\trenderUrlToMarkdown\n\t}\n}\n"]}

package/dist/content-type.d.ts

@@ -0,0 +1,9 @@
+/** The upload descriptor for a binary we can convert, or null for text/assets. */
+export declare function convertibleDocumentType(contentType: string | null): {
+    extension: string;
+    mimeType: string;
+} | null;
+export type ContentKind = 'markdown' | 'html' | 'text' | 'document' | 'asset';
+/** What the fetch layer should do with a body, decided from the header alone. */
+export declare function classifyContentType(contentType: string | null): ContentKind;
+//# sourceMappingURL=content-type.d.ts.map

package/dist/content-type.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-type.d.ts","sourceRoot":"","sources":["../src/content-type.ts"],"names":[],"mappings":"AA2DA,kFAAkF;AAClF,wBAAgB,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;;;SAIjE;AAED,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,OAAO,CAAA;AAE7E,iFAAiF;AACjF,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,GAAG,WAAW,CAO3E"}

package/dist/content-type.js

@@ -0,0 +1,71 @@
+const MARKDOWN_CONTENT_TYPES = new Set([
+    'text/markdown',
+    'text/x-markdown',
+    'application/markdown'
+]);
+const HTML_CONTENT_TYPES = new Set(['text/html', 'application/xhtml+xml']);
+const TEXTUAL_CONTENT_TYPES = new Set([
+    'application/xml',
+    'application/json',
+    'application/rss+xml',
+    'application/atom+xml'
+]);
+/**
+ * The formats Workers AI `toMarkdown` accepts as documents (its `/supported`
+ * list, read 2026-09-08). Images are accepted there too but are deliberately
+ * absent: this list decides what the CRAWLER downloads and converts, and a
+ * site's gallery must stay an asset. Uploads accept images on their own list.
+ */
+const CONVERTIBLE_DOCUMENT_EXTENSIONS = new Map([
+    ['application/pdf', 'pdf'],
+    [
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        'docx'
+    ],
+    ['application/vnd.oasis.opendocument.text', 'odt'],
+    ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'xlsx'],
+    ['application/vnd.ms-excel.sheet.macroenabled.12', 'xlsm'],
+    ['application/vnd.ms-excel.sheet.binary.macroenabled.12', 'xlsb'],
+    ['application/vnd.ms-excel', 'xls'],
+    ['application/vnd.oasis.opendocument.spreadsheet', 'ods'],
+    [
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+        'pptx'
+    ],
+    [
+        'application/vnd.openxmlformats-officedocument.presentationml.slideshow',
+        'ppsx'
+    ],
+    ['application/vnd.oasis.opendocument.presentation', 'odp']
+]);
+function normalizeContentType(contentType) {
+    return contentType?.split(';')[0]?.trim().toLowerCase() ?? '';
+}
+function isTextualContentType(type) {
+    return (type === '' ||
+        type.startsWith('text/') ||
+        TEXTUAL_CONTENT_TYPES.has(type) ||
+        // `application/ld+json`, `application/rdf+xml`, … but not `image/svg+xml`.
+        (type.startsWith('application/') &&
+            (type.endsWith('+json') || type.endsWith('+xml'))));
+}
+/** The upload descriptor for a binary we can convert, or null for text/assets. */
+export function convertibleDocumentType(contentType) {
+    const mimeType = normalizeContentType(contentType);
+    const extension = CONVERTIBLE_DOCUMENT_EXTENSIONS.get(mimeType);
+    return extension ? { extension, mimeType } : null;
+}
+/** What the fetch layer should do with a body, decided from the header alone. */
+export function classifyContentType(contentType) {
+    const type = normalizeContentType(contentType);
+    if (MARKDOWN_CONTENT_TYPES.has(type))
+        return 'markdown';
+    if (HTML_CONTENT_TYPES.has(type))
+        return 'html';
+    if (CONVERTIBLE_DOCUMENT_EXTENSIONS.has(type))
+        return 'document';
+    if (isTextualContentType(type))
+        return 'text';
+    return 'asset';
+}
+//# sourceMappingURL=content-type.js.map

package/dist/content-type.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-type.js","sourceRoot":"","sources":["../src/content-type.ts"],"names":[],"mappings":"AAAA,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACtC,eAAe;IACf,iBAAiB;IACjB,sBAAsB;CACtB,CAAC,CAAA;AAEF,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,uBAAuB,CAAC,CAAC,CAAA;AAE1E,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC;IACrC,iBAAiB;IACjB,kBAAkB;IAClB,qBAAqB;IACrB,sBAAsB;CACtB,CAAC,CAAA;AAEF;;;;;GAKG;AACH,MAAM,+BAA+B,GAAG,IAAI,GAAG,CAAC;IAC/C,CAAC,iBAAiB,EAAE,KAAK,CAAC;IAC1B;QACC,yEAAyE;QACzE,MAAM;KACN;IACD,CAAC,yCAAyC,EAAE,KAAK,CAAC;IAClD,CAAC,mEAAmE,EAAE,MAAM,CAAC;IAC7E,CAAC,gDAAgD,EAAE,MAAM,CAAC;IAC1D,CAAC,uDAAuD,EAAE,MAAM,CAAC;IACjE,CAAC,0BAA0B,EAAE,KAAK,CAAC;IACnC,CAAC,gDAAgD,EAAE,KAAK,CAAC;IACzD;QACC,2EAA2E;QAC3E,MAAM;KACN;IACD;QACC,wEAAwE;QACxE,MAAM;KACN;IACD,CAAC,iDAAiD,EAAE,KAAK,CAAC;CAC1D,CAAC,CAAA;AAEF,SAAS,oBAAoB,CAAC,WAAsC;IACnE,OAAO,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAA;AAC9D,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACzC,OAAO,CACN,IAAI,KAAK,EAAE;QACX,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QACxB,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC;QAC/B,2EAA2E;QAC3E,CAAC,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC;YAC/B,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CACnD,CAAA;AACF,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,uBAAuB,CAAC,WAA0B;IACjE,MAAM,QAAQ,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAA;IAClD,MAAM,SAAS,GAAG,+BAA+B,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAC/D,OAAO,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AAClD,CAAC;AAID,iFAAiF;AACjF,MAAM,UAAU,mBAAmB,CAAC,WAA0B;IAC7D,MAAM,IAAI,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAA;IAC9C,IAAI,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IACvD,IAAI,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAA;IAC/C,IAAI,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAChE,IAAI,oBAAoB,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAA;IAC7C,OAAO,OAAO,CAAA;AACf,CAAC","sourcesContent":["const MARKDOWN_CONTENT_TYPES = new Set([\n\t'text/markdown',\n\t'text/x-markdown',\n\t'application/markdown'\n])\n\nconst HTML_CONTENT_TYPES = new Set(['text/html', 'application/xhtml+xml'])\n\nconst TEXTUAL_CONTENT_TYPES = new Set([\n\t'application/xml',\n\t'application/json',\n\t'application/rss+xml',\n\t'application/atom+xml'\n])\n\n/**\n * The formats Workers AI `toMarkdown` accepts as documents (its `/supported`\n * list, read 2026-09-08). Images are accepted there too but are deliberately\n * absent: this list decides what the CRAWLER downloads and converts, and a\n * site's gallery must stay an asset. Uploads accept images on their own list.\n */\nconst CONVERTIBLE_DOCUMENT_EXTENSIONS = new Map([\n\t['application/pdf', 'pdf'],\n\t[\n\t\t'application/vnd.openxmlformats-officedocument.wordprocessingml.document',\n\t\t'docx'\n\t],\n\t['application/vnd.oasis.opendocument.text', 'odt'],\n\t['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'xlsx'],\n\t['application/vnd.ms-excel.sheet.macroenabled.12', 'xlsm'],\n\t['application/vnd.ms-excel.sheet.binary.macroenabled.12', 'xlsb'],\n\t['application/vnd.ms-excel', 'xls'],\n\t['application/vnd.oasis.opendocument.spreadsheet', 'ods'],\n\t[\n\t\t'application/vnd.openxmlformats-officedocument.presentationml.presentation',\n\t\t'pptx'\n\t],\n\t[\n\t\t'application/vnd.openxmlformats-officedocument.presentationml.slideshow',\n\t\t'ppsx'\n\t],\n\t['application/vnd.oasis.opendocument.presentation', 'odp']\n])\n\nfunction normalizeContentType(contentType: string | null | undefined) {\n\treturn contentType?.split(';')[0]?.trim().toLowerCase() ?? ''\n}\n\nfunction isTextualContentType(type: string) {\n\treturn (\n\t\ttype === '' ||\n\t\ttype.startsWith('text/') ||\n\t\tTEXTUAL_CONTENT_TYPES.has(type) ||\n\t\t// `application/ld+json`, `application/rdf+xml`, … but not `image/svg+xml`.\n\t\t(type.startsWith('application/') &&\n\t\t\t(type.endsWith('+json') || type.endsWith('+xml')))\n\t)\n}\n\n/** The upload descriptor for a binary we can convert, or null for text/assets. */\nexport function convertibleDocumentType(contentType: string | null) {\n\tconst mimeType = normalizeContentType(contentType)\n\tconst extension = CONVERTIBLE_DOCUMENT_EXTENSIONS.get(mimeType)\n\treturn extension ? { extension, mimeType } : null\n}\n\nexport type ContentKind = 'markdown' | 'html' | 'text' | 'document' | 'asset'\n\n/** What the fetch layer should do with a body, decided from the header alone. */\nexport function classifyContentType(contentType: string | null): ContentKind {\n\tconst type = normalizeContentType(contentType)\n\tif (MARKDOWN_CONTENT_TYPES.has(type)) return 'markdown'\n\tif (HTML_CONTENT_TYPES.has(type)) return 'html'\n\tif (CONVERTIBLE_DOCUMENT_EXTENSIONS.has(type)) return 'document'\n\tif (isTextualContentType(type)) return 'text'\n\treturn 'asset'\n}\n"]}

package/dist/crawl-snapshot.d.ts

@@ -0,0 +1,38 @@
+import type { Logger } from './logger.js';
+/**
+ * A crawl recorded once and replayed, for benchmarks. Two runs of a pipeline
+ * then read the same pages, so a difference between them is the code: not a
+ * site that published a page in between, a 429, or a host that timed out on
+ * one run and not the other.
+ *
+ * Recorded: the network reads - pages as `fetchPageAsMarkdown` returns them,
+ * robots.txt and sitemaps - plus anything the caller routes through
+ * `throughCrawlSnapshot`. Everything else runs live on every replay, which is
+ * the point.
+ *
+ * Set per job through `withCrawlSnapshot`, never globally: a worker drains
+ * other tenants' jobs on the same lane. Never in production.
+ */
+export type CrawlSnapshotInput = {
+    mode: 'record' | 'replay';
+    /** `<snapshot>/<host>`: the bench's snapshot label, then the site. */
+    name: string;
+};
+/** The name becomes a path, so it may not climb out of `root`. */
+export declare const CRAWL_SNAPSHOT_NAME_PATTERN: RegExp;
+export declare function withCrawlSnapshot<T>(input: CrawlSnapshotInput | null | undefined, run: () => Promise<T>, options: {
+    /** Directory the snapshots live under. */
+    root: string;
+    /** Told about replay misses. */
+    logger?: Logger;
+}): Promise<T>;
+/**
+ * `read` through the active snapshot. Recording stores what the network
+ * returned; replaying returns what was stored. A read the recording never made
+ * (a code change that discovers a new URL) goes to the network and is added,
+ * with a warning, so the snapshot stays complete without hiding the drift.
+ */
+export declare function throughCrawlSnapshot<T>(key: string, read: () => Promise<T>): Promise<T>;
+/** The snapshot this job reads or writes, to hand on to the job it queues. */
+export declare function currentCrawlSnapshot(): CrawlSnapshotInput | null;
+//# sourceMappingURL=crawl-snapshot.d.ts.map

package/dist/crawl-snapshot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl-snapshot.d.ts","sourceRoot":"","sources":["../src/crawl-snapshot.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAEzC;;;;;;;;;;;;;GAaG;AAEH,MAAM,MAAM,kBAAkB,GAAG;IAChC,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACzB,sEAAsE;IACtE,IAAI,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,kEAAkE;AAClE,eAAO,MAAM,2BAA2B,QACE,CAAA;AAS1C,wBAAgB,iBAAiB,CAAC,CAAC,EAClC,KAAK,EAAE,kBAAkB,GAAG,IAAI,GAAG,SAAS,EAC5C,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACrB,OAAO,EAAE;IACR,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAA;IACZ,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAA;CACf,cAmBD;AAED;;;;;GAKG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC3C,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,CAAC,CAeZ;AAED,8EAA8E;AAC9E,wBAAgB,oBAAoB,IAAI,kBAAkB,GAAG,IAAI,CAGhE"}

package/dist/crawl-snapshot.js

@@ -0,0 +1,51 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { sha256Hex } from './hash.js';
+/** The name becomes a path, so it may not climb out of `root`. */
+export const CRAWL_SNAPSHOT_NAME_PATTERN = /^[a-z0-9._-]{1,80}\/[a-z0-9._-]{1,120}$/;
+const snapshot = new AsyncLocalStorage();
+export function withCrawlSnapshot(input, run, options) {
+    if (!input)
+        return run();
+    if (process.env.NODE_ENV === 'production') {
+        throw new Error('A crawl snapshot is a local bench tool');
+    }
+    if (!CRAWL_SNAPSHOT_NAME_PATTERN.test(input.name)) {
+        throw new Error(`Invalid crawl snapshot name: ${input.name}`);
+    }
+    return snapshot.run({
+        mode: input.mode,
+        name: input.name,
+        directory: resolve(options.root, input.name),
+        logger: options.logger
+    }, run);
+}
+/**
+ * `read` through the active snapshot. Recording stores what the network
+ * returned; replaying returns what was stored. A read the recording never made
+ * (a code change that discovers a new URL) goes to the network and is added,
+ * with a warning, so the snapshot stays complete without hiding the drift.
+ */
+export async function throughCrawlSnapshot(key, read) {
+    const active = snapshot.getStore();
+    if (!active)
+        return read();
+    const file = resolve(active.directory, `${sha256Hex(key).slice(0, 32)}.json`);
+    if (active.mode === 'replay') {
+        const stored = await readFile(file, 'utf8').catch(() => null);
+        if (stored !== null)
+            return JSON.parse(stored).value;
+        active.logger?.warn('crawl-snapshot.miss', { key });
+    }
+    const value = await read();
+    await mkdir(active.directory, { recursive: true });
+    await writeFile(file, JSON.stringify({ key, value }));
+    return value;
+}
+/** The snapshot this job reads or writes, to hand on to the job it queues. */
+export function currentCrawlSnapshot() {
+    const active = snapshot.getStore();
+    return active ? { mode: active.mode, name: active.name } : null;
+}
+//# sourceMappingURL=crawl-snapshot.js.map

package/dist/crawl-snapshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl-snapshot.js","sourceRoot":"","sources":["../src/crawl-snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAwBrC,kEAAkE;AAClE,MAAM,CAAC,MAAM,2BAA2B,GACvC,yCAAyC,CAAA;AAO1C,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAiB,CAAA;AAEvD,MAAM,UAAU,iBAAiB,CAChC,KAA4C,EAC5C,GAAqB,EACrB,OAKC;IAED,IAAI,CAAC,KAAK;QAAE,OAAO,GAAG,EAAE,CAAA;IACxB,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAC1D,CAAC;IACD,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,gCAAgC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IAC9D,CAAC;IAED,OAAO,QAAQ,CAAC,GAAG,CAClB;QACC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;KACtB,EACD,GAAG,CACH,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,GAAW,EACX,IAAsB;IAEtB,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;IAClC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,EAAE,CAAA;IAE1B,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAA;IAC7E,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;QAC7D,IAAI,MAAM,KAAK,IAAI;YAAE,OAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,CAAkB,CAAC,KAAK,CAAA;QACtE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAA;IAC1B,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAClD,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IACrD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,oBAAoB;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;IAClC,OAAO,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AAChE,CAAC","sourcesContent":["import { AsyncLocalStorage } from 'node:async_hooks'\nimport { mkdir, readFile, writeFile } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport { sha256Hex } from './hash.js'\nimport type { Logger } from './logger.js'\n\n/**\n * A crawl recorded once and replayed, for benchmarks. Two runs of a pipeline\n * then read the same pages, so a difference between them is the code: not a\n * site that published a page in between, a 429, or a host that timed out on\n * one run and not the other.\n *\n * Recorded: the network reads - pages as `fetchPageAsMarkdown` returns them,\n * robots.txt and sitemaps - plus anything the caller routes through\n * `throughCrawlSnapshot`. Everything else runs live on every replay, which is\n * the point.\n *\n * Set per job through `withCrawlSnapshot`, never globally: a worker drains\n * other tenants' jobs on the same lane. Never in production.\n */\n\nexport type CrawlSnapshotInput = {\n\tmode: 'record' | 'replay'\n\t/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */\n\tname: string\n}\n\n/** The name becomes a path, so it may not climb out of `root`. */\nexport const CRAWL_SNAPSHOT_NAME_PATTERN =\n\t/^[a-z0-9._-]{1,80}\\/[a-z0-9._-]{1,120}$/\n\ntype CrawlSnapshot = CrawlSnapshotInput & {\n\tdirectory: string\n\tlogger: Logger | undefined\n}\n\nconst snapshot = new AsyncLocalStorage<CrawlSnapshot>()\n\nexport function withCrawlSnapshot<T>(\n\tinput: CrawlSnapshotInput | null | undefined,\n\trun: () => Promise<T>,\n\toptions: {\n\t\t/** Directory the snapshots live under. */\n\t\troot: string\n\t\t/** Told about replay misses. */\n\t\tlogger?: Logger\n\t}\n) {\n\tif (!input) return run()\n\tif (process.env.NODE_ENV === 'production') {\n\t\tthrow new Error('A crawl snapshot is a local bench tool')\n\t}\n\tif (!CRAWL_SNAPSHOT_NAME_PATTERN.test(input.name)) {\n\t\tthrow new Error(`Invalid crawl snapshot name: ${input.name}`)\n\t}\n\n\treturn snapshot.run(\n\t\t{\n\t\t\tmode: input.mode,\n\t\t\tname: input.name,\n\t\t\tdirectory: resolve(options.root, input.name),\n\t\t\tlogger: options.logger\n\t\t},\n\t\trun\n\t)\n}\n\n/**\n * `read` through the active snapshot. Recording stores what the network\n * returned; replaying returns what was stored. A read the recording never made\n * (a code change that discovers a new URL) goes to the network and is added,\n * with a warning, so the snapshot stays complete without hiding the drift.\n */\nexport async function throughCrawlSnapshot<T>(\n\tkey: string,\n\tread: () => Promise<T>\n): Promise<T> {\n\tconst active = snapshot.getStore()\n\tif (!active) return read()\n\n\tconst file = resolve(active.directory, `${sha256Hex(key).slice(0, 32)}.json`)\n\tif (active.mode === 'replay') {\n\t\tconst stored = await readFile(file, 'utf8').catch(() => null)\n\t\tif (stored !== null) return (JSON.parse(stored) as { value: T }).value\n\t\tactive.logger?.warn('crawl-snapshot.miss', { key })\n\t}\n\n\tconst value = await read()\n\tawait mkdir(active.directory, { recursive: true })\n\tawait writeFile(file, JSON.stringify({ key, value }))\n\treturn value\n}\n\n/** The snapshot this job reads or writes, to hand on to the job it queues. */\nexport function currentCrawlSnapshot(): CrawlSnapshotInput | null {\n\tconst active = snapshot.getStore()\n\treturn active ? { mode: active.mode, name: active.name } : null\n}\n"]}

package/dist/detect-markdown.d.ts

@@ -0,0 +1,12 @@
+/** A server that mislabels HTML as `text/plain` still needs the HTML tier. */
+export declare function looksLikeHtml(body: string): boolean;
+/**
+ * Deliberately "empty", not "short": a real 300-character opening-hours page
+ * must not be promoted to Browser Rendering, which is the slow, rate-limited
+ * tier and the only one that ever timed out in the probe.
+ */
+export declare function isBlankMarkdown(markdown: string): boolean;
+/** Require a JS bootstrap signal AND missing meaningful prose before rendering.
+ * Short opening-hours pages remain on the fast path. */
+export declare function isApplicationShell(html: string, markdown: string): boolean;
+//# sourceMappingURL=detect-markdown.d.ts.map

package/dist/detect-markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"detect-markdown.d.ts","sourceRoot":"","sources":["../src/detect-markdown.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,WAEzC;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,WAE/C;AAOD;wDACwD;AACxD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,WAyBhE"}