website-to-markdown 1.0.1 → 1.1.0

package/README.md

@@ -40,9 +40,18 @@ per-host limits.
 | `cloudflare` | required | `{ apiToken, accountId }` |
 | `createLimit` | `createMemoryConcurrencyLimit` | Builds every pool. Pass a shared (Redis) limiter when several processes share the Cloudflare quota. |
 | `onApiError` | none | Workers AI client errors (hanzio `ApiError`) other than 429s. |
+| `reportTimeouts` | `true` | `false` keeps Workers AI timeouts from `onApiError`; the page falls through to the next tier anyway. |
 | `logger` | silent | `{ info, warn }` |
 | `parsePdf` | none | OCR tier for a PDF Workers AI returned nothing for. |
 | `isHostAllowed` | public addresses only | SSRF guard run before every origin request. |
+| `crawlSnapshot` | none | `{ root, logger? }` for the returned `withCrawlSnapshot`. |
+
+Besides the converters, the factory returns:
+
+- `isHostAllowed(hostname)`: the SSRF guard the converter runs, bound to its
+  `logger`. Use it for any other fetch of a customer-supplied URL.
+- `withCrawlSnapshot(input, run)`: [crawl snapshots](#crawl-snapshots) under
+  `crawlSnapshot.root`. A `null` input just runs `run`.
 
 A limiter is a function `run => Promise` with `tripCooldown(ms)` and
 `dispose()`:
@@ -51,6 +60,25 @@ A limiter is a function `run => Promise` with `tripCooldown(ms)` and
 import type { CreateConcurrencyLimit } from 'website-to-markdown'
 ```
 
+`hanzio/redis` has one that shares the pools across processes; bind its
+`send`:
+
+```ts
+import { createRedisConcurrencyLimit } from 'hanzio/redis'
+
+const converter = createWebsiteToMarkdown({
+	cloudflare,
+	createLimit: (options) =>
+		createRedisConcurrencyLimit({
+			...options,
+			send: (command, args) => redis.send(command, args)
+		})
+})
+```
+
+`options.metricLabel` names the pool a per-host limiter belongs to; report
+metrics under it rather than under `key`, which is one per customer host.
+
 ### Cancellation
 
 Pass `signal`, or run inside `withOperationSignal(signal, run)` so every read
@@ -60,7 +88,20 @@ in a job inherits it.
 
 `withCrawlSnapshot({ mode: 'record' | 'replay', name: 'label/host' }, run, { root })`
 records every page and sitemap read under `root` and replays them, so two
-benchmark runs read the same site. Never in production.
+benchmark runs read the same site. Never in production. Pass
+`crawlSnapshot: { root }` to the factory to get it bound as
+`converter.withCrawlSnapshot(input, run)`.
+
+A shared schema can check a name without importing the runtime:
+
+```ts
+import { CRAWL_SNAPSHOT_NAME_PATTERN } from 'website-to-markdown/schema'
+
+const CrawlSnapshotInput = z.object({
+	mode: z.enum(['record', 'replay']),
+	name: z.string().regex(CRAWL_SNAPSHOT_NAME_PATTERN)
+})
+```
 
 ## Sitemaps
 
@@ -77,3 +118,14 @@ const readable = pages.filter((page) => robotsAllows(robots.rules, page.url))
 ```sh
 CLOUDFLARE_API_TOKEN=… CLOUDFLARE_ACCOUNT_ID=… bunx website-to-markdown https://example.com [--no-render]
 ```
+
+`--env-file <path>` reads the two variables from a dotenv file instead; a
+variable already in the environment wins. With a secrets manager, export to a
+file or run the CLI under it:
+
+```sh
+infisical export --format=dotenv > .env.cloudflare
+bunx website-to-markdown https://example.com --env-file .env.cloudflare
+
+infisical run -- bunx website-to-markdown https://example.com
+```

package/dist/cli.d.ts

@@ -1,8 +1,4 @@
 #!/usr/bin/env bun
-/**
- * Fetch a public URL as markdown and print the result. Page metadata goes to
- * stderr; markdown goes to stdout.
- */
 import { type CliIO } from 'hanzio/cli';
 export declare const cli: import("hanzio/cli").Cli;
 /** The bin: 1.0 printed the help and exited 1 when no URL was given, even with `--help`. */

package/dist/cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,EAAE,KAAK,KAAK,EAAsC,MAAM,YAAY,CAAA;AAI3E,eAAO,MAAM,GAAG,0BAiEd,CAAA;AAEF,4FAA4F;AAC5F,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,mBAI7D"}
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAOA,OAAO,EAAE,KAAK,KAAK,EAAsC,MAAM,YAAY,CAAA;AAI3E,eAAO,MAAM,GAAG,0BAyEd,CAAA;AAQF,4FAA4F;AAC5F,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,mBAI7D"}

package/dist/cli.js

@@ -3,6 +3,8 @@
  * Fetch a public URL as markdown and print the result. Page metadata goes to
  * stderr; markdown goes to stdout.
  */
+import { readFile } from 'node:fs/promises';
+import { parseEnv } from 'node:util';
 import { CliError, defineCli, defineCommand } from 'hanzio/cli';
 import { z } from 'zod';
 import { createWebsiteToMarkdown } from './fetch-page.js';
@@ -18,10 +20,16 @@ export const cli = defineCli({
             render: z
                 .boolean()
                 .default(true)
-                .describe('Use the Browser Rendering tier; --no-render keeps to HTML tiers.')
+                .describe('Use the Browser Rendering tier; --no-render keeps to HTML tiers.'),
+            envFile: z
+                .string()
+                .optional()
+                .describe('Read CLOUDFLARE_* from a dotenv file; the environment wins.')
         }),
-        run: async ({ url, render }, ctx) => {
-            const { CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID } = ctx.env;
+        run: async ({ url, render, envFile }, ctx) => {
+            const file = envFile ? parseEnv(await readEnvFile(envFile)) : {};
+            const CLOUDFLARE_API_TOKEN = ctx.env.CLOUDFLARE_API_TOKEN || file.CLOUDFLARE_API_TOKEN;
+            const CLOUDFLARE_ACCOUNT_ID = ctx.env.CLOUDFLARE_ACCOUNT_ID || file.CLOUDFLARE_ACCOUNT_ID;
             if (!CLOUDFLARE_API_TOKEN || !CLOUDFLARE_ACCOUNT_ID)
                 throw new CliError('Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.');
             const { fetchPageAsMarkdown } = createWebsiteToMarkdown({
@@ -57,6 +65,11 @@ export const cli = defineCli({
         }
     })
 });
+function readEnvFile(path) {
+    return readFile(path, 'utf8').catch(() => {
+        throw new CliError(`Cannot read --env-file ${path}.`);
+    });
+}
 /** The bin: 1.0 printed the help and exited 1 when no URL was given, even with `--help`. */
 export async function main(argv, io) {
     if (argv.some((arg) => !arg.startsWith('-')))

package/dist/cli.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,EAAc,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC3E,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AAEzD,MAAM,CAAC,MAAM,GAAG,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,qBAAqB;IAC3B,QAAQ,EAAE,aAAa,CAAC;QACvB,WAAW,EACV,uFAAuF;QACxF,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,GAAG,EAAE,CAAC;iBACJ,MAAM,EAAE;iBACR,QAAQ,CAAC,gCAAgC,CAAC;iBAC1C,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC;iBACP,OAAO,EAAE;iBACT,OAAO,CAAC,IAAI,CAAC;iBACb,QAAQ,CACR,kEAAkE,CAClE;SACF,CAAC;QACF,GAAG,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE;YACnC,MAAM,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,GAAG,GAAG,CAAC,GAAG,CAAA;YAC/D,IAAI,CAAC,oBAAoB,IAAI,CAAC,qBAAqB;gBAClD,MAAM,IAAI,QAAQ,CACjB,qDAAqD,CACrD,CAAA;YAEF,MAAM,EAAE,mBAAmB,EAAE,GAAG,uBAAuB,CAAC;gBACvD,UAAU,EAAE;oBACX,QAAQ,EAAE,oBAAoB;oBAC9B,SAAS,EAAE,qBAAqB;iBAChC;aACD,CAAC,CAAA;YAEF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,CAAA;YAEtC,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAA;YAEvD,GAAG,CAAC,MAAM,CAAC,KAAK,CACf,GAAG,IAAI,CAAC,SAAS,CAChB;gBACC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAChC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACxB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBACnC,UAAU,EAAE,IAAI,CAAC,UAAU;aAC3B,EACD,IAAI,EACJ,CAAC,CACD,IAAI,CACL,CAAA;YAED,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;gBAAE,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAA;YAC3C,OAAO,IAAI,CAAC,QAAQ,CAAA;QACrB,CAAC;KACD,CAAC;CACF,CAAC,CAAA;AAEF,4FAA4F;AAC5F,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAuB,EAAE,EAAU;IAC7D,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IACtE,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAA;IAC7B,OAAO,CAAC,CAAA;AACT,CAAC;AAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI;IAAE,OAAO,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA","sourcesContent":["#!/usr/bin/env bun\n/**\n * Fetch a public URL as markdown and print the result. Page metadata goes to\n * stderr; markdown goes to stdout.\n */\nimport { type CliIO, CliError, defineCli, defineCommand } from 'hanzio/cli'\nimport { z } from 'zod'\nimport { createWebsiteToMarkdown } from './fetch-page.js'\n\nexport const cli = defineCli({\n\tname: 'website-to-markdown',\n\tcommands: defineCommand({\n\t\tdescription:\n\t\t\t'Fetch a public URL as markdown. Needs CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.',\n\t\tinput: z.object({\n\t\t\turl: z\n\t\t\t\t.string()\n\t\t\t\t.describe('e.g. https://example.com/about')\n\t\t\t\t.meta({ positional: true }),\n\t\t\trender: z\n\t\t\t\t.boolean()\n\t\t\t\t.default(true)\n\t\t\t\t.describe(\n\t\t\t\t\t'Use the Browser Rendering tier; --no-render keeps to HTML tiers.'\n\t\t\t\t)\n\t\t}),\n\t\trun: async ({ url, render }, ctx) => {\n\t\t\tconst { CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID } = ctx.env\n\t\t\tif (!CLOUDFLARE_API_TOKEN || !CLOUDFLARE_ACCOUNT_ID)\n\t\t\t\tthrow new CliError(\n\t\t\t\t\t'Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.'\n\t\t\t\t)\n\n\t\t\tconst { fetchPageAsMarkdown } = createWebsiteToMarkdown({\n\t\t\t\tcloudflare: {\n\t\t\t\t\tapiToken: CLOUDFLARE_API_TOKEN,\n\t\t\t\t\taccountId: CLOUDFLARE_ACCOUNT_ID\n\t\t\t\t}\n\t\t\t})\n\n\t\t\tctx.stderr.write(`Fetching ${url}…\\n`)\n\n\t\t\tconst page = await fetchPageAsMarkdown({ url, render })\n\n\t\t\tctx.stderr.write(\n\t\t\t\t`${JSON.stringify(\n\t\t\t\t\t{\n\t\t\t\t\t\toutcome: page.outcome,\n\t\t\t\t\t\tstatus: page.status,\n\t\t\t\t\t\tsource: page.source,\n\t\t\t\t\t\turl: page.url,\n\t\t\t\t\t\tfinalUrl: page.finalUrl,\n\t\t\t\t\t\tcontentType: page.contentType,\n\t\t\t\t\t\tetag: page.etag,\n\t\t\t\t\t\tlastModified: page.lastModified,\n\t\t\t\t\t\ttitle: page.title,\n\t\t\t\t\t\tlang: page.lang,\n\t\t\t\t\t\tcanonical: page.canonical,\n\t\t\t\t\t\tmetaDescription: page.metaDescription,\n\t\t\t\t\t\tjsonLdBlocks: page.jsonLd.length,\n\t\t\t\t\t\tlinks: page.links.length,\n\t\t\t\t\t\timages: page.images.length,\n\t\t\t\t\t\tmarkdownChars: page.markdown.length,\n\t\t\t\t\t\tdurationMs: page.durationMs\n\t\t\t\t\t},\n\t\t\t\t\tnull,\n\t\t\t\t\t2\n\t\t\t\t)}\\n`\n\t\t\t)\n\n\t\t\tif (page.outcome !== 'ok') ctx.exitCode = 1\n\t\t\treturn page.markdown\n\t\t}\n\t})\n})\n\n/** The bin: 1.0 printed the help and exited 1 when no URL was given, even with `--help`. */\nexport async function main(argv: readonly string[], io?: CliIO) {\n\tif (argv.some((arg) => !arg.startsWith('-'))) return cli.run(argv, io)\n\tawait cli.run(['--help'], io)\n\treturn 1\n}\n\nif (import.meta.main) process.exitCode = await main(process.argv.slice(2))\n"]}
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,EAAc,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC3E,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAA;AAEzD,MAAM,CAAC,MAAM,GAAG,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,qBAAqB;IAC3B,QAAQ,EAAE,aAAa,CAAC;QACvB,WAAW,EACV,uFAAuF;QACxF,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,GAAG,EAAE,CAAC;iBACJ,MAAM,EAAE;iBACR,QAAQ,CAAC,gCAAgC,CAAC;iBAC1C,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC;iBACP,OAAO,EAAE;iBACT,OAAO,CAAC,IAAI,CAAC;iBACb,QAAQ,CACR,kEAAkE,CAClE;YACF,OAAO,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,EAAE;iBACV,QAAQ,CAAC,6DAA6D,CAAC;SACzE,CAAC;QACF,GAAG,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,EAAE;YAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;YAChE,MAAM,oBAAoB,GACzB,GAAG,CAAC,GAAG,CAAC,oBAAoB,IAAI,IAAI,CAAC,oBAAoB,CAAA;YAC1D,MAAM,qBAAqB,GAC1B,GAAG,CAAC,GAAG,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB,CAAA;YAC5D,IAAI,CAAC,oBAAoB,IAAI,CAAC,qBAAqB;gBAClD,MAAM,IAAI,QAAQ,CACjB,qDAAqD,CACrD,CAAA;YAEF,MAAM,EAAE,mBAAmB,EAAE,GAAG,uBAAuB,CAAC;gBACvD,UAAU,EAAE;oBACX,QAAQ,EAAE,oBAAoB;oBAC9B,SAAS,EAAE,qBAAqB;iBAChC;aACD,CAAC,CAAA;YAEF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,CAAA;YAEtC,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAA;YAEvD,GAAG,CAAC,MAAM,CAAC,KAAK,CACf,GAAG,IAAI,CAAC,SAAS,CAChB;gBACC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAChC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACxB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBACnC,UAAU,EAAE,IAAI,CAAC,UAAU;aAC3B,EACD,IAAI,EACJ,CAAC,CACD,IAAI,CACL,CAAA;YAED,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;gBAAE,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAA;YAC3C,OAAO,IAAI,CAAC,QAAQ,CAAA;QACrB,CAAC;KACD,CAAC;CACF,CAAC,CAAA;AAEF,SAAS,WAAW,CAAC,IAAY;IAChC,OAAO,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;QACxC,MAAM,IAAI,QAAQ,CAAC,0BAA0B,IAAI,GAAG,CAAC,CAAA;IACtD,CAAC,CAAC,CAAA;AACH,CAAC;AAED,4FAA4F;AAC5F,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAuB,EAAE,EAAU;IAC7D,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IACtE,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAA;IAC7B,OAAO,CAAC,CAAA;AACT,CAAC;AAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI;IAAE,OAAO,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA","sourcesContent":["#!/usr/bin/env bun\n/**\n * Fetch a public URL as markdown and print the result. Page metadata goes to\n * stderr; markdown goes to stdout.\n */\nimport { readFile } from 'node:fs/promises'\nimport { parseEnv } from 'node:util'\nimport { type CliIO, CliError, defineCli, defineCommand } from 'hanzio/cli'\nimport { z } from 'zod'\nimport { createWebsiteToMarkdown } from './fetch-page.js'\n\nexport const cli = defineCli({\n\tname: 'website-to-markdown',\n\tcommands: defineCommand({\n\t\tdescription:\n\t\t\t'Fetch a public URL as markdown. Needs CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.',\n\t\tinput: z.object({\n\t\t\turl: z\n\t\t\t\t.string()\n\t\t\t\t.describe('e.g. https://example.com/about')\n\t\t\t\t.meta({ positional: true }),\n\t\t\trender: z\n\t\t\t\t.boolean()\n\t\t\t\t.default(true)\n\t\t\t\t.describe(\n\t\t\t\t\t'Use the Browser Rendering tier; --no-render keeps to HTML tiers.'\n\t\t\t\t),\n\t\t\tenvFile: z\n\t\t\t\t.string()\n\t\t\t\t.optional()\n\t\t\t\t.describe('Read CLOUDFLARE_* from a dotenv file; the environment wins.')\n\t\t}),\n\t\trun: async ({ url, render, envFile }, ctx) => {\n\t\t\tconst file = envFile ? parseEnv(await readEnvFile(envFile)) : {}\n\t\t\tconst CLOUDFLARE_API_TOKEN =\n\t\t\t\tctx.env.CLOUDFLARE_API_TOKEN || file.CLOUDFLARE_API_TOKEN\n\t\t\tconst CLOUDFLARE_ACCOUNT_ID =\n\t\t\t\tctx.env.CLOUDFLARE_ACCOUNT_ID || file.CLOUDFLARE_ACCOUNT_ID\n\t\t\tif (!CLOUDFLARE_API_TOKEN || !CLOUDFLARE_ACCOUNT_ID)\n\t\t\t\tthrow new CliError(\n\t\t\t\t\t'Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.'\n\t\t\t\t)\n\n\t\t\tconst { fetchPageAsMarkdown } = createWebsiteToMarkdown({\n\t\t\t\tcloudflare: {\n\t\t\t\t\tapiToken: CLOUDFLARE_API_TOKEN,\n\t\t\t\t\taccountId: CLOUDFLARE_ACCOUNT_ID\n\t\t\t\t}\n\t\t\t})\n\n\t\t\tctx.stderr.write(`Fetching ${url}…\\n`)\n\n\t\t\tconst page = await fetchPageAsMarkdown({ url, render })\n\n\t\t\tctx.stderr.write(\n\t\t\t\t`${JSON.stringify(\n\t\t\t\t\t{\n\t\t\t\t\t\toutcome: page.outcome,\n\t\t\t\t\t\tstatus: page.status,\n\t\t\t\t\t\tsource: page.source,\n\t\t\t\t\t\turl: page.url,\n\t\t\t\t\t\tfinalUrl: page.finalUrl,\n\t\t\t\t\t\tcontentType: page.contentType,\n\t\t\t\t\t\tetag: page.etag,\n\t\t\t\t\t\tlastModified: page.lastModified,\n\t\t\t\t\t\ttitle: page.title,\n\t\t\t\t\t\tlang: page.lang,\n\t\t\t\t\t\tcanonical: page.canonical,\n\t\t\t\t\t\tmetaDescription: page.metaDescription,\n\t\t\t\t\t\tjsonLdBlocks: page.jsonLd.length,\n\t\t\t\t\t\tlinks: page.links.length,\n\t\t\t\t\t\timages: page.images.length,\n\t\t\t\t\t\tmarkdownChars: page.markdown.length,\n\t\t\t\t\t\tdurationMs: page.durationMs\n\t\t\t\t\t},\n\t\t\t\t\tnull,\n\t\t\t\t\t2\n\t\t\t\t)}\\n`\n\t\t\t)\n\n\t\t\tif (page.outcome !== 'ok') ctx.exitCode = 1\n\t\t\treturn page.markdown\n\t\t}\n\t})\n})\n\nfunction readEnvFile(path: string) {\n\treturn readFile(path, 'utf8').catch(() => {\n\t\tthrow new CliError(`Cannot read --env-file ${path}.`)\n\t})\n}\n\n/** The bin: 1.0 printed the help and exited 1 when no URL was given, even with `--help`. */\nexport async function main(argv: readonly string[], io?: CliIO) {\n\tif (argv.some((arg) => !arg.startsWith('-'))) return cli.run(argv, io)\n\tawait cli.run(['--help'], io)\n\treturn 1\n}\n\nif (import.meta.main) process.exitCode = await main(process.argv.slice(2))\n"]}

package/dist/client.d.ts

@@ -27,6 +27,8 @@ export declare function createCloudflareClient(options: {
     createLimit: CreateConcurrencyLimit;
     /** Every Workers AI client error except a 429, which is retried instead. */
     onApiError?: ((error: ApiError) => void) | undefined;
+    /** Pass timeouts to `onApiError`; default true. */
+    reportTimeouts?: boolean | undefined;
     logger: Logger;
 }): {
     convertFileWithWorkersAi: (input: ConvertFileInput) => Promise<string | null>;

package/dist/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AACA,OAAO,EACN,KAAK,QAAQ,EAGb,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAEN,KAAK,sBAAsB,EAE3B,MAAM,YAAY,CAAA;AACnB,OAAO,EAAsB,KAAK,MAAM,EAAE,MAAM,aAAa,CAAA;AAC7D,OAAO,EACN,0BAA0B,EAE1B,MAAM,uBAAuB,CAAA;AAK9B,OAAO,EAAE,0BAA0B,EAAE,CAAA;AAErC,8DAA8D;AAC9D,MAAM,MAAM,qBAAqB,GAAG;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;CACjB,CAAA;AASD,MAAM,MAAM,gBAAgB,GAAG;IAC9B,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;IAChC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAA;IAC1B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAC7B,CAAA;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE;IAC/C,WAAW,EAAE,qBAAqB,CAAA;IAClC,WAAW,EAAE,sBAAsB,CAAA;IACnC,4EAA4E;IAC5E,UAAU,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC,GAAG,SAAS,CAAA;IACpD,MAAM,EAAE,MAAM,CAAA;CACd;sCAkF+C,gBAAgB;sCAmCtB;QACxC,IAAI,EAAE,MAAM,CAAA;QACZ,QAAQ,EAAE,MAAM,CAAA;QAChB,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;KAChC;+BAUuC,MAAM,WAAW,WAAW;EA+BpE"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,QAAQ,EAAmB,MAAM,oBAAoB,CAAA;AACnE,OAAO,EAEN,KAAK,sBAAsB,EAE3B,MAAM,YAAY,CAAA;AACnB,OAAO,EAAsB,KAAK,MAAM,EAAE,MAAM,aAAa,CAAA;AAC7D,OAAO,EACN,0BAA0B,EAE1B,MAAM,uBAAuB,CAAA;AAM9B,OAAO,EAAE,0BAA0B,EAAE,CAAA;AAErC,8DAA8D;AAC9D,MAAM,MAAM,qBAAqB,GAAG;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;CACjB,CAAA;AASD,MAAM,MAAM,gBAAgB,GAAG;IAC9B,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;IAChC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAA;IAC1B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAC7B,CAAA;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE;IAC/C,WAAW,EAAE,qBAAqB,CAAA;IAClC,WAAW,EAAE,sBAAsB,CAAA;IACnC,4EAA4E;IAC5E,UAAU,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC,GAAG,SAAS,CAAA;IACpD,mDAAmD;IACnD,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IACpC,MAAM,EAAE,MAAM,CAAA;CACd;sCAiF+C,gBAAgB;sCAmCtB;QACxC,IAAI,EAAE,MAAM,CAAA;QACZ,QAAQ,EAAE,MAAM,CAAA;QAChB,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;KAChC;+BAUuC,MAAM,WAAW,WAAW;EA+BpE"}

package/dist/client.js

@@ -1,8 +1,9 @@
 import Cloudflare from 'cloudflare';
-import { createApiClient, HttpResponseError } from 'hanzio/api-wrapper';
+import { createApiClient } from 'hanzio/api-wrapper';
 import { createCloudflareLimits } from './limit.js';
 import { errorLogAttributes } from './logger.js';
 import { CloudflareRateLimitedError, withCloudflareRateLimitRetry } from './rate-limit-retry.js';
+import { isReportableApiError } from './report-api-error.js';
 import { rateLimitFromError } from './retry-after.js';
 import { toMarkdown } from './to-markdown.js';
 import { BROWSER_USER_AGENT } from './user-agent.js';
@@ -21,12 +22,10 @@ const BROWSER_RENDERING_SETTLE_MS = 1_500;
 export function createCloudflareClient(options) {
     const { credentials, logger } = options;
     const limits = createCloudflareLimits(options.createLimit);
-    function reportUnlessRateLimited(error) {
-        // Expected transient throttles: we retry and trip the shared cooldown.
-        // Reporting every 429 floods the error tracker without adding signal.
-        if (error instanceof HttpResponseError && error.status === 429)
-            return;
-        options.onApiError?.(error);
+    function report(error) {
+        if (isReportableApiError(error, options.reportTimeouts)) {
+            options.onApiError?.(error);
+        }
     }
     let cloudflareSdk;
     let toMarkdownApi;
@@ -41,7 +40,7 @@ export function createCloudflareClient(options) {
             },
             timeoutMs: 30_000,
             retries: 0,
-            onError: reportUnlessRateLimited,
+            onError: report,
             endpoints: {
                 toMarkdown
             }

package/dist/client.js.map

@@ -1 +1 @@
-{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,YAAY,CAAA;AACnC,OAAO,EAEN,eAAe,EACf,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAGN,sBAAsB,EACtB,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAA;AAC7D,OAAO,EACN,0BAA0B,EAC1B,4BAA4B,EAC5B,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAEpD,OAAO,EAAE,0BAA0B,EAAE,CAAA;AAQrC,MAAM,4BAA4B,GAAG,MAAM,CAAA;AAC3C;;;GAGG;AACH,MAAM,2BAA2B,GAAG,KAAK,CAAA;AAYzC;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAMtC;IACA,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACvC,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAE1D,SAAS,uBAAuB,CAAC,KAAe;QAC/C,uEAAuE;QACvE,sEAAsE;QACtE,IAAI,KAAK,YAAY,iBAAiB,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG;YAAE,OAAM;QACtE,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAA;IAC5B,CAAC;IAED,IAAI,aAAqC,CAAA;IACzC,IAAI,aAAiE,CAAA;IAErE,SAAS,mBAAmB;QAC3B,OAAO,eAAe,CAAC;YACtB,IAAI,EAAE,yBAAyB;YAC/B,WAAW,EAAE;gBACZ,OAAO,EAAE,sCAAsC;aAC/C;YACD,cAAc,EAAE;gBACf,aAAa,EAAE,UAAU,WAAW,CAAC,QAAQ,EAAE;aAC/C;YACD,SAAS,EAAE,MAAM;YACjB,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,uBAAuB;YAChC,SAAS,EAAE;gBACV,UAAU;aACV;SACD,CAAC,CAAA;IACH,CAAC;IAED,SAAS,MAAM;QACd,aAAa,KAAK,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAA;QACpE,OAAO,aAAa,CAAA;IACrB,CAAC;IAED,KAAK,UAAU,SAAS,CAAC,IAAsB;QAC9C,aAAa,KAAK,mBAAmB,EAAE,CAAA;QACvC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,UAAU,CAAC;YAC1C,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3C,SAAS,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE;YAC/C,OAAO,EAAE;gBACR,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC3C,IAAI,EAAE,IAAI,CAAC,QAAQ;iBACnB,CAAC;gBACF,iEAAiE;gBACjE,sDAAsD;gBACtD,iBAAiB,EAAE,IAAI,CAAC,QAAQ;oBAC/B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;oBACvD,CAAC,CAAC,SAAS;aACZ;SACD,CAAC,CAAA;QACF,OAAO,GAAG,CAAC,IAAI,CAAA;IAChB,CAAC;IAED,KAAK,UAAU,0BAA0B,CAAI,KAK5C;QACA,OAAO,4BAA4B,CAAC;YACnC,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,kBAAkB;YAClB,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC;YAC1D,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE;gBACpC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,eAAe,EAAE;oBAC1C,OAAO;oBACP,OAAO,EAAE,MAAM;iBACf,CAAC,CAAA;YACH,CAAC;YACD,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC;SACjC,CAAC,CAAA;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,UAAU,wBAAwB,CAAC,KAAuB;QAC9D,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBACjD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,MAAM,CAAC,SAAS;gBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,GAAG,EAAE;oBACT,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;oBAC9B,OAAO,SAAS,CAAC,KAAK,CAAC,CAAA;gBACxB,CAAC;aACD,CAAC,CAAA;YACF,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;YAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YAChC,IACC,CAAC,QAAQ,CAAC,OAAO;gBACjB,CAAC,KAAK;gBACN,KAAK,CAAC,MAAM,KAAK,OAAO;gBACxB,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,EAClB,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;oBAC9C,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,EAAE;oBAC3B,SAAS,EAAE,KAAK,CAAC,QAAQ;oBACzB,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;iBAChC,CAAC,CAAA;gBACF,OAAO,IAAI,CAAA;YACZ,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,0BAA0B;gBAAE,MAAM,KAAK,CAAA;YAC5D,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YACvE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,SAAS,wBAAwB,CAAC,KAIjC;QACA,OAAO,wBAAwB,CAAC;YAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,UAAU,mBAAmB,CAAC,GAAW,EAAE,MAAoB;QACnE,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBACjD,KAAK,EAAE,mBAAmB;gBAC1B,KAAK,EAAE,MAAM,CAAC,gBAAgB;gBAC9B,MAAM;gBACN,GAAG,EAAE,GAAG,EAAE,CACT,MAAM,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CACxC;oBACC,UAAU,EAAE,WAAW,CAAC,SAAS;oBACjC,GAAG;oBACH,SAAS,EAAE,kBAAkB;oBAC7B,WAAW,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE;oBAClC,cAAc,EAAE,2BAA2B;iBAC3C,EACD,EAAE,OAAO,EAAE,4BAA4B,EAAE,MAAM,EAAE,CACjD;aACF,CAAC,CAAA;YACF,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,IAAI,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,0BAA0B;gBAAE,MAAM,KAAK,CAAA;YAC5D,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YAClE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO;QACN,wBAAwB;QACxB,wBAAwB;QACxB,mBAAmB;KACnB,CAAA;AACF,CAAC","sourcesContent":["import Cloudflare from 'cloudflare'\nimport {\n\ttype ApiError,\n\tcreateApiClient,\n\tHttpResponseError\n} from 'hanzio/api-wrapper'\nimport {\n\ttype ConcurrencyLimit,\n\ttype CreateConcurrencyLimit,\n\tcreateCloudflareLimits\n} from './limit.js'\nimport { errorLogAttributes, type Logger } from './logger.js'\nimport {\n\tCloudflareRateLimitedError,\n\twithCloudflareRateLimitRetry\n} from './rate-limit-retry.js'\nimport { rateLimitFromError } from './retry-after.js'\nimport { toMarkdown } from './to-markdown.js'\nimport { BROWSER_USER_AGENT } from './user-agent.js'\n\nexport { CloudflareRateLimitedError }\n\n/** Read on first use, so getters over a secret store work. */\nexport type CloudflareCredentials = {\n\tapiToken: string\n\taccountId: string\n}\n\nconst BROWSER_RENDERING_TIMEOUT_MS = 60_000\n/**\n * `networkidle0` never settles on Squarespace commerce pages and burned 30s\n * timeouts in the probe; `load` plus a short settle gets the same markdown.\n */\nconst BROWSER_RENDERING_SETTLE_MS = 1_500\n\nexport type ConvertFileInput = {\n\tsignal?: AbortSignal | undefined\n\tbody: string | ArrayBuffer\n\t/** Cloudflare picks its converter from the extension. */\n\tfilename: string\n\tmimeType: string\n\t/** HTML only: base host for relative links. */\n\thostname?: string | undefined\n}\n\n/**\n * The two Cloudflare tiers behind their pools. Rate-limit retries live in\n * `withCloudflareRateLimitRetry` so a 429 releases the slot and trips the\n * shared cooldown before the next try.\n */\nexport function createCloudflareClient(options: {\n\tcredentials: CloudflareCredentials\n\tcreateLimit: CreateConcurrencyLimit\n\t/** Every Workers AI client error except a 429, which is retried instead. */\n\tonApiError?: ((error: ApiError) => void) | undefined\n\tlogger: Logger\n}) {\n\tconst { credentials, logger } = options\n\tconst limits = createCloudflareLimits(options.createLimit)\n\n\tfunction reportUnlessRateLimited(error: ApiError) {\n\t\t// Expected transient throttles: we retry and trip the shared cooldown.\n\t\t// Reporting every 429 floods the error tracker without adding signal.\n\t\tif (error instanceof HttpResponseError && error.status === 429) return\n\t\toptions.onApiError?.(error)\n\t}\n\n\tlet cloudflareSdk: Cloudflare | undefined\n\tlet toMarkdownApi: ReturnType<typeof createToMarkdownApi> | undefined\n\n\tfunction createToMarkdownApi() {\n\t\treturn createApiClient({\n\t\t\tname: 'cloudflareToMarkdownApi',\n\t\t\tbaseApiUrls: {\n\t\t\t\tdefault: 'https://api.cloudflare.com/client/v4'\n\t\t\t},\n\t\t\tdefaultHeaders: {\n\t\t\t\tAuthorization: `Bearer ${credentials.apiToken}`\n\t\t\t},\n\t\t\ttimeoutMs: 30_000,\n\t\t\tretries: 0,\n\t\t\tonError: reportUnlessRateLimited,\n\t\t\tendpoints: {\n\t\t\t\ttoMarkdown\n\t\t\t}\n\t\t})\n\t}\n\n\tfunction getSdk() {\n\t\tcloudflareSdk ??= new Cloudflare({ apiToken: credentials.apiToken })\n\t\treturn cloudflareSdk\n\t}\n\n\tasync function transform(file: ConvertFileInput) {\n\t\ttoMarkdownApi ??= createToMarkdownApi()\n\t\tconst res = await toMarkdownApi.toMarkdown({\n\t\t\t...(file.signal && { signal: file.signal }),\n\t\t\treqParams: { accountId: credentials.accountId },\n\t\t\treqBody: {\n\t\t\t\tfiles: new File([file.body], file.filename, {\n\t\t\t\t\ttype: file.mimeType\n\t\t\t\t}),\n\t\t\t\t// Only the HTML converter takes options; it uses the hostname to\n\t\t\t\t// resolve relative links while it walks the document.\n\t\t\t\tconversionOptions: file.hostname\n\t\t\t\t\t? JSON.stringify({ html: { hostname: file.hostname } })\n\t\t\t\t\t: undefined\n\t\t\t}\n\t\t})\n\t\treturn res.data\n\t}\n\n\tasync function runWithCloudflareRateLimit<T>(input: {\n\t\tevent: string\n\t\tlimit: ConcurrencyLimit\n\t\tsignal?: AbortSignal | undefined\n\t\trun: () => Promise<T>\n\t}) {\n\t\treturn withCloudflareRateLimitRetry({\n\t\t\tevent: input.event,\n\t\t\tsignal: input.signal,\n\t\t\trateLimitFromError,\n\t\t\ttripCooldown: (waitMs) => input.limit.tripCooldown(waitMs),\n\t\t\tonRateLimit: ({ attempt, waitMs }) => {\n\t\t\t\tlogger.warn(`${input.event}.rate_limited`, {\n\t\t\t\t\tattempt,\n\t\t\t\t\twait_ms: waitMs\n\t\t\t\t})\n\t\t\t},\n\t\t\trun: () => input.limit(input.run)\n\t\t})\n\t}\n\n\t/**\n\t * Workers AI `toMarkdown` over any file it accepts - HTML, PDF, docx, xlsx,\n\t * pptx. Returns null for a soft tier miss. Throws `CloudflareRateLimitedError`\n\t * when 429 retries are exhausted so callers can surface a clear capture issue.\n\t */\n\tasync function convertFileWithWorkersAi(input: ConvertFileInput) {\n\t\ttry {\n\t\t\tconst envelope = await runWithCloudflareRateLimit({\n\t\t\t\tevent: 'workers-ai.to-markdown',\n\t\t\t\tlimit: limits.workersAi,\n\t\t\t\tsignal: input.signal,\n\t\t\t\trun: () => {\n\t\t\t\t\tinput.signal?.throwIfAborted()\n\t\t\t\t\treturn transform(input)\n\t\t\t\t}\n\t\t\t})\n\t\t\tinput.signal?.throwIfAborted()\n\t\t\tconst first = envelope.result[0]\n\t\t\tif (\n\t\t\t\t!envelope.success ||\n\t\t\t\t!first ||\n\t\t\t\tfirst.format === 'error' ||\n\t\t\t\t!first.data?.trim()\n\t\t\t) {\n\t\t\t\tlogger.warn('workers-ai.to-markdown.unusable', {\n\t\t\t\t\tsuccess: envelope.success,\n\t\t\t\t\tformat: first?.format ?? '',\n\t\t\t\t\tmime_type: input.mimeType,\n\t\t\t\t\thas_error: Boolean(first?.error)\n\t\t\t\t})\n\t\t\t\treturn null\n\t\t\t}\n\t\t\treturn first.data\n\t\t} catch (error) {\n\t\t\tif (error instanceof CloudflareRateLimitedError) throw error\n\t\t\tlogger.warn('workers-ai.to-markdown.failed', errorLogAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\tfunction convertHtmlWithWorkersAi(input: {\n\t\thtml: string\n\t\thostname: string\n\t\tsignal?: AbortSignal | undefined\n\t}) {\n\t\treturn convertFileWithWorkersAi({\n\t\t\tbody: input.html,\n\t\t\tfilename: 'page.html',\n\t\t\tmimeType: 'text/html',\n\t\t\thostname: input.hostname,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\tasync function renderUrlToMarkdown(url: string, signal?: AbortSignal) {\n\t\ttry {\n\t\t\tconst markdown = await runWithCloudflareRateLimit({\n\t\t\t\tevent: 'browser-rendering',\n\t\t\t\tlimit: limits.browserRendering,\n\t\t\t\tsignal,\n\t\t\t\trun: () =>\n\t\t\t\t\tgetSdk().browserRendering.markdown.create(\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\taccount_id: credentials.accountId,\n\t\t\t\t\t\t\turl,\n\t\t\t\t\t\t\tuserAgent: BROWSER_USER_AGENT,\n\t\t\t\t\t\t\tgotoOptions: { waitUntil: 'load' },\n\t\t\t\t\t\t\twaitForTimeout: BROWSER_RENDERING_SETTLE_MS\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{ timeout: BROWSER_RENDERING_TIMEOUT_MS, signal }\n\t\t\t\t\t)\n\t\t\t})\n\t\t\treturn markdown.trim() || null\n\t\t} catch (error) {\n\t\t\tif (error instanceof CloudflareRateLimitedError) throw error\n\t\t\tlogger.warn('browser-rendering.failed', errorLogAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\treturn {\n\t\tconvertFileWithWorkersAi,\n\t\tconvertHtmlWithWorkersAi,\n\t\trenderUrlToMarkdown\n\t}\n}\n"]}
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,YAAY,CAAA;AACnC,OAAO,EAAiB,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACnE,OAAO,EAGN,sBAAsB,EACtB,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAA;AAC7D,OAAO,EACN,0BAA0B,EAC1B,4BAA4B,EAC5B,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAEpD,OAAO,EAAE,0BAA0B,EAAE,CAAA;AAQrC,MAAM,4BAA4B,GAAG,MAAM,CAAA;AAC3C;;;GAGG;AACH,MAAM,2BAA2B,GAAG,KAAK,CAAA;AAYzC;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAQtC;IACA,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACvC,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAE1D,SAAS,MAAM,CAAC,KAAe;QAC9B,IAAI,oBAAoB,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YACzD,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAA;QAC5B,CAAC;IACF,CAAC;IAED,IAAI,aAAqC,CAAA;IACzC,IAAI,aAAiE,CAAA;IAErE,SAAS,mBAAmB;QAC3B,OAAO,eAAe,CAAC;YACtB,IAAI,EAAE,yBAAyB;YAC/B,WAAW,EAAE;gBACZ,OAAO,EAAE,sCAAsC;aAC/C;YACD,cAAc,EAAE;gBACf,aAAa,EAAE,UAAU,WAAW,CAAC,QAAQ,EAAE;aAC/C;YACD,SAAS,EAAE,MAAM;YACjB,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,MAAM;YACf,SAAS,EAAE;gBACV,UAAU;aACV;SACD,CAAC,CAAA;IACH,CAAC;IAED,SAAS,MAAM;QACd,aAAa,KAAK,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAA;QACpE,OAAO,aAAa,CAAA;IACrB,CAAC;IAED,KAAK,UAAU,SAAS,CAAC,IAAsB;QAC9C,aAAa,KAAK,mBAAmB,EAAE,CAAA;QACvC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,UAAU,CAAC;YAC1C,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3C,SAAS,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE;YAC/C,OAAO,EAAE;gBACR,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC3C,IAAI,EAAE,IAAI,CAAC,QAAQ;iBACnB,CAAC;gBACF,iEAAiE;gBACjE,sDAAsD;gBACtD,iBAAiB,EAAE,IAAI,CAAC,QAAQ;oBAC/B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;oBACvD,CAAC,CAAC,SAAS;aACZ;SACD,CAAC,CAAA;QACF,OAAO,GAAG,CAAC,IAAI,CAAA;IAChB,CAAC;IAED,KAAK,UAAU,0BAA0B,CAAI,KAK5C;QACA,OAAO,4BAA4B,CAAC;YACnC,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,kBAAkB;YAClB,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC;YAC1D,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE;gBACpC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,eAAe,EAAE;oBAC1C,OAAO;oBACP,OAAO,EAAE,MAAM;iBACf,CAAC,CAAA;YACH,CAAC;YACD,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC;SACjC,CAAC,CAAA;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,UAAU,wBAAwB,CAAC,KAAuB;QAC9D,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBACjD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,MAAM,CAAC,SAAS;gBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,GAAG,EAAE;oBACT,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;oBAC9B,OAAO,SAAS,CAAC,KAAK,CAAC,CAAA;gBACxB,CAAC;aACD,CAAC,CAAA;YACF,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;YAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YAChC,IACC,CAAC,QAAQ,CAAC,OAAO;gBACjB,CAAC,KAAK;gBACN,KAAK,CAAC,MAAM,KAAK,OAAO;gBACxB,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,EAClB,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;oBAC9C,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,EAAE;oBAC3B,SAAS,EAAE,KAAK,CAAC,QAAQ;oBACzB,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;iBAChC,CAAC,CAAA;gBACF,OAAO,IAAI,CAAA;YACZ,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,0BAA0B;gBAAE,MAAM,KAAK,CAAA;YAC5D,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YACvE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,SAAS,wBAAwB,CAAC,KAIjC;QACA,OAAO,wBAAwB,CAAC;YAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,UAAU,mBAAmB,CAAC,GAAW,EAAE,MAAoB;QACnE,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;gBACjD,KAAK,EAAE,mBAAmB;gBAC1B,KAAK,EAAE,MAAM,CAAC,gBAAgB;gBAC9B,MAAM;gBACN,GAAG,EAAE,GAAG,EAAE,CACT,MAAM,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CACxC;oBACC,UAAU,EAAE,WAAW,CAAC,SAAS;oBACjC,GAAG;oBACH,SAAS,EAAE,kBAAkB;oBAC7B,WAAW,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE;oBAClC,cAAc,EAAE,2BAA2B;iBAC3C,EACD,EAAE,OAAO,EAAE,4BAA4B,EAAE,MAAM,EAAE,CACjD;aACF,CAAC,CAAA;YACF,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,IAAI,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,0BAA0B;gBAAE,MAAM,KAAK,CAAA;YAC5D,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YAClE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO;QACN,wBAAwB;QACxB,wBAAwB;QACxB,mBAAmB;KACnB,CAAA;AACF,CAAC","sourcesContent":["import Cloudflare from 'cloudflare'\nimport { type ApiError, createApiClient } from 'hanzio/api-wrapper'\nimport {\n\ttype ConcurrencyLimit,\n\ttype CreateConcurrencyLimit,\n\tcreateCloudflareLimits\n} from './limit.js'\nimport { errorLogAttributes, type Logger } from './logger.js'\nimport {\n\tCloudflareRateLimitedError,\n\twithCloudflareRateLimitRetry\n} from './rate-limit-retry.js'\nimport { isReportableApiError } from './report-api-error.js'\nimport { rateLimitFromError } from './retry-after.js'\nimport { toMarkdown } from './to-markdown.js'\nimport { BROWSER_USER_AGENT } from './user-agent.js'\n\nexport { CloudflareRateLimitedError }\n\n/** Read on first use, so getters over a secret store work. */\nexport type CloudflareCredentials = {\n\tapiToken: string\n\taccountId: string\n}\n\nconst BROWSER_RENDERING_TIMEOUT_MS = 60_000\n/**\n * `networkidle0` never settles on Squarespace commerce pages and burned 30s\n * timeouts in the probe; `load` plus a short settle gets the same markdown.\n */\nconst BROWSER_RENDERING_SETTLE_MS = 1_500\n\nexport type ConvertFileInput = {\n\tsignal?: AbortSignal | undefined\n\tbody: string | ArrayBuffer\n\t/** Cloudflare picks its converter from the extension. */\n\tfilename: string\n\tmimeType: string\n\t/** HTML only: base host for relative links. */\n\thostname?: string | undefined\n}\n\n/**\n * The two Cloudflare tiers behind their pools. Rate-limit retries live in\n * `withCloudflareRateLimitRetry` so a 429 releases the slot and trips the\n * shared cooldown before the next try.\n */\nexport function createCloudflareClient(options: {\n\tcredentials: CloudflareCredentials\n\tcreateLimit: CreateConcurrencyLimit\n\t/** Every Workers AI client error except a 429, which is retried instead. */\n\tonApiError?: ((error: ApiError) => void) | undefined\n\t/** Pass timeouts to `onApiError`; default true. */\n\treportTimeouts?: boolean | undefined\n\tlogger: Logger\n}) {\n\tconst { credentials, logger } = options\n\tconst limits = createCloudflareLimits(options.createLimit)\n\n\tfunction report(error: ApiError) {\n\t\tif (isReportableApiError(error, options.reportTimeouts)) {\n\t\t\toptions.onApiError?.(error)\n\t\t}\n\t}\n\n\tlet cloudflareSdk: Cloudflare | undefined\n\tlet toMarkdownApi: ReturnType<typeof createToMarkdownApi> | undefined\n\n\tfunction createToMarkdownApi() {\n\t\treturn createApiClient({\n\t\t\tname: 'cloudflareToMarkdownApi',\n\t\t\tbaseApiUrls: {\n\t\t\t\tdefault: 'https://api.cloudflare.com/client/v4'\n\t\t\t},\n\t\t\tdefaultHeaders: {\n\t\t\t\tAuthorization: `Bearer ${credentials.apiToken}`\n\t\t\t},\n\t\t\ttimeoutMs: 30_000,\n\t\t\tretries: 0,\n\t\t\tonError: report,\n\t\t\tendpoints: {\n\t\t\t\ttoMarkdown\n\t\t\t}\n\t\t})\n\t}\n\n\tfunction getSdk() {\n\t\tcloudflareSdk ??= new Cloudflare({ apiToken: credentials.apiToken })\n\t\treturn cloudflareSdk\n\t}\n\n\tasync function transform(file: ConvertFileInput) {\n\t\ttoMarkdownApi ??= createToMarkdownApi()\n\t\tconst res = await toMarkdownApi.toMarkdown({\n\t\t\t...(file.signal && { signal: file.signal }),\n\t\t\treqParams: { accountId: credentials.accountId },\n\t\t\treqBody: {\n\t\t\t\tfiles: new File([file.body], file.filename, {\n\t\t\t\t\ttype: file.mimeType\n\t\t\t\t}),\n\t\t\t\t// Only the HTML converter takes options; it uses the hostname to\n\t\t\t\t// resolve relative links while it walks the document.\n\t\t\t\tconversionOptions: file.hostname\n\t\t\t\t\t? JSON.stringify({ html: { hostname: file.hostname } })\n\t\t\t\t\t: undefined\n\t\t\t}\n\t\t})\n\t\treturn res.data\n\t}\n\n\tasync function runWithCloudflareRateLimit<T>(input: {\n\t\tevent: string\n\t\tlimit: ConcurrencyLimit\n\t\tsignal?: AbortSignal | undefined\n\t\trun: () => Promise<T>\n\t}) {\n\t\treturn withCloudflareRateLimitRetry({\n\t\t\tevent: input.event,\n\t\t\tsignal: input.signal,\n\t\t\trateLimitFromError,\n\t\t\ttripCooldown: (waitMs) => input.limit.tripCooldown(waitMs),\n\t\t\tonRateLimit: ({ attempt, waitMs }) => {\n\t\t\t\tlogger.warn(`${input.event}.rate_limited`, {\n\t\t\t\t\tattempt,\n\t\t\t\t\twait_ms: waitMs\n\t\t\t\t})\n\t\t\t},\n\t\t\trun: () => input.limit(input.run)\n\t\t})\n\t}\n\n\t/**\n\t * Workers AI `toMarkdown` over any file it accepts - HTML, PDF, docx, xlsx,\n\t * pptx. Returns null for a soft tier miss. Throws `CloudflareRateLimitedError`\n\t * when 429 retries are exhausted so callers can surface a clear capture issue.\n\t */\n\tasync function convertFileWithWorkersAi(input: ConvertFileInput) {\n\t\ttry {\n\t\t\tconst envelope = await runWithCloudflareRateLimit({\n\t\t\t\tevent: 'workers-ai.to-markdown',\n\t\t\t\tlimit: limits.workersAi,\n\t\t\t\tsignal: input.signal,\n\t\t\t\trun: () => {\n\t\t\t\t\tinput.signal?.throwIfAborted()\n\t\t\t\t\treturn transform(input)\n\t\t\t\t}\n\t\t\t})\n\t\t\tinput.signal?.throwIfAborted()\n\t\t\tconst first = envelope.result[0]\n\t\t\tif (\n\t\t\t\t!envelope.success ||\n\t\t\t\t!first ||\n\t\t\t\tfirst.format === 'error' ||\n\t\t\t\t!first.data?.trim()\n\t\t\t) {\n\t\t\t\tlogger.warn('workers-ai.to-markdown.unusable', {\n\t\t\t\t\tsuccess: envelope.success,\n\t\t\t\t\tformat: first?.format ?? '',\n\t\t\t\t\tmime_type: input.mimeType,\n\t\t\t\t\thas_error: Boolean(first?.error)\n\t\t\t\t})\n\t\t\t\treturn null\n\t\t\t}\n\t\t\treturn first.data\n\t\t} catch (error) {\n\t\t\tif (error instanceof CloudflareRateLimitedError) throw error\n\t\t\tlogger.warn('workers-ai.to-markdown.failed', errorLogAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\tfunction convertHtmlWithWorkersAi(input: {\n\t\thtml: string\n\t\thostname: string\n\t\tsignal?: AbortSignal | undefined\n\t}) {\n\t\treturn convertFileWithWorkersAi({\n\t\t\tbody: input.html,\n\t\t\tfilename: 'page.html',\n\t\t\tmimeType: 'text/html',\n\t\t\thostname: input.hostname,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\tasync function renderUrlToMarkdown(url: string, signal?: AbortSignal) {\n\t\ttry {\n\t\t\tconst markdown = await runWithCloudflareRateLimit({\n\t\t\t\tevent: 'browser-rendering',\n\t\t\t\tlimit: limits.browserRendering,\n\t\t\t\tsignal,\n\t\t\t\trun: () =>\n\t\t\t\t\tgetSdk().browserRendering.markdown.create(\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\taccount_id: credentials.accountId,\n\t\t\t\t\t\t\turl,\n\t\t\t\t\t\t\tuserAgent: BROWSER_USER_AGENT,\n\t\t\t\t\t\t\tgotoOptions: { waitUntil: 'load' },\n\t\t\t\t\t\t\twaitForTimeout: BROWSER_RENDERING_SETTLE_MS\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{ timeout: BROWSER_RENDERING_TIMEOUT_MS, signal }\n\t\t\t\t\t)\n\t\t\t})\n\t\t\treturn markdown.trim() || null\n\t\t} catch (error) {\n\t\t\tif (error instanceof CloudflareRateLimitedError) throw error\n\t\t\tlogger.warn('browser-rendering.failed', errorLogAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\treturn {\n\t\tconvertFileWithWorkersAi,\n\t\tconvertHtmlWithWorkersAi,\n\t\trenderUrlToMarkdown\n\t}\n}\n"]}

package/dist/crawl-snapshot.d.ts

@@ -1,31 +1,13 @@
 import type { Logger } from './logger.js';
-/**
- * A crawl recorded once and replayed, for benchmarks. Two runs of a pipeline
- * then read the same pages, so a difference between them is the code: not a
- * site that published a page in between, a 429, or a host that timed out on
- * one run and not the other.
- *
- * Recorded: the network reads - pages as `fetchPageAsMarkdown` returns them,
- * robots.txt and sitemaps - plus anything the caller routes through
- * `throughCrawlSnapshot`. Everything else runs live on every replay, which is
- * the point.
- *
- * Set per job through `withCrawlSnapshot`, never globally: a worker drains
- * other tenants' jobs on the same lane. Never in production.
- */
-export type CrawlSnapshotInput = {
-    mode: 'record' | 'replay';
-    /** `<snapshot>/<host>`: the bench's snapshot label, then the site. */
-    name: string;
-};
-/** The name becomes a path, so it may not climb out of `root`. */
-export declare const CRAWL_SNAPSHOT_NAME_PATTERN: RegExp;
-export declare function withCrawlSnapshot<T>(input: CrawlSnapshotInput | null | undefined, run: () => Promise<T>, options: {
+import { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput } from './schema/index.js';
+export { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput };
+export type CrawlSnapshotOptions = {
     /** Directory the snapshots live under. */
     root: string;
     /** Told about replay misses. */
-    logger?: Logger;
-}): Promise<T>;
+    logger?: Logger | undefined;
+};
+export declare function withCrawlSnapshot<T>(input: CrawlSnapshotInput | null | undefined, run: () => Promise<T>, options: CrawlSnapshotOptions): Promise<T>;
 /**
  * `read` through the active snapshot. Recording stores what the network
  * returned; replaying returns what was stored. A read the recording never made

package/dist/crawl-snapshot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"crawl-snapshot.d.ts","sourceRoot":"","sources":["../src/crawl-snapshot.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAEzC;;;;;;;;;;;;;GAaG;AAEH,MAAM,MAAM,kBAAkB,GAAG;IAChC,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACzB,sEAAsE;IACtE,IAAI,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,kEAAkE;AAClE,eAAO,MAAM,2BAA2B,QACE,CAAA;AAS1C,wBAAgB,iBAAiB,CAAC,CAAC,EAClC,KAAK,EAAE,kBAAkB,GAAG,IAAI,GAAG,SAAS,EAC5C,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACrB,OAAO,EAAE;IACR,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAA;IACZ,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAA;CACf,cAmBD;AAED;;;;;GAKG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC3C,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,CAAC,CAeZ;AAED,8EAA8E;AAC9E,wBAAgB,oBAAoB,IAAI,kBAAkB,GAAG,IAAI,CAGhE"}
+{"version":3,"file":"crawl-snapshot.d.ts","sourceRoot":"","sources":["../src/crawl-snapshot.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EACN,2BAA2B,EAC3B,KAAK,kBAAkB,EACvB,MAAM,mBAAmB,CAAA;AAE1B,OAAO,EAAE,2BAA2B,EAAE,KAAK,kBAAkB,EAAE,CAAA;AAsB/D,MAAM,MAAM,oBAAoB,GAAG;IAClC,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAA;IACZ,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAC3B,CAAA;AAID,wBAAgB,iBAAiB,CAAC,CAAC,EAClC,KAAK,EAAE,kBAAkB,GAAG,IAAI,GAAG,SAAS,EAC5C,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACrB,OAAO,EAAE,oBAAoB,cAmB7B;AAED;;;;;GAKG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC3C,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,CAAC,CAeZ;AAED,8EAA8E;AAC9E,wBAAgB,oBAAoB,IAAI,kBAAkB,GAAG,IAAI,CAGhE"}

package/dist/crawl-snapshot.js

@@ -2,8 +2,8 @@ import { AsyncLocalStorage } from 'node:async_hooks';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 import { sha256Hex } from './hash.js';
-/** The name becomes a path, so it may not climb out of `root`. */
-export const CRAWL_SNAPSHOT_NAME_PATTERN = /^[a-z0-9._-]{1,80}\/[a-z0-9._-]{1,120}$/;
+import { CRAWL_SNAPSHOT_NAME_PATTERN } from './schema/index.js';
+export { CRAWL_SNAPSHOT_NAME_PATTERN };
 const snapshot = new AsyncLocalStorage();
 export function withCrawlSnapshot(input, run, options) {
     if (!input)

package/dist/crawl-snapshot.js.map

@@ -1 +1 @@
-{"version":3,"file":"crawl-snapshot.js","sourceRoot":"","sources":["../src/crawl-snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAwBrC,kEAAkE;AAClE,MAAM,CAAC,MAAM,2BAA2B,GACvC,yCAAyC,CAAA;AAO1C,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAiB,CAAA;AAEvD,MAAM,UAAU,iBAAiB,CAChC,KAA4C,EAC5C,GAAqB,EACrB,OAKC;IAED,IAAI,CAAC,KAAK;QAAE,OAAO,GAAG,EAAE,CAAA;IACxB,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAC1D,CAAC;IACD,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,gCAAgC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IAC9D,CAAC;IAED,OAAO,QAAQ,CAAC,GAAG,CAClB;QACC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;KACtB,EACD,GAAG,CACH,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,GAAW,EACX,IAAsB;IAEtB,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;IAClC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,EAAE,CAAA;IAE1B,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAA;IAC7E,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;QAC7D,IAAI,MAAM,KAAK,IAAI;YAAE,OAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,CAAkB,CAAC,KAAK,CAAA;QACtE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAA;IAC1B,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAClD,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IACrD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,oBAAoB;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;IAClC,OAAO,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AAChE,CAAC","sourcesContent":["import { AsyncLocalStorage } from 'node:async_hooks'\nimport { mkdir, readFile, writeFile } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport { sha256Hex } from './hash.js'\nimport type { Logger } from './logger.js'\n\n/**\n * A crawl recorded once and replayed, for benchmarks. Two runs of a pipeline\n * then read the same pages, so a difference between them is the code: not a\n * site that published a page in between, a 429, or a host that timed out on\n * one run and not the other.\n *\n * Recorded: the network reads - pages as `fetchPageAsMarkdown` returns them,\n * robots.txt and sitemaps - plus anything the caller routes through\n * `throughCrawlSnapshot`. Everything else runs live on every replay, which is\n * the point.\n *\n * Set per job through `withCrawlSnapshot`, never globally: a worker drains\n * other tenants' jobs on the same lane. Never in production.\n */\n\nexport type CrawlSnapshotInput = {\n\tmode: 'record' | 'replay'\n\t/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */\n\tname: string\n}\n\n/** The name becomes a path, so it may not climb out of `root`. */\nexport const CRAWL_SNAPSHOT_NAME_PATTERN =\n\t/^[a-z0-9._-]{1,80}\\/[a-z0-9._-]{1,120}$/\n\ntype CrawlSnapshot = CrawlSnapshotInput & {\n\tdirectory: string\n\tlogger: Logger | undefined\n}\n\nconst snapshot = new AsyncLocalStorage<CrawlSnapshot>()\n\nexport function withCrawlSnapshot<T>(\n\tinput: CrawlSnapshotInput | null | undefined,\n\trun: () => Promise<T>,\n\toptions: {\n\t\t/** Directory the snapshots live under. */\n\t\troot: string\n\t\t/** Told about replay misses. */\n\t\tlogger?: Logger\n\t}\n) {\n\tif (!input) return run()\n\tif (process.env.NODE_ENV === 'production') {\n\t\tthrow new Error('A crawl snapshot is a local bench tool')\n\t}\n\tif (!CRAWL_SNAPSHOT_NAME_PATTERN.test(input.name)) {\n\t\tthrow new Error(`Invalid crawl snapshot name: ${input.name}`)\n\t}\n\n\treturn snapshot.run(\n\t\t{\n\t\t\tmode: input.mode,\n\t\t\tname: input.name,\n\t\t\tdirectory: resolve(options.root, input.name),\n\t\t\tlogger: options.logger\n\t\t},\n\t\trun\n\t)\n}\n\n/**\n * `read` through the active snapshot. Recording stores what the network\n * returned; replaying returns what was stored. A read the recording never made\n * (a code change that discovers a new URL) goes to the network and is added,\n * with a warning, so the snapshot stays complete without hiding the drift.\n */\nexport async function throughCrawlSnapshot<T>(\n\tkey: string,\n\tread: () => Promise<T>\n): Promise<T> {\n\tconst active = snapshot.getStore()\n\tif (!active) return read()\n\n\tconst file = resolve(active.directory, `${sha256Hex(key).slice(0, 32)}.json`)\n\tif (active.mode === 'replay') {\n\t\tconst stored = await readFile(file, 'utf8').catch(() => null)\n\t\tif (stored !== null) return (JSON.parse(stored) as { value: T }).value\n\t\tactive.logger?.warn('crawl-snapshot.miss', { key })\n\t}\n\n\tconst value = await read()\n\tawait mkdir(active.directory, { recursive: true })\n\tawait writeFile(file, JSON.stringify({ key, value }))\n\treturn value\n}\n\n/** The snapshot this job reads or writes, to hand on to the job it queues. */\nexport function currentCrawlSnapshot(): CrawlSnapshotInput | null {\n\tconst active = snapshot.getStore()\n\treturn active ? { mode: active.mode, name: active.name } : null\n}\n"]}
+{"version":3,"file":"crawl-snapshot.js","sourceRoot":"","sources":["../src/crawl-snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EACN,2BAA2B,EAE3B,MAAM,mBAAmB,CAAA;AAE1B,OAAO,EAAE,2BAA2B,EAA2B,CAAA;AA6B/D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAiB,CAAA;AAEvD,MAAM,UAAU,iBAAiB,CAChC,KAA4C,EAC5C,GAAqB,EACrB,OAA6B;IAE7B,IAAI,CAAC,KAAK;QAAE,OAAO,GAAG,EAAE,CAAA;IACxB,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAC1D,CAAC;IACD,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,gCAAgC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IAC9D,CAAC;IAED,OAAO,QAAQ,CAAC,GAAG,CAClB;QACC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;KACtB,EACD,GAAG,CACH,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,GAAW,EACX,IAAsB;IAEtB,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;IAClC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,EAAE,CAAA;IAE1B,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAA;IAC7E,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;QAC7D,IAAI,MAAM,KAAK,IAAI;YAAE,OAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,CAAkB,CAAC,KAAK,CAAA;QACtE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAA;IAC1B,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAClD,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IACrD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,oBAAoB;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;IAClC,OAAO,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AAChE,CAAC","sourcesContent":["import { AsyncLocalStorage } from 'node:async_hooks'\nimport { mkdir, readFile, writeFile } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport { sha256Hex } from './hash.js'\nimport type { Logger } from './logger.js'\nimport {\n\tCRAWL_SNAPSHOT_NAME_PATTERN,\n\ttype CrawlSnapshotInput\n} from './schema/index.js'\n\nexport { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput }\n\n/**\n * A crawl recorded once and replayed, for benchmarks. Two runs of a pipeline\n * then read the same pages, so a difference between them is the code: not a\n * site that published a page in between, a 429, or a host that timed out on\n * one run and not the other.\n *\n * Recorded: the network reads - pages as `fetchPageAsMarkdown` returns them,\n * robots.txt and sitemaps - plus anything the caller routes through\n * `throughCrawlSnapshot`. Everything else runs live on every replay, which is\n * the point.\n *\n * Set per job through `withCrawlSnapshot`, never globally: a worker drains\n * other tenants' jobs on the same lane. Never in production.\n */\n\ntype CrawlSnapshot = CrawlSnapshotInput & {\n\tdirectory: string\n\tlogger: Logger | undefined\n}\n\nexport type CrawlSnapshotOptions = {\n\t/** Directory the snapshots live under. */\n\troot: string\n\t/** Told about replay misses. */\n\tlogger?: Logger | undefined\n}\n\nconst snapshot = new AsyncLocalStorage<CrawlSnapshot>()\n\nexport function withCrawlSnapshot<T>(\n\tinput: CrawlSnapshotInput | null | undefined,\n\trun: () => Promise<T>,\n\toptions: CrawlSnapshotOptions\n) {\n\tif (!input) return run()\n\tif (process.env.NODE_ENV === 'production') {\n\t\tthrow new Error('A crawl snapshot is a local bench tool')\n\t}\n\tif (!CRAWL_SNAPSHOT_NAME_PATTERN.test(input.name)) {\n\t\tthrow new Error(`Invalid crawl snapshot name: ${input.name}`)\n\t}\n\n\treturn snapshot.run(\n\t\t{\n\t\t\tmode: input.mode,\n\t\t\tname: input.name,\n\t\t\tdirectory: resolve(options.root, input.name),\n\t\t\tlogger: options.logger\n\t\t},\n\t\trun\n\t)\n}\n\n/**\n * `read` through the active snapshot. Recording stores what the network\n * returned; replaying returns what was stored. A read the recording never made\n * (a code change that discovers a new URL) goes to the network and is added,\n * with a warning, so the snapshot stays complete without hiding the drift.\n */\nexport async function throughCrawlSnapshot<T>(\n\tkey: string,\n\tread: () => Promise<T>\n): Promise<T> {\n\tconst active = snapshot.getStore()\n\tif (!active) return read()\n\n\tconst file = resolve(active.directory, `${sha256Hex(key).slice(0, 32)}.json`)\n\tif (active.mode === 'replay') {\n\t\tconst stored = await readFile(file, 'utf8').catch(() => null)\n\t\tif (stored !== null) return (JSON.parse(stored) as { value: T }).value\n\t\tactive.logger?.warn('crawl-snapshot.miss', { key })\n\t}\n\n\tconst value = await read()\n\tawait mkdir(active.directory, { recursive: true })\n\tawait writeFile(file, JSON.stringify({ key, value }))\n\treturn value\n}\n\n/** The snapshot this job reads or writes, to hand on to the job it queues. */\nexport function currentCrawlSnapshot(): CrawlSnapshotInput | null {\n\tconst active = snapshot.getStore()\n\treturn active ? { mode: active.mode, name: active.name } : null\n}\n"]}

package/dist/fetch-page.d.ts

@@ -1,5 +1,6 @@
 import type { ApiError } from 'hanzio/api-wrapper';
 import { type CloudflareCredentials } from './client.js';
+import { type CrawlSnapshotInput, type CrawlSnapshotOptions } from './crawl-snapshot.js';
 import { type CreateConcurrencyLimit } from './limit.js';
 import { type Logger } from './logger.js';
 /** Stored markdown ceiling; a page over this is a listing or an upload, not knowledge. */
@@ -99,7 +100,18 @@ export type WebsiteToMarkdownOptions = {
     createLimit?: CreateConcurrencyLimit;
     /** Workers AI client errors other than 429s, for an error tracker. */
     onApiError?: (error: ApiError) => void;
+    /**
+     * Pass Workers AI timeouts to `onApiError`. A timed-out page falls through
+     * to the next tier, so `false` keeps import bursts out of error tracking.
+     * Default true.
+     */
+    reportTimeouts?: boolean;
     logger?: Logger;
+    /**
+     * Where the returned `withCrawlSnapshot` keeps snapshots. Its logger
+     * defaults to `logger`.
+     */
+    crawlSnapshot?: CrawlSnapshotOptions;
     /** The OCR tier for a PDF Workers AI returned nothing for. */
     parsePdf?: PdfParser;
     /**
@@ -128,6 +140,10 @@ export declare function createWebsiteToMarkdown(options: WebsiteToMarkdownOption
     }>;
     convertFileWithWorkersAi: (input: import("./client.js").ConvertFileInput) => Promise<string | null>;
     renderUrlToMarkdown: (url: string, signal?: AbortSignal) => Promise<string | null>;
+    /** The SSRF guard the converter runs, bound to its `logger`. */
+    isHostAllowed: (hostname: string) => Promise<boolean>;
+    /** `withCrawlSnapshot` bound to the `crawlSnapshot` option. */
+    withCrawlSnapshot<T>(input: CrawlSnapshotInput | null | undefined, run: () => Promise<T>): Promise<T>;
 };
 export type WebsiteToMarkdown = ReturnType<typeof createWebsiteToMarkdown>;
 //# sourceMappingURL=fetch-page.d.ts.map

package/dist/fetch-page.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-page.d.ts","sourceRoot":"","sources":["../src/fetch-page.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAElD,OAAO,EACN,KAAK,qBAAqB,EAG1B,MAAM,aAAa,CAAA;AAcpB,OAAO,EAAE,KAAK,sBAAsB,EAAoB,MAAM,YAAY,CAAA;AAC1E,OAAO,EAAsB,KAAK,MAAM,EAAgB,MAAM,aAAa,CAAA;AAsB3E,0FAA0F;AAC1F,eAAO,MAAM,kBAAkB,SAAU,CAAA;AAEzC,MAAM,MAAM,YAAY,GACrB,IAAI,GACJ,WAAW,GACX,MAAM,GACN,SAAS,GACT,WAAW,GACX,aAAa,CAAA;AAChB,MAAM,MAAM,WAAW,GACpB,qBAAqB,GACrB,YAAY,GACZ,mBAAmB,GACnB,cAAc,GACd,iBAAiB,CAAA;AAEpB,MAAM,MAAM,WAAW,GAAG;IACzB,wEAAwE;IACxE,GAAG,EAAE,MAAM,CAAA;IACX,mDAAmD;IACnD,QAAQ,EAAE,MAAM,CAAA;IAChB,iFAAiF;IACjF,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,YAAY,CAAA;IACrB,MAAM,EAAE,WAAW,GAAG,IAAI,CAAA;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,CAAA;IAChB,2EAA2E;IAC3E,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,yDAAyD;IACzD,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,wCAAwC;IACxC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,iIAAiI;IACjI,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;IACrB,oEAAoE;IACpE,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC3C,uDAAuD;IACvD,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,sHAAsH;IACtH,MAAM,EAAE,OAAO,EAAE,CAAA;IACjB;;;;OAIG;IACH,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,0DAA0D;IAC1D,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACxC;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;CACpC,CAAA;AAYD,MAAM,MAAM,QAAQ,GAAG;IACtB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACxC,OAAO,EAAE,YAAY,CAAA;IACrB,MAAM,EAAE,WAAW,GAAG,IAAI,CAAA;IAC1B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACpC,gFAAgF;IAChF,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;CAChC,CAAA;AAyTD,MAAM,MAAM,cAAc,GAAG;IAC5B,GAAG,EAAE,MAAM,CAAA;IACX,8EAA8E;IAC9E,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IAChC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACxC,sDAAsD;IACtD,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IAC5B;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACnC,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACpC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAChC,CAAA;AAED,MAAM,MAAM,SAAS,GAAG,CAAC,KAAK,EAAE;IAC/B,KAAK,EAAE,WAAW,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAChC,KAAK,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAA;AAE5D,MAAM,MAAM,wBAAwB,GAAG;IACtC,4EAA4E;IAC5E,UAAU,EAAE,qBAAqB,CAAA;IACjC;;;;OAIG;IACH,WAAW,CAAC,EAAE,sBAAsB,CAAA;IACpC,sEAAsE;IACtE,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAA;IACtC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,SAAS,CAAA;IACpB;;;OAGG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CACtD,CAAA;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,wBAAwB;iCA6UhE,cAAc,KACnB,OAAO,CAAC,WAAW,CAAC;kCA7QoB;QAC1C,KAAK,EAAE,WAAW,CAAA;QAClB,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;QACtC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;KAChC,KAAG,OAAO,CAAC,QAAQ,CAAC;+BA4amB,MAAM;;;;;;EAkB9C;AAED,MAAM,MAAM,iBAAiB,GAAG,UAAU,CAAC,OAAO,uBAAuB,CAAC,CAAA"}
+{"version":3,"file":"fetch-page.d.ts","sourceRoot":"","sources":["../src/fetch-page.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAElD,OAAO,EACN,KAAK,qBAAqB,EAG1B,MAAM,aAAa,CAAA;AAMpB,OAAO,EACN,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EAIzB,MAAM,qBAAqB,CAAA;AAQ5B,OAAO,EAAE,KAAK,sBAAsB,EAAoB,MAAM,YAAY,CAAA;AAC1E,OAAO,EAAsB,KAAK,MAAM,EAAgB,MAAM,aAAa,CAAA;AAsB3E,0FAA0F;AAC1F,eAAO,MAAM,kBAAkB,SAAU,CAAA;AAEzC,MAAM,MAAM,YAAY,GACrB,IAAI,GACJ,WAAW,GACX,MAAM,GACN,SAAS,GACT,WAAW,GACX,aAAa,CAAA;AAChB,MAAM,MAAM,WAAW,GACpB,qBAAqB,GACrB,YAAY,GACZ,mBAAmB,GACnB,cAAc,GACd,iBAAiB,CAAA;AAEpB,MAAM,MAAM,WAAW,GAAG;IACzB,wEAAwE;IACxE,GAAG,EAAE,MAAM,CAAA;IACX,mDAAmD;IACnD,QAAQ,EAAE,MAAM,CAAA;IAChB,iFAAiF;IACjF,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,YAAY,CAAA;IACrB,MAAM,EAAE,WAAW,GAAG,IAAI,CAAA;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,CAAA;IAChB,2EAA2E;IAC3E,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,yDAAyD;IACzD,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,wCAAwC;IACxC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,iIAAiI;IACjI,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;IACrB,oEAAoE;IACpE,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC3C,uDAAuD;IACvD,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,sHAAsH;IACtH,MAAM,EAAE,OAAO,EAAE,CAAA;IACjB;;;;OAIG;IACH,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,0DAA0D;IAC1D,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACxC;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;CACpC,CAAA;AAYD,MAAM,MAAM,QAAQ,GAAG;IACtB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACxC,OAAO,EAAE,YAAY,CAAA;IACrB,MAAM,EAAE,WAAW,GAAG,IAAI,CAAA;IAC1B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACpC,gFAAgF;IAChF,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;CAChC,CAAA;AAyTD,MAAM,MAAM,cAAc,GAAG;IAC5B,GAAG,EAAE,MAAM,CAAA;IACX,8EAA8E;IAC9E,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IAChC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACxC,sDAAsD;IACtD,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IAC5B;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACnC,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACpC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAChC,CAAA;AAED,MAAM,MAAM,SAAS,GAAG,CAAC,KAAK,EAAE;IAC/B,KAAK,EAAE,WAAW,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAChC,KAAK,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAA;AAE5D,MAAM,MAAM,wBAAwB,GAAG;IACtC,4EAA4E;IAC5E,UAAU,EAAE,qBAAqB,CAAA;IACjC;;;;OAIG;IACH,WAAW,CAAC,EAAE,sBAAsB,CAAA;IACpC,sEAAsE;IACtE,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAA;IACtC;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,aAAa,CAAC,EAAE,oBAAoB,CAAA;IACpC,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,SAAS,CAAA;IACpB;;;OAGG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CACtD,CAAA;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,wBAAwB;iCA8UhE,cAAc,KACnB,OAAO,CAAC,WAAW,CAAC;kCA7QoB;QAC1C,KAAK,EAAE,WAAW,CAAA;QAClB,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;QACtC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;KAChC,KAAG,OAAO,CAAC,QAAQ,CAAC;+BA4amB,MAAM;;;;;;IAiB7C,gEAAgE;8BA7gBtC,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC;IA+gBrD,+DAA+D;sBAC7C,CAAC,SACX,kBAAkB,GAAG,IAAI,GAAG,SAAS,OACvC,MAAM,OAAO,CAAC,CAAC,CAAC;EAYvB;AAED,MAAM,MAAM,iBAAiB,GAAG,UAAU,CAAC,OAAO,uBAAuB,CAAC,CAAA"}

package/dist/fetch-page.js

@@ -1,7 +1,7 @@
 import { parseRetryAfter } from 'hanzio';
 import { CloudflareRateLimitedError, createCloudflareClient } from './client.js';
 import { classifyContentType, convertibleDocumentType } from './content-type.js';
-import { currentCrawlSnapshot, throughCrawlSnapshot } from './crawl-snapshot.js';
+import { currentCrawlSnapshot, throughCrawlSnapshot, withCrawlSnapshot } from './crawl-snapshot.js';
 import { isApplicationShell, isBlankMarkdown, looksLikeHtml } from './detect-markdown.js';
 import { sha256Hex } from './hash.js';
 import { extractHtmlMetadata } from './html-metadata.js';
@@ -305,6 +305,7 @@ export function createWebsiteToMarkdown(options) {
         credentials: options.cloudflare,
         createLimit,
         onApiError: options.onApiError,
+        reportTimeouts: options.reportTimeouts,
         logger
     });
     const hostLimits = createHostLimits(createLimit);
@@ -738,7 +739,21 @@ export function createWebsiteToMarkdown(options) {
         convertDocumentBytes,
         urlToMarkdown,
         convertFileWithWorkersAi: client.convertFileWithWorkersAi,
-        renderUrlToMarkdown: client.renderUrlToMarkdown
+        renderUrlToMarkdown: client.renderUrlToMarkdown,
+        /** The SSRF guard the converter runs, bound to its `logger`. */
+        isHostAllowed: allowHost,
+        /** `withCrawlSnapshot` bound to the `crawlSnapshot` option. */
+        withCrawlSnapshot(input, run) {
+            if (!input)
+                return run();
+            if (!options.crawlSnapshot) {
+                throw new Error('Set crawlSnapshot.root to use crawl snapshots');
+            }
+            return withCrawlSnapshot(input, run, {
+                root: options.crawlSnapshot.root,
+                logger: options.crawlSnapshot.logger ?? logger
+            });
+        }
     };
 }
 //# sourceMappingURL=fetch-page.js.map