website-to-markdown 1.1.0 → 2.0.0

package/src/hash.ts

@@ -1,13 +1,9 @@
 import { createHash } from 'node:crypto'
 
 /**
- * Hex SHA-256 of a UTF-8 string or of raw bytes - the repo's one content-hash
- * function.
- *
- * Used as a change detector (has this text moved since we last derived from
- * it?), as an API-secret digest, and to fingerprint fetched binaries. The value
- * is a stored identity in several tables, so it must stay byte-for-byte what it
- * has always been.
+ * Hex SHA-256 of a UTF-8 string or of raw bytes. `htmlHash` and snapshot file
+ * names are built on it, and callers store them, so it must stay byte-for-byte
+ * what it has always been.
  */
 export function sha256Hex(data: string | Uint8Array): string {
 	return typeof data === 'string'

package/src/html-metadata.ts

@@ -1,3 +1,9 @@
+import {
+	getUniqueValues,
+	isOneOf,
+	normalizeWhitespace,
+	safeJsonParse
+} from 'hanzio'
 import { decodeHtmlEntities } from './html-entities.js'
 
 const TITLE_RE = /<title[^>]*>([\s\S]*?)<\/title>/i
@@ -29,17 +35,15 @@ function attributesOf(tag: string) {
 }
 
 function cleanText(value: string) {
-	return decodeHtmlEntities(value.replace(HTML_TAG_RE, ' '))
-		.replace(/\s+/g, ' ')
-		.trim()
+	return normalizeWhitespace(
+		decodeHtmlEntities(value.replace(HTML_TAG_RE, ' '))
+	)
 }
 
 function absolute(url: string, baseUrl: string) {
 	try {
 		const resolved = new URL(url, baseUrl)
-		if (resolved.protocol !== 'https:' && resolved.protocol !== 'http:') {
-			return null
-		}
+		if (!isOneOf(['https:', 'http:'], resolved.protocol)) return null
 		return resolved.href
 	} catch {
 		return null
@@ -51,15 +55,13 @@ function parseJsonLd(html: string) {
 	for (const match of html.matchAll(JSON_LD_RE)) {
 		const body = match[1]?.trim()
 		if (!body) continue
-		try {
-			const parsed: unknown = JSON.parse(body)
-			// A single script may hold an array of nodes; flatten so the profile
-			// layer can look for `@type` without walking two shapes.
-			if (Array.isArray(parsed)) blocks.push(...parsed)
-			else blocks.push(parsed)
-		} catch {
-			// Hand-written JSON-LD with a trailing comma is common; skip the block.
-		}
+		const parsed = safeJsonParse(body)
+		// Hand-written JSON-LD with a trailing comma is common; skip the block.
+		if (!parsed.ok) continue
+		// A single script may hold an array of nodes; flatten so the profile
+		// layer can look for `@type` without walking two shapes.
+		if (Array.isArray(parsed.data)) blocks.push(...parsed.data)
+		else blocks.push(parsed.data)
 	}
 	return blocks
 }
@@ -82,7 +84,7 @@ export function extractHtmlMetadata(html: string, baseUrl: string) {
 	// `<link rel="alternate" hreflang>`: the site's own declaration of which
 	// URLs are translations of this page. `x-default` is a language picker,
 	// not a translation.
-	const alternates = new Set<string>()
+	const alternates: string[] = []
 	/** Each alternate's language (`da` of `da-DK`); what lets a crawl skip a mirror. */
 	const alternateLanguages: Record<string, string> = {}
 	for (const match of html.matchAll(LINK_TAG_RE)) {
@@ -97,7 +99,7 @@ export function extractHtmlMetadata(html: string, baseUrl: string) {
 		if (rel.includes('alternate') && hreflang && hreflang !== 'x-default') {
 			const href = absolute(attributes.href, baseUrl)
 			const language = hreflang.split('-')[0] ?? ''
-			if (href) alternates.add(href)
+			if (href) alternates.push(href)
 			if (href && /^[a-z]{2}$/.test(language))
 				alternateLanguages[href] = language
 		}
@@ -118,7 +120,7 @@ export function extractHtmlMetadata(html: string, baseUrl: string) {
 		title,
 		lang,
 		canonical,
-		alternates: [...alternates],
+		alternates: getUniqueValues(alternates),
 		alternateLanguages,
 		metaDescription: metaDescription || ogDescription || null,
 		jsonLd: parseJsonLd(html)

package/src/index.ts

@@ -17,6 +17,7 @@ export {
 	withCrawlSnapshot
 } from './crawl-snapshot.js'
 export {
+	type CaptureIssue,
 	createWebsiteToMarkdown,
 	type FetchedPage,
 	type FetchOutcome,
@@ -38,6 +39,7 @@ export {
 } from './limit.js'
 export type { LogAttributes, Logger } from './logger.js'
 export {
+	cleanConvertedMarkdown,
 	htmlImages,
 	htmlLinks,
 	markdownRefs,
@@ -47,6 +49,11 @@ export {
 } from './markdown-refs.js'
 export { createMemoryConcurrencyLimit } from './memory-limit.js'
 export {
+	type Fetch,
+	type GuardedFetchOptions,
+	guardedFetch,
+	type HostGuardOptions,
+	HostNotAllowedError,
 	isBlockedAddress,
 	isHostAllowed,
 	isLocalhostHostname

package/src/limit.ts

@@ -90,7 +90,7 @@ export const URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY = 16
 
 /**
  * A host that answered 429 is read this narrowly from then on. At 16 in
- * flight Shopify refused 117 of Good Habit's 244 reads even after each had
+ * flight Shopify refused 117 of one store's 244 reads even after each had
  * waited out its pause: the pause only delayed the next burst. Calibration
  * knobs - set them from the crawl log's `rate_limited`, not by theory.
  */

package/src/markdown-refs.ts

@@ -1,3 +1,4 @@
+import { compact, getUniqueValues, isOneOf } from 'hanzio'
 import { decodeHtmlEntities } from './html-entities.js'
 
 /**
@@ -31,10 +32,9 @@ export function stripFrontMatter(markdown: string) {
  * Workers AI `toMarkdown` opens every converted PDF and office file with the
  * file name as an H1, a `## Metadata` list (creation dates, uuids,
  * `PDFFormatVersion=1.7`) and a `## Contents` heading, then labels every page
- * `### Page N`. None of it is what the document says, and the fact guard
- * demanded those dates and numbers of every written document - which is how
- * Hotel Hvide Klit's terms came back as 13,000 characters of raw text with
- * the PDF metadata inside.
+ * `### Page N`. None of it is what the document says: a hotel's terms PDF
+ * came back as 13,000 characters with its creation dates and uuids inside,
+ * and a reader of the markdown takes those for facts of the document.
  */
 const DOCUMENT_TITLE_RE =
 	/^#\s+[^\n]*\.(?:pdf|docx?|xlsx?|pptx?|odt|rtf)\s*\r?\n/i
@@ -60,6 +60,94 @@ export function stripDocumentPreamble(markdown: string): string {
 		.trim()
 }
 
+/**
+ * Cloudflare's e-mail obfuscation rewrites every address it finds into
+ * `[\[email protected\]](/cdn-cgi/l/email-protection#<hex>)`, so a converted
+ * page behind it shows no readable address at all. The hex is the address
+ * XOR-ed with its first byte; decoded only when the result is shaped like an
+ * address.
+ */
+const CF_EMAIL_LINK_RE =
+	/\[(?:[^[\]\\]|\\.|\[[^\]]*\])*\]\((?:https?:\/\/[^/)]+)?\/cdn-cgi\/l\/email-protection#([0-9a-fA-F]{8,})\)/g
+
+function decodeCloudflareEmail(hex: string) {
+	const key = Number.parseInt(hex.slice(0, 2), 16)
+	let address = ''
+	for (let index = 2; index + 1 < hex.length; index += 2) {
+		address += String.fromCharCode(
+			Number.parseInt(hex.slice(index, index + 2), 16) ^ key
+		)
+	}
+	return /^[^\s@]+@[^\s@]+\.[^\s@.]+$/.test(address) ? address : null
+}
+
+/**
+ * `<script type="application/ld+json">` survives conversion as a fenced code
+ * block on Yoast and Squarespace sites. It is never prose, and the page already
+ * carries it parsed as `jsonLd`.
+ */
+const FENCED_BLOCK_RE = /^```[^\n]*\n([\s\S]*?)^```[ \t]*$/gm
+const JSON_LD_RE = /schema\.org|["']@context["']\s*:/
+
+/** A page that inlines an SVG file gets its prolog back from the converter as text. */
+const MARKUP_DECLARATION_RE = /<\?xml[^>]*\?>|<!DOCTYPE[^>]*>/gi
+
+/**
+ * A label rendered next to its own value with no space between -
+ * `[Saleshello@example.com](mailto:hello@example.com)`,
+ * `[Support+45 22 81 82 95](tel:+4522818295)` - reads as an invented address.
+ * Matched on the value's shape, so the label's language never matters.
+ */
+const VALUE_LINK_RE = /\[([^\]]+)\]\((mailto|tel):([^)\s]+)\)/gi
+
+function emailStart(label: string, target: string) {
+	const address = (target.split('?')[0] ?? '').toLowerCase()
+	return address && label.toLowerCase().endsWith(address)
+		? label.length - address.length
+		: -1
+}
+
+function phoneStart(label: string, target: string) {
+	const digits = target.replace(/\D/g, '')
+	if (digits.length < 6) return -1
+	const first = label.search(/[+\d]/)
+	if (first < 0) return -1
+	const rest = label.slice(first)
+	return /^[+\d\s().-]+$/.test(rest) && rest.replace(/\D/g, '') === digits
+		? first
+		: -1
+}
+
+/**
+ * Removes what Cloudflare's conversion leaves behind and is never page text:
+ * obfuscated e-mail links are decoded, JSON-LD code fences and markup
+ * declarations dropped, and labels glued to their `mailto:`/`tel:` value
+ * split. Applied to every fetched page; idempotent, so markdown stored by an
+ * older version can be run through it too.
+ */
+export function cleanConvertedMarkdown(markdown: string) {
+	return markdown
+		.replace(CF_EMAIL_LINK_RE, (link, hex: string) => {
+			const address = decodeCloudflareEmail(hex)
+			return address ? `[${address}](mailto:${address})` : link
+		})
+		.replace(FENCED_BLOCK_RE, (block, body: string) =>
+			JSON_LD_RE.test(body) ? '' : block
+		)
+		.replace(MARKUP_DECLARATION_RE, '')
+		.replace(
+			VALUE_LINK_RE,
+			(link, label: string, scheme: string, target: string) => {
+				const start =
+					scheme.toLowerCase() === 'mailto'
+						? emailStart(label, target)
+						: phoneStart(label, target)
+				if (start <= 0 || !/\p{L}/u.test(label[start - 1] ?? '')) return link
+				return `[${label.slice(0, start)} ${label.slice(start)}](${scheme}:${target})`
+			}
+		)
+}
+
 export function titleFromMarkdown(markdown: string) {
 	const heading = markdown.match(/^#\s+(.+)$/m)
 	return heading?.[1]?.trim() || null
@@ -92,13 +180,11 @@ function resolveTarget(target: string, baseUrl: string) {
 		const resolved = new URL(raw, baseUrl)
 		// Drops `mailto:`, `tel:`, `javascript:`, `data:` and every other scheme
 		// discovery cannot crawl; they stay in the markdown itself.
-		if (resolved.protocol !== 'https:' && resolved.protocol !== 'http:') {
-			return null
-		}
+		if (!isOneOf(['https:', 'http:'], resolved.protocol)) return null
 		// Cloudflare's reserved prefix is plumbing, never a page: the e-mail
 		// obfuscation endpoint under it answers 410, which used to count as a
-		// page this site failed to deliver. `clean-pages.ts` decodes the address
-		// out of the link before it is dropped here.
+		// page this site failed to deliver. `cleanConvertedMarkdown` has already
+		// decoded the address out of the link.
 		if (resolved.pathname.startsWith('/cdn-cgi/')) return null
 		return resolved.href
 	} catch {
@@ -106,37 +192,32 @@ function resolveTarget(target: string, baseUrl: string) {
 	}
 }
 
+/** Resolved targets, deduped, in the order they were found. */
+function resolveUnique(targets: (string | undefined)[], baseUrl: string) {
+	return getUniqueValues(
+		compact(
+			targets.map((target) => (target ? resolveTarget(target, baseUrl) : null))
+		)
+	)
+}
+
 /**
  * Absolute, deduped, document-order links and images. Discovery crawls these,
  * so relative targets (`/kontakt`) and CMS-escaped query strings
  * (`?image=x&amp;width=200`) have to come out resolved and decoded.
  */
 export function markdownRefs(markdown: string, baseUrl: string) {
-	const links: string[] = []
-	const images: string[] = []
-	const seenLinks = new Set<string>()
-	const seenImages = new Set<string>()
-
-	const push = (isImage: boolean, target: string) => {
-		const resolved = resolveTarget(target, baseUrl)
-		if (!resolved) return
-		const seen = isImage ? seenImages : seenLinks
-		if (seen.has(resolved)) return
-		seen.add(resolved)
-		;(isImage ? images : links).push(resolved)
-	}
-
-	for (const match of markdown.matchAll(IMAGE_RE)) {
-		if (match[1]) push(true, match[1])
-	}
-	for (const match of markdown.matchAll(LINK_RE)) {
-		if (match[1] !== '!' && match[2]) push(false, match[2])
-	}
-	for (const match of markdown.matchAll(AUTOLINK_RE)) {
-		if (match[1]) push(false, match[1])
+	const images = [...markdown.matchAll(IMAGE_RE)].map((match) => match[1])
+	const links = [
+		...[...markdown.matchAll(LINK_RE)].map((match) =>
+			match[1] === '!' ? undefined : match[2]
+		),
+		...[...markdown.matchAll(AUTOLINK_RE)].map((match) => match[1])
+	]
+	return {
+		links: resolveUnique(links, baseUrl),
+		images: resolveUnique(images, baseUrl)
 	}
-
-	return { links, images }
 }
 
 const ANCHOR_HREF_RE = /<a\b[^>]*?\bhref\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi
@@ -160,38 +241,29 @@ const NON_MARKUP_RE = /<(script|style|template)\b[^>]*>[\s\S]*?<\/\1\s*>/gi
  * reachable if ANY representation mentions it.
  */
 export function htmlLinks(html: string, baseUrl: string): string[] {
-	const links: string[] = []
-	const seen = new Set<string>()
-	for (const match of html
-		.replace(NON_MARKUP_RE, ' ')
-		.matchAll(ANCHOR_HREF_RE)) {
+	const targets = [
+		...html.replace(NON_MARKUP_RE, ' ').matchAll(ANCHOR_HREF_RE)
+	].map((match) => {
 		const raw = match[1]
-		if (!raw) continue
+		if (!raw) return undefined
 		const first = raw[0]
 		const target =
 			(first === '"' || first === "'") && raw.endsWith(first)
 				? raw.slice(1, -1)
 				: raw
-		const resolved = resolveTarget(decodeHtmlEntities(target), baseUrl)
-		if (!resolved || seen.has(resolved)) continue
-		seen.add(resolved)
-		links.push(resolved)
-	}
-	return links
+		return decodeHtmlEntities(target)
+	})
+	return resolveUnique(targets, baseUrl)
 }
 
 /** Conversion may omit an image-only menu; retain original and lazy asset URLs. */
 export function htmlImages(html: string, baseUrl: string): string[] {
-	const images = new Set<string>()
-	for (const tag of html.matchAll(/<img\b[^>]*>/gi)) {
-		for (const attribute of tag[0].matchAll(
-			/\b(?:src|data-src|data-original)\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi
-		)) {
-			const raw = attribute[1]
-			if (!raw) continue
-			const url = resolveTarget(raw.replace(/^["']|["']$/g, ''), baseUrl)
-			if (url) images.add(url)
-		}
-	}
-	return [...images]
+	const targets = [...html.matchAll(/<img\b[^>]*>/gi)].flatMap((tag) =>
+		[
+			...tag[0].matchAll(
+				/\b(?:src|data-src|data-original)\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi
+			)
+		].map((attribute) => attribute[1]?.replace(/^["']|["']$/g, ''))
+	)
+	return resolveUnique(targets, baseUrl)
 }

package/src/memory-limit.ts

@@ -7,7 +7,7 @@ const START_WINDOW_MS = 1_000
  * starts in any second, and a `tripCooldown` pause before the next start.
  *
  * ponytail: per-process only. Several processes sharing one Cloudflare quota
- * need a shared limiter (hellu passes its Redis Lua limiter).
+ * need a shared limiter, e.g. `hanzio/redis`'s `createRedisConcurrencyLimit`.
  */
 export function createMemoryConcurrencyLimit(
 	options: ConcurrencyLimitOptions

package/src/net-guard.ts

@@ -1,4 +1,5 @@
 import { lookup } from 'node:dns/promises'
+import { createCache, isOneOf } from 'hanzio'
 import { errorLogAttributes, type Logger, silentLogger } from './logger.js'
 
 /** One crawl hits the same host hundreds of times; DNS is not cached by Node. */
@@ -116,32 +117,50 @@ export function isBlockedAddress(address: string): boolean {
 
 export function isLocalhostHostname(hostname: string) {
 	const host = hostname.toLowerCase().replace(/^\[|\]$/g, '')
-	return host === 'localhost' || host === '127.0.0.1' || host === '::1'
+	return isOneOf(['localhost', '127.0.0.1', '::1'], host)
 }
 
-const resolutionCache = new Map<string, { allowed: boolean; atMs: number }>()
+const resolutionCache = createCache({
+	defaultTtlMs: RESOLUTION_TTL_MS,
+	maxEntries: RESOLUTION_CACHE_MAX
+})
+
+export type HostGuardOptions = {
+	logger?: Logger | undefined
+	/** Allow `localhost`, `127.0.0.1` and `::1`, e.g. for a dev server. Default false. */
+	allowLocalhost?: boolean | undefined
+}
 
 /**
  * SSRF guard: a customer can put any hostname in the website field, and an
  * internal address behind it would turn the importer into a proxy for our own
- * network. Localhost stays reachable outside production for the dev server.
+ * network. An IP literal (`[fd00::1]` included) is judged as written, without
+ * a DNS lookup.
  */
 export async function isHostAllowed(
 	hostname: string,
-	logger: Logger = silentLogger
+	options: HostGuardOptions = {}
 ) {
-	if (isLocalhostHostname(hostname)) {
-		return process.env.NODE_ENV !== 'production'
-	}
+	const logger = options.logger ?? silentLogger
+	const host = hostname.toLowerCase().replace(/^\[|\]$/g, '')
+	if (isLocalhostHostname(host)) return options.allowLocalhost === true
 
-	const cached = resolutionCache.get(hostname)
-	if (cached && Date.now() - cached.atMs < RESOLUTION_TTL_MS) {
-		return cached.allowed
+	// `dns.lookup` fails on a bracketed literal, and a failed lookup is let
+	// through below, so a literal never reaches it.
+	if (parseIpv4(host) || host.includes(':')) {
+		const allowed = !isBlockedAddress(host)
+		if (!allowed) {
+			logger.warn('net-guard.blocked_address', { hostname, addresses: host })
+		}
+		return allowed
 	}
 
+	const cached = resolutionCache.get<boolean>(host)
+	if (cached !== undefined) return cached
+
 	let allowed = false
 	try {
-		const addresses = await lookup(hostname, { all: true })
+		const addresses = await lookup(host, { all: true })
 		allowed =
 			addresses.length > 0 &&
 			addresses.every((entry) => !isBlockedAddress(entry.address))
@@ -161,10 +180,101 @@ export async function isHostAllowed(
 		return true
 	}
 
-	if (resolutionCache.size >= RESOLUTION_CACHE_MAX) {
-		const oldest = resolutionCache.keys().next().value
-		if (oldest) resolutionCache.delete(oldest)
-	}
-	resolutionCache.set(hostname, { allowed, atMs: Date.now() })
+	resolutionCache.set(host, allowed)
 	return allowed
 }
+
+/** The `fetch` shape the guard wraps and the library accepts as an option. */
+export type Fetch = (
+	input: string | URL,
+	init?: RequestInit
+) => Promise<Response>
+
+/** A request, or one of its redirects, went to a host the guard refuses. */
+export class HostNotAllowedError extends Error {
+	override readonly name = 'HostNotAllowedError'
+
+	constructor(readonly url: string) {
+		super(`Host not allowed: ${url}`)
+	}
+}
+
+export type GuardedFetchOptions = {
+	/** Run on the URL and on every redirect target. Default {@link isHostAllowed}. */
+	isHostAllowed?: ((hostname: string) => Promise<boolean>) | undefined
+	/** Default `globalThis.fetch`, read at call time. */
+	fetch?: Fetch | undefined
+	/** Default 10. */
+	maxRedirects?: number | undefined
+}
+
+const REDIRECT_STATUSES = [301, 302, 303, 307, 308] as const
+const DEFAULT_MAX_REDIRECTS = 10
+
+/**
+ * `fetch` that runs the guard on every hop: `redirect: 'follow'` would take a
+ * public host's 302 to `http://169.254.169.254/` without asking. Returns the
+ * last response and the URL it came from.
+ */
+export async function followRedirects(
+	input: string | URL,
+	init: RequestInit = {},
+	options: GuardedFetchOptions = {}
+): Promise<{ response: Response; url: URL }> {
+	const allow =
+		options.isHostAllowed ?? ((hostname: string) => isHostAllowed(hostname))
+	const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS
+	const follow = (init.redirect ?? 'follow') === 'follow'
+	let url = new URL(input)
+	let request: RequestInit = follow ? { ...init, redirect: 'manual' } : init
+
+	for (let hop = 0; ; hop++) {
+		if (
+			!isOneOf(['http:', 'https:'], url.protocol) ||
+			!(await allow(url.hostname))
+		) {
+			throw new HostNotAllowedError(url.href)
+		}
+		const response = await (options.fetch ?? globalThis.fetch)(url, request)
+		const location = response.headers.get('location')
+		if (!follow || !location || !isOneOf(REDIRECT_STATUSES, response.status)) {
+			return { response, url }
+		}
+		void response.body?.cancel().catch(() => {})
+		if (hop >= maxRedirects) {
+			throw new TypeError(`Too many redirects from ${String(input)}`)
+		}
+
+		const next = new URL(location, url)
+		// What `fetch` itself does on a redirect: credentials stay with their
+		// origin, and a 303 (or a POST answered with 301/302) becomes a GET.
+		if (next.origin !== url.origin) {
+			const headers = new Headers(request.headers)
+			headers.delete('authorization')
+			headers.delete('cookie')
+			request = { ...request, headers }
+		}
+		const method = (request.method ?? 'GET').toUpperCase()
+		if (
+			(response.status === 303 && !isOneOf(['GET', 'HEAD'], method)) ||
+			(isOneOf([301, 302], response.status) && method === 'POST')
+		) {
+			const headers = new Headers(request.headers)
+			headers.delete('content-type')
+			request = { ...request, method: 'GET', body: null, headers }
+		}
+		url = next
+	}
+}
+
+/**
+ * {@link followRedirects} as a drop-in `fetch`. Throws
+ * {@link HostNotAllowedError} when the URL or a redirect is refused.
+ */
+export async function guardedFetch(
+	input: string | URL,
+	init?: RequestInit,
+	options?: GuardedFetchOptions
+): Promise<Response> {
+	return (await followRedirects(input, init, options)).response
+}

package/src/rate-limit-retry.ts

@@ -1,3 +1,12 @@
+import {
+	exponentialBackoff,
+	getAtPath,
+	isNumber,
+	isRecord,
+	isString,
+	safeJsonParse
+} from 'hanzio'
+
 /**
  * Bounded retries for Cloudflare 429s (Workers AI code 971 / Browser Rendering).
  *
@@ -27,28 +36,15 @@ const THROTTLE_971_BACKOFF_MS = 5_000
 const MAX_BACKOFF_MS = 20_000
 
 export function cloudflareErrorCode(error: unknown): number | null {
-	if (typeof error !== 'object' || !error) return null
-	const body =
-		'body' in error && typeof error.body === 'string' ? error.body : null
+	if (!isRecord(error)) return null
 	const bodyJson =
-		'bodyJson' in error && error.bodyJson && typeof error.bodyJson === 'object'
+		error.bodyJson && typeof error.bodyJson === 'object'
 			? error.bodyJson
-			: body
-				? safeJson(body)
+			: isString(error.body) && error.body
+				? safeJsonParse(error.body).data
 				: null
-	if (!bodyJson || typeof bodyJson !== 'object') return null
-	const errors = (bodyJson as { errors?: unknown }).errors
-	if (!Array.isArray(errors) || errors.length === 0) return null
-	const code = (errors[0] as { code?: unknown }).code
-	return typeof code === 'number' ? code : null
-}
-
-function safeJson(raw: string) {
-	try {
-		return JSON.parse(raw) as unknown
-	} catch {
-		return null
-	}
+	const code = getAtPath(bodyJson, ['errors', 0, 'code'])
+	return isNumber(code) ? code : null
 }
 
 /**
@@ -66,7 +62,11 @@ export function rateLimitBackoffMs(
 		cloudflareErrorCode(error) === 971
 			? THROTTLE_971_BACKOFF_MS
 			: DEFAULT_BACKOFF_MS
-	const exponential = floor * 2 ** Math.max(0, attempt - 1)
+	const exponential = exponentialBackoff(attempt, {
+		baseMs: floor,
+		maxMs: MAX_BACKOFF_MS,
+		jitter: false
+	})
 	const base = Math.min(MAX_BACKOFF_MS, retryAfterMs ?? exponential)
 	const jitter = Math.floor(random() * Math.max(250, base * 0.2))
 	return base + jitter

package/src/read-body.ts

@@ -0,0 +1,38 @@
+/**
+ * At most `maxBytes` of a response body, streamed, so a chunked body is bounded
+ * as well as a declared `Content-Length`. `truncated` says the body went on;
+ * the rest is cancelled, never downloaded.
+ */
+export async function readBody(response: Response, maxBytes: number) {
+	const reader = response.body?.getReader()
+	if (!reader) return { bytes: new Uint8Array(0), truncated: false }
+	const chunks: Uint8Array[] = []
+	let size = 0
+	let truncated = false
+	while (true) {
+		const next = await reader.read()
+		if (next.done) break
+		const room = maxBytes - size
+		if (next.value.byteLength > room) {
+			chunks.push(next.value.subarray(0, room))
+			size = maxBytes
+			truncated = true
+			await reader.cancel()
+			break
+		}
+		chunks.push(next.value)
+		size += next.value.byteLength
+	}
+	const bytes = new Uint8Array(size)
+	let offset = 0
+	for (const chunk of chunks) {
+		bytes.set(chunk, offset)
+		offset += chunk.byteLength
+	}
+	return { bytes, truncated }
+}
+
+/** UTF-8, as `Response.text()` decodes whatever the charset header says. */
+export function decodeText(bytes: Uint8Array) {
+	return new TextDecoder().decode(bytes)
+}