website-to-markdown 1.1.1 → 2.0.0

package/dist/net-guard.js.map

@@ -1 +1 @@
-{"version":3,"file":"net-guard.js","sourceRoot":"","sources":["../src/net-guard.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAA;AAC1C,OAAO,EAAE,kBAAkB,EAAe,YAAY,EAAE,MAAM,aAAa,CAAA;AAE3E,iFAAiF;AACjF,MAAM,iBAAiB,GAAG,MAAM,CAAA;AAChC,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAEhC,SAAS,SAAS,CAAC,OAAe;IACjC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACnC,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;QAC1B,IAAI,KAAK,GAAG,GAAG;YAAE,OAAO,IAAI,CAAA;QAC5B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACnB,CAAC;IACD,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,aAAa,CAAC,MAAgB;IACtC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAA;IAC7B,OAAO,CACN,CAAC,KAAK,CAAC,IAAI,iBAAiB;QAC5B,CAAC,KAAK,EAAE;QACR,CAAC,KAAK,GAAG;QACT,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,kBAAkB;QACxD,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,aAAa;QACzC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QACjC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,4BAA4B;QACtD,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;QACxB,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,eAAe;QACxD,CAAC,IAAI,GAAG,CAAC,mCAAmC;KAC5C,CAAA;AACF,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IACnC,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IAC9B,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,OAAO;QACN,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;KAC1C,CAAA;AACF,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IAClC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAA;IACpB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC7B,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7C,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;YACrE,IAAI,CAAC,IAAI;gBAAE,OAAO,IAAI,CAAA;YACtB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;YACpB,SAAQ;QACT,CAAC;QACD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QAC/C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAA;IACvC,CAAC;IACD,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,SAAS,CAAC,OAAe;IACjC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACxC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC/B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAElC,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACvE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAChC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;IAE/D,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAA;IAC7C,IAAI,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC3B,OAAO,CAAC,GAAG,IAAI,EAAE,GAAG,IAAI,KAAK,CAAS,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAA;AACjE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IACtD,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,CAAA;IAC/B,IAAI,IAAI;QAAE,OAAO,aAAa,CAAC,IAAI,CAAC,CAAA;IAEpC,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAA;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IAExB,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,MAAM,CAAA;IACtC,0EAA0E;IAC1E,MAAM,QAAQ,GACb,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC;QAChD,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;IAC1C,IAAI,QAAQ,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG;YAChB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;YACrB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI;YACvB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;YACrB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI;SACvB,CAAA;QACD,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAA,CAAC,aAAa;QACrE,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;QAC5E,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,CACN,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,wBAAwB;QACvD,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,uBAAuB;QACtD,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,qBAAqB;QACpD,CAAC,KAAK,KAAK,MAAM,IAAI,MAAM,KAAK,MAAM,CAAC,CAAC,sBAAsB;KAC9D,CAAA;AACF,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IACnD,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IAC3D,OAAO,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,CAAA;AACtE,CAAC;AAED,MAAM,eAAe,GAAG,IAAI,GAAG,EAA8C,CAAA;AAE7E;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,QAAgB,EAChB,SAAiB,YAAY;IAE7B,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAA;IAC7C,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAC5C,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,IAAI,GAAG,iBAAiB,EAAE,CAAC;QAC5D,OAAO,MAAM,CAAC,OAAO,CAAA;IACtB,CAAC;IAED,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,CAAC;QACJ,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;QACvD,OAAO;YACN,SAAS,CAAC,MAAM,GAAG,CAAC;gBACpB,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAA;QAC7D,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACxC,QAAQ;gBACR,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;aAC5D,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,uEAAuE;QACvE,+CAA+C;QAC/C,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;YACtC,QAAQ;YACR,GAAG,kBAAkB,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAA;QACF,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,IAAI,eAAe,CAAC,IAAI,IAAI,oBAAoB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAA;QAClD,IAAI,MAAM;YAAE,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC3C,CAAC;IACD,eAAe,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;IAC5D,OAAO,OAAO,CAAA;AACf,CAAC","sourcesContent":["import { lookup } from 'node:dns/promises'\nimport { errorLogAttributes, type Logger, silentLogger } from './logger.js'\n\n/** One crawl hits the same host hundreds of times; DNS is not cached by Node. */\nconst RESOLUTION_TTL_MS = 60_000\nconst RESOLUTION_CACHE_MAX = 512\n\nfunction parseIpv4(address: string): number[] | null {\n\tconst parts = address.split('.')\n\tif (parts.length !== 4) return null\n\tconst octets: number[] = []\n\tfor (const part of parts) {\n\t\tif (!/^\\d{1,3}$/.test(part)) return null\n\t\tconst value = Number(part)\n\t\tif (value > 255) return null\n\t\toctets.push(value)\n\t}\n\treturn octets\n}\n\nfunction isBlockedIpv4(octets: number[]) {\n\tconst [a = 0, b = 0] = octets\n\treturn (\n\t\ta === 0 || // \"this network\"\n\t\ta === 10 ||\n\t\ta === 127 ||\n\t\t(a === 100 && b >= 64 && b <= 127) || // CGNAT 100.64/10\n\t\t(a === 169 && b === 254) || // link-local\n\t\t(a === 172 && b >= 16 && b <= 31) ||\n\t\t(a === 192 && b === 0) || // IETF protocol assignments\n\t\t(a === 192 && b === 168) ||\n\t\t(a === 198 && (b === 18 || b === 19)) || // benchmarking\n\t\ta >= 224 // multicast + reserved + broadcast\n\t)\n}\n\nfunction ipv4TailGroups(part: string): number[] | null {\n\tconst octets = parseIpv4(part)\n\tif (!octets) return null\n\treturn [\n\t\t((octets[0] ?? 0) << 8) | (octets[1] ?? 0),\n\t\t((octets[2] ?? 0) << 8) | (octets[3] ?? 0)\n\t]\n}\n\nfunction parseIpv6Side(side: string): number[] | null {\n\tif (!side) return []\n\tconst parts = side.split(':')\n\tconst groups: number[] = []\n\tfor (const [index, part] of parts.entries()) {\n\t\tif (part.includes('.')) {\n\t\t\tconst tail = index === parts.length - 1 ? ipv4TailGroups(part) : null\n\t\t\tif (!tail) return null\n\t\t\tgroups.push(...tail)\n\t\t\tcontinue\n\t\t}\n\t\tif (!/^[0-9a-f]{1,4}$/i.test(part)) return null\n\t\tgroups.push(Number.parseInt(part, 16))\n\t}\n\treturn groups\n}\n\nfunction parseIpv6(address: string): number[] | null {\n\tconst head = address.split('%')[0] ?? ''\n\tif (!head.includes(':')) return null\n\tconst halves = head.split('::')\n\tif (halves.length > 2) return null\n\n\tconst left = parseIpv6Side(halves[0] ?? '')\n\tconst right = halves.length === 2 ? parseIpv6Side(halves[1] ?? '') : []\n\tif (!left || !right) return null\n\tif (halves.length === 1) return left.length === 8 ? left : null\n\n\tconst filled = 8 - left.length - right.length\n\tif (filled < 1) return null\n\treturn [...left, ...new Array<number>(filled).fill(0), ...right]\n}\n\n/**\n * Refuse anything that is not a routable public address. Fails closed: an\n * address we cannot parse is treated as blocked, because the only reason to\n * fetch it is a customer website and those resolve to public addresses.\n */\nexport function isBlockedAddress(address: string): boolean {\n\tconst trimmed = address.trim().replace(/^\\[|\\]$/g, '')\n\tconst ipv4 = parseIpv4(trimmed)\n\tif (ipv4) return isBlockedIpv4(ipv4)\n\n\tconst groups = parseIpv6(trimmed)\n\tif (!groups) return true\n\n\tconst [first = 0, second = 0] = groups\n\t// ::ffff:a.b.c.d and ::a.b.c.d carry an IPv4 address; judge that instead.\n\tconst isMapped =\n\t\tgroups.slice(0, 5).every((group) => group === 0) &&\n\t\t(groups[5] === 0xffff || groups[5] === 0)\n\tif (isMapped) {\n\t\tconst embedded = [\n\t\t\t(groups[6] ?? 0) >> 8,\n\t\t\t(groups[6] ?? 0) & 0xff,\n\t\t\t(groups[7] ?? 0) >> 8,\n\t\t\t(groups[7] ?? 0) & 0xff\n\t\t]\n\t\tif (embedded.every((octet) => octet === 0)) return true // :: and ::1\n\t\tif (embedded[0] === 0 && embedded[1] === 0 && embedded[2] === 0) return true\n\t\treturn isBlockedIpv4(embedded)\n\t}\n\n\treturn (\n\t\t(first & 0xfe00) === 0xfc00 || // fc00::/7 unique local\n\t\t(first & 0xffc0) === 0xfe80 || // fe80::/10 link-local\n\t\t(first & 0xff00) === 0xff00 || // ff00::/8 multicast\n\t\t(first === 0x2001 && second === 0x0db8) // documentation range\n\t)\n}\n\nexport function isLocalhostHostname(hostname: string) {\n\tconst host = hostname.toLowerCase().replace(/^\\[|\\]$/g, '')\n\treturn host === 'localhost' || host === '127.0.0.1' || host === '::1'\n}\n\nconst resolutionCache = new Map<string, { allowed: boolean; atMs: number }>()\n\n/**\n * SSRF guard: a customer can put any hostname in the website field, and an\n * internal address behind it would turn the importer into a proxy for our own\n * network. Localhost stays reachable outside production for the dev server.\n */\nexport async function isHostAllowed(\n\thostname: string,\n\tlogger: Logger = silentLogger\n) {\n\tif (isLocalhostHostname(hostname)) {\n\t\treturn process.env.NODE_ENV !== 'production'\n\t}\n\n\tconst cached = resolutionCache.get(hostname)\n\tif (cached && Date.now() - cached.atMs < RESOLUTION_TTL_MS) {\n\t\treturn cached.allowed\n\t}\n\n\tlet allowed = false\n\ttry {\n\t\tconst addresses = await lookup(hostname, { all: true })\n\t\tallowed =\n\t\t\taddresses.length > 0 &&\n\t\t\taddresses.every((entry) => !isBlockedAddress(entry.address))\n\t\tif (!allowed) {\n\t\t\tlogger.warn('net-guard.blocked_address', {\n\t\t\t\thostname,\n\t\t\t\taddresses: addresses.map((entry) => entry.address).join(',')\n\t\t\t})\n\t\t}\n\t} catch (error) {\n\t\t// DNS failure is a transient network condition, not a policy decision:\n\t\t// let the fetch run and report the real error.\n\t\tlogger.warn('net-guard.lookup_failed', {\n\t\t\thostname,\n\t\t\t...errorLogAttributes(error)\n\t\t})\n\t\treturn true\n\t}\n\n\tif (resolutionCache.size >= RESOLUTION_CACHE_MAX) {\n\t\tconst oldest = resolutionCache.keys().next().value\n\t\tif (oldest) resolutionCache.delete(oldest)\n\t}\n\tresolutionCache.set(hostname, { allowed, atMs: Date.now() })\n\treturn allowed\n}\n"]}
+{"version":3,"file":"net-guard.js","sourceRoot":"","sources":["../src/net-guard.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAA;AAC1C,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAA;AAC7C,OAAO,EAAE,kBAAkB,EAAe,YAAY,EAAE,MAAM,aAAa,CAAA;AAE3E,iFAAiF;AACjF,MAAM,iBAAiB,GAAG,MAAM,CAAA;AAChC,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAEhC,SAAS,SAAS,CAAC,OAAe;IACjC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACnC,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;QAC1B,IAAI,KAAK,GAAG,GAAG;YAAE,OAAO,IAAI,CAAA;QAC5B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACnB,CAAC;IACD,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,aAAa,CAAC,MAAgB;IACtC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAA;IAC7B,OAAO,CACN,CAAC,KAAK,CAAC,IAAI,iBAAiB;QAC5B,CAAC,KAAK,EAAE;QACR,CAAC,KAAK,GAAG;QACT,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,kBAAkB;QACxD,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,aAAa;QACzC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QACjC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,4BAA4B;QACtD,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;QACxB,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,eAAe;QACxD,CAAC,IAAI,GAAG,CAAC,mCAAmC;KAC5C,CAAA;AACF,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IACnC,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IAC9B,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,OAAO;QACN,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;KAC1C,CAAA;AACF,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IAClC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAA;IACpB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC7B,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7C,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;YACrE,IAAI,CAAC,IAAI;gBAAE,OAAO,IAAI,CAAA;YACtB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;YACpB,SAAQ;QACT,CAAC;QACD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QAC/C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAA;IACvC,CAAC;IACD,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,SAAS,CAAC,OAAe;IACjC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACxC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC/B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAElC,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACvE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAChC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;IAE/D,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAA;IAC7C,IAAI,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC3B,OAAO,CAAC,GAAG,IAAI,EAAE,GAAG,IAAI,KAAK,CAAS,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAA;AACjE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IACtD,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,CAAA;IAC/B,IAAI,IAAI;QAAE,OAAO,aAAa,CAAC,IAAI,CAAC,CAAA;IAEpC,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAA;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IAExB,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,MAAM,CAAA;IACtC,0EAA0E;IAC1E,MAAM,QAAQ,GACb,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC;QAChD,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;IAC1C,IAAI,QAAQ,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG;YAChB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;YACrB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI;YACvB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;YACrB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI;SACvB,CAAA;QACD,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAA,CAAC,aAAa;QACrE,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;QAC5E,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,CACN,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,wBAAwB;QACvD,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,uBAAuB;QACtD,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,qBAAqB;QACpD,CAAC,KAAK,KAAK,MAAM,IAAI,MAAM,KAAK,MAAM,CAAC,CAAC,sBAAsB;KAC9D,CAAA;AACF,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IACnD,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IAC3D,OAAO,OAAO,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,KAAK,CAAC,EAAE,IAAI,CAAC,CAAA;AACxD,CAAC;AAED,MAAM,eAAe,GAAG,WAAW,CAAC;IACnC,YAAY,EAAE,iBAAiB;IAC/B,UAAU,EAAE,oBAAoB;CAChC,CAAC,CAAA;AAQF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,QAAgB,EAChB,UAA4B,EAAE;IAE9B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,YAAY,CAAA;IAC7C,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IAC3D,IAAI,mBAAmB,CAAC,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC,cAAc,KAAK,IAAI,CAAA;IAErE,wEAAwE;IACxE,gDAAgD;IAChD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAA;QACvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACxE,CAAC;QACD,OAAO,OAAO,CAAA;IACf,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAU,IAAI,CAAC,CAAA;IACjD,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAA;IAEvC,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,CAAC;QACJ,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;QACnD,OAAO;YACN,SAAS,CAAC,MAAM,GAAG,CAAC;gBACpB,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAA;QAC7D,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACxC,QAAQ;gBACR,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;aAC5D,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,uEAAuE;QACvE,+CAA+C;QAC/C,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;YACtC,QAAQ;YACR,GAAG,kBAAkB,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAA;QACF,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAClC,OAAO,OAAO,CAAA;AACf,CAAC;AAQD,4EAA4E;AAC5E,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAGxB;IAFH,IAAI,GAAG,qBAAqB,CAAA;IAE9C,YAAqB,GAAW;QAC/B,KAAK,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAA;QADb,QAAG,GAAH,GAAG,CAAQ;IAEhC,CAAC;CACD;AAWD,MAAM,iBAAiB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAU,CAAA;AAC5D,MAAM,qBAAqB,GAAG,EAAE,CAAA;AAEhC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,KAAmB,EACnB,OAAoB,EAAE,EACtB,UAA+B,EAAE;IAEjC,MAAM,KAAK,GACV,OAAO,CAAC,aAAa,IAAI,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzE,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,qBAAqB,CAAA;IAClE,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,QAAQ,CAAA;IACvD,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAA;IACxB,IAAI,OAAO,GAAgB,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;IAE1E,KAAK,IAAI,GAAG,GAAG,CAAC,GAAI,GAAG,EAAE,EAAE,CAAC;QAC3B,IACC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC;YAC3C,CAAC,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAC3B,CAAC;YACF,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACxC,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACxE,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QACjD,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1E,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAA;QACzB,CAAC;QACD,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC5C,IAAI,GAAG,IAAI,YAAY,EAAE,CAAC;YACzB,MAAM,IAAI,SAAS,CAAC,2BAA2B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAChE,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QACnC,sEAAsE;QACtE,qEAAqE;QACrE,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YAC5C,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAA;YAC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YACxB,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,CAAA;QAClC,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;QACtD,IACC,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;YAC9D,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,MAAM,CAAC,EAC1D,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YAC5C,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;YAC9B,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;QAC7D,CAAC;QACD,GAAG,GAAG,IAAI,CAAA;IACX,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,KAAmB,EACnB,IAAkB,EAClB,OAA6B;IAE7B,OAAO,CAAC,MAAM,eAAe,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAA;AAC9D,CAAC","sourcesContent":["import { lookup } from 'node:dns/promises'\nimport { createCache, isOneOf } from 'hanzio'\nimport { errorLogAttributes, type Logger, silentLogger } from './logger.js'\n\n/** One crawl hits the same host hundreds of times; DNS is not cached by Node. */\nconst RESOLUTION_TTL_MS = 60_000\nconst RESOLUTION_CACHE_MAX = 512\n\nfunction parseIpv4(address: string): number[] | null {\n\tconst parts = address.split('.')\n\tif (parts.length !== 4) return null\n\tconst octets: number[] = []\n\tfor (const part of parts) {\n\t\tif (!/^\\d{1,3}$/.test(part)) return null\n\t\tconst value = Number(part)\n\t\tif (value > 255) return null\n\t\toctets.push(value)\n\t}\n\treturn octets\n}\n\nfunction isBlockedIpv4(octets: number[]) {\n\tconst [a = 0, b = 0] = octets\n\treturn (\n\t\ta === 0 || // \"this network\"\n\t\ta === 10 ||\n\t\ta === 127 ||\n\t\t(a === 100 && b >= 64 && b <= 127) || // CGNAT 100.64/10\n\t\t(a === 169 && b === 254) || // link-local\n\t\t(a === 172 && b >= 16 && b <= 31) ||\n\t\t(a === 192 && b === 0) || // IETF protocol assignments\n\t\t(a === 192 && b === 168) ||\n\t\t(a === 198 && (b === 18 || b === 19)) || // benchmarking\n\t\ta >= 224 // multicast + reserved + broadcast\n\t)\n}\n\nfunction ipv4TailGroups(part: string): number[] | null {\n\tconst octets = parseIpv4(part)\n\tif (!octets) return null\n\treturn [\n\t\t((octets[0] ?? 0) << 8) | (octets[1] ?? 0),\n\t\t((octets[2] ?? 0) << 8) | (octets[3] ?? 0)\n\t]\n}\n\nfunction parseIpv6Side(side: string): number[] | null {\n\tif (!side) return []\n\tconst parts = side.split(':')\n\tconst groups: number[] = []\n\tfor (const [index, part] of parts.entries()) {\n\t\tif (part.includes('.')) {\n\t\t\tconst tail = index === parts.length - 1 ? ipv4TailGroups(part) : null\n\t\t\tif (!tail) return null\n\t\t\tgroups.push(...tail)\n\t\t\tcontinue\n\t\t}\n\t\tif (!/^[0-9a-f]{1,4}$/i.test(part)) return null\n\t\tgroups.push(Number.parseInt(part, 16))\n\t}\n\treturn groups\n}\n\nfunction parseIpv6(address: string): number[] | null {\n\tconst head = address.split('%')[0] ?? ''\n\tif (!head.includes(':')) return null\n\tconst halves = head.split('::')\n\tif (halves.length > 2) return null\n\n\tconst left = parseIpv6Side(halves[0] ?? '')\n\tconst right = halves.length === 2 ? parseIpv6Side(halves[1] ?? '') : []\n\tif (!left || !right) return null\n\tif (halves.length === 1) return left.length === 8 ? left : null\n\n\tconst filled = 8 - left.length - right.length\n\tif (filled < 1) return null\n\treturn [...left, ...new Array<number>(filled).fill(0), ...right]\n}\n\n/**\n * Refuse anything that is not a routable public address. Fails closed: an\n * address we cannot parse is treated as blocked, because the only reason to\n * fetch it is a customer website and those resolve to public addresses.\n */\nexport function isBlockedAddress(address: string): boolean {\n\tconst trimmed = address.trim().replace(/^\\[|\\]$/g, '')\n\tconst ipv4 = parseIpv4(trimmed)\n\tif (ipv4) return isBlockedIpv4(ipv4)\n\n\tconst groups = parseIpv6(trimmed)\n\tif (!groups) return true\n\n\tconst [first = 0, second = 0] = groups\n\t// ::ffff:a.b.c.d and ::a.b.c.d carry an IPv4 address; judge that instead.\n\tconst isMapped =\n\t\tgroups.slice(0, 5).every((group) => group === 0) &&\n\t\t(groups[5] === 0xffff || groups[5] === 0)\n\tif (isMapped) {\n\t\tconst embedded = [\n\t\t\t(groups[6] ?? 0) >> 8,\n\t\t\t(groups[6] ?? 0) & 0xff,\n\t\t\t(groups[7] ?? 0) >> 8,\n\t\t\t(groups[7] ?? 0) & 0xff\n\t\t]\n\t\tif (embedded.every((octet) => octet === 0)) return true // :: and ::1\n\t\tif (embedded[0] === 0 && embedded[1] === 0 && embedded[2] === 0) return true\n\t\treturn isBlockedIpv4(embedded)\n\t}\n\n\treturn (\n\t\t(first & 0xfe00) === 0xfc00 || // fc00::/7 unique local\n\t\t(first & 0xffc0) === 0xfe80 || // fe80::/10 link-local\n\t\t(first & 0xff00) === 0xff00 || // ff00::/8 multicast\n\t\t(first === 0x2001 && second === 0x0db8) // documentation range\n\t)\n}\n\nexport function isLocalhostHostname(hostname: string) {\n\tconst host = hostname.toLowerCase().replace(/^\\[|\\]$/g, '')\n\treturn isOneOf(['localhost', '127.0.0.1', '::1'], host)\n}\n\nconst resolutionCache = createCache({\n\tdefaultTtlMs: RESOLUTION_TTL_MS,\n\tmaxEntries: RESOLUTION_CACHE_MAX\n})\n\nexport type HostGuardOptions = {\n\tlogger?: Logger | undefined\n\t/** Allow `localhost`, `127.0.0.1` and `::1`, e.g. for a dev server. Default false. */\n\tallowLocalhost?: boolean | undefined\n}\n\n/**\n * SSRF guard: a customer can put any hostname in the website field, and an\n * internal address behind it would turn the importer into a proxy for our own\n * network. An IP literal (`[fd00::1]` included) is judged as written, without\n * a DNS lookup.\n */\nexport async function isHostAllowed(\n\thostname: string,\n\toptions: HostGuardOptions = {}\n) {\n\tconst logger = options.logger ?? silentLogger\n\tconst host = hostname.toLowerCase().replace(/^\\[|\\]$/g, '')\n\tif (isLocalhostHostname(host)) return options.allowLocalhost === true\n\n\t// `dns.lookup` fails on a bracketed literal, and a failed lookup is let\n\t// through below, so a literal never reaches it.\n\tif (parseIpv4(host) || host.includes(':')) {\n\t\tconst allowed = !isBlockedAddress(host)\n\t\tif (!allowed) {\n\t\t\tlogger.warn('net-guard.blocked_address', { hostname, addresses: host })\n\t\t}\n\t\treturn allowed\n\t}\n\n\tconst cached = resolutionCache.get<boolean>(host)\n\tif (cached !== undefined) return cached\n\n\tlet allowed = false\n\ttry {\n\t\tconst addresses = await lookup(host, { all: true })\n\t\tallowed =\n\t\t\taddresses.length > 0 &&\n\t\t\taddresses.every((entry) => !isBlockedAddress(entry.address))\n\t\tif (!allowed) {\n\t\t\tlogger.warn('net-guard.blocked_address', {\n\t\t\t\thostname,\n\t\t\t\taddresses: addresses.map((entry) => entry.address).join(',')\n\t\t\t})\n\t\t}\n\t} catch (error) {\n\t\t// DNS failure is a transient network condition, not a policy decision:\n\t\t// let the fetch run and report the real error.\n\t\tlogger.warn('net-guard.lookup_failed', {\n\t\t\thostname,\n\t\t\t...errorLogAttributes(error)\n\t\t})\n\t\treturn true\n\t}\n\n\tresolutionCache.set(host, allowed)\n\treturn allowed\n}\n\n/** The `fetch` shape the guard wraps and the library accepts as an option. */\nexport type Fetch = (\n\tinput: string | URL,\n\tinit?: RequestInit\n) => Promise<Response>\n\n/** A request, or one of its redirects, went to a host the guard refuses. */\nexport class HostNotAllowedError extends Error {\n\toverride readonly name = 'HostNotAllowedError'\n\n\tconstructor(readonly url: string) {\n\t\tsuper(`Host not allowed: ${url}`)\n\t}\n}\n\nexport type GuardedFetchOptions = {\n\t/** Run on the URL and on every redirect target. Default {@link isHostAllowed}. */\n\tisHostAllowed?: ((hostname: string) => Promise<boolean>) | undefined\n\t/** Default `globalThis.fetch`, read at call time. */\n\tfetch?: Fetch | undefined\n\t/** Default 10. */\n\tmaxRedirects?: number | undefined\n}\n\nconst REDIRECT_STATUSES = [301, 302, 303, 307, 308] as const\nconst DEFAULT_MAX_REDIRECTS = 10\n\n/**\n * `fetch` that runs the guard on every hop: `redirect: 'follow'` would take a\n * public host's 302 to `http://169.254.169.254/` without asking. Returns the\n * last response and the URL it came from.\n */\nexport async function followRedirects(\n\tinput: string | URL,\n\tinit: RequestInit = {},\n\toptions: GuardedFetchOptions = {}\n): Promise<{ response: Response; url: URL }> {\n\tconst allow =\n\t\toptions.isHostAllowed ?? ((hostname: string) => isHostAllowed(hostname))\n\tconst maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS\n\tconst follow = (init.redirect ?? 'follow') === 'follow'\n\tlet url = new URL(input)\n\tlet request: RequestInit = follow ? { ...init, redirect: 'manual' } : init\n\n\tfor (let hop = 0; ; hop++) {\n\t\tif (\n\t\t\t!isOneOf(['http:', 'https:'], url.protocol) ||\n\t\t\t!(await allow(url.hostname))\n\t\t) {\n\t\t\tthrow new HostNotAllowedError(url.href)\n\t\t}\n\t\tconst response = await (options.fetch ?? globalThis.fetch)(url, request)\n\t\tconst location = response.headers.get('location')\n\t\tif (!follow || !location || !isOneOf(REDIRECT_STATUSES, response.status)) {\n\t\t\treturn { response, url }\n\t\t}\n\t\tvoid response.body?.cancel().catch(() => {})\n\t\tif (hop >= maxRedirects) {\n\t\t\tthrow new TypeError(`Too many redirects from ${String(input)}`)\n\t\t}\n\n\t\tconst next = new URL(location, url)\n\t\t// What `fetch` itself does on a redirect: credentials stay with their\n\t\t// origin, and a 303 (or a POST answered with 301/302) becomes a GET.\n\t\tif (next.origin !== url.origin) {\n\t\t\tconst headers = new Headers(request.headers)\n\t\t\theaders.delete('authorization')\n\t\t\theaders.delete('cookie')\n\t\t\trequest = { ...request, headers }\n\t\t}\n\t\tconst method = (request.method ?? 'GET').toUpperCase()\n\t\tif (\n\t\t\t(response.status === 303 && !isOneOf(['GET', 'HEAD'], method)) ||\n\t\t\t(isOneOf([301, 302], response.status) && method === 'POST')\n\t\t) {\n\t\t\tconst headers = new Headers(request.headers)\n\t\t\theaders.delete('content-type')\n\t\t\trequest = { ...request, method: 'GET', body: null, headers }\n\t\t}\n\t\turl = next\n\t}\n}\n\n/**\n * {@link followRedirects} as a drop-in `fetch`. Throws\n * {@link HostNotAllowedError} when the URL or a redirect is refused.\n */\nexport async function guardedFetch(\n\tinput: string | URL,\n\tinit?: RequestInit,\n\toptions?: GuardedFetchOptions\n): Promise<Response> {\n\treturn (await followRedirects(input, init, options)).response\n}\n"]}

package/dist/rate-limit-retry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rate-limit-retry.d.ts","sourceRoot":"","sources":["../src/rate-limit-retry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,qBAAa,0BAA2B,SAAQ,KAAK;IAInD,QAAQ,CAAC,KAAK,EAAE,MAAM;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM;IAJ1B,SAAkB,IAAI,gCAA+B;gBAG3C,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM;CAI1B;AAED,eAAO,MAAM,kCAAkC,IAAI,CAAA;AACnD,yEAAyE;AACzE,eAAO,MAAM,+BAA+B,QAAS,CAAA;AAKrD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAejE;AAUD;;;;GAIG;AACH,wBAAgB,kBAAkB,CACjC,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,MAAM,GAAE,MAAM,MAAoB,UAUlC;AAED,wBAAsB,4BAA4B,CAAC,CAAC,EAAE,KAAK,EAAE;IAC5D,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;IAChC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK;QAAE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAA;IAC9E,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,CAAA;IACrB,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE;QACpB,OAAO,EAAE,MAAM,CAAA;QACf,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,OAAO,CAAA;KACd,KAAK,IAAI,CAAA;IACV,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CAClB,GAAG,OAAO,CAAC,CAAC,CAAC,CAsCb"}
+{"version":3,"file":"rate-limit-retry.d.ts","sourceRoot":"","sources":["../src/rate-limit-retry.ts"],"names":[],"mappings":"AASA;;;;;;;;GAQG;AAEH,qBAAa,0BAA2B,SAAQ,KAAK;IAInD,QAAQ,CAAC,KAAK,EAAE,MAAM;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM;IAJ1B,SAAkB,IAAI,gCAA+B;gBAG3C,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM;CAI1B;AAED,eAAO,MAAM,kCAAkC,IAAI,CAAA;AACnD,yEAAyE;AACzE,eAAO,MAAM,+BAA+B,QAAS,CAAA;AAKrD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAUjE;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CACjC,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,MAAM,GAAE,MAAM,MAAoB,UAclC;AAED,wBAAsB,4BAA4B,CAAC,CAAC,EAAE,KAAK,EAAE;IAC5D,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;IAChC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK;QAAE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAA;IAC9E,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,CAAA;IACrB,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE;QACpB,OAAO,EAAE,MAAM,CAAA;QACf,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,OAAO,CAAA;KACd,KAAK,IAAI,CAAA;IACV,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CAClB,GAAG,OAAO,CAAC,CAAC,CAAC,CAsCb"}

package/dist/rate-limit-retry.js

@@ -1,3 +1,4 @@
+import { exponentialBackoff, getAtPath, isNumber, isRecord, isString, safeJsonParse } from 'hanzio';
 /**
  * Bounded retries for Cloudflare 429s (Workers AI code 971 / Browser Rendering).
  *
@@ -24,29 +25,15 @@ const DEFAULT_BACKOFF_MS = 2_000;
 const THROTTLE_971_BACKOFF_MS = 5_000;
 const MAX_BACKOFF_MS = 20_000;
 export function cloudflareErrorCode(error) {
-    if (typeof error !== 'object' || !error)
+    if (!isRecord(error))
         return null;
-    const body = 'body' in error && typeof error.body === 'string' ? error.body : null;
-    const bodyJson = 'bodyJson' in error && error.bodyJson && typeof error.bodyJson === 'object'
+    const bodyJson = error.bodyJson && typeof error.bodyJson === 'object'
         ? error.bodyJson
-        : body
-            ? safeJson(body)
+        : isString(error.body) && error.body
+            ? safeJsonParse(error.body).data
             : null;
-    if (!bodyJson || typeof bodyJson !== 'object')
-        return null;
-    const errors = bodyJson.errors;
-    if (!Array.isArray(errors) || errors.length === 0)
-        return null;
-    const code = errors[0].code;
-    return typeof code === 'number' ? code : null;
-}
-function safeJson(raw) {
-    try {
-        return JSON.parse(raw);
-    }
-    catch {
-        return null;
-    }
+    const code = getAtPath(bodyJson, ['errors', 0, 'code']);
+    return isNumber(code) ? code : null;
 }
 /**
  * Backoff for a 429: honor `Retry-After` when present, otherwise exponential
@@ -57,7 +44,11 @@ export function rateLimitBackoffMs(error, attempt, retryAfterMs, random = Math.r
     const floor = cloudflareErrorCode(error) === 971
         ? THROTTLE_971_BACKOFF_MS
         : DEFAULT_BACKOFF_MS;
-    const exponential = floor * 2 ** Math.max(0, attempt - 1);
+    const exponential = exponentialBackoff(attempt, {
+        baseMs: floor,
+        maxMs: MAX_BACKOFF_MS,
+        jitter: false
+    });
     const base = Math.min(MAX_BACKOFF_MS, retryAfterMs ?? exponential);
     const jitter = Math.floor(random() * Math.max(250, base * 0.2));
     return base + jitter;

package/dist/rate-limit-retry.js.map

@@ -1 +1 @@
-{"version":3,"file":"rate-limit-retry.js","sourceRoot":"","sources":["../src/rate-limit-retry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,OAAO,0BAA2B,SAAQ,KAAK;IAI1C;IACA;IAJQ,IAAI,GAAG,4BAA4B,CAAA;IAErD,YACU,KAAa,EACb,QAAgB;QAEzB,KAAK,CAAC,GAAG,KAAK,+BAA+B,QAAQ,aAAa,CAAC,CAAA;QAH1D,UAAK,GAAL,KAAK,CAAQ;QACb,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;CACD;AAED,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAA;AACnD,yEAAyE;AACzE,MAAM,CAAC,MAAM,+BAA+B,GAAG,MAAM,CAAA;AACrD,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAChC,MAAM,uBAAuB,GAAG,KAAK,CAAA;AACrC,MAAM,cAAc,GAAG,MAAM,CAAA;AAE7B,MAAM,UAAU,mBAAmB,CAAC,KAAc;IACjD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IACpD,MAAM,IAAI,GACT,MAAM,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;IACtE,MAAM,QAAQ,GACb,UAAU,IAAI,KAAK,IAAI,KAAK,CAAC,QAAQ,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ;QAC1E,CAAC,CAAC,KAAK,CAAC,QAAQ;QAChB,CAAC,CAAC,IAAI;YACL,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC,IAAI,CAAA;IACT,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IAC1D,MAAM,MAAM,GAAI,QAAiC,CAAC,MAAM,CAAA;IACxD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IAC9D,MAAM,IAAI,GAAI,MAAM,CAAC,CAAC,CAAwB,CAAC,IAAI,CAAA;IACnD,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;AAC9C,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC5B,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,CAAA;IAClC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CACjC,KAAc,EACd,OAAe,EACf,YAA2B,EAC3B,SAAuB,IAAI,CAAC,MAAM;IAElC,MAAM,KAAK,GACV,mBAAmB,CAAC,KAAK,CAAC,KAAK,GAAG;QACjC,CAAC,CAAC,uBAAuB;QACzB,CAAC,CAAC,kBAAkB,CAAA;IACtB,MAAM,WAAW,GAAG,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAA;IACzD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,IAAI,WAAW,CAAC,CAAA;IAClE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,GAAG,CAAC,CAAC,CAAA;IAC/D,OAAO,IAAI,GAAG,MAAM,CAAA;AACrB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAI,KAmBrD;IACA,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,kCAAkC,CAAA;IAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,+BAA+B,CAAA;IAClE,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAA;IACjC,MAAM,SAAS,GAAG,GAAG,EAAE,CAAA;IACvB,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,OAAO,IAAI,EAAE,CAAC;QACb,OAAO,EAAE,CAAA;QACT,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;QAC9B,IAAI,CAAC;YACJ,OAAO,MAAM,KAAK,CAAC,GAAG,EAAE,CAAA;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,KAAK,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAA;YACjD,IAAI,CAAC,SAAS;gBAAE,MAAM,KAAK,CAAA;YAE3B,MAAM,MAAM,GAAG,kBAAkB,CAChC,KAAK,EACL,OAAO,EACP,SAAS,CAAC,YAAY,EACtB,KAAK,CAAC,MAAM,CACZ,CAAA;YACD,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;YAC/C,MAAM,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;YAEhC,MAAM,SAAS,GAAG,GAAG,EAAE,GAAG,SAAS,CAAA;YACnC,MAAM,YAAY,GAAG,QAAQ,GAAG,SAAS,CAAA;YACzC,IACC,OAAO,IAAI,WAAW;gBACtB,YAAY,GAAG,MAAM;gBACrB,KAAK,CAAC,MAAM,EAAE,OAAO,EACpB,CAAC;gBACF,MAAM,IAAI,0BAA0B,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;YAC3D,CAAC;YACD,uEAAuE;YACvE,+DAA+D;QAChE,CAAC;IACF,CAAC;AACF,CAAC","sourcesContent":["/**\n * Bounded retries for Cloudflare 429s (Workers AI code 971 / Browser Rendering).\n *\n * The Redis concurrency limiter already waits out a shared cooldown once\n * `tripCooldown` is called. This helper trips that cooldown, then re-enters the\n * limited call so the next attempt only starts after the fleet-wide pause.\n * Hanzio's own retries stay off for these clients: retrying while still holding\n * a slot stampedes the upstream before the cooldown can take effect.\n */\n\nexport class CloudflareRateLimitedError extends Error {\n\toverride readonly name = 'CloudflareRateLimitedError'\n\n\tconstructor(\n\t\treadonly event: string,\n\t\treadonly attempts: number\n\t) {\n\t\tsuper(`${event} rate limit exhausted after ${attempts} attempt(s)`)\n\t}\n}\n\nexport const CLOUDFLARE_RATE_LIMIT_MAX_ATTEMPTS = 4\n/** Wall-clock budget for waiting out 429s inside one conversion call. */\nexport const CLOUDFLARE_RATE_LIMIT_BUDGET_MS = 45_000\nconst DEFAULT_BACKOFF_MS = 2_000\nconst THROTTLE_971_BACKOFF_MS = 5_000\nconst MAX_BACKOFF_MS = 20_000\n\nexport function cloudflareErrorCode(error: unknown): number | null {\n\tif (typeof error !== 'object' || !error) return null\n\tconst body =\n\t\t'body' in error && typeof error.body === 'string' ? error.body : null\n\tconst bodyJson =\n\t\t'bodyJson' in error && error.bodyJson && typeof error.bodyJson === 'object'\n\t\t\t? error.bodyJson\n\t\t\t: body\n\t\t\t\t? safeJson(body)\n\t\t\t\t: null\n\tif (!bodyJson || typeof bodyJson !== 'object') return null\n\tconst errors = (bodyJson as { errors?: unknown }).errors\n\tif (!Array.isArray(errors) || errors.length === 0) return null\n\tconst code = (errors[0] as { code?: unknown }).code\n\treturn typeof code === 'number' ? code : null\n}\n\nfunction safeJson(raw: string) {\n\ttry {\n\t\treturn JSON.parse(raw) as unknown\n\t} catch {\n\t\treturn null\n\t}\n}\n\n/**\n * Backoff for a 429: honor `Retry-After` when present, otherwise exponential\n * with jitter. Cloudflare code 971 (\"throttle your request speed\") gets a\n * longer floor than a bare 429 with no header.\n */\nexport function rateLimitBackoffMs(\n\terror: unknown,\n\tattempt: number,\n\tretryAfterMs: number | null,\n\trandom: () => number = Math.random\n) {\n\tconst floor =\n\t\tcloudflareErrorCode(error) === 971\n\t\t\t? THROTTLE_971_BACKOFF_MS\n\t\t\t: DEFAULT_BACKOFF_MS\n\tconst exponential = floor * 2 ** Math.max(0, attempt - 1)\n\tconst base = Math.min(MAX_BACKOFF_MS, retryAfterMs ?? exponential)\n\tconst jitter = Math.floor(random() * Math.max(250, base * 0.2))\n\treturn base + jitter\n}\n\nexport async function withCloudflareRateLimitRetry<T>(input: {\n\tevent: string\n\tsignal?: AbortSignal | undefined\n\tmaxAttempts?: number\n\tbudgetMs?: number\n\t/**\n\t * `null` = not a rate limit. Otherwise `retryAfterMs` is the parsed\n\t * `Retry-After` header, or `null` when the 429 omitted one.\n\t */\n\trateLimitFromError: (error: unknown) => { retryAfterMs: number | null } | null\n\ttripCooldown: (waitMs: number) => Promise<void>\n\trun: () => Promise<T>\n\tonRateLimit?: (info: {\n\t\tattempt: number\n\t\twaitMs: number\n\t\terror: unknown\n\t}) => void\n\trandom?: () => number\n\tnow?: () => number\n}): Promise<T> {\n\tconst maxAttempts = input.maxAttempts ?? CLOUDFLARE_RATE_LIMIT_MAX_ATTEMPTS\n\tconst budgetMs = input.budgetMs ?? CLOUDFLARE_RATE_LIMIT_BUDGET_MS\n\tconst now = input.now ?? Date.now\n\tconst startedAt = now()\n\tlet attempt = 0\n\n\twhile (true) {\n\t\tattempt++\n\t\tinput.signal?.throwIfAborted()\n\t\ttry {\n\t\t\treturn await input.run()\n\t\t} catch (error) {\n\t\t\tconst rateLimit = input.rateLimitFromError(error)\n\t\t\tif (!rateLimit) throw error\n\n\t\t\tconst waitMs = rateLimitBackoffMs(\n\t\t\t\terror,\n\t\t\t\tattempt,\n\t\t\t\trateLimit.retryAfterMs,\n\t\t\t\tinput.random\n\t\t\t)\n\t\t\tinput.onRateLimit?.({ attempt, waitMs, error })\n\t\t\tawait input.tripCooldown(waitMs)\n\n\t\t\tconst elapsedMs = now() - startedAt\n\t\t\tconst budgetLeftMs = budgetMs - elapsedMs\n\t\t\tif (\n\t\t\t\tattempt >= maxAttempts ||\n\t\t\t\tbudgetLeftMs < waitMs ||\n\t\t\t\tinput.signal?.aborted\n\t\t\t) {\n\t\t\t\tthrow new CloudflareRateLimitedError(input.event, attempt)\n\t\t\t}\n\t\t\t// The next `run()` re-enters the Redis limiter, which blocks until the\n\t\t\t// cooldown we just tripped expires (plus start-rate headroom).\n\t\t}\n\t}\n}\n"]}
+{"version":3,"file":"rate-limit-retry.js","sourceRoot":"","sources":["../src/rate-limit-retry.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,kBAAkB,EAClB,SAAS,EACT,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,MAAM,QAAQ,CAAA;AAEf;;;;;;;;GAQG;AAEH,MAAM,OAAO,0BAA2B,SAAQ,KAAK;IAI1C;IACA;IAJQ,IAAI,GAAG,4BAA4B,CAAA;IAErD,YACU,KAAa,EACb,QAAgB;QAEzB,KAAK,CAAC,GAAG,KAAK,+BAA+B,QAAQ,aAAa,CAAC,CAAA;QAH1D,UAAK,GAAL,KAAK,CAAQ;QACb,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;CACD;AAED,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,CAAA;AACnD,yEAAyE;AACzE,MAAM,CAAC,MAAM,+BAA+B,GAAG,MAAM,CAAA;AACrD,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAChC,MAAM,uBAAuB,GAAG,KAAK,CAAA;AACrC,MAAM,cAAc,GAAG,MAAM,CAAA;AAE7B,MAAM,UAAU,mBAAmB,CAAC,KAAc;IACjD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACjC,MAAM,QAAQ,GACb,KAAK,CAAC,QAAQ,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ;QACnD,CAAC,CAAC,KAAK,CAAC,QAAQ;QAChB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI;YACnC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI;YAChC,CAAC,CAAC,IAAI,CAAA;IACT,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAA;IACvD,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;AACpC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CACjC,KAAc,EACd,OAAe,EACf,YAA2B,EAC3B,SAAuB,IAAI,CAAC,MAAM;IAElC,MAAM,KAAK,GACV,mBAAmB,CAAC,KAAK,CAAC,KAAK,GAAG;QACjC,CAAC,CAAC,uBAAuB;QACzB,CAAC,CAAC,kBAAkB,CAAA;IACtB,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,EAAE;QAC/C,MAAM,EAAE,KAAK;QACb,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,KAAK;KACb,CAAC,CAAA;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,IAAI,WAAW,CAAC,CAAA;IAClE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,GAAG,CAAC,CAAC,CAAA;IAC/D,OAAO,IAAI,GAAG,MAAM,CAAA;AACrB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAI,KAmBrD;IACA,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,kCAAkC,CAAA;IAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,+BAA+B,CAAA;IAClE,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAA;IACjC,MAAM,SAAS,GAAG,GAAG,EAAE,CAAA;IACvB,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,OAAO,IAAI,EAAE,CAAC;QACb,OAAO,EAAE,CAAA;QACT,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAA;QAC9B,IAAI,CAAC;YACJ,OAAO,MAAM,KAAK,CAAC,GAAG,EAAE,CAAA;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,KAAK,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAA;YACjD,IAAI,CAAC,SAAS;gBAAE,MAAM,KAAK,CAAA;YAE3B,MAAM,MAAM,GAAG,kBAAkB,CAChC,KAAK,EACL,OAAO,EACP,SAAS,CAAC,YAAY,EACtB,KAAK,CAAC,MAAM,CACZ,CAAA;YACD,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;YAC/C,MAAM,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;YAEhC,MAAM,SAAS,GAAG,GAAG,EAAE,GAAG,SAAS,CAAA;YACnC,MAAM,YAAY,GAAG,QAAQ,GAAG,SAAS,CAAA;YACzC,IACC,OAAO,IAAI,WAAW;gBACtB,YAAY,GAAG,MAAM;gBACrB,KAAK,CAAC,MAAM,EAAE,OAAO,EACpB,CAAC;gBACF,MAAM,IAAI,0BAA0B,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;YAC3D,CAAC;YACD,uEAAuE;YACvE,+DAA+D;QAChE,CAAC;IACF,CAAC;AACF,CAAC","sourcesContent":["import {\n\texponentialBackoff,\n\tgetAtPath,\n\tisNumber,\n\tisRecord,\n\tisString,\n\tsafeJsonParse\n} from 'hanzio'\n\n/**\n * Bounded retries for Cloudflare 429s (Workers AI code 971 / Browser Rendering).\n *\n * The Redis concurrency limiter already waits out a shared cooldown once\n * `tripCooldown` is called. This helper trips that cooldown, then re-enters the\n * limited call so the next attempt only starts after the fleet-wide pause.\n * Hanzio's own retries stay off for these clients: retrying while still holding\n * a slot stampedes the upstream before the cooldown can take effect.\n */\n\nexport class CloudflareRateLimitedError extends Error {\n\toverride readonly name = 'CloudflareRateLimitedError'\n\n\tconstructor(\n\t\treadonly event: string,\n\t\treadonly attempts: number\n\t) {\n\t\tsuper(`${event} rate limit exhausted after ${attempts} attempt(s)`)\n\t}\n}\n\nexport const CLOUDFLARE_RATE_LIMIT_MAX_ATTEMPTS = 4\n/** Wall-clock budget for waiting out 429s inside one conversion call. */\nexport const CLOUDFLARE_RATE_LIMIT_BUDGET_MS = 45_000\nconst DEFAULT_BACKOFF_MS = 2_000\nconst THROTTLE_971_BACKOFF_MS = 5_000\nconst MAX_BACKOFF_MS = 20_000\n\nexport function cloudflareErrorCode(error: unknown): number | null {\n\tif (!isRecord(error)) return null\n\tconst bodyJson =\n\t\terror.bodyJson && typeof error.bodyJson === 'object'\n\t\t\t? error.bodyJson\n\t\t\t: isString(error.body) && error.body\n\t\t\t\t? safeJsonParse(error.body).data\n\t\t\t\t: null\n\tconst code = getAtPath(bodyJson, ['errors', 0, 'code'])\n\treturn isNumber(code) ? code : null\n}\n\n/**\n * Backoff for a 429: honor `Retry-After` when present, otherwise exponential\n * with jitter. Cloudflare code 971 (\"throttle your request speed\") gets a\n * longer floor than a bare 429 with no header.\n */\nexport function rateLimitBackoffMs(\n\terror: unknown,\n\tattempt: number,\n\tretryAfterMs: number | null,\n\trandom: () => number = Math.random\n) {\n\tconst floor =\n\t\tcloudflareErrorCode(error) === 971\n\t\t\t? THROTTLE_971_BACKOFF_MS\n\t\t\t: DEFAULT_BACKOFF_MS\n\tconst exponential = exponentialBackoff(attempt, {\n\t\tbaseMs: floor,\n\t\tmaxMs: MAX_BACKOFF_MS,\n\t\tjitter: false\n\t})\n\tconst base = Math.min(MAX_BACKOFF_MS, retryAfterMs ?? exponential)\n\tconst jitter = Math.floor(random() * Math.max(250, base * 0.2))\n\treturn base + jitter\n}\n\nexport async function withCloudflareRateLimitRetry<T>(input: {\n\tevent: string\n\tsignal?: AbortSignal | undefined\n\tmaxAttempts?: number\n\tbudgetMs?: number\n\t/**\n\t * `null` = not a rate limit. Otherwise `retryAfterMs` is the parsed\n\t * `Retry-After` header, or `null` when the 429 omitted one.\n\t */\n\trateLimitFromError: (error: unknown) => { retryAfterMs: number | null } | null\n\ttripCooldown: (waitMs: number) => Promise<void>\n\trun: () => Promise<T>\n\tonRateLimit?: (info: {\n\t\tattempt: number\n\t\twaitMs: number\n\t\terror: unknown\n\t}) => void\n\trandom?: () => number\n\tnow?: () => number\n}): Promise<T> {\n\tconst maxAttempts = input.maxAttempts ?? CLOUDFLARE_RATE_LIMIT_MAX_ATTEMPTS\n\tconst budgetMs = input.budgetMs ?? CLOUDFLARE_RATE_LIMIT_BUDGET_MS\n\tconst now = input.now ?? Date.now\n\tconst startedAt = now()\n\tlet attempt = 0\n\n\twhile (true) {\n\t\tattempt++\n\t\tinput.signal?.throwIfAborted()\n\t\ttry {\n\t\t\treturn await input.run()\n\t\t} catch (error) {\n\t\t\tconst rateLimit = input.rateLimitFromError(error)\n\t\t\tif (!rateLimit) throw error\n\n\t\t\tconst waitMs = rateLimitBackoffMs(\n\t\t\t\terror,\n\t\t\t\tattempt,\n\t\t\t\trateLimit.retryAfterMs,\n\t\t\t\tinput.random\n\t\t\t)\n\t\t\tinput.onRateLimit?.({ attempt, waitMs, error })\n\t\t\tawait input.tripCooldown(waitMs)\n\n\t\t\tconst elapsedMs = now() - startedAt\n\t\t\tconst budgetLeftMs = budgetMs - elapsedMs\n\t\t\tif (\n\t\t\t\tattempt >= maxAttempts ||\n\t\t\t\tbudgetLeftMs < waitMs ||\n\t\t\t\tinput.signal?.aborted\n\t\t\t) {\n\t\t\t\tthrow new CloudflareRateLimitedError(input.event, attempt)\n\t\t\t}\n\t\t\t// The next `run()` re-enters the Redis limiter, which blocks until the\n\t\t\t// cooldown we just tripped expires (plus start-rate headroom).\n\t\t}\n\t}\n}\n"]}

package/dist/read-body.d.ts

@@ -0,0 +1,12 @@
+/**
+ * At most `maxBytes` of a response body, streamed, so a chunked body is bounded
+ * as well as a declared `Content-Length`. `truncated` says the body went on;
+ * the rest is cancelled, never downloaded.
+ */
+export declare function readBody(response: Response, maxBytes: number): Promise<{
+    bytes: Uint8Array<ArrayBuffer>;
+    truncated: boolean;
+}>;
+/** UTF-8, as `Response.text()` decodes whatever the charset header says. */
+export declare function decodeText(bytes: Uint8Array): string;
+//# sourceMappingURL=read-body.d.ts.map

package/dist/read-body.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-body.d.ts","sourceRoot":"","sources":["../src/read-body.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM;;;GA2BlE;AAED,4EAA4E;AAC5E,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,UAE3C"}

package/dist/read-body.js

@@ -0,0 +1,40 @@
+/**
+ * At most `maxBytes` of a response body, streamed, so a chunked body is bounded
+ * as well as a declared `Content-Length`. `truncated` says the body went on;
+ * the rest is cancelled, never downloaded.
+ */
+export async function readBody(response, maxBytes) {
+    const reader = response.body?.getReader();
+    if (!reader)
+        return { bytes: new Uint8Array(0), truncated: false };
+    const chunks = [];
+    let size = 0;
+    let truncated = false;
+    while (true) {
+        const next = await reader.read();
+        if (next.done)
+            break;
+        const room = maxBytes - size;
+        if (next.value.byteLength > room) {
+            chunks.push(next.value.subarray(0, room));
+            size = maxBytes;
+            truncated = true;
+            await reader.cancel();
+            break;
+        }
+        chunks.push(next.value);
+        size += next.value.byteLength;
+    }
+    const bytes = new Uint8Array(size);
+    let offset = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return { bytes, truncated };
+}
+/** UTF-8, as `Response.text()` decodes whatever the charset header says. */
+export function decodeText(bytes) {
+    return new TextDecoder().decode(bytes);
+}
+//# sourceMappingURL=read-body.js.map

package/dist/read-body.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-body.js","sourceRoot":"","sources":["../src/read-body.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAAkB,EAAE,QAAgB;IAClE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAA;IACzC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;IAClE,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,SAAS,GAAG,KAAK,CAAA;IACrB,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;QAChC,IAAI,IAAI,CAAC,IAAI;YAAE,MAAK;QACpB,MAAM,IAAI,GAAG,QAAQ,GAAG,IAAI,CAAA;QAC5B,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,EAAE,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAA;YACzC,IAAI,GAAG,QAAQ,CAAA;YACf,SAAS,GAAG,IAAI,CAAA;YAChB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAA;YACrB,MAAK;QACN,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACvB,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAA;IAC9B,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAA;IAClC,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACxB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAA;IAC3B,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAA;AAC5B,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,UAAU,CAAC,KAAiB;IAC3C,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AACvC,CAAC","sourcesContent":["/**\n * At most `maxBytes` of a response body, streamed, so a chunked body is bounded\n * as well as a declared `Content-Length`. `truncated` says the body went on;\n * the rest is cancelled, never downloaded.\n */\nexport async function readBody(response: Response, maxBytes: number) {\n\tconst reader = response.body?.getReader()\n\tif (!reader) return { bytes: new Uint8Array(0), truncated: false }\n\tconst chunks: Uint8Array[] = []\n\tlet size = 0\n\tlet truncated = false\n\twhile (true) {\n\t\tconst next = await reader.read()\n\t\tif (next.done) break\n\t\tconst room = maxBytes - size\n\t\tif (next.value.byteLength > room) {\n\t\t\tchunks.push(next.value.subarray(0, room))\n\t\t\tsize = maxBytes\n\t\t\ttruncated = true\n\t\t\tawait reader.cancel()\n\t\t\tbreak\n\t\t}\n\t\tchunks.push(next.value)\n\t\tsize += next.value.byteLength\n\t}\n\tconst bytes = new Uint8Array(size)\n\tlet offset = 0\n\tfor (const chunk of chunks) {\n\t\tbytes.set(chunk, offset)\n\t\toffset += chunk.byteLength\n\t}\n\treturn { bytes, truncated }\n}\n\n/** UTF-8, as `Response.text()` decodes whatever the charset header says. */\nexport function decodeText(bytes: Uint8Array) {\n\treturn new TextDecoder().decode(bytes)\n}\n"]}

package/dist/retry-after.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"retry-after.d.ts","sourceRoot":"","sources":["../src/retry-after.ts"],"names":[],"mappings":"AAcA;;;;GAIG;AACH,wBAAgB,kBAAkB,CACjC,KAAK,EAAE,OAAO,GACZ;IAAE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,CAmBxC;AAED,0EAA0E;AAC1E,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,iBAOnD"}
+{"version":3,"file":"retry-after.d.ts","sourceRoot":"","sources":["../src/retry-after.ts"],"names":[],"mappings":"AAYA;;;;GAIG;AACH,wBAAgB,kBAAkB,CACjC,KAAK,EAAE,OAAO,GACZ;IAAE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,CAaxC;AAED,0EAA0E;AAC1E,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,iBAQnD"}

package/dist/retry-after.js

@@ -1,12 +1,11 @@
-import { parseRetryAfter } from 'hanzio';
+import { clamp, isFunction, isRecord, parseRetryAfter } from 'hanzio';
 const DEFAULT_RETRY_AFTER_MS = 1_000;
 const MAX_RETRY_AFTER_MS = 5 * 60 * 1_000;
 function headerGet(headers, name) {
-    if (!headers || typeof headers !== 'object')
+    if (!isRecord(headers))
         return null;
-    if (typeof headers.get === 'function') {
+    if (isFunction(headers.get))
         return headers.get(name);
-    }
     const record = headers;
     return record[name] ?? record[name.toLowerCase()] ?? null;
 }
@@ -16,18 +15,15 @@ function headerGet(headers, name) {
  * that need a wait should fall back to exponential backoff).
  */
 export function rateLimitFromError(error) {
-    if (typeof error !== 'object' || !error)
+    if (!isRecord(error) || error.status !== 429)
         return null;
-    const status = 'status' in error && typeof error.status === 'number' ? error.status : null;
-    if (status !== 429)
-        return null;
-    const raw = headerGet('headers' in error ? error.headers : null, 'retry-after') ??
-        headerGet('responseHeaders' in error ? error.responseHeaders : null, 'retry-after');
+    const raw = headerGet(error.headers, 'retry-after') ??
+        headerGet(error.responseHeaders, 'retry-after');
     const parsed = parseRetryAfter(raw);
     return {
         retryAfterMs: parsed === undefined
             ? null
-            : Math.min(MAX_RETRY_AFTER_MS, Math.ceil(parsed))
+            : clamp(Math.ceil(parsed), 0, MAX_RETRY_AFTER_MS)
     };
 }
 /** Wait ms for a 429, with a 1s default when `Retry-After` is missing. */
@@ -35,6 +31,6 @@ export function retryAfterMsFromError(error) {
     const info = rateLimitFromError(error);
     if (!info)
         return null;
-    return Math.min(MAX_RETRY_AFTER_MS, Math.ceil(info.retryAfterMs ?? DEFAULT_RETRY_AFTER_MS));
+    return clamp(Math.ceil(info.retryAfterMs ?? DEFAULT_RETRY_AFTER_MS), 0, MAX_RETRY_AFTER_MS);
 }
 //# sourceMappingURL=retry-after.js.map

package/dist/retry-after.js.map

@@ -1 +1 @@
-{"version":3,"file":"retry-after.js","sourceRoot":"","sources":["../src/retry-after.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAA;AAExC,MAAM,sBAAsB,GAAG,KAAK,CAAA;AACpC,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;AAEzC,SAAS,SAAS,CAAC,OAAgB,EAAE,IAAY;IAChD,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IACxD,IAAI,OAAQ,OAA6B,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QAC9D,OAAQ,OAAmD,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACtE,CAAC;IACD,MAAM,MAAM,GAAG,OAAiC,CAAA;IAChD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,CAAA;AAC1D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CACjC,KAAc;IAEd,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IACpD,MAAM,MAAM,GACX,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;IAC5E,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,IAAI,CAAA;IAE/B,MAAM,GAAG,GACR,SAAS,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,aAAa,CAAC;QACnE,SAAS,CACR,iBAAiB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EACzD,aAAa,CACb,CAAA;IACF,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAA;IACnC,OAAO;QACN,YAAY,EACX,MAAM,KAAK,SAAS;YACnB,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;KACnD,CAAA;AACF,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,qBAAqB,CAAC,KAAc;IACnD,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAA;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,OAAO,IAAI,CAAC,GAAG,CACd,kBAAkB,EAClB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,sBAAsB,CAAC,CACtD,CAAA;AACF,CAAC","sourcesContent":["import { parseRetryAfter } from 'hanzio'\n\nconst DEFAULT_RETRY_AFTER_MS = 1_000\nconst MAX_RETRY_AFTER_MS = 5 * 60 * 1_000\n\nfunction headerGet(headers: unknown, name: string): string | null {\n\tif (!headers || typeof headers !== 'object') return null\n\tif (typeof (headers as { get?: unknown }).get === 'function') {\n\t\treturn (headers as { get: (key: string) => string | null }).get(name)\n\t}\n\tconst record = headers as Record<string, string>\n\treturn record[name] ?? record[name.toLowerCase()] ?? null\n}\n\n/**\n * `null` when the error is not an HTTP 429. Otherwise `retryAfterMs` is the\n * parsed `Retry-After` header, or `null` when the 429 omitted one (callers\n * that need a wait should fall back to exponential backoff).\n */\nexport function rateLimitFromError(\n\terror: unknown\n): { retryAfterMs: number | null } | null {\n\tif (typeof error !== 'object' || !error) return null\n\tconst status =\n\t\t'status' in error && typeof error.status === 'number' ? error.status : null\n\tif (status !== 429) return null\n\n\tconst raw =\n\t\theaderGet('headers' in error ? error.headers : null, 'retry-after') ??\n\t\theaderGet(\n\t\t\t'responseHeaders' in error ? error.responseHeaders : null,\n\t\t\t'retry-after'\n\t\t)\n\tconst parsed = parseRetryAfter(raw)\n\treturn {\n\t\tretryAfterMs:\n\t\t\tparsed === undefined\n\t\t\t\t? null\n\t\t\t\t: Math.min(MAX_RETRY_AFTER_MS, Math.ceil(parsed))\n\t}\n}\n\n/** Wait ms for a 429, with a 1s default when `Retry-After` is missing. */\nexport function retryAfterMsFromError(error: unknown) {\n\tconst info = rateLimitFromError(error)\n\tif (!info) return null\n\treturn Math.min(\n\t\tMAX_RETRY_AFTER_MS,\n\t\tMath.ceil(info.retryAfterMs ?? DEFAULT_RETRY_AFTER_MS)\n\t)\n}\n"]}
+{"version":3,"file":"retry-after.js","sourceRoot":"","sources":["../src/retry-after.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAA;AAErE,MAAM,sBAAsB,GAAG,KAAK,CAAA;AACpC,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;AAEzC,SAAS,SAAS,CAAC,OAAgB,EAAE,IAAY;IAChD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAA;IACnC,IAAI,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,OAAiC,CAAA;IAChD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,CAAA;AAC1D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CACjC,KAAc;IAEd,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,IAAI,CAAA;IAEzD,MAAM,GAAG,GACR,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,aAAa,CAAC;QACvC,SAAS,CAAC,KAAK,CAAC,eAAe,EAAE,aAAa,CAAC,CAAA;IAChD,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAA;IACnC,OAAO;QACN,YAAY,EACX,MAAM,KAAK,SAAS;YACnB,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,kBAAkB,CAAC;KACnD,CAAA;AACF,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,qBAAqB,CAAC,KAAc;IACnD,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAA;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,OAAO,KAAK,CACX,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,sBAAsB,CAAC,EACtD,CAAC,EACD,kBAAkB,CAClB,CAAA;AACF,CAAC","sourcesContent":["import { clamp, isFunction, isRecord, parseRetryAfter } from 'hanzio'\n\nconst DEFAULT_RETRY_AFTER_MS = 1_000\nconst MAX_RETRY_AFTER_MS = 5 * 60 * 1_000\n\nfunction headerGet(headers: unknown, name: string): string | null {\n\tif (!isRecord(headers)) return null\n\tif (isFunction(headers.get)) return headers.get(name)\n\tconst record = headers as Record<string, string>\n\treturn record[name] ?? record[name.toLowerCase()] ?? null\n}\n\n/**\n * `null` when the error is not an HTTP 429. Otherwise `retryAfterMs` is the\n * parsed `Retry-After` header, or `null` when the 429 omitted one (callers\n * that need a wait should fall back to exponential backoff).\n */\nexport function rateLimitFromError(\n\terror: unknown\n): { retryAfterMs: number | null } | null {\n\tif (!isRecord(error) || error.status !== 429) return null\n\n\tconst raw =\n\t\theaderGet(error.headers, 'retry-after') ??\n\t\theaderGet(error.responseHeaders, 'retry-after')\n\tconst parsed = parseRetryAfter(raw)\n\treturn {\n\t\tretryAfterMs:\n\t\t\tparsed === undefined\n\t\t\t\t? null\n\t\t\t\t: clamp(Math.ceil(parsed), 0, MAX_RETRY_AFTER_MS)\n\t}\n}\n\n/** Wait ms for a 429, with a 1s default when `Retry-After` is missing. */\nexport function retryAfterMsFromError(error: unknown) {\n\tconst info = rateLimitFromError(error)\n\tif (!info) return null\n\treturn clamp(\n\t\tMath.ceil(info.retryAfterMs ?? DEFAULT_RETRY_AFTER_MS),\n\t\t0,\n\t\tMAX_RETRY_AFTER_MS\n\t)\n}\n"]}

package/dist/schema/index.d.ts

@@ -9,4 +9,13 @@ export type CrawlSnapshotInput = {
     /** `<snapshot>/<host>`: the bench's snapshot label, then the site. */
     name: string;
 };
+/** Every `FetchedPage.outcome`, for narrowing a value read back from storage. */
+export declare const FETCH_OUTCOMES: readonly ["ok", "unchanged", "gone", "blocked", "transient", "unsupported"];
+export type FetchOutcome = (typeof FETCH_OUTCOMES)[number];
+/** Every `FetchedPage.source`. `firecrawl-parse` is the `parsePdf` tier, whatever parser it is. */
+export declare const FETCH_SOURCES: readonly ["content-negotiation", "workers-ai", "browser-rendering", "not-modified", "firecrawl-parse"];
+export type FetchSource = (typeof FETCH_SOURCES)[number];
+/** Every `FetchedPage.captureIssue`: why a page came back incomplete or not at all. */
+export declare const CAPTURE_ISSUES: readonly ["markdown_limit", "html_byte_limit", "document_byte_limit", "workers_ai_rate_limited", "browser_rendering_rate_limited", "render_incomplete", "pdf_page_limit_unverified", "pdf_extraction_failed"];
+export type CaptureIssue = (typeof CAPTURE_ISSUES)[number];
 //# sourceMappingURL=index.d.ts.map

package/dist/schema/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/schema/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,0GAA0G;AAC1G,eAAO,MAAM,2BAA2B,QACE,CAAA;AAE1C,MAAM,MAAM,kBAAkB,GAAG;IAChC,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACzB,sEAAsE;IACtE,IAAI,EAAE,MAAM,CAAA;CACZ,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/schema/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,0GAA0G;AAC1G,eAAO,MAAM,2BAA2B,QACE,CAAA;AAE1C,MAAM,MAAM,kBAAkB,GAAG;IAChC,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACzB,sEAAsE;IACtE,IAAI,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,iFAAiF;AACjF,eAAO,MAAM,cAAc,6EAOjB,CAAA;AACV,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAA;AAE1D,mGAAmG;AACnG,eAAO,MAAM,aAAa,wGAMhB,CAAA;AACV,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAA;AAExD,uFAAuF;AACvF,eAAO,MAAM,cAAc,+MASjB,CAAA;AACV,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAA"}

package/dist/schema/index.js

@@ -4,4 +4,32 @@
  */
 /** A crawl snapshot name, `<snapshot>/<host>`; it becomes a path, so it may not climb out of the root. */
 export const CRAWL_SNAPSHOT_NAME_PATTERN = /^[a-z0-9._-]{1,80}\/[a-z0-9._-]{1,120}$/;
+/** Every `FetchedPage.outcome`, for narrowing a value read back from storage. */
+export const FETCH_OUTCOMES = [
+    'ok',
+    'unchanged',
+    'gone',
+    'blocked',
+    'transient',
+    'unsupported'
+];
+/** Every `FetchedPage.source`. `firecrawl-parse` is the `parsePdf` tier, whatever parser it is. */
+export const FETCH_SOURCES = [
+    'content-negotiation',
+    'workers-ai',
+    'browser-rendering',
+    'not-modified',
+    'firecrawl-parse'
+];
+/** Every `FetchedPage.captureIssue`: why a page came back incomplete or not at all. */
+export const CAPTURE_ISSUES = [
+    'markdown_limit',
+    'html_byte_limit',
+    'document_byte_limit',
+    'workers_ai_rate_limited',
+    'browser_rendering_rate_limited',
+    'render_incomplete',
+    'pdf_page_limit_unverified',
+    'pdf_extraction_failed'
+];
 //# sourceMappingURL=index.js.map

package/dist/schema/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/schema/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,0GAA0G;AAC1G,MAAM,CAAC,MAAM,2BAA2B,GACvC,yCAAyC,CAAA","sourcesContent":["/**\n * Values a shared schema can import without the runtime: no Node built-ins,\n * no peer libraries.\n */\n\n/** A crawl snapshot name, `<snapshot>/<host>`; it becomes a path, so it may not climb out of the root. */\nexport const CRAWL_SNAPSHOT_NAME_PATTERN =\n\t/^[a-z0-9._-]{1,80}\\/[a-z0-9._-]{1,120}$/\n\nexport type CrawlSnapshotInput = {\n\tmode: 'record' | 'replay'\n\t/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */\n\tname: string\n}\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/schema/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,0GAA0G;AAC1G,MAAM,CAAC,MAAM,2BAA2B,GACvC,yCAAyC,CAAA;AAQ1C,iFAAiF;AACjF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC7B,IAAI;IACJ,WAAW;IACX,MAAM;IACN,SAAS;IACT,WAAW;IACX,aAAa;CACJ,CAAA;AAGV,mGAAmG;AACnG,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B,qBAAqB;IACrB,YAAY;IACZ,mBAAmB;IACnB,cAAc;IACd,iBAAiB;CACR,CAAA;AAGV,uFAAuF;AACvF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC7B,gBAAgB;IAChB,iBAAiB;IACjB,qBAAqB;IACrB,yBAAyB;IACzB,gCAAgC;IAChC,mBAAmB;IACnB,2BAA2B;IAC3B,uBAAuB;CACd,CAAA","sourcesContent":["/**\n * Values a shared schema can import without the runtime: no Node built-ins,\n * no peer libraries.\n */\n\n/** A crawl snapshot name, `<snapshot>/<host>`; it becomes a path, so it may not climb out of the root. */\nexport const CRAWL_SNAPSHOT_NAME_PATTERN =\n\t/^[a-z0-9._-]{1,80}\\/[a-z0-9._-]{1,120}$/\n\nexport type CrawlSnapshotInput = {\n\tmode: 'record' | 'replay'\n\t/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */\n\tname: string\n}\n\n/** Every `FetchedPage.outcome`, for narrowing a value read back from storage. */\nexport const FETCH_OUTCOMES = [\n\t'ok',\n\t'unchanged',\n\t'gone',\n\t'blocked',\n\t'transient',\n\t'unsupported'\n] as const\nexport type FetchOutcome = (typeof FETCH_OUTCOMES)[number]\n\n/** Every `FetchedPage.source`. `firecrawl-parse` is the `parsePdf` tier, whatever parser it is. */\nexport const FETCH_SOURCES = [\n\t'content-negotiation',\n\t'workers-ai',\n\t'browser-rendering',\n\t'not-modified',\n\t'firecrawl-parse'\n] as const\nexport type FetchSource = (typeof FETCH_SOURCES)[number]\n\n/** Every `FetchedPage.captureIssue`: why a page came back incomplete or not at all. */\nexport const CAPTURE_ISSUES = [\n\t'markdown_limit',\n\t'html_byte_limit',\n\t'document_byte_limit',\n\t'workers_ai_rate_limited',\n\t'browser_rendering_rate_limited',\n\t'render_incomplete',\n\t'pdf_page_limit_unverified',\n\t'pdf_extraction_failed'\n] as const\nexport type CaptureIssue = (typeof CAPTURE_ISSUES)[number]\n"]}

package/dist/sitemap/index.d.ts

@@ -1,3 +1,13 @@
+import { type Fetch } from '../net-guard.js';
+export type { Fetch };
+export type SitemapOptions = {
+    /**
+     * The `fetch` robots.txt and sitemaps go through. Default
+     * {@link guardedFetch} with the default guard (public addresses only, every
+     * redirect checked); pass `converter.guardedFetch` to use the converter's.
+     */
+    fetch?: Fetch | undefined;
+};
 /** Parses one sitemap document into page entries and nested sitemap URLs. */
 export declare function parseSitemapXml(xml: string): {
     pages: {
@@ -12,7 +22,7 @@ export declare function parseSitemapXml(xml: string): {
  * here), and its rules. An HTML body or any non-200 means "no robots.txt";
  * never throws.
  */
-export declare function fetchRobots(websiteUrl: string): Promise<{
+export declare function fetchRobots(websiteUrl: string, options?: SitemapOptions): Promise<{
     sitemaps: string[];
     rules: RobotsRule[];
 }>;
@@ -30,8 +40,7 @@ export declare function parseRobotsRules(body: string): RobotsRule[];
 /**
  * Whether robots.txt lets us read this URL: the longest matching rule wins and
  * `Allow` wins a tie, as Google reads it. Shopify disallows `/cart/`, and the
- * crawler had fetched `/cart/add?id=…&quantity=1` - a GET that fills a cart
- * (Good Habit audit §3.5).
+ * crawler had fetched `/cart/add?id=…&quantity=1` - a GET that fills a cart.
  */
 export declare function robotsAllows(rules: RobotsRule[], url: string): boolean;
 /**
@@ -41,7 +50,7 @@ export declare function robotsAllows(rules: RobotsRule[], url: string): boolean;
  * an error: link-following discovery is the source, sitemaps a supplement.
  * Never throws.
  */
-export declare function discoverSitemapUrls(websiteUrl: string, robotsSitemaps: string[]): Promise<{
+export declare function discoverSitemapUrls(websiteUrl: string, robotsSitemaps: string[], options?: SitemapOptions): Promise<{
     url: string;
     lastmod: string | null;
 }[]>;

package/dist/sitemap/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sitemap/index.ts"],"names":[],"mappings":"AAsCA,6EAA6E;AAC7E,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM;;aACtB,MAAM;iBAAW,MAAM,GAAG,IAAI;;;EAwBlD;AAgCD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM;;;GAuBnD;AAED,MAAM,MAAM,UAAU,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAA;AAE5D;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,gBAyB5C;AAqBD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,WAiB5D;AAoDD;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACxC,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EAAE;SAIa,MAAM;aAAW,MAAM,GAAG,IAAI;KAiBnE;AAED,gEAAgE;AAChE,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,UAQ9C"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sitemap/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,KAAK,EAAgB,MAAM,iBAAiB,CAAA;AAI1D,YAAY,EAAE,KAAK,EAAE,CAAA;AAarB,MAAM,MAAM,cAAc,GAAG;IAC5B;;;;OAIG;IACH,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAA;CACzB,CAAA;AAiCD,6EAA6E;AAC7E,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM;;aACtB,MAAM;iBAAW,MAAM,GAAG,IAAI;;;EAwBlD;AA6CD;;;;;GAKG;AACH,wBAAsB,WAAW,CAChC,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE,cAAmB;;;GA4B5B;AAED,MAAM,MAAM,UAAU,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAA;AAE5D;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,EAAE,CAM3D;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,WAQ5D;AAqDD;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACxC,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EAAE,EACxB,OAAO,GAAE,cAAmB;SAKS,MAAM;aAAW,MAAM,GAAG,IAAI;KAiBnE;AAED,gEAAgE;AAChE,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,UAQ9C"}