website-to-markdown 1.1.1 → 2.0.0

package/dist/limit.js.map

@@ -1 +1 @@
-{"version":3,"file":"limit.js","sourceRoot":"","sources":["../src/limit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAgChE;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,EAAE,CAAA;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,EAAE,CAAA;AACtD,MAAM,CAAC,MAAM,4CAA4C,GAAG,EAAE,CAAA;AAE9D;;;;GAIG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,CAAA;AAC/C,MAAM,CAAC,MAAM,qCAAqC,GAAG,EAAE,CAAA;AAEvD,yEAAyE;AACzE,MAAM,UAAU,sBAAsB,CAAC,WAAmC;IACzE,OAAO;QACN,gBAAgB,EAAE,WAAW,CAAC;YAC7B,GAAG,EAAE,8BAA8B;YACnC,GAAG,EAAE,4CAA4C;YACjD,kBAAkB,EAAE,oCAAoC;YACxD,6EAA6E;YAC7E,SAAS,EAAE,MAAM;SACjB,CAAC;QACF,SAAS,EAAE,WAAW,CAAC;YACtB,GAAG,EAAE,uBAAuB;YAC5B,GAAG,EAAE,qCAAqC;YAC1C,kBAAkB,EAAE,6BAA6B;YACjD,SAAS,EAAE,MAAM;SACjB,CAAC;KACF,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,EAAE,CAAA;AAEtD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8CAA8C,GAAG,CAAC,CAAA;AAC/D,MAAM,gCAAgC,GAAG,CAAC,CAAA;AAE1C,oFAAoF;AACpF,MAAM,gBAAgB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAEvC;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,sBAAsB,CAAA;AAEhD,+EAA+E;AAC/E,MAAM,CAAC,MAAM,oCAAoC,GAAG,GAAG,CAAA;AAUvD,qFAAqF;AACrF,MAAM,UAAU,gBAAgB,CAC/B,cAAsC,4BAA4B;IAElE,uEAAuE;IACvE,uEAAuE;IACvE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAA;IAE5C,SAAS,SAAS;QACjB,IAAI,OAAO,CAAC,IAAI,GAAG,oCAAoC;YAAE,OAAM;QAC/D,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC;gBAAE,SAAQ;YAChC,mEAAmE;YACnE,wEAAwE;YACxE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAA;YACrB,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAA;YACzB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACpB,OAAM;QACP,CAAC;QACD,oEAAoE;QACpE,kEAAkE;IACnE,CAAC;IAED,SAAS,QAAQ,CAAC,QAAgB;QACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAA;QAClC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC/B,IAAI,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YACxB,OAAO,MAAM,CAAA;QACd,CAAC;QACD,SAAS,EAAE,CAAA;QACX,MAAM,KAAK,GAAc;YACxB,KAAK,EAAE,WAAW,CAAC;gBAClB,GAAG,EAAE,wBAAwB,GAAG,EAAE;gBAClC,WAAW,EAAE,iBAAiB;gBAC9B,GAAG,EAAE,oCAAoC;gBACzC,oEAAoE;gBACpE,SAAS,EAAE,MAAM;aACjB,CAAC;YACF,QAAQ,EAAE,CAAC;YACX,QAAQ,EAAE,IAAI;YACd,cAAc,EAAE,CAAC;SACjB,CAAA;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,OAAO,KAAK,CAAA;IACb,CAAC;IAED,OAAO;QACN,KAAK,CAAC,aAAa,CAAI,QAAgB,EAAE,GAAqB;YAC7D,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAChC,wEAAwE;YACxE,6DAA6D;YAC7D,KAAK,CAAC,QAAQ,EAAE,CAAA;YAChB,IAAI,CAAC;gBACJ,OAAO,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC7B,2EAA2E;oBAC3E,MAAM,QAAQ,GACb,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAA;oBAC1D,OAAO,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;gBACxC,CAAC,CAAC,CAAA;YACH,CAAC;oBAAS,CAAC;gBACV,KAAK,CAAC,QAAQ,EAAE,CAAA;YACjB,CAAC;QACF,CAAC;QAED;;;;;WAKG;QACH,KAAK,CAAC,YAAY,CAAC,QAAgB,EAAE,MAAc;YAClD,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,cAAc,CAAA;YACnD,KAAK,CAAC,QAAQ,KAAK,WAAW,CAAC;gBAC9B,GAAG,EAAE,wBAAwB,QAAQ,CAAC,WAAW,EAAE,YAAY;gBAC/D,WAAW,EAAE,iBAAiB;gBAC9B,GAAG,EAAE,8CAA8C;gBACnD,kBAAkB,EAAE,gCAAgC;gBACpD,SAAS,EAAE,MAAM;aACjB,CAAC,CAAA;YACF,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAA;YACpD,MAAM,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;YACzC,OAAO,QAAQ,CAAA;QAChB,CAAC;KACD,CAAA;AACF,CAAC","sourcesContent":["import { createMemoryConcurrencyLimit } from './memory-limit.js'\n\n/**\n * A semaphore: at most `max` callbacks in flight. `hanzio/redis`'s\n * `createRedisConcurrencyLimit` returns this shape (plus `queueDepth`, which is\n * not needed here) once `send` is bound; the default is\n * {@link createMemoryConcurrencyLimit}.\n */\nexport type ConcurrencyLimit = {\n\t<T>(run: () => Promise<T>, signal?: AbortSignal): Promise<T>\n\t/** Pause every start for `waitMs`, e.g. from an upstream `Retry-After`. */\n\ttripCooldown: (waitMs: number) => Promise<void>\n\t/** Drop the limiter's timers and metric registrations; it is discarded. */\n\tdispose: () => void\n}\n\nexport type ConcurrencyLimitOptions = {\n\t/** Shared limiters key their state on this; one per pool or host. */\n\tkey: string\n\tmax: number\n\t/** How long a slot held by a crashed process stays taken. */\n\tslotTtlMs: number\n\t/** A separate sliding one-second window of starts (provider rps quotas). */\n\tmaxStartsPerSecond?: number\n\t/** The pool a per-host limiter reports metrics under, instead of its key. */\n\tmetricLabel?: string\n}\n\nexport type CreateConcurrencyLimit = (\n\toptions: ConcurrencyLimitOptions\n) => ConcurrencyLimit\n\n/**\n * Page-level fetch parallelism. Content negotiation is a plain GET and is not\n * a Cloudflare API call, so this can sit above the Workers Paid start rates.\n * Workers AI and Browser Rendering still queue on their own pools.\n */\nexport const URL_TO_MARKDOWN_PAGE_CONCURRENCY = 80\n\n/**\n * Workers Paid Quick Actions are 30 rps, not 30 in-flight. Leave a little\n * headroom so two processes cannot land a burst on the same clock second.\n * Concurrency is higher so a 10-20s render does not leave the rps budget idle.\n */\nexport const CLOUDFLARE_BROWSER_RENDERING_MAX_RPS = 28\nexport const CLOUDFLARE_BROWSER_RENDERING_MAX_CONCURRENCY = 80\n\n/**\n * toMarkdown has no published rps cap. Stay below the Client API 200/s IP\n * burst and let `tripCooldown` pull the fleet back if the 1200/5min token\n * window applies.\n */\nexport const CLOUDFLARE_WORKERS_AI_MAX_RPS = 12\nexport const CLOUDFLARE_WORKERS_AI_MAX_CONCURRENCY = 24\n\n/** The two Cloudflare pools; a page can wait on both and on its host. */\nexport function createCloudflareLimits(createLimit: CreateConcurrencyLimit) {\n\treturn {\n\t\tbrowserRendering: createLimit({\n\t\t\tkey: 'cloudflare-browser-rendering',\n\t\t\tmax: CLOUDFLARE_BROWSER_RENDERING_MAX_CONCURRENCY,\n\t\t\tmaxStartsPerSecond: CLOUDFLARE_BROWSER_RENDERING_MAX_RPS,\n\t\t\t// Browser Rendering can run up to 60s; TTL is crash detection, not a budget.\n\t\t\tslotTtlMs: 90_000\n\t\t}),\n\t\tworkersAi: createLimit({\n\t\t\tkey: 'cloudflare-workers-ai',\n\t\t\tmax: CLOUDFLARE_WORKERS_AI_MAX_CONCURRENCY,\n\t\t\tmaxStartsPerSecond: CLOUDFLARE_WORKERS_AI_MAX_RPS,\n\t\t\tslotTtlMs: 45_000\n\t\t})\n\t}\n}\n\n/**\n * Politeness cap per customer host. This is the only thing standing\n * between an 80-wide page pool and one small WordPress box, and it applies to a\n * SOLO import as much as a concurrent one: two orgs are two domains and never\n * contend, so the burst this prevents is a single site's own crawl.\n *\n * Deliberately generous rather than cautious - a healthy origin should be read\n * at speed, and 500 pages at 16 in flight is well inside the run timeout. It\n * stays a cap because the failure it prevents is silent and un-retryable: a WAF\n * that decides we are an attack blocks the whole import, and the customer-site\n * probe already found an origin that 403s on User-Agent alone.\n *\n * Separate from the Cloudflare pools - a page can wait on both.\n */\nexport const URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY = 16\n\n/**\n * A host that answered 429 is read this narrowly from then on. At 16 in\n * flight Shopify refused 117 of Good Habit's 244 reads even after each had\n * waited out its pause: the pause only delayed the next burst. Calibration\n * knobs - set them from the crawl log's `rate_limited`, not by theory.\n */\nexport const URL_TO_MARKDOWN_THROTTLED_HOST_MAX_CONCURRENCY = 2\nconst THROTTLED_HOST_STARTS_PER_SECOND = 2\n\n/** A throttled host is read at the full cap again after this long without a 429. */\nconst HOST_THROTTLE_MS = 15 * 60 * 1000\n\n/**\n * Every host limiter meters under this one label. Per-host series would grow\n * with the customer base and never shrink, and the question the metric answers\n * - is the crawler backed up behind origins - is a pool-wide question anyway.\n */\nconst HOST_METRIC_LABEL = 'url-to-markdown:host'\n\n/** Ceiling on live host limiters, and so on the closures the gauge retains. */\nexport const URL_TO_MARKDOWN_HOST_LIMIT_CACHE_MAX = 512\n\ntype HostEntry = {\n\tlimit: ConcurrencyLimit\n\tinFlight: number\n\t/** Set by the host's first 429 (`coolDownHost`); every read passes it too. */\n\tthrottle: ConcurrencyLimit | null\n\tthrottledUntil: number\n}\n\n/** Limiters are per host and created on demand; one per request would leak pumps. */\nexport function createHostLimits(\n\tcreateLimit: CreateConcurrencyLimit = createMemoryConcurrencyLimit\n) {\n\t// Insertion order is maintained as LRU order: a hit re-inserts, so the\n\t// eviction scan starts at the host we have not touched in the longest.\n\tconst entries = new Map<string, HostEntry>()\n\n\tfunction evictIdle() {\n\t\tif (entries.size < URL_TO_MARKDOWN_HOST_LIMIT_CACHE_MAX) return\n\t\tfor (const [host, entry] of entries) {\n\t\t\tif (entry.inFlight > 0) continue\n\t\t\t// Disposing drops the metric registration, and with it the closure\n\t\t\t// that would otherwise retain this limiter for the life of the process.\n\t\t\tentry.limit.dispose()\n\t\t\tentry.throttle?.dispose()\n\t\t\tentries.delete(host)\n\t\t\treturn\n\t\t}\n\t\t// Every cached host is busy. Running over the cap beats disposing a\n\t\t// limiter whose slots are still held and still being heartbeated.\n\t}\n\n\tfunction getEntry(hostname: string) {\n\t\tconst key = hostname.toLowerCase()\n\t\tconst cached = entries.get(key)\n\t\tif (cached) {\n\t\t\tentries.delete(key)\n\t\t\tentries.set(key, cached)\n\t\t\treturn cached\n\t\t}\n\t\tevictIdle()\n\t\tconst entry: HostEntry = {\n\t\t\tlimit: createLimit({\n\t\t\t\tkey: `url-to-markdown:host:${key}`,\n\t\t\t\tmetricLabel: HOST_METRIC_LABEL,\n\t\t\t\tmax: URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY,\n\t\t\t\t// Origin fetches time out at 30s; the TTL only has to outlive that.\n\t\t\t\tslotTtlMs: 45_000\n\t\t\t}),\n\t\t\tinFlight: 0,\n\t\t\tthrottle: null,\n\t\t\tthrottledUntil: 0\n\t\t}\n\t\tentries.set(key, entry)\n\t\treturn entry\n\t}\n\n\treturn {\n\t\tasync withHostLimit<T>(hostname: string, run: () => Promise<T>) {\n\t\t\tconst entry = getEntry(hostname)\n\t\t\t// Counted around the whole wait+run so eviction can tell a limiter that\n\t\t\t// is merely cached from one that is holding slots right now.\n\t\t\tentry.inFlight++\n\t\t\ttry {\n\t\t\t\treturn await entry.limit(() => {\n\t\t\t\t\t// A lapsed throttle is kept for the host's next 429; eviction disposes it.\n\t\t\t\t\tconst throttle =\n\t\t\t\t\t\tDate.now() < entry.throttledUntil ? entry.throttle : null\n\t\t\t\t\treturn throttle ? throttle(run) : run()\n\t\t\t\t})\n\t\t\t} finally {\n\t\t\t\tentry.inFlight--\n\t\t\t}\n\t\t},\n\n\t\t/**\n\t\t * The host answered 429: from now on it is read at the throttled cap, and\n\t\t * nothing is sent to it for `waitMs`. A shared limiter shares the pause, so\n\t\t * every worker reading the host waits it out. True when this 429 is what\n\t\t * narrowed the host.\n\t\t */\n\t\tasync coolDownHost(hostname: string, waitMs: number) {\n\t\t\tconst entry = getEntry(hostname)\n\t\t\tconst narrowed = Date.now() >= entry.throttledUntil\n\t\t\tentry.throttle ??= createLimit({\n\t\t\t\tkey: `url-to-markdown:host:${hostname.toLowerCase()}:throttled`,\n\t\t\t\tmetricLabel: HOST_METRIC_LABEL,\n\t\t\t\tmax: URL_TO_MARKDOWN_THROTTLED_HOST_MAX_CONCURRENCY,\n\t\t\t\tmaxStartsPerSecond: THROTTLED_HOST_STARTS_PER_SECOND,\n\t\t\t\tslotTtlMs: 45_000\n\t\t\t})\n\t\t\tentry.throttledUntil = Date.now() + HOST_THROTTLE_MS\n\t\t\tawait entry.throttle.tripCooldown(waitMs)\n\t\t\treturn narrowed\n\t\t}\n\t}\n}\n"]}
+{"version":3,"file":"limit.js","sourceRoot":"","sources":["../src/limit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAgChE;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,EAAE,CAAA;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,EAAE,CAAA;AACtD,MAAM,CAAC,MAAM,4CAA4C,GAAG,EAAE,CAAA;AAE9D;;;;GAIG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,CAAA;AAC/C,MAAM,CAAC,MAAM,qCAAqC,GAAG,EAAE,CAAA;AAEvD,yEAAyE;AACzE,MAAM,UAAU,sBAAsB,CAAC,WAAmC;IACzE,OAAO;QACN,gBAAgB,EAAE,WAAW,CAAC;YAC7B,GAAG,EAAE,8BAA8B;YACnC,GAAG,EAAE,4CAA4C;YACjD,kBAAkB,EAAE,oCAAoC;YACxD,6EAA6E;YAC7E,SAAS,EAAE,MAAM;SACjB,CAAC;QACF,SAAS,EAAE,WAAW,CAAC;YACtB,GAAG,EAAE,uBAAuB;YAC5B,GAAG,EAAE,qCAAqC;YAC1C,kBAAkB,EAAE,6BAA6B;YACjD,SAAS,EAAE,MAAM;SACjB,CAAC;KACF,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,EAAE,CAAA;AAEtD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8CAA8C,GAAG,CAAC,CAAA;AAC/D,MAAM,gCAAgC,GAAG,CAAC,CAAA;AAE1C,oFAAoF;AACpF,MAAM,gBAAgB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAEvC;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,sBAAsB,CAAA;AAEhD,+EAA+E;AAC/E,MAAM,CAAC,MAAM,oCAAoC,GAAG,GAAG,CAAA;AAUvD,qFAAqF;AACrF,MAAM,UAAU,gBAAgB,CAC/B,cAAsC,4BAA4B;IAElE,uEAAuE;IACvE,uEAAuE;IACvE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAA;IAE5C,SAAS,SAAS;QACjB,IAAI,OAAO,CAAC,IAAI,GAAG,oCAAoC;YAAE,OAAM;QAC/D,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC;gBAAE,SAAQ;YAChC,mEAAmE;YACnE,wEAAwE;YACxE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAA;YACrB,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAA;YACzB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACpB,OAAM;QACP,CAAC;QACD,oEAAoE;QACpE,kEAAkE;IACnE,CAAC;IAED,SAAS,QAAQ,CAAC,QAAgB;QACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAA;QAClC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC/B,IAAI,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YACxB,OAAO,MAAM,CAAA;QACd,CAAC;QACD,SAAS,EAAE,CAAA;QACX,MAAM,KAAK,GAAc;YACxB,KAAK,EAAE,WAAW,CAAC;gBAClB,GAAG,EAAE,wBAAwB,GAAG,EAAE;gBAClC,WAAW,EAAE,iBAAiB;gBAC9B,GAAG,EAAE,oCAAoC;gBACzC,oEAAoE;gBACpE,SAAS,EAAE,MAAM;aACjB,CAAC;YACF,QAAQ,EAAE,CAAC;YACX,QAAQ,EAAE,IAAI;YACd,cAAc,EAAE,CAAC;SACjB,CAAA;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,OAAO,KAAK,CAAA;IACb,CAAC;IAED,OAAO;QACN,KAAK,CAAC,aAAa,CAAI,QAAgB,EAAE,GAAqB;YAC7D,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAChC,wEAAwE;YACxE,6DAA6D;YAC7D,KAAK,CAAC,QAAQ,EAAE,CAAA;YAChB,IAAI,CAAC;gBACJ,OAAO,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC7B,2EAA2E;oBAC3E,MAAM,QAAQ,GACb,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAA;oBAC1D,OAAO,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;gBACxC,CAAC,CAAC,CAAA;YACH,CAAC;oBAAS,CAAC;gBACV,KAAK,CAAC,QAAQ,EAAE,CAAA;YACjB,CAAC;QACF,CAAC;QAED;;;;;WAKG;QACH,KAAK,CAAC,YAAY,CAAC,QAAgB,EAAE,MAAc;YAClD,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,cAAc,CAAA;YACnD,KAAK,CAAC,QAAQ,KAAK,WAAW,CAAC;gBAC9B,GAAG,EAAE,wBAAwB,QAAQ,CAAC,WAAW,EAAE,YAAY;gBAC/D,WAAW,EAAE,iBAAiB;gBAC9B,GAAG,EAAE,8CAA8C;gBACnD,kBAAkB,EAAE,gCAAgC;gBACpD,SAAS,EAAE,MAAM;aACjB,CAAC,CAAA;YACF,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAA;YACpD,MAAM,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;YACzC,OAAO,QAAQ,CAAA;QAChB,CAAC;KACD,CAAA;AACF,CAAC","sourcesContent":["import { createMemoryConcurrencyLimit } from './memory-limit.js'\n\n/**\n * A semaphore: at most `max` callbacks in flight. `hanzio/redis`'s\n * `createRedisConcurrencyLimit` returns this shape (plus `queueDepth`, which is\n * not needed here) once `send` is bound; the default is\n * {@link createMemoryConcurrencyLimit}.\n */\nexport type ConcurrencyLimit = {\n\t<T>(run: () => Promise<T>, signal?: AbortSignal): Promise<T>\n\t/** Pause every start for `waitMs`, e.g. from an upstream `Retry-After`. */\n\ttripCooldown: (waitMs: number) => Promise<void>\n\t/** Drop the limiter's timers and metric registrations; it is discarded. */\n\tdispose: () => void\n}\n\nexport type ConcurrencyLimitOptions = {\n\t/** Shared limiters key their state on this; one per pool or host. */\n\tkey: string\n\tmax: number\n\t/** How long a slot held by a crashed process stays taken. */\n\tslotTtlMs: number\n\t/** A separate sliding one-second window of starts (provider rps quotas). */\n\tmaxStartsPerSecond?: number\n\t/** The pool a per-host limiter reports metrics under, instead of its key. */\n\tmetricLabel?: string\n}\n\nexport type CreateConcurrencyLimit = (\n\toptions: ConcurrencyLimitOptions\n) => ConcurrencyLimit\n\n/**\n * Page-level fetch parallelism. Content negotiation is a plain GET and is not\n * a Cloudflare API call, so this can sit above the Workers Paid start rates.\n * Workers AI and Browser Rendering still queue on their own pools.\n */\nexport const URL_TO_MARKDOWN_PAGE_CONCURRENCY = 80\n\n/**\n * Workers Paid Quick Actions are 30 rps, not 30 in-flight. Leave a little\n * headroom so two processes cannot land a burst on the same clock second.\n * Concurrency is higher so a 10-20s render does not leave the rps budget idle.\n */\nexport const CLOUDFLARE_BROWSER_RENDERING_MAX_RPS = 28\nexport const CLOUDFLARE_BROWSER_RENDERING_MAX_CONCURRENCY = 80\n\n/**\n * toMarkdown has no published rps cap. Stay below the Client API 200/s IP\n * burst and let `tripCooldown` pull the fleet back if the 1200/5min token\n * window applies.\n */\nexport const CLOUDFLARE_WORKERS_AI_MAX_RPS = 12\nexport const CLOUDFLARE_WORKERS_AI_MAX_CONCURRENCY = 24\n\n/** The two Cloudflare pools; a page can wait on both and on its host. */\nexport function createCloudflareLimits(createLimit: CreateConcurrencyLimit) {\n\treturn {\n\t\tbrowserRendering: createLimit({\n\t\t\tkey: 'cloudflare-browser-rendering',\n\t\t\tmax: CLOUDFLARE_BROWSER_RENDERING_MAX_CONCURRENCY,\n\t\t\tmaxStartsPerSecond: CLOUDFLARE_BROWSER_RENDERING_MAX_RPS,\n\t\t\t// Browser Rendering can run up to 60s; TTL is crash detection, not a budget.\n\t\t\tslotTtlMs: 90_000\n\t\t}),\n\t\tworkersAi: createLimit({\n\t\t\tkey: 'cloudflare-workers-ai',\n\t\t\tmax: CLOUDFLARE_WORKERS_AI_MAX_CONCURRENCY,\n\t\t\tmaxStartsPerSecond: CLOUDFLARE_WORKERS_AI_MAX_RPS,\n\t\t\tslotTtlMs: 45_000\n\t\t})\n\t}\n}\n\n/**\n * Politeness cap per customer host. This is the only thing standing\n * between an 80-wide page pool and one small WordPress box, and it applies to a\n * SOLO import as much as a concurrent one: two orgs are two domains and never\n * contend, so the burst this prevents is a single site's own crawl.\n *\n * Deliberately generous rather than cautious - a healthy origin should be read\n * at speed, and 500 pages at 16 in flight is well inside the run timeout. It\n * stays a cap because the failure it prevents is silent and un-retryable: a WAF\n * that decides we are an attack blocks the whole import, and the customer-site\n * probe already found an origin that 403s on User-Agent alone.\n *\n * Separate from the Cloudflare pools - a page can wait on both.\n */\nexport const URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY = 16\n\n/**\n * A host that answered 429 is read this narrowly from then on. At 16 in\n * flight Shopify refused 117 of one store's 244 reads even after each had\n * waited out its pause: the pause only delayed the next burst. Calibration\n * knobs - set them from the crawl log's `rate_limited`, not by theory.\n */\nexport const URL_TO_MARKDOWN_THROTTLED_HOST_MAX_CONCURRENCY = 2\nconst THROTTLED_HOST_STARTS_PER_SECOND = 2\n\n/** A throttled host is read at the full cap again after this long without a 429. */\nconst HOST_THROTTLE_MS = 15 * 60 * 1000\n\n/**\n * Every host limiter meters under this one label. Per-host series would grow\n * with the customer base and never shrink, and the question the metric answers\n * - is the crawler backed up behind origins - is a pool-wide question anyway.\n */\nconst HOST_METRIC_LABEL = 'url-to-markdown:host'\n\n/** Ceiling on live host limiters, and so on the closures the gauge retains. */\nexport const URL_TO_MARKDOWN_HOST_LIMIT_CACHE_MAX = 512\n\ntype HostEntry = {\n\tlimit: ConcurrencyLimit\n\tinFlight: number\n\t/** Set by the host's first 429 (`coolDownHost`); every read passes it too. */\n\tthrottle: ConcurrencyLimit | null\n\tthrottledUntil: number\n}\n\n/** Limiters are per host and created on demand; one per request would leak pumps. */\nexport function createHostLimits(\n\tcreateLimit: CreateConcurrencyLimit = createMemoryConcurrencyLimit\n) {\n\t// Insertion order is maintained as LRU order: a hit re-inserts, so the\n\t// eviction scan starts at the host we have not touched in the longest.\n\tconst entries = new Map<string, HostEntry>()\n\n\tfunction evictIdle() {\n\t\tif (entries.size < URL_TO_MARKDOWN_HOST_LIMIT_CACHE_MAX) return\n\t\tfor (const [host, entry] of entries) {\n\t\t\tif (entry.inFlight > 0) continue\n\t\t\t// Disposing drops the metric registration, and with it the closure\n\t\t\t// that would otherwise retain this limiter for the life of the process.\n\t\t\tentry.limit.dispose()\n\t\t\tentry.throttle?.dispose()\n\t\t\tentries.delete(host)\n\t\t\treturn\n\t\t}\n\t\t// Every cached host is busy. Running over the cap beats disposing a\n\t\t// limiter whose slots are still held and still being heartbeated.\n\t}\n\n\tfunction getEntry(hostname: string) {\n\t\tconst key = hostname.toLowerCase()\n\t\tconst cached = entries.get(key)\n\t\tif (cached) {\n\t\t\tentries.delete(key)\n\t\t\tentries.set(key, cached)\n\t\t\treturn cached\n\t\t}\n\t\tevictIdle()\n\t\tconst entry: HostEntry = {\n\t\t\tlimit: createLimit({\n\t\t\t\tkey: `url-to-markdown:host:${key}`,\n\t\t\t\tmetricLabel: HOST_METRIC_LABEL,\n\t\t\t\tmax: URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY,\n\t\t\t\t// Origin fetches time out at 30s; the TTL only has to outlive that.\n\t\t\t\tslotTtlMs: 45_000\n\t\t\t}),\n\t\t\tinFlight: 0,\n\t\t\tthrottle: null,\n\t\t\tthrottledUntil: 0\n\t\t}\n\t\tentries.set(key, entry)\n\t\treturn entry\n\t}\n\n\treturn {\n\t\tasync withHostLimit<T>(hostname: string, run: () => Promise<T>) {\n\t\t\tconst entry = getEntry(hostname)\n\t\t\t// Counted around the whole wait+run so eviction can tell a limiter that\n\t\t\t// is merely cached from one that is holding slots right now.\n\t\t\tentry.inFlight++\n\t\t\ttry {\n\t\t\t\treturn await entry.limit(() => {\n\t\t\t\t\t// A lapsed throttle is kept for the host's next 429; eviction disposes it.\n\t\t\t\t\tconst throttle =\n\t\t\t\t\t\tDate.now() < entry.throttledUntil ? entry.throttle : null\n\t\t\t\t\treturn throttle ? throttle(run) : run()\n\t\t\t\t})\n\t\t\t} finally {\n\t\t\t\tentry.inFlight--\n\t\t\t}\n\t\t},\n\n\t\t/**\n\t\t * The host answered 429: from now on it is read at the throttled cap, and\n\t\t * nothing is sent to it for `waitMs`. A shared limiter shares the pause, so\n\t\t * every worker reading the host waits it out. True when this 429 is what\n\t\t * narrowed the host.\n\t\t */\n\t\tasync coolDownHost(hostname: string, waitMs: number) {\n\t\t\tconst entry = getEntry(hostname)\n\t\t\tconst narrowed = Date.now() >= entry.throttledUntil\n\t\t\tentry.throttle ??= createLimit({\n\t\t\t\tkey: `url-to-markdown:host:${hostname.toLowerCase()}:throttled`,\n\t\t\t\tmetricLabel: HOST_METRIC_LABEL,\n\t\t\t\tmax: URL_TO_MARKDOWN_THROTTLED_HOST_MAX_CONCURRENCY,\n\t\t\t\tmaxStartsPerSecond: THROTTLED_HOST_STARTS_PER_SECOND,\n\t\t\t\tslotTtlMs: 45_000\n\t\t\t})\n\t\t\tentry.throttledUntil = Date.now() + HOST_THROTTLE_MS\n\t\t\tawait entry.throttle.tripCooldown(waitMs)\n\t\t\treturn narrowed\n\t\t}\n\t}\n}\n"]}

package/dist/markdown-refs.d.ts

@@ -4,6 +4,14 @@ export declare function stripFrontMatter(markdown: string): {
     description: string | null;
 };
 export declare function stripDocumentPreamble(markdown: string): string;
+/**
+ * Removes what Cloudflare's conversion leaves behind and is never page text:
+ * obfuscated e-mail links are decoded, JSON-LD code fences and markup
+ * declarations dropped, and labels glued to their `mailto:`/`tel:` value
+ * split. Applied to every fetched page; idempotent, so markdown stored by an
+ * older version can be run through it too.
+ */
+export declare function cleanConvertedMarkdown(markdown: string): string;
 export declare function titleFromMarkdown(markdown: string): string | null;
 /**
  * Absolute, deduped, document-order links and images. Discovery crawls these,

package/dist/markdown-refs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-refs.d.ts","sourceRoot":"","sources":["../src/markdown-refs.ts"],"names":[],"mappings":"AAkBA,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM;;;;EAShD;AAkBD,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAe9D;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,iBAGjD;AA2CD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;;;EA0B7D;AAWD;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAmBjE;AAED,mFAAmF;AACnF,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAalE"}
+{"version":3,"file":"markdown-refs.d.ts","sourceRoot":"","sources":["../src/markdown-refs.ts"],"names":[],"mappings":"AAmBA,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM;;;;EAShD;AAiBD,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAe9D;AA4DD;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,UAqBtD;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,iBAGjD;AAkDD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;;;EAY7D;AAWD;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAcjE;AAED,mFAAmF;AACnF,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CASlE"}

package/dist/markdown-refs.js

@@ -1,3 +1,4 @@
+import { compact, getUniqueValues, isOneOf } from 'hanzio';
 import { decodeHtmlEntities } from './html-entities.js';
 /**
  * Workers AI `toMarkdown` prefixes every conversion with a YAML block:
@@ -29,10 +30,9 @@ export function stripFrontMatter(markdown) {
  * Workers AI `toMarkdown` opens every converted PDF and office file with the
  * file name as an H1, a `## Metadata` list (creation dates, uuids,
  * `PDFFormatVersion=1.7`) and a `## Contents` heading, then labels every page
- * `### Page N`. None of it is what the document says, and the fact guard
- * demanded those dates and numbers of every written document - which is how
- * Hotel Hvide Klit's terms came back as 13,000 characters of raw text with
- * the PDF metadata inside.
+ * `### Page N`. None of it is what the document says: a hotel's terms PDF
+ * came back as 13,000 characters with its creation dates and uuids inside,
+ * and a reader of the markdown takes those for facts of the document.
  */
 const DOCUMENT_TITLE_RE = /^#\s+[^\n]*\.(?:pdf|docx?|xlsx?|pptx?|odt|rtf)\s*\r?\n/i;
 const DOCUMENT_METADATA_RE = /^##\s+Metadata\s*\r?\n(?:(?:-\s[^\n]*|\s*)\r?\n)*/i;
@@ -54,6 +54,80 @@ export function stripDocumentPreamble(markdown) {
         .replace(/\n{3,}/g, '\n\n')
         .trim();
 }
+/**
+ * Cloudflare's e-mail obfuscation rewrites every address it finds into
+ * `[\[email protected\]](/cdn-cgi/l/email-protection#<hex>)`, so a converted
+ * page behind it shows no readable address at all. The hex is the address
+ * XOR-ed with its first byte; decoded only when the result is shaped like an
+ * address.
+ */
+const CF_EMAIL_LINK_RE = /\[(?:[^[\]\\]|\\.|\[[^\]]*\])*\]\((?:https?:\/\/[^/)]+)?\/cdn-cgi\/l\/email-protection#([0-9a-fA-F]{8,})\)/g;
+function decodeCloudflareEmail(hex) {
+    const key = Number.parseInt(hex.slice(0, 2), 16);
+    let address = '';
+    for (let index = 2; index + 1 < hex.length; index += 2) {
+        address += String.fromCharCode(Number.parseInt(hex.slice(index, index + 2), 16) ^ key);
+    }
+    return /^[^\s@]+@[^\s@]+\.[^\s@.]+$/.test(address) ? address : null;
+}
+/**
+ * `<script type="application/ld+json">` survives conversion as a fenced code
+ * block on Yoast and Squarespace sites. It is never prose, and the page already
+ * carries it parsed as `jsonLd`.
+ */
+const FENCED_BLOCK_RE = /^```[^\n]*\n([\s\S]*?)^```[ \t]*$/gm;
+const JSON_LD_RE = /schema\.org|["']@context["']\s*:/;
+/** A page that inlines an SVG file gets its prolog back from the converter as text. */
+const MARKUP_DECLARATION_RE = /<\?xml[^>]*\?>|<!DOCTYPE[^>]*>/gi;
+/**
+ * A label rendered next to its own value with no space between -
+ * `[Saleshello@example.com](mailto:hello@example.com)`,
+ * `[Support+45 22 81 82 95](tel:+4522818295)` - reads as an invented address.
+ * Matched on the value's shape, so the label's language never matters.
+ */
+const VALUE_LINK_RE = /\[([^\]]+)\]\((mailto|tel):([^)\s]+)\)/gi;
+function emailStart(label, target) {
+    const address = (target.split('?')[0] ?? '').toLowerCase();
+    return address && label.toLowerCase().endsWith(address)
+        ? label.length - address.length
+        : -1;
+}
+function phoneStart(label, target) {
+    const digits = target.replace(/\D/g, '');
+    if (digits.length < 6)
+        return -1;
+    const first = label.search(/[+\d]/);
+    if (first < 0)
+        return -1;
+    const rest = label.slice(first);
+    return /^[+\d\s().-]+$/.test(rest) && rest.replace(/\D/g, '') === digits
+        ? first
+        : -1;
+}
+/**
+ * Removes what Cloudflare's conversion leaves behind and is never page text:
+ * obfuscated e-mail links are decoded, JSON-LD code fences and markup
+ * declarations dropped, and labels glued to their `mailto:`/`tel:` value
+ * split. Applied to every fetched page; idempotent, so markdown stored by an
+ * older version can be run through it too.
+ */
+export function cleanConvertedMarkdown(markdown) {
+    return markdown
+        .replace(CF_EMAIL_LINK_RE, (link, hex) => {
+        const address = decodeCloudflareEmail(hex);
+        return address ? `[${address}](mailto:${address})` : link;
+    })
+        .replace(FENCED_BLOCK_RE, (block, body) => JSON_LD_RE.test(body) ? '' : block)
+        .replace(MARKUP_DECLARATION_RE, '')
+        .replace(VALUE_LINK_RE, (link, label, scheme, target) => {
+        const start = scheme.toLowerCase() === 'mailto'
+            ? emailStart(label, target)
+            : phoneStart(label, target);
+        if (start <= 0 || !/\p{L}/u.test(label[start - 1] ?? ''))
+            return link;
+        return `[${label.slice(0, start)} ${label.slice(start)}](${scheme}:${target})`;
+    });
+}
 export function titleFromMarkdown(markdown) {
     const heading = markdown.match(/^#\s+(.+)$/m);
     return heading?.[1]?.trim() || null;
@@ -76,13 +150,12 @@ function resolveTarget(target, baseUrl) {
         const resolved = new URL(raw, baseUrl);
         // Drops `mailto:`, `tel:`, `javascript:`, `data:` and every other scheme
         // discovery cannot crawl; they stay in the markdown itself.
-        if (resolved.protocol !== 'https:' && resolved.protocol !== 'http:') {
+        if (!isOneOf(['https:', 'http:'], resolved.protocol))
             return null;
-        }
         // Cloudflare's reserved prefix is plumbing, never a page: the e-mail
         // obfuscation endpoint under it answers 410, which used to count as a
-        // page this site failed to deliver. `clean-pages.ts` decodes the address
-        // out of the link before it is dropped here.
+        // page this site failed to deliver. `cleanConvertedMarkdown` has already
+        // decoded the address out of the link.
         if (resolved.pathname.startsWith('/cdn-cgi/'))
             return null;
         return resolved.href;
@@ -91,39 +164,25 @@ function resolveTarget(target, baseUrl) {
         return null;
     }
 }
+/** Resolved targets, deduped, in the order they were found. */
+function resolveUnique(targets, baseUrl) {
+    return getUniqueValues(compact(targets.map((target) => (target ? resolveTarget(target, baseUrl) : null))));
+}
 /**
  * Absolute, deduped, document-order links and images. Discovery crawls these,
  * so relative targets (`/kontakt`) and CMS-escaped query strings
  * (`?image=x&amp;width=200`) have to come out resolved and decoded.
  */
 export function markdownRefs(markdown, baseUrl) {
-    const links = [];
-    const images = [];
-    const seenLinks = new Set();
-    const seenImages = new Set();
-    const push = (isImage, target) => {
-        const resolved = resolveTarget(target, baseUrl);
-        if (!resolved)
-            return;
-        const seen = isImage ? seenImages : seenLinks;
-        if (seen.has(resolved))
-            return;
-        seen.add(resolved);
-        (isImage ? images : links).push(resolved);
+    const images = [...markdown.matchAll(IMAGE_RE)].map((match) => match[1]);
+    const links = [
+        ...[...markdown.matchAll(LINK_RE)].map((match) => match[1] === '!' ? undefined : match[2]),
+        ...[...markdown.matchAll(AUTOLINK_RE)].map((match) => match[1])
+    ];
+    return {
+        links: resolveUnique(links, baseUrl),
+        images: resolveUnique(images, baseUrl)
     };
-    for (const match of markdown.matchAll(IMAGE_RE)) {
-        if (match[1])
-            push(true, match[1]);
-    }
-    for (const match of markdown.matchAll(LINK_RE)) {
-        if (match[1] !== '!' && match[2])
-            push(false, match[2]);
-    }
-    for (const match of markdown.matchAll(AUTOLINK_RE)) {
-        if (match[1])
-            push(false, match[1]);
-    }
-    return { links, images };
 }
 const ANCHOR_HREF_RE = /<a\b[^>]*?\bhref\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi;
 /**
@@ -145,39 +204,25 @@ const NON_MARKUP_RE = /<(script|style|template)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
  * reachable if ANY representation mentions it.
  */
 export function htmlLinks(html, baseUrl) {
-    const links = [];
-    const seen = new Set();
-    for (const match of html
-        .replace(NON_MARKUP_RE, ' ')
-        .matchAll(ANCHOR_HREF_RE)) {
+    const targets = [
+        ...html.replace(NON_MARKUP_RE, ' ').matchAll(ANCHOR_HREF_RE)
+    ].map((match) => {
         const raw = match[1];
         if (!raw)
-            continue;
+            return undefined;
         const first = raw[0];
         const target = (first === '"' || first === "'") && raw.endsWith(first)
             ? raw.slice(1, -1)
             : raw;
-        const resolved = resolveTarget(decodeHtmlEntities(target), baseUrl);
-        if (!resolved || seen.has(resolved))
-            continue;
-        seen.add(resolved);
-        links.push(resolved);
-    }
-    return links;
+        return decodeHtmlEntities(target);
+    });
+    return resolveUnique(targets, baseUrl);
 }
 /** Conversion may omit an image-only menu; retain original and lazy asset URLs. */
 export function htmlImages(html, baseUrl) {
-    const images = new Set();
-    for (const tag of html.matchAll(/<img\b[^>]*>/gi)) {
-        for (const attribute of tag[0].matchAll(/\b(?:src|data-src|data-original)\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi)) {
-            const raw = attribute[1];
-            if (!raw)
-                continue;
-            const url = resolveTarget(raw.replace(/^["']|["']$/g, ''), baseUrl);
-            if (url)
-                images.add(url);
-        }
-    }
-    return [...images];
+    const targets = [...html.matchAll(/<img\b[^>]*>/gi)].flatMap((tag) => [
+        ...tag[0].matchAll(/\b(?:src|data-src|data-original)\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi)
+    ].map((attribute) => attribute[1]?.replace(/^["']|["']$/g, '')));
+    return resolveUnique(targets, baseUrl);
 }
 //# sourceMappingURL=markdown-refs.js.map

package/dist/markdown-refs.js.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-refs.js","sourceRoot":"","sources":["../src/markdown-refs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AAEvD;;;;;GAKG;AACH,MAAM,eAAe,GAAG,gDAAgD,CAAA;AAExE,SAAS,gBAAgB,CAAC,KAAa,EAAE,GAAW;IACnD,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,eAAe,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACjE,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;IAC7B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IACrB,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAA;IAC5D,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,IAAI,CAAA;AAC/B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAChD,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5C,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;IAC/D,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC5B,OAAO;QACN,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAC7D,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC;QACvC,WAAW,EAAE,gBAAgB,CAAC,KAAK,EAAE,aAAa,CAAC;KACnD,CAAA;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,iBAAiB,GACtB,yDAAyD,CAAA;AAC1D,MAAM,oBAAoB,GACzB,oDAAoD,CAAA;AACrD,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AACtD,MAAM,wBAAwB,GAAG,6BAA6B,CAAA;AAE9D,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACrD,IAAI,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,WAAW,GAAG,IAAI,CAAA;IACxB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IAC9D,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACjE,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACjE,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,IAAI,KAAK,WAAW,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAClE,OAAO,QAAQ,CAAA;IAChB,CAAC;IACD,OAAO,IAAI;SACT,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC;SACrC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAA;AACT,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;IAC7C,OAAO,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAA;AACpC,CAAC;AAED,MAAM,UAAU,GAAG,0CAA0C,CAAA;AAC7D,MAAM,WAAW,GAAG,6BAA6B,UAAU,SAAS,CAAA;AAEpE,oFAAoF;AACpF,MAAM,QAAQ,GAAG,IAAI,MAAM,CAC1B,kCAAkC,WAAW,EAAE,EAC/C,GAAG,CACH,CAAA;AACD,oEAAoE;AACpE,MAAM,OAAO,GAAG,IAAI,MAAM,CACzB,mDAAmD,WAAW,EAAE,EAChE,GAAG,CACH,CAAA;AACD,MAAM,WAAW,GAAG,2BAA2B,CAAA;AAE/C,SAAS,aAAa,CAAC,MAAc,EAAE,OAAe;IACrD,MAAM,GAAG,GAAG,kBAAkB,CAC7B,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAC7C,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;QAC5B,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAChB,CAAA;IACD,oDAAoD;IACpD,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC5C,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACtC,yEAAyE;QACzE,4DAA4D;QAC5D,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACrE,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,qEAAqE;QACrE,sEAAsE;QACtE,yEAAyE;QACzE,6CAA6C;QAC7C,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC;YAAE,OAAO,IAAI,CAAA;QAC1D,OAAO,QAAQ,CAAC,IAAI,CAAA;IACrB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,OAAe;IAC7D,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAA;IACnC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;IAEpC,MAAM,IAAI,GAAG,CAAC,OAAgB,EAAE,MAAc,EAAE,EAAE;QACjD,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAC/C,IAAI,CAAC,QAAQ;YAAE,OAAM;QACrB,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAA;QAC7C,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,OAAM;QAC9B,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CACjB;QAAA,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC3C,CAAC,CAAA;IAED,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjD,IAAI,KAAK,CAAC,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IACnC,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC;YAAE,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IACxD,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACpD,IAAI,KAAK,CAAC,CAAC,CAAC;YAAE,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IACpC,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAED,MAAM,cAAc,GAAG,oDAAoD,CAAA;AAC3E;;;;;GAKG;AACH,MAAM,aAAa,GAAG,qDAAqD,CAAA;AAE3E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,OAAe;IACtD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,KAAK,IAAI,IAAI;SACtB,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACpB,IAAI,CAAC,GAAG;YAAE,SAAQ;QAClB,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAA;QACpB,MAAM,MAAM,GACX,CAAC,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;YACtD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,GAAG,CAAA;QACP,MAAM,QAAQ,GAAG,aAAa,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAA;QACnE,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,SAAQ;QAC7C,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAClB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACrB,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAAe;IACvD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAA;IAChC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACnD,KAAK,MAAM,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CACtC,oEAAoE,CACpE,EAAE,CAAC;YACH,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;YACxB,IAAI,CAAC,GAAG;gBAAE,SAAQ;YAClB,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,EAAE,OAAO,CAAC,CAAA;YACnE,IAAI,GAAG;gBAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAA;AACnB,CAAC","sourcesContent":["import { decodeHtmlEntities } from './html-entities.js'\n\n/**\n * Workers AI `toMarkdown` prefixes every conversion with a YAML block:\n * `---\\ntitle: …\\ndescription: …\\n---`. It is a conversion artefact, not page\n * content, so it never reaches a document - but its two fields are the best\n * fallback we have for pages whose HTML we never saw.\n */\nconst FRONT_MATTER_RE = /^?---\\r?\\n([\\s\\S]*?)\\r?\\n---[ \\t]*(?:\\r?\\n|$)/\n\nfunction frontMatterValue(block: string, key: string) {\n\tconst line = new RegExp(`^${key}:[ \\\\t]*(.*)$`, 'im').exec(block)\n\tconst raw = line?.[1]?.trim()\n\tif (!raw) return null\n\tconst unquoted = /^([\"'])([\\s\\S]*)\\1$/.exec(raw)?.[2] ?? raw\n\treturn unquoted.trim() || null\n}\n\nexport function stripFrontMatter(markdown: string) {\n\tconst match = FRONT_MATTER_RE.exec(markdown)\n\tif (!match) return { markdown, title: null, description: null }\n\tconst block = match[1] ?? ''\n\treturn {\n\t\tmarkdown: markdown.slice(match[0].length).replace(/^\\s+/, ''),\n\t\ttitle: frontMatterValue(block, 'title'),\n\t\tdescription: frontMatterValue(block, 'description')\n\t}\n}\n\n/**\n * Workers AI `toMarkdown` opens every converted PDF and office file with the\n * file name as an H1, a `## Metadata` list (creation dates, uuids,\n * `PDFFormatVersion=1.7`) and a `## Contents` heading, then labels every page\n * `### Page N`. None of it is what the document says, and the fact guard\n * demanded those dates and numbers of every written document - which is how\n * Hotel Hvide Klit's terms came back as 13,000 characters of raw text with\n * the PDF metadata inside.\n */\nconst DOCUMENT_TITLE_RE =\n\t/^#\\s+[^\\n]*\\.(?:pdf|docx?|xlsx?|pptx?|odt|rtf)\\s*\\r?\\n/i\nconst DOCUMENT_METADATA_RE =\n\t/^##\\s+Metadata\\s*\\r?\\n(?:(?:-\\s[^\\n]*|\\s*)\\r?\\n)*/i\nconst DOCUMENT_CONTENTS_RE = /^##\\s+Contents\\s*\\r?\\n/i\nconst DOCUMENT_PAGE_HEADING_RE = /^#{1,6}\\s+Page\\s+\\d+\\s*$/gim\n\nexport function stripDocumentPreamble(markdown: string): string {\n\tlet text = markdown.replace(/^\\s+/, '')\n\tconst beforeTitle = text\n\ttext = text.replace(DOCUMENT_TITLE_RE, '').replace(/^\\s+/, '')\n\ttext = text.replace(DOCUMENT_METADATA_RE, '').replace(/^\\s+/, '')\n\ttext = text.replace(DOCUMENT_CONTENTS_RE, '').replace(/^\\s+/, '')\n\t// A file that carried none of the preamble is a plain conversion; the page\n\t// labels are still Workers AI's, so they go regardless.\n\tif (text === beforeTitle && !DOCUMENT_PAGE_HEADING_RE.test(text)) {\n\t\treturn markdown\n\t}\n\treturn text\n\t\t.replace(DOCUMENT_PAGE_HEADING_RE, '')\n\t\t.replace(/\\n{3,}/g, '\\n\\n')\n\t\t.trim()\n}\n\nexport function titleFromMarkdown(markdown: string) {\n\tconst heading = markdown.match(/^#\\s+(.+)$/m)\n\treturn heading?.[1]?.trim() || null\n}\n\nconst TITLE_PART = `(?:\\\\s+(?:\"[^\"]*\"|'[^']*'|\\\\([^)]*\\\\)))?`\nconst TARGET_PART = `\\\\(\\\\s*(<[^>]*>|[^()\\\\s]+)${TITLE_PART}\\\\s*\\\\)`\n\n/** `![alt](url \"title\")`. Scanned separately so `[![logo](img)](/)` yields both. */\nconst IMAGE_RE = new RegExp(\n\t`!\\\\[(?:[^\\\\[\\\\]\\\\\\\\]|\\\\\\\\.)*\\\\]${TARGET_PART}`,\n\t'g'\n)\n/** `[text](url)`; the leading `!` marks an image and is skipped. */\nconst LINK_RE = new RegExp(\n\t`(!?)\\\\[(?:[^\\\\[\\\\]\\\\\\\\]|\\\\\\\\.|\\\\[[^\\\\]]*\\\\])*\\\\]${TARGET_PART}`,\n\t'g'\n)\nconst AUTOLINK_RE = /<(https?:\\/\\/[^\\s<>]+)>/gi\n\nfunction resolveTarget(target: string, baseUrl: string) {\n\tconst raw = decodeHtmlEntities(\n\t\ttarget.startsWith('<') && target.endsWith('>')\n\t\t\t? target.slice(1, -1).trim()\n\t\t\t: target.trim()\n\t)\n\t// Pure fragments point at the page we already have.\n\tif (!raw || raw.startsWith('#')) return null\n\ttry {\n\t\tconst resolved = new URL(raw, baseUrl)\n\t\t// Drops `mailto:`, `tel:`, `javascript:`, `data:` and every other scheme\n\t\t// discovery cannot crawl; they stay in the markdown itself.\n\t\tif (resolved.protocol !== 'https:' && resolved.protocol !== 'http:') {\n\t\t\treturn null\n\t\t}\n\t\t// Cloudflare's reserved prefix is plumbing, never a page: the e-mail\n\t\t// obfuscation endpoint under it answers 410, which used to count as a\n\t\t// page this site failed to deliver. `clean-pages.ts` decodes the address\n\t\t// out of the link before it is dropped here.\n\t\tif (resolved.pathname.startsWith('/cdn-cgi/')) return null\n\t\treturn resolved.href\n\t} catch {\n\t\treturn null\n\t}\n}\n\n/**\n * Absolute, deduped, document-order links and images. Discovery crawls these,\n * so relative targets (`/kontakt`) and CMS-escaped query strings\n * (`?image=x&amp;width=200`) have to come out resolved and decoded.\n */\nexport function markdownRefs(markdown: string, baseUrl: string) {\n\tconst links: string[] = []\n\tconst images: string[] = []\n\tconst seenLinks = new Set<string>()\n\tconst seenImages = new Set<string>()\n\n\tconst push = (isImage: boolean, target: string) => {\n\t\tconst resolved = resolveTarget(target, baseUrl)\n\t\tif (!resolved) return\n\t\tconst seen = isImage ? seenImages : seenLinks\n\t\tif (seen.has(resolved)) return\n\t\tseen.add(resolved)\n\t\t;(isImage ? images : links).push(resolved)\n\t}\n\n\tfor (const match of markdown.matchAll(IMAGE_RE)) {\n\t\tif (match[1]) push(true, match[1])\n\t}\n\tfor (const match of markdown.matchAll(LINK_RE)) {\n\t\tif (match[1] !== '!' && match[2]) push(false, match[2])\n\t}\n\tfor (const match of markdown.matchAll(AUTOLINK_RE)) {\n\t\tif (match[1]) push(false, match[1])\n\t}\n\n\treturn { links, images }\n}\n\nconst ANCHOR_HREF_RE = /<a\\b[^>]*?\\bhref\\s*=\\s*(\"[^\"]*\"|'[^']*'|[^\\s>]+)/gi\n/**\n * Script, style and template bodies are not markup. A theme that builds its\n * cards in JavaScript (`'<a href=\"'+dipi_permalink+'\">'`) otherwise hands\n * discovery hundreds of URLs ending in `+dipi_permalink+` or `'+n+'`, and\n * every one of them is a 404 the crawl waits in line for.\n */\nconst NON_MARKUP_RE = /<(script|style|template)\\b[^>]*>[\\s\\S]*?<\\/\\1\\s*>/gi\n\n/**\n * Every `<a href>` of the page, absolute and deduped.\n *\n * Markdown conversion is lossy about navigation: an anchor wrapped around an\n * image, a `<nav>` the converter folded away, or a menu rendered as list items\n * all lose their target. Measured on one customer homepage: 16 internal hrefs\n * in the HTML, 4 in the markdown - and one of the twelve missing links was the\n * only route to a page holding prices and opening hours. Discovery unions this\n * with the markdown links whenever an HTML tier read the page, so a page is\n * reachable if ANY representation mentions it.\n */\nexport function htmlLinks(html: string, baseUrl: string): string[] {\n\tconst links: string[] = []\n\tconst seen = new Set<string>()\n\tfor (const match of html\n\t\t.replace(NON_MARKUP_RE, ' ')\n\t\t.matchAll(ANCHOR_HREF_RE)) {\n\t\tconst raw = match[1]\n\t\tif (!raw) continue\n\t\tconst first = raw[0]\n\t\tconst target =\n\t\t\t(first === '\"' || first === \"'\") && raw.endsWith(first)\n\t\t\t\t? raw.slice(1, -1)\n\t\t\t\t: raw\n\t\tconst resolved = resolveTarget(decodeHtmlEntities(target), baseUrl)\n\t\tif (!resolved || seen.has(resolved)) continue\n\t\tseen.add(resolved)\n\t\tlinks.push(resolved)\n\t}\n\treturn links\n}\n\n/** Conversion may omit an image-only menu; retain original and lazy asset URLs. */\nexport function htmlImages(html: string, baseUrl: string): string[] {\n\tconst images = new Set<string>()\n\tfor (const tag of html.matchAll(/<img\\b[^>]*>/gi)) {\n\t\tfor (const attribute of tag[0].matchAll(\n\t\t\t/\\b(?:src|data-src|data-original)\\s*=\\s*(\"[^\"]*\"|'[^']*'|[^\\s>]+)/gi\n\t\t)) {\n\t\t\tconst raw = attribute[1]\n\t\t\tif (!raw) continue\n\t\t\tconst url = resolveTarget(raw.replace(/^[\"']|[\"']$/g, ''), baseUrl)\n\t\t\tif (url) images.add(url)\n\t\t}\n\t}\n\treturn [...images]\n}\n"]}
+{"version":3,"file":"markdown-refs.js","sourceRoot":"","sources":["../src/markdown-refs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAA;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AAEvD;;;;;GAKG;AACH,MAAM,eAAe,GAAG,gDAAgD,CAAA;AAExE,SAAS,gBAAgB,CAAC,KAAa,EAAE,GAAW;IACnD,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,eAAe,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACjE,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;IAC7B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IACrB,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAA;IAC5D,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,IAAI,CAAA;AAC/B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAChD,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5C,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;IAC/D,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC5B,OAAO;QACN,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAC7D,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC;QACvC,WAAW,EAAE,gBAAgB,CAAC,KAAK,EAAE,aAAa,CAAC;KACnD,CAAA;AACF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,iBAAiB,GACtB,yDAAyD,CAAA;AAC1D,MAAM,oBAAoB,GACzB,oDAAoD,CAAA;AACrD,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AACtD,MAAM,wBAAwB,GAAG,6BAA6B,CAAA;AAE9D,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACrD,IAAI,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,WAAW,GAAG,IAAI,CAAA;IACxB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IAC9D,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACjE,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACjE,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,IAAI,KAAK,WAAW,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAClE,OAAO,QAAQ,CAAA;IAChB,CAAC;IACD,OAAO,IAAI;SACT,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC;SACrC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAA;AACT,CAAC;AAED;;;;;;GAMG;AACH,MAAM,gBAAgB,GACrB,6GAA6G,CAAA;AAE9G,SAAS,qBAAqB,CAAC,GAAW;IACzC,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAChD,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,OAAO,IAAI,MAAM,CAAC,YAAY,CAC7B,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CACtD,CAAA;IACF,CAAC;IACD,OAAO,6BAA6B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;AACpE,CAAC;AAED;;;;GAIG;AACH,MAAM,eAAe,GAAG,qCAAqC,CAAA;AAC7D,MAAM,UAAU,GAAG,kCAAkC,CAAA;AAErD,uFAAuF;AACvF,MAAM,qBAAqB,GAAG,kCAAkC,CAAA;AAEhE;;;;;GAKG;AACH,MAAM,aAAa,GAAG,0CAA0C,CAAA;AAEhE,SAAS,UAAU,CAAC,KAAa,EAAE,MAAc;IAChD,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;IAC1D,OAAO,OAAO,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;QACtD,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;QAC/B,CAAC,CAAC,CAAC,CAAC,CAAA;AACN,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,MAAc;IAChD,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;IACxC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAA;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACnC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAA;IACxB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAC/B,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,MAAM;QACvE,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,CAAC,CAAC,CAAA;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAgB;IACtD,OAAO,QAAQ;SACb,OAAO,CAAC,gBAAgB,EAAE,CAAC,IAAI,EAAE,GAAW,EAAE,EAAE;QAChD,MAAM,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAA;QAC1C,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,YAAY,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1D,CAAC,CAAC;SACD,OAAO,CAAC,eAAe,EAAE,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE,CACjD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAClC;SACA,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC;SAClC,OAAO,CACP,aAAa,EACb,CAAC,IAAI,EAAE,KAAa,EAAE,MAAc,EAAE,MAAc,EAAE,EAAE;QACvD,MAAM,KAAK,GACV,MAAM,CAAC,WAAW,EAAE,KAAK,QAAQ;YAChC,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC;YAC3B,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QAC7B,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAAE,OAAO,IAAI,CAAA;QACrE,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,CAAA;IAC/E,CAAC,CACD,CAAA;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;IAC7C,OAAO,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAA;AACpC,CAAC;AAED,MAAM,UAAU,GAAG,0CAA0C,CAAA;AAC7D,MAAM,WAAW,GAAG,6BAA6B,UAAU,SAAS,CAAA;AAEpE,oFAAoF;AACpF,MAAM,QAAQ,GAAG,IAAI,MAAM,CAC1B,kCAAkC,WAAW,EAAE,EAC/C,GAAG,CACH,CAAA;AACD,oEAAoE;AACpE,MAAM,OAAO,GAAG,IAAI,MAAM,CACzB,mDAAmD,WAAW,EAAE,EAChE,GAAG,CACH,CAAA;AACD,MAAM,WAAW,GAAG,2BAA2B,CAAA;AAE/C,SAAS,aAAa,CAAC,MAAc,EAAE,OAAe;IACrD,MAAM,GAAG,GAAG,kBAAkB,CAC7B,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAC7C,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;QAC5B,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAChB,CAAA;IACD,oDAAoD;IACpD,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC5C,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACtC,yEAAyE;QACzE,4DAA4D;QAC5D,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAA;QACjE,qEAAqE;QACrE,sEAAsE;QACtE,yEAAyE;QACzE,uCAAuC;QACvC,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC;YAAE,OAAO,IAAI,CAAA;QAC1D,OAAO,QAAQ,CAAC,IAAI,CAAA;IACrB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED,+DAA+D;AAC/D,SAAS,aAAa,CAAC,OAA+B,EAAE,OAAe;IACtE,OAAO,eAAe,CACrB,OAAO,CACN,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CACzE,CACD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,OAAe;IAC7D,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IACxE,MAAM,KAAK,GAAG;QACb,GAAG,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAChD,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CACvC;QACD,GAAG,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KAC/D,CAAA;IACD,OAAO;QACN,KAAK,EAAE,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC;QACpC,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC;KACtC,CAAA;AACF,CAAC;AAED,MAAM,cAAc,GAAG,oDAAoD,CAAA;AAC3E;;;;;GAKG;AACH,MAAM,aAAa,GAAG,qDAAqD,CAAA;AAE3E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,OAAe;IACtD,MAAM,OAAO,GAAG;QACf,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC;KAC5D,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACpB,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAA;QAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAA;QACpB,MAAM,MAAM,GACX,CAAC,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;YACtD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,GAAG,CAAA;QACP,OAAO,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC,CAAC,CAAA;IACF,OAAO,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;AACvC,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAAe;IACvD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CACpE;QACC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CACjB,oEAAoE,CACpE;KACD,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,CAC/D,CAAA;IACD,OAAO,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;AACvC,CAAC","sourcesContent":["import { compact, getUniqueValues, isOneOf } from 'hanzio'\nimport { decodeHtmlEntities } from './html-entities.js'\n\n/**\n * Workers AI `toMarkdown` prefixes every conversion with a YAML block:\n * `---\\ntitle: …\\ndescription: …\\n---`. It is a conversion artefact, not page\n * content, so it never reaches a document - but its two fields are the best\n * fallback we have for pages whose HTML we never saw.\n */\nconst FRONT_MATTER_RE = /^?---\\r?\\n([\\s\\S]*?)\\r?\\n---[ \\t]*(?:\\r?\\n|$)/\n\nfunction frontMatterValue(block: string, key: string) {\n\tconst line = new RegExp(`^${key}:[ \\\\t]*(.*)$`, 'im').exec(block)\n\tconst raw = line?.[1]?.trim()\n\tif (!raw) return null\n\tconst unquoted = /^([\"'])([\\s\\S]*)\\1$/.exec(raw)?.[2] ?? raw\n\treturn unquoted.trim() || null\n}\n\nexport function stripFrontMatter(markdown: string) {\n\tconst match = FRONT_MATTER_RE.exec(markdown)\n\tif (!match) return { markdown, title: null, description: null }\n\tconst block = match[1] ?? ''\n\treturn {\n\t\tmarkdown: markdown.slice(match[0].length).replace(/^\\s+/, ''),\n\t\ttitle: frontMatterValue(block, 'title'),\n\t\tdescription: frontMatterValue(block, 'description')\n\t}\n}\n\n/**\n * Workers AI `toMarkdown` opens every converted PDF and office file with the\n * file name as an H1, a `## Metadata` list (creation dates, uuids,\n * `PDFFormatVersion=1.7`) and a `## Contents` heading, then labels every page\n * `### Page N`. None of it is what the document says: a hotel's terms PDF\n * came back as 13,000 characters with its creation dates and uuids inside,\n * and a reader of the markdown takes those for facts of the document.\n */\nconst DOCUMENT_TITLE_RE =\n\t/^#\\s+[^\\n]*\\.(?:pdf|docx?|xlsx?|pptx?|odt|rtf)\\s*\\r?\\n/i\nconst DOCUMENT_METADATA_RE =\n\t/^##\\s+Metadata\\s*\\r?\\n(?:(?:-\\s[^\\n]*|\\s*)\\r?\\n)*/i\nconst DOCUMENT_CONTENTS_RE = /^##\\s+Contents\\s*\\r?\\n/i\nconst DOCUMENT_PAGE_HEADING_RE = /^#{1,6}\\s+Page\\s+\\d+\\s*$/gim\n\nexport function stripDocumentPreamble(markdown: string): string {\n\tlet text = markdown.replace(/^\\s+/, '')\n\tconst beforeTitle = text\n\ttext = text.replace(DOCUMENT_TITLE_RE, '').replace(/^\\s+/, '')\n\ttext = text.replace(DOCUMENT_METADATA_RE, '').replace(/^\\s+/, '')\n\ttext = text.replace(DOCUMENT_CONTENTS_RE, '').replace(/^\\s+/, '')\n\t// A file that carried none of the preamble is a plain conversion; the page\n\t// labels are still Workers AI's, so they go regardless.\n\tif (text === beforeTitle && !DOCUMENT_PAGE_HEADING_RE.test(text)) {\n\t\treturn markdown\n\t}\n\treturn text\n\t\t.replace(DOCUMENT_PAGE_HEADING_RE, '')\n\t\t.replace(/\\n{3,}/g, '\\n\\n')\n\t\t.trim()\n}\n\n/**\n * Cloudflare's e-mail obfuscation rewrites every address it finds into\n * `[\\[email protected\\]](/cdn-cgi/l/email-protection#<hex>)`, so a converted\n * page behind it shows no readable address at all. The hex is the address\n * XOR-ed with its first byte; decoded only when the result is shaped like an\n * address.\n */\nconst CF_EMAIL_LINK_RE =\n\t/\\[(?:[^[\\]\\\\]|\\\\.|\\[[^\\]]*\\])*\\]\\((?:https?:\\/\\/[^/)]+)?\\/cdn-cgi\\/l\\/email-protection#([0-9a-fA-F]{8,})\\)/g\n\nfunction decodeCloudflareEmail(hex: string) {\n\tconst key = Number.parseInt(hex.slice(0, 2), 16)\n\tlet address = ''\n\tfor (let index = 2; index + 1 < hex.length; index += 2) {\n\t\taddress += String.fromCharCode(\n\t\t\tNumber.parseInt(hex.slice(index, index + 2), 16) ^ key\n\t\t)\n\t}\n\treturn /^[^\\s@]+@[^\\s@]+\\.[^\\s@.]+$/.test(address) ? address : null\n}\n\n/**\n * `<script type=\"application/ld+json\">` survives conversion as a fenced code\n * block on Yoast and Squarespace sites. It is never prose, and the page already\n * carries it parsed as `jsonLd`.\n */\nconst FENCED_BLOCK_RE = /^```[^\\n]*\\n([\\s\\S]*?)^```[ \\t]*$/gm\nconst JSON_LD_RE = /schema\\.org|[\"']@context[\"']\\s*:/\n\n/** A page that inlines an SVG file gets its prolog back from the converter as text. */\nconst MARKUP_DECLARATION_RE = /<\\?xml[^>]*\\?>|<!DOCTYPE[^>]*>/gi\n\n/**\n * A label rendered next to its own value with no space between -\n * `[Saleshello@example.com](mailto:hello@example.com)`,\n * `[Support+45 22 81 82 95](tel:+4522818295)` - reads as an invented address.\n * Matched on the value's shape, so the label's language never matters.\n */\nconst VALUE_LINK_RE = /\\[([^\\]]+)\\]\\((mailto|tel):([^)\\s]+)\\)/gi\n\nfunction emailStart(label: string, target: string) {\n\tconst address = (target.split('?')[0] ?? '').toLowerCase()\n\treturn address && label.toLowerCase().endsWith(address)\n\t\t? label.length - address.length\n\t\t: -1\n}\n\nfunction phoneStart(label: string, target: string) {\n\tconst digits = target.replace(/\\D/g, '')\n\tif (digits.length < 6) return -1\n\tconst first = label.search(/[+\\d]/)\n\tif (first < 0) return -1\n\tconst rest = label.slice(first)\n\treturn /^[+\\d\\s().-]+$/.test(rest) && rest.replace(/\\D/g, '') === digits\n\t\t? first\n\t\t: -1\n}\n\n/**\n * Removes what Cloudflare's conversion leaves behind and is never page text:\n * obfuscated e-mail links are decoded, JSON-LD code fences and markup\n * declarations dropped, and labels glued to their `mailto:`/`tel:` value\n * split. Applied to every fetched page; idempotent, so markdown stored by an\n * older version can be run through it too.\n */\nexport function cleanConvertedMarkdown(markdown: string) {\n\treturn markdown\n\t\t.replace(CF_EMAIL_LINK_RE, (link, hex: string) => {\n\t\t\tconst address = decodeCloudflareEmail(hex)\n\t\t\treturn address ? `[${address}](mailto:${address})` : link\n\t\t})\n\t\t.replace(FENCED_BLOCK_RE, (block, body: string) =>\n\t\t\tJSON_LD_RE.test(body) ? '' : block\n\t\t)\n\t\t.replace(MARKUP_DECLARATION_RE, '')\n\t\t.replace(\n\t\t\tVALUE_LINK_RE,\n\t\t\t(link, label: string, scheme: string, target: string) => {\n\t\t\t\tconst start =\n\t\t\t\t\tscheme.toLowerCase() === 'mailto'\n\t\t\t\t\t\t? emailStart(label, target)\n\t\t\t\t\t\t: phoneStart(label, target)\n\t\t\t\tif (start <= 0 || !/\\p{L}/u.test(label[start - 1] ?? '')) return link\n\t\t\t\treturn `[${label.slice(0, start)} ${label.slice(start)}](${scheme}:${target})`\n\t\t\t}\n\t\t)\n}\n\nexport function titleFromMarkdown(markdown: string) {\n\tconst heading = markdown.match(/^#\\s+(.+)$/m)\n\treturn heading?.[1]?.trim() || null\n}\n\nconst TITLE_PART = `(?:\\\\s+(?:\"[^\"]*\"|'[^']*'|\\\\([^)]*\\\\)))?`\nconst TARGET_PART = `\\\\(\\\\s*(<[^>]*>|[^()\\\\s]+)${TITLE_PART}\\\\s*\\\\)`\n\n/** `![alt](url \"title\")`. Scanned separately so `[![logo](img)](/)` yields both. */\nconst IMAGE_RE = new RegExp(\n\t`!\\\\[(?:[^\\\\[\\\\]\\\\\\\\]|\\\\\\\\.)*\\\\]${TARGET_PART}`,\n\t'g'\n)\n/** `[text](url)`; the leading `!` marks an image and is skipped. */\nconst LINK_RE = new RegExp(\n\t`(!?)\\\\[(?:[^\\\\[\\\\]\\\\\\\\]|\\\\\\\\.|\\\\[[^\\\\]]*\\\\])*\\\\]${TARGET_PART}`,\n\t'g'\n)\nconst AUTOLINK_RE = /<(https?:\\/\\/[^\\s<>]+)>/gi\n\nfunction resolveTarget(target: string, baseUrl: string) {\n\tconst raw = decodeHtmlEntities(\n\t\ttarget.startsWith('<') && target.endsWith('>')\n\t\t\t? target.slice(1, -1).trim()\n\t\t\t: target.trim()\n\t)\n\t// Pure fragments point at the page we already have.\n\tif (!raw || raw.startsWith('#')) return null\n\ttry {\n\t\tconst resolved = new URL(raw, baseUrl)\n\t\t// Drops `mailto:`, `tel:`, `javascript:`, `data:` and every other scheme\n\t\t// discovery cannot crawl; they stay in the markdown itself.\n\t\tif (!isOneOf(['https:', 'http:'], resolved.protocol)) return null\n\t\t// Cloudflare's reserved prefix is plumbing, never a page: the e-mail\n\t\t// obfuscation endpoint under it answers 410, which used to count as a\n\t\t// page this site failed to deliver. `cleanConvertedMarkdown` has already\n\t\t// decoded the address out of the link.\n\t\tif (resolved.pathname.startsWith('/cdn-cgi/')) return null\n\t\treturn resolved.href\n\t} catch {\n\t\treturn null\n\t}\n}\n\n/** Resolved targets, deduped, in the order they were found. */\nfunction resolveUnique(targets: (string | undefined)[], baseUrl: string) {\n\treturn getUniqueValues(\n\t\tcompact(\n\t\t\ttargets.map((target) => (target ? resolveTarget(target, baseUrl) : null))\n\t\t)\n\t)\n}\n\n/**\n * Absolute, deduped, document-order links and images. Discovery crawls these,\n * so relative targets (`/kontakt`) and CMS-escaped query strings\n * (`?image=x&amp;width=200`) have to come out resolved and decoded.\n */\nexport function markdownRefs(markdown: string, baseUrl: string) {\n\tconst images = [...markdown.matchAll(IMAGE_RE)].map((match) => match[1])\n\tconst links = [\n\t\t...[...markdown.matchAll(LINK_RE)].map((match) =>\n\t\t\tmatch[1] === '!' ? undefined : match[2]\n\t\t),\n\t\t...[...markdown.matchAll(AUTOLINK_RE)].map((match) => match[1])\n\t]\n\treturn {\n\t\tlinks: resolveUnique(links, baseUrl),\n\t\timages: resolveUnique(images, baseUrl)\n\t}\n}\n\nconst ANCHOR_HREF_RE = /<a\\b[^>]*?\\bhref\\s*=\\s*(\"[^\"]*\"|'[^']*'|[^\\s>]+)/gi\n/**\n * Script, style and template bodies are not markup. A theme that builds its\n * cards in JavaScript (`'<a href=\"'+dipi_permalink+'\">'`) otherwise hands\n * discovery hundreds of URLs ending in `+dipi_permalink+` or `'+n+'`, and\n * every one of them is a 404 the crawl waits in line for.\n */\nconst NON_MARKUP_RE = /<(script|style|template)\\b[^>]*>[\\s\\S]*?<\\/\\1\\s*>/gi\n\n/**\n * Every `<a href>` of the page, absolute and deduped.\n *\n * Markdown conversion is lossy about navigation: an anchor wrapped around an\n * image, a `<nav>` the converter folded away, or a menu rendered as list items\n * all lose their target. Measured on one customer homepage: 16 internal hrefs\n * in the HTML, 4 in the markdown - and one of the twelve missing links was the\n * only route to a page holding prices and opening hours. Discovery unions this\n * with the markdown links whenever an HTML tier read the page, so a page is\n * reachable if ANY representation mentions it.\n */\nexport function htmlLinks(html: string, baseUrl: string): string[] {\n\tconst targets = [\n\t\t...html.replace(NON_MARKUP_RE, ' ').matchAll(ANCHOR_HREF_RE)\n\t].map((match) => {\n\t\tconst raw = match[1]\n\t\tif (!raw) return undefined\n\t\tconst first = raw[0]\n\t\tconst target =\n\t\t\t(first === '\"' || first === \"'\") && raw.endsWith(first)\n\t\t\t\t? raw.slice(1, -1)\n\t\t\t\t: raw\n\t\treturn decodeHtmlEntities(target)\n\t})\n\treturn resolveUnique(targets, baseUrl)\n}\n\n/** Conversion may omit an image-only menu; retain original and lazy asset URLs. */\nexport function htmlImages(html: string, baseUrl: string): string[] {\n\tconst targets = [...html.matchAll(/<img\\b[^>]*>/gi)].flatMap((tag) =>\n\t\t[\n\t\t\t...tag[0].matchAll(\n\t\t\t\t/\\b(?:src|data-src|data-original)\\s*=\\s*(\"[^\"]*\"|'[^']*'|[^\\s>]+)/gi\n\t\t\t)\n\t\t].map((attribute) => attribute[1]?.replace(/^[\"']|[\"']$/g, ''))\n\t)\n\treturn resolveUnique(targets, baseUrl)\n}\n"]}

package/dist/memory-limit.d.ts

@@ -4,7 +4,7 @@ import type { ConcurrencyLimit, ConcurrencyLimitOptions } from './limit.js';
  * starts in any second, and a `tripCooldown` pause before the next start.
  *
  * ponytail: per-process only. Several processes sharing one Cloudflare quota
- * need a shared limiter (hellu passes its Redis Lua limiter).
+ * need a shared limiter, e.g. `hanzio/redis`'s `createRedisConcurrencyLimit`.
  */
 export declare function createMemoryConcurrencyLimit(options: ConcurrencyLimitOptions): ConcurrencyLimit;
 //# sourceMappingURL=memory-limit.d.ts.map

package/dist/memory-limit.js

@@ -4,7 +4,7 @@ const START_WINDOW_MS = 1_000;
  * starts in any second, and a `tripCooldown` pause before the next start.
  *
  * ponytail: per-process only. Several processes sharing one Cloudflare quota
- * need a shared limiter (hellu passes its Redis Lua limiter).
+ * need a shared limiter, e.g. `hanzio/redis`'s `createRedisConcurrencyLimit`.
  */
 export function createMemoryConcurrencyLimit(options) {
     let active = 0;

package/dist/memory-limit.js.map

@@ -1 +1 @@
-{"version":3,"file":"memory-limit.js","sourceRoot":"","sources":["../src/memory-limit.ts"],"names":[],"mappings":"AAEA,MAAM,eAAe,GAAG,KAAK,CAAA;AAE7B;;;;;;GAMG;AACH,MAAM,UAAU,4BAA4B,CAC3C,OAAgC;IAEhC,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAc,CAAA;IAEpC,SAAS,OAAO;QACf,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC;YAAE,IAAI,EAAE,CAAA;IACvC,CAAC;IAED,yEAAyE;IACzE,SAAS,KAAK,CAAC,EAAiB,EAAE,MAAoB;QACrD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACpC,MAAM,KAAK,GAAG,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YAC5D,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YAChB,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;YACvD,SAAS,IAAI;gBACZ,YAAY,CAAC,KAAK,CAAC,CAAA;gBACnB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACnB,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;gBAC1C,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC;IAED,2EAA2E;IAC3E,SAAS,MAAM,CAAC,GAAW;QAC1B,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,eAAe;YACnE,MAAM,CAAC,KAAK,EAAE,CAAA;QACf,IAAI,aAAa,GAAG,GAAG;YAAE,OAAO,aAAa,GAAG,GAAG,CAAA;QACnD,IAAI,MAAM,IAAI,OAAO,CAAC,GAAG;YAAE,OAAO,IAAI,CAAA;QACtC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;QACxB,IACC,OAAO,CAAC,kBAAkB;YAC1B,MAAM,KAAK,SAAS;YACpB,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,kBAAkB,EAC1C,CAAC;YACF,OAAO,MAAM,GAAG,eAAe,GAAG,GAAG,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,CAAA;IACT,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,EAAK,GAAqB,EAAE,MAAoB,EAAE,EAAE;QACtE,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,EAAE,cAAc,EAAE,CAAA;YACxB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YACtB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;YACxB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBAChB,MAAM,EAAE,CAAA;gBACR,IAAI,OAAO,CAAC,kBAAkB;oBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBAChD,MAAK;YACN,CAAC;YACD,MAAM,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QAC1B,CAAC;QACD,IAAI,CAAC;YACJ,OAAO,MAAM,GAAG,EAAE,CAAA;QACnB,CAAC;gBAAS,CAAC;YACV,MAAM,EAAE,CAAA;YACR,OAAO,EAAE,CAAA;QACV,CAAC;IACF,CAAC,CAAA;IACD,KAAK,CAAC,YAAY,GAAG,KAAK,EAAE,MAAc,EAAE,EAAE;QAC7C,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAA;IAC7D,CAAC,CAAA;IACD,KAAK,CAAC,OAAO,GAAG,GAAG,EAAE,GAAE,CAAC,CAAA;IACxB,OAAO,KAAK,CAAA;AACb,CAAC","sourcesContent":["import type { ConcurrencyLimit, ConcurrencyLimitOptions } from './limit.js'\n\nconst START_WINDOW_MS = 1_000\n\n/**\n * The default limiter: `max` callbacks in flight, at most `maxStartsPerSecond`\n * starts in any second, and a `tripCooldown` pause before the next start.\n *\n * ponytail: per-process only. Several processes sharing one Cloudflare quota\n * need a shared limiter (hellu passes its Redis Lua limiter).\n */\nexport function createMemoryConcurrencyLimit(\n\toptions: ConcurrencyLimitOptions\n): ConcurrencyLimit {\n\tlet active = 0\n\tlet cooldownUntil = 0\n\tconst starts: number[] = []\n\tconst wakers = new Set<() => void>()\n\n\tfunction wakeAll() {\n\t\tfor (const wake of [...wakers]) wake()\n\t}\n\n\t/** Resolves after `ms`, on the next release (`ms` null), or on abort. */\n\tfunction sleep(ms: number | null, signal?: AbortSignal) {\n\t\treturn new Promise<void>((resolve) => {\n\t\t\tconst timer = ms === null ? undefined : setTimeout(done, ms)\n\t\t\twakers.add(done)\n\t\t\tsignal?.addEventListener('abort', done, { once: true })\n\t\t\tfunction done() {\n\t\t\t\tclearTimeout(timer)\n\t\t\t\twakers.delete(done)\n\t\t\t\tsignal?.removeEventListener('abort', done)\n\t\t\t\tresolve()\n\t\t\t}\n\t\t})\n\t}\n\n\t/** 0 when a slot can be taken now, null to wait for a release, else ms. */\n\tfunction waitMs(now: number) {\n\t\twhile (starts[0] !== undefined && starts[0] <= now - START_WINDOW_MS)\n\t\t\tstarts.shift()\n\t\tif (cooldownUntil > now) return cooldownUntil - now\n\t\tif (active >= options.max) return null\n\t\tconst oldest = starts[0]\n\t\tif (\n\t\t\toptions.maxStartsPerSecond &&\n\t\t\toldest !== undefined &&\n\t\t\tstarts.length >= options.maxStartsPerSecond\n\t\t) {\n\t\t\treturn oldest + START_WINDOW_MS - now\n\t\t}\n\t\treturn 0\n\t}\n\n\tconst limit = async <T>(run: () => Promise<T>, signal?: AbortSignal) => {\n\t\twhile (true) {\n\t\t\tsignal?.throwIfAborted()\n\t\t\tconst now = Date.now()\n\t\t\tconst wait = waitMs(now)\n\t\t\tif (wait === 0) {\n\t\t\t\tactive++\n\t\t\t\tif (options.maxStartsPerSecond) starts.push(now)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tawait sleep(wait, signal)\n\t\t}\n\t\ttry {\n\t\t\treturn await run()\n\t\t} finally {\n\t\t\tactive--\n\t\t\twakeAll()\n\t\t}\n\t}\n\tlimit.tripCooldown = async (waitMs: number) => {\n\t\tcooldownUntil = Math.max(cooldownUntil, Date.now() + waitMs)\n\t}\n\tlimit.dispose = () => {}\n\treturn limit\n}\n"]}
+{"version":3,"file":"memory-limit.js","sourceRoot":"","sources":["../src/memory-limit.ts"],"names":[],"mappings":"AAEA,MAAM,eAAe,GAAG,KAAK,CAAA;AAE7B;;;;;;GAMG;AACH,MAAM,UAAU,4BAA4B,CAC3C,OAAgC;IAEhC,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAc,CAAA;IAEpC,SAAS,OAAO;QACf,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC;YAAE,IAAI,EAAE,CAAA;IACvC,CAAC;IAED,yEAAyE;IACzE,SAAS,KAAK,CAAC,EAAiB,EAAE,MAAoB;QACrD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACpC,MAAM,KAAK,GAAG,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YAC5D,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YAChB,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;YACvD,SAAS,IAAI;gBACZ,YAAY,CAAC,KAAK,CAAC,CAAA;gBACnB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACnB,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;gBAC1C,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC;IAED,2EAA2E;IAC3E,SAAS,MAAM,CAAC,GAAW;QAC1B,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,eAAe;YACnE,MAAM,CAAC,KAAK,EAAE,CAAA;QACf,IAAI,aAAa,GAAG,GAAG;YAAE,OAAO,aAAa,GAAG,GAAG,CAAA;QACnD,IAAI,MAAM,IAAI,OAAO,CAAC,GAAG;YAAE,OAAO,IAAI,CAAA;QACtC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;QACxB,IACC,OAAO,CAAC,kBAAkB;YAC1B,MAAM,KAAK,SAAS;YACpB,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,kBAAkB,EAC1C,CAAC;YACF,OAAO,MAAM,GAAG,eAAe,GAAG,GAAG,CAAA;QACtC,CAAC;QACD,OAAO,CAAC,CAAA;IACT,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,EAAK,GAAqB,EAAE,MAAoB,EAAE,EAAE;QACtE,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,EAAE,cAAc,EAAE,CAAA;YACxB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YACtB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;YACxB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBAChB,MAAM,EAAE,CAAA;gBACR,IAAI,OAAO,CAAC,kBAAkB;oBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBAChD,MAAK;YACN,CAAC;YACD,MAAM,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QAC1B,CAAC;QACD,IAAI,CAAC;YACJ,OAAO,MAAM,GAAG,EAAE,CAAA;QACnB,CAAC;gBAAS,CAAC;YACV,MAAM,EAAE,CAAA;YACR,OAAO,EAAE,CAAA;QACV,CAAC;IACF,CAAC,CAAA;IACD,KAAK,CAAC,YAAY,GAAG,KAAK,EAAE,MAAc,EAAE,EAAE;QAC7C,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAA;IAC7D,CAAC,CAAA;IACD,KAAK,CAAC,OAAO,GAAG,GAAG,EAAE,GAAE,CAAC,CAAA;IACxB,OAAO,KAAK,CAAA;AACb,CAAC","sourcesContent":["import type { ConcurrencyLimit, ConcurrencyLimitOptions } from './limit.js'\n\nconst START_WINDOW_MS = 1_000\n\n/**\n * The default limiter: `max` callbacks in flight, at most `maxStartsPerSecond`\n * starts in any second, and a `tripCooldown` pause before the next start.\n *\n * ponytail: per-process only. Several processes sharing one Cloudflare quota\n * need a shared limiter, e.g. `hanzio/redis`'s `createRedisConcurrencyLimit`.\n */\nexport function createMemoryConcurrencyLimit(\n\toptions: ConcurrencyLimitOptions\n): ConcurrencyLimit {\n\tlet active = 0\n\tlet cooldownUntil = 0\n\tconst starts: number[] = []\n\tconst wakers = new Set<() => void>()\n\n\tfunction wakeAll() {\n\t\tfor (const wake of [...wakers]) wake()\n\t}\n\n\t/** Resolves after `ms`, on the next release (`ms` null), or on abort. */\n\tfunction sleep(ms: number | null, signal?: AbortSignal) {\n\t\treturn new Promise<void>((resolve) => {\n\t\t\tconst timer = ms === null ? undefined : setTimeout(done, ms)\n\t\t\twakers.add(done)\n\t\t\tsignal?.addEventListener('abort', done, { once: true })\n\t\t\tfunction done() {\n\t\t\t\tclearTimeout(timer)\n\t\t\t\twakers.delete(done)\n\t\t\t\tsignal?.removeEventListener('abort', done)\n\t\t\t\tresolve()\n\t\t\t}\n\t\t})\n\t}\n\n\t/** 0 when a slot can be taken now, null to wait for a release, else ms. */\n\tfunction waitMs(now: number) {\n\t\twhile (starts[0] !== undefined && starts[0] <= now - START_WINDOW_MS)\n\t\t\tstarts.shift()\n\t\tif (cooldownUntil > now) return cooldownUntil - now\n\t\tif (active >= options.max) return null\n\t\tconst oldest = starts[0]\n\t\tif (\n\t\t\toptions.maxStartsPerSecond &&\n\t\t\toldest !== undefined &&\n\t\t\tstarts.length >= options.maxStartsPerSecond\n\t\t) {\n\t\t\treturn oldest + START_WINDOW_MS - now\n\t\t}\n\t\treturn 0\n\t}\n\n\tconst limit = async <T>(run: () => Promise<T>, signal?: AbortSignal) => {\n\t\twhile (true) {\n\t\t\tsignal?.throwIfAborted()\n\t\t\tconst now = Date.now()\n\t\t\tconst wait = waitMs(now)\n\t\t\tif (wait === 0) {\n\t\t\t\tactive++\n\t\t\t\tif (options.maxStartsPerSecond) starts.push(now)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tawait sleep(wait, signal)\n\t\t}\n\t\ttry {\n\t\t\treturn await run()\n\t\t} finally {\n\t\t\tactive--\n\t\t\twakeAll()\n\t\t}\n\t}\n\tlimit.tripCooldown = async (waitMs: number) => {\n\t\tcooldownUntil = Math.max(cooldownUntil, Date.now() + waitMs)\n\t}\n\tlimit.dispose = () => {}\n\treturn limit\n}\n"]}

package/dist/net-guard.d.ts

@@ -6,10 +6,46 @@ import { type Logger } from './logger.js';
  */
 export declare function isBlockedAddress(address: string): boolean;
 export declare function isLocalhostHostname(hostname: string): boolean;
+export type HostGuardOptions = {
+    logger?: Logger | undefined;
+    /** Allow `localhost`, `127.0.0.1` and `::1`, e.g. for a dev server. Default false. */
+    allowLocalhost?: boolean | undefined;
+};
 /**
  * SSRF guard: a customer can put any hostname in the website field, and an
  * internal address behind it would turn the importer into a proxy for our own
- * network. Localhost stays reachable outside production for the dev server.
+ * network. An IP literal (`[fd00::1]` included) is judged as written, without
+ * a DNS lookup.
  */
-export declare function isHostAllowed(hostname: string, logger?: Logger): Promise<boolean>;
+export declare function isHostAllowed(hostname: string, options?: HostGuardOptions): Promise<boolean>;
+/** The `fetch` shape the guard wraps and the library accepts as an option. */
+export type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
+/** A request, or one of its redirects, went to a host the guard refuses. */
+export declare class HostNotAllowedError extends Error {
+    readonly url: string;
+    readonly name = "HostNotAllowedError";
+    constructor(url: string);
+}
+export type GuardedFetchOptions = {
+    /** Run on the URL and on every redirect target. Default {@link isHostAllowed}. */
+    isHostAllowed?: ((hostname: string) => Promise<boolean>) | undefined;
+    /** Default `globalThis.fetch`, read at call time. */
+    fetch?: Fetch | undefined;
+    /** Default 10. */
+    maxRedirects?: number | undefined;
+};
+/**
+ * `fetch` that runs the guard on every hop: `redirect: 'follow'` would take a
+ * public host's 302 to `http://169.254.169.254/` without asking. Returns the
+ * last response and the URL it came from.
+ */
+export declare function followRedirects(input: string | URL, init?: RequestInit, options?: GuardedFetchOptions): Promise<{
+    response: Response;
+    url: URL;
+}>;
+/**
+ * {@link followRedirects} as a drop-in `fetch`. Throws
+ * {@link HostNotAllowedError} when the URL or a redirect is refused.
+ */
+export declare function guardedFetch(input: string | URL, init?: RequestInit, options?: GuardedFetchOptions): Promise<Response>;
 //# sourceMappingURL=net-guard.d.ts.map

package/dist/net-guard.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"net-guard.d.ts","sourceRoot":"","sources":["../src/net-guard.ts"],"names":[],"mappings":"AACA,OAAO,EAAsB,KAAK,MAAM,EAAgB,MAAM,aAAa,CAAA;AA6E3E;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CA+BzD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,WAGnD;AAID;;;;GAIG;AACH,wBAAsB,aAAa,CAClC,QAAQ,EAAE,MAAM,EAChB,MAAM,GAAE,MAAqB,oBAuC7B"}
+{"version":3,"file":"net-guard.d.ts","sourceRoot":"","sources":["../src/net-guard.ts"],"names":[],"mappings":"AAEA,OAAO,EAAsB,KAAK,MAAM,EAAgB,MAAM,aAAa,CAAA;AA6E3E;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CA+BzD;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,WAGnD;AAOD,MAAM,MAAM,gBAAgB,GAAG;IAC9B,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC3B,sFAAsF;IACtF,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;CACpC,CAAA;AAED;;;;;GAKG;AACH,wBAAsB,aAAa,CAClC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,gBAAqB,oBA2C9B;AAED,8EAA8E;AAC9E,MAAM,MAAM,KAAK,GAAG,CACnB,KAAK,EAAE,MAAM,GAAG,GAAG,EACnB,IAAI,CAAC,EAAE,WAAW,KACd,OAAO,CAAC,QAAQ,CAAC,CAAA;AAEtB,4EAA4E;AAC5E,qBAAa,mBAAoB,SAAQ,KAAK;IAGjC,QAAQ,CAAC,GAAG,EAAE,MAAM;IAFhC,SAAkB,IAAI,yBAAwB;gBAEzB,GAAG,EAAE,MAAM;CAGhC;AAED,MAAM,MAAM,mBAAmB,GAAG;IACjC,kFAAkF;IAClF,aAAa,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,SAAS,CAAA;IACpE,qDAAqD;IACrD,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAA;IACzB,kBAAkB;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACjC,CAAA;AAKD;;;;GAIG;AACH,wBAAsB,eAAe,CACpC,KAAK,EAAE,MAAM,GAAG,GAAG,EACnB,IAAI,GAAE,WAAgB,EACtB,OAAO,GAAE,mBAAwB,GAC/B,OAAO,CAAC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,GAAG,CAAA;CAAE,CAAC,CA6C3C;AAED;;;GAGG;AACH,wBAAsB,YAAY,CACjC,KAAK,EAAE,MAAM,GAAG,GAAG,EACnB,IAAI,CAAC,EAAE,WAAW,EAClB,OAAO,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,QAAQ,CAAC,CAEnB"}

package/dist/net-guard.js

@@ -1,4 +1,5 @@
 import { lookup } from 'node:dns/promises';
+import { createCache, isOneOf } from 'hanzio';
 import { errorLogAttributes, silentLogger } from './logger.js';
 /** One crawl hits the same host hundreds of times; DNS is not cached by Node. */
 const RESOLUTION_TTL_MS = 60_000;
@@ -116,25 +117,38 @@ export function isBlockedAddress(address) {
 }
 export function isLocalhostHostname(hostname) {
     const host = hostname.toLowerCase().replace(/^\[|\]$/g, '');
-    return host === 'localhost' || host === '127.0.0.1' || host === '::1';
+    return isOneOf(['localhost', '127.0.0.1', '::1'], host);
 }
-const resolutionCache = new Map();
+const resolutionCache = createCache({
+    defaultTtlMs: RESOLUTION_TTL_MS,
+    maxEntries: RESOLUTION_CACHE_MAX
+});
 /**
  * SSRF guard: a customer can put any hostname in the website field, and an
  * internal address behind it would turn the importer into a proxy for our own
- * network. Localhost stays reachable outside production for the dev server.
+ * network. An IP literal (`[fd00::1]` included) is judged as written, without
+ * a DNS lookup.
  */
-export async function isHostAllowed(hostname, logger = silentLogger) {
-    if (isLocalhostHostname(hostname)) {
-        return process.env.NODE_ENV !== 'production';
-    }
-    const cached = resolutionCache.get(hostname);
-    if (cached && Date.now() - cached.atMs < RESOLUTION_TTL_MS) {
-        return cached.allowed;
+export async function isHostAllowed(hostname, options = {}) {
+    const logger = options.logger ?? silentLogger;
+    const host = hostname.toLowerCase().replace(/^\[|\]$/g, '');
+    if (isLocalhostHostname(host))
+        return options.allowLocalhost === true;
+    // `dns.lookup` fails on a bracketed literal, and a failed lookup is let
+    // through below, so a literal never reaches it.
+    if (parseIpv4(host) || host.includes(':')) {
+        const allowed = !isBlockedAddress(host);
+        if (!allowed) {
+            logger.warn('net-guard.blocked_address', { hostname, addresses: host });
+        }
+        return allowed;
     }
+    const cached = resolutionCache.get(host);
+    if (cached !== undefined)
+        return cached;
     let allowed = false;
     try {
-        const addresses = await lookup(hostname, { all: true });
+        const addresses = await lookup(host, { all: true });
         allowed =
             addresses.length > 0 &&
                 addresses.every((entry) => !isBlockedAddress(entry.address));
@@ -154,12 +168,69 @@ export async function isHostAllowed(hostname, logger = silentLogger) {
         });
         return true;
     }
-    if (resolutionCache.size >= RESOLUTION_CACHE_MAX) {
-        const oldest = resolutionCache.keys().next().value;
-        if (oldest)
-            resolutionCache.delete(oldest);
-    }
-    resolutionCache.set(hostname, { allowed, atMs: Date.now() });
+    resolutionCache.set(host, allowed);
     return allowed;
 }
+/** A request, or one of its redirects, went to a host the guard refuses. */
+export class HostNotAllowedError extends Error {
+    url;
+    name = 'HostNotAllowedError';
+    constructor(url) {
+        super(`Host not allowed: ${url}`);
+        this.url = url;
+    }
+}
+const REDIRECT_STATUSES = [301, 302, 303, 307, 308];
+const DEFAULT_MAX_REDIRECTS = 10;
+/**
+ * `fetch` that runs the guard on every hop: `redirect: 'follow'` would take a
+ * public host's 302 to `http://169.254.169.254/` without asking. Returns the
+ * last response and the URL it came from.
+ */
+export async function followRedirects(input, init = {}, options = {}) {
+    const allow = options.isHostAllowed ?? ((hostname) => isHostAllowed(hostname));
+    const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
+    const follow = (init.redirect ?? 'follow') === 'follow';
+    let url = new URL(input);
+    let request = follow ? { ...init, redirect: 'manual' } : init;
+    for (let hop = 0;; hop++) {
+        if (!isOneOf(['http:', 'https:'], url.protocol) ||
+            !(await allow(url.hostname))) {
+            throw new HostNotAllowedError(url.href);
+        }
+        const response = await (options.fetch ?? globalThis.fetch)(url, request);
+        const location = response.headers.get('location');
+        if (!follow || !location || !isOneOf(REDIRECT_STATUSES, response.status)) {
+            return { response, url };
+        }
+        void response.body?.cancel().catch(() => { });
+        if (hop >= maxRedirects) {
+            throw new TypeError(`Too many redirects from ${String(input)}`);
+        }
+        const next = new URL(location, url);
+        // What `fetch` itself does on a redirect: credentials stay with their
+        // origin, and a 303 (or a POST answered with 301/302) becomes a GET.
+        if (next.origin !== url.origin) {
+            const headers = new Headers(request.headers);
+            headers.delete('authorization');
+            headers.delete('cookie');
+            request = { ...request, headers };
+        }
+        const method = (request.method ?? 'GET').toUpperCase();
+        if ((response.status === 303 && !isOneOf(['GET', 'HEAD'], method)) ||
+            (isOneOf([301, 302], response.status) && method === 'POST')) {
+            const headers = new Headers(request.headers);
+            headers.delete('content-type');
+            request = { ...request, method: 'GET', body: null, headers };
+        }
+        url = next;
+    }
+}
+/**
+ * {@link followRedirects} as a drop-in `fetch`. Throws
+ * {@link HostNotAllowedError} when the URL or a redirect is refused.
+ */
+export async function guardedFetch(input, init, options) {
+    return (await followRedirects(input, init, options)).response;
+}
 //# sourceMappingURL=net-guard.js.map