polite-crawler 0.0.0-stage → 0.1.0

package/build/browser-gJ_h4AhB.mjs

@@ -0,0 +1,511 @@
+import { basename, dirname, join } from "node:path";
+import { existsSync } from "node:fs";
+//#region \0rolldown/runtime.js
+var __defProp = Object.defineProperty;
+var __exportAll = (all, no_symbols) => {
+	let target = {};
+	for (var name in all) __defProp(target, name, {
+		get: all[name],
+		enumerable: true
+	});
+	if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
+	return target;
+};
+//#endregion
+//#region src/render.ts
+/**
+* This error tells that the renderer did not render the page, and that the page or the site is
+* not at fault. The reason is one of these:
+*
+* - `refused`: `allowRequest` refused the first navigation, a step of its redirects, or a later
+*   navigation of the page that left it on an error page. `url` names the refused URL.
+* - `download`: the URL gives a file, not a page.
+* - `scheme`: the URL is not an http or https URL.
+* - `browser`: the renderer has no browser (`playwright-core` or its browser is not installed). The
+*   crawler then renders no more pages, and gives each one the note at once.
+*
+* A crawler does not count this error as a failure of the host.
+*/
+var RenderRefused = class extends Error {
+	reason;
+	/** This URL is the request that `allowRequest` or the limit of redirects refused, when the reason is `refused`. */
+	url;
+	constructor(reason, message, url) {
+		super(message);
+		this.name = "RenderRefused";
+		this.reason = reason;
+		this.url = url;
+	}
+};
+/** The browser does not ask for these. The text of a page needs none of them. */
+const BLOCKED_TYPES = /* @__PURE__ */ new Set([
+	"image",
+	"font",
+	"media"
+]);
+/** These statuses send the request on to the URL in `Location` (the Fetch standard). */
+const REDIRECTS = /* @__PURE__ */ new Set([
+	301,
+	302,
+	303,
+	307,
+	308
+]);
+const MAX_REDIRECTS = 5;
+/** The text is stable when it does not grow for this time. */
+const SETTLED_MS = 1e3;
+const POLL_MS = 250;
+const SETTLE_CAP_MS = 12e3;
+/** A page with no text after this time has nothing more to show. */
+const EMPTY_SETTLE_MS = 3e3;
+/** The part of the deadline that is kept to read the page after the wait. */
+const READ_RESERVE_MS = 2e3;
+/** A context that does not close in this time takes its browser with it. */
+const CLOSE_MS = 5e3;
+/** A browser that does not open a context in this time is replaced. */
+const OPEN_MS = 5e3;
+/** The text of the error when the optional package `playwright-core` is not installed. */
+const NO_PLAYWRIGHT = "rendering needs the package playwright-core and a browser: npm install playwright-core, then npx playwright-core install chromium-headless-shell. For the MCP server with npx, start it with npx -y -p polite-crawler -p playwright-core polite-crawler mcp";
+const launch = async () => {
+	let chromium;
+	try {
+		({chromium} = await import("playwright-core"));
+	} catch (cause) {
+		throw new Error(NO_PLAYWRIGHT, { cause });
+	}
+	return chromium.launch({ headless: true });
+};
+/** This script gives the length of the text that the page shows: its body, its open shadow roots and its frames of the same origin. */
+const MEASURE = `(() => {
+  let total = document.body ? document.body.innerText.length : 0;
+  const visit = (root) => {
+    for (const element of root.querySelectorAll("*")) {
+      if (element.shadowRoot) {
+        total += (element.shadowRoot.textContent || "").length;
+        visit(element.shadowRoot);
+      }
+    }
+  };
+  visit(document);
+  for (const frame of document.querySelectorAll("iframe, frame")) {
+    try {
+      const inner = frame.contentDocument;
+      if (inner && inner.body) total += inner.body.innerText.length;
+    } catch (error) {}
+  }
+  return total;
+})()`;
+/** This script gives the HTML of the page with its open shadow roots, and the body of each frame of the same origin. */
+const SERIALIZE = `(() => {
+  const roots = [];
+  const visit = (root) => {
+    for (const element of root.querySelectorAll("*")) {
+      if (element.shadowRoot) {
+        roots.push(element.shadowRoot);
+        visit(element.shadowRoot);
+      }
+    }
+  };
+  visit(document);
+  const root = document.documentElement;
+  if (!root) return { html: "", frames: [] };
+  const quote = (value) => value.replace(/&/g, "&").replace(/"/g, """);
+  const open = "<html" + Array.from(root.attributes, (item) => " " + item.name + '="' + quote(item.value) + '"').join("") + ">";
+  const inner =
+    roots.length > 0 && typeof root.getHTML === "function"
+      ? root.getHTML({ serializableShadowRoots: true, shadowRoots: roots })
+      : root.innerHTML;
+  const frames = [];
+  for (const frame of document.querySelectorAll("iframe, frame")) {
+    try {
+      const doc = frame.contentDocument;
+      if (doc && doc.body) frames.push({ src: frame.src || "", html: doc.body.innerHTML });
+    } catch (error) {}
+  }
+  return { html: "<!DOCTYPE html>" + open + inner + "</html>", frames };
+})()`;
+/** This function puts the body of each frame at the end of the page, in a section that names the frame. */
+const withFrames = ({ html, frames }) => {
+	if (frames.length === 0) return html;
+	const sections = frames.map(({ src, html: body }) => `<section data-polite-frame="${src.replace(/&/g, "&amp;").replace(/"/g, "&quot;")}">${body}</section>`).join("");
+	const end = html.toLowerCase().lastIndexOf("</body>");
+	return end === -1 ? `${html}${sections}` : `${html.slice(0, end)}${sections}${html.slice(end)}`;
+};
+const messageOf = (error) => error instanceof Error ? error.message : String(error);
+const sleep = (ms) => new Promise((resolve) => {
+	setTimeout(resolve, Math.max(0, ms)).unref();
+});
+/** This function gives the value of a promise, or undefined when the time ends first. */
+const within = async (promise, ms) => {
+	let timer;
+	const late = new Promise((resolve) => {
+		timer = setTimeout(() => resolve(void 0), Math.max(0, ms));
+	});
+	try {
+		return await Promise.race([promise, late]);
+	} finally {
+		clearTimeout(timer);
+	}
+};
+const playwrightRenderer = (options = {}) => {
+	const idleMs = options.idleMs ?? 12e4;
+	const start = options.launch ?? launch;
+	let browser;
+	let inFlight = 0;
+	let idle;
+	const shutSoon = () => {
+		if (idle !== void 0) clearTimeout(idle);
+		idle = setTimeout(() => {
+			idle = void 0;
+			if (inFlight > 0 || browser === void 0) return;
+			const closing = browser;
+			browser = void 0;
+			closing.then((running) => running.close()).catch(() => void 0);
+		}, idleMs);
+		idle.unref();
+	};
+	const acquire = async () => {
+		for (let attempt = 0;; attempt += 1) {
+			if (browser === void 0) {
+				const launching = start().then((running) => {
+					running.on("disconnected", () => {
+						if (browser === launching) browser = void 0;
+					});
+					return running;
+				});
+				launching.catch(() => {
+					if (browser === launching) browser = void 0;
+				});
+				browser = launching;
+			}
+			const held = browser;
+			const running = await held;
+			if (running.isConnected() || attempt > 0) return running;
+			if (browser === held) browser = void 0;
+		}
+	};
+	/** This function gives up on a browser that does not answer. The next page launches a new one. */
+	const forget = (running) => {
+		const held = browser;
+		if (held !== void 0) held.then((current) => {
+			if (current === running && browser === held) browser = void 0;
+		}, () => void 0);
+		running.close().catch(() => void 0);
+	};
+	const renderPage = async (url, { userAgent, timeoutMs, headers, allowRequest, maxRedirects = MAX_REDIRECTS, signal }) => {
+		if (signal?.aborted === true) throw signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("the render was cancelled");
+		const deadline = Date.now() + timeoutMs;
+		const left = () => Math.max(1, deadline - Date.now());
+		let running;
+		let context;
+		let shownPage;
+		/** The last navigation of the page that the route refused: the reason, and the refused URL. */
+		let refused;
+		/** The last navigation of the page that the route answered: the URL that the browser asked for, and the URL of the answer. */
+		let navigated;
+		const extra = Object.fromEntries(Object.entries(headers ?? {}).map(([name, value]) => [name.toLowerCase(), value]));
+		const contextOptions = {
+			userAgent,
+			serviceWorkers: "block",
+			acceptDownloads: false,
+			viewport: {
+				width: 1280,
+				height: 900
+			}
+		};
+		const ask = async (url) => allowRequest === void 0 ? true : Promise.resolve().then(() => allowRequest(url)).catch(() => false);
+		/**
+		* This function sends one request of the page and follows its redirects. Each step is a new
+		* decision of `allowRequest`, and a step past `maxRedirects` is refused. It gives the answer,
+		* or the reason of a refusal.
+		*
+		* These rules are the rules of a browser. Each step gets the cookies of its own URL from the
+		* context. A credential does not go to a different origin. A 303, or a 301 or 302 after a POST,
+		* changes the request to a GET.
+		*/
+		const follow = async (route, sent) => {
+			const request = route.request();
+			let url = request.url();
+			let method = request.method();
+			let headers = {
+				...request.headers(),
+				...extra
+			};
+			let response = await route.fetch({
+				maxRedirects: 0,
+				timeout: left(),
+				...sent === void 0 ? {} : { headers: sent }
+			});
+			for (let hop = 0;; hop += 1) {
+				const location = REDIRECTS.has(response.status()) ? response.headers()["location"] : void 0;
+				if (location === void 0) return {
+					response,
+					url
+				};
+				let next;
+				try {
+					next = new URL(location, url);
+				} catch {
+					return {
+						refusal: `${url} redirects to a URL that does not parse`,
+						url
+					};
+				}
+				if (hop >= maxRedirects) return {
+					refusal: `${request.url()} redirects more than ${String(maxRedirects)} times`,
+					url: next.href
+				};
+				if (!await ask(next.href)) return {
+					refusal: `the request for ${next.href}, a redirect of ${url}, was refused`,
+					url: next.href
+				};
+				const status = response.status();
+				const get = status === 303 || (status === 301 || status === 302) && method === "POST";
+				const crossOrigin = next.origin !== new URL(url).origin;
+				headers = Object.fromEntries(Object.entries(headers).filter(([name]) => name !== "cookie" && !(crossOrigin && name === "authorization") && !(get && (name === "content-type" || name === "content-length"))));
+				if (get) method = "GET";
+				const body = get ? void 0 : request.postDataBuffer() ?? void 0;
+				response = await context.request.fetch(next.href, {
+					method,
+					headers,
+					maxRedirects: 0,
+					timeout: left(),
+					...body === void 0 ? {} : { data: body }
+				});
+				url = next.href;
+			}
+		};
+		/** This function tells if a request is a navigation of the page itself, not of a frame or a popup. */
+		const ofPage = (request) => {
+			if (!request.isNavigationRequest()) return false;
+			try {
+				return request.frame().parentFrame() === null && request.frame().page() === shownPage;
+			} catch {
+				return false;
+			}
+		};
+		const decide = async (route) => {
+			const request = route.request();
+			const main = ofPage(request);
+			try {
+				if (BLOCKED_TYPES.has(request.resourceType())) {
+					await route.abort("blockedbyclient");
+					return;
+				}
+				if (!await ask(request.url())) {
+					if (main) refused = {
+						message: `the request for ${request.url()} was refused`,
+						url: request.url()
+					};
+					await route.abort("blockedbyclient");
+					return;
+				}
+				const sent = Object.keys(extra).length === 0 ? void 0 : {
+					...Object.fromEntries(Object.entries(await request.allHeaders()).filter(([name]) => !name.startsWith(":") && name !== "host")),
+					...extra
+				};
+				if (allowRequest === void 0) {
+					await route.continue(sent === void 0 ? void 0 : { headers: sent });
+					return;
+				}
+				const answer = await follow(route, sent);
+				if ("refusal" in answer) {
+					if (main) refused = {
+						message: answer.refusal,
+						url: answer.url
+					};
+					await route.abort("blockedbyclient");
+					return;
+				}
+				if (main) navigated = {
+					asked: request.url(),
+					final: answer.url
+				};
+				await route.fulfill({ response: answer.response });
+			} catch {}
+		};
+		const work = (async () => {
+			running = await acquire();
+			const opened = await within(running.newContext(contextOptions).catch((error) => {
+				if (!/closed|disconnected/i.test(messageOf(error))) throw error;
+			}), Math.min(OPEN_MS, left()));
+			if (opened === void 0) {
+				forget(running);
+				running = await acquire();
+				context = await running.newContext(contextOptions);
+			} else context = opened;
+			await context.route("**/*", decide);
+			if (allowRequest !== void 0 && typeof context.routeWebSocket === "function") await context.routeWebSocket(/.*/, async (socket) => {
+				if (await Promise.resolve().then(() => allowRequest(socket.url().replace(/^ws/i, "http"))).catch(() => false)) socket.connectToServer();
+				else await socket.close({
+					code: 1008,
+					reason: "refused"
+				}).catch(() => void 0);
+			});
+			const page = await context.newPage();
+			shownPage = page;
+			context.on("page", (other) => {
+				if (other !== page) other.close().catch(() => void 0);
+			});
+			page.on("dialog", (dialog) => {
+				dialog.dismiss().catch(() => void 0);
+			});
+			let response;
+			try {
+				response = await page.goto(url, {
+					waitUntil: "domcontentloaded",
+					timeout: Math.max(1, left() - READ_RESERVE_MS)
+				});
+			} catch (error) {
+				if (refused !== void 0) throw new RenderRefused("refused", refused.message, refused.url);
+				if (/Download is starting/i.test(messageOf(error))) throw new RenderRefused("download", `${url} gives a file, not a page`);
+				throw error;
+			}
+			const settleEnd = Math.min(deadline - READ_RESERVE_MS, Date.now() + SETTLE_CAP_MS);
+			const settleStart = Date.now();
+			let last = -1;
+			let since = Date.now();
+			while (Date.now() < settleEnd) {
+				const measured = page.evaluate(MEASURE).catch(async (error) => {
+					if (!/destroyed|navigat/i.test(messageOf(error))) throw error;
+					await page.waitForLoadState("domcontentloaded", { timeout: left() }).catch(() => void 0);
+					return NaN;
+				});
+				const length = await within(measured, settleEnd - Date.now());
+				if (length === void 0) break;
+				if (length !== last) {
+					last = length;
+					since = Date.now();
+				} else if (Date.now() - since >= SETTLED_MS && (length > 0 || Date.now() - settleStart >= EMPTY_SETTLE_MS)) break;
+				await sleep(Math.min(POLL_MS, settleEnd - Date.now()));
+			}
+			let serialized;
+			while (serialized === void 0) try {
+				serialized = await page.evaluate(SERIALIZE);
+			} catch (error) {
+				if (!/destroyed|navigat/i.test(messageOf(error)) || Date.now() >= deadline) throw error;
+				await page.waitForLoadState("domcontentloaded", { timeout: left() }).catch(() => void 0);
+			}
+			if (refused !== void 0 && page.url().startsWith("chrome-error:")) throw new RenderRefused("refused", `the page moved: ${refused.message}`, refused.url);
+			const shown = page.url().replace(/#.*$/s, "");
+			return {
+				html: withFrames(serialized),
+				finalUrl: navigated !== void 0 && shown === navigated.asked ? navigated.final : page.url(),
+				status: response?.status() ?? null,
+				headers: response?.headers() ?? {}
+			};
+		})();
+		work.catch(() => void 0);
+		let timer;
+		let cancel;
+		const expired = new Promise((_, reject) => {
+			timer = setTimeout(() => reject(/* @__PURE__ */ new Error(`render timed out after ${String(timeoutMs)} ms`)), timeoutMs);
+			if (signal !== void 0) {
+				cancel = () => reject(signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("the render was cancelled"));
+				signal.addEventListener("abort", cancel, { once: true });
+			}
+		});
+		const closeContext = async () => {
+			const open = context;
+			if (open === void 0) return;
+			if (await within(open.close().then(() => true, () => true), CLOSE_MS) === void 0 && running !== void 0) forget(running);
+		};
+		try {
+			return await Promise.race([work, expired]);
+		} finally {
+			clearTimeout(timer);
+			if (cancel !== void 0) signal?.removeEventListener("abort", cancel);
+			if (context !== void 0) await closeContext();
+			else work.then(closeContext, closeContext).catch(() => void 0);
+		}
+	};
+	return {
+		open: () => browser !== void 0,
+		async render(url, renderOptions) {
+			let target;
+			try {
+				target = new URL(url);
+			} catch {
+				throw new RenderRefused("scheme", `${url} is not a URL`);
+			}
+			if (target.protocol !== "http:" && target.protocol !== "https:") throw new RenderRefused("scheme", `the renderer reads http and https pages, not ${target.protocol}`);
+			if (idle !== void 0) {
+				clearTimeout(idle);
+				idle = void 0;
+			}
+			inFlight += 1;
+			try {
+				return await renderPage(target.href, renderOptions);
+			} finally {
+				inFlight -= 1;
+				if (inFlight === 0) shutSoon();
+			}
+		},
+		async close() {
+			if (idle !== void 0) clearTimeout(idle);
+			idle = void 0;
+			const launched = browser;
+			browser = void 0;
+			if (launched !== void 0) await launched.then((open) => open.close(), () => void 0);
+		}
+	};
+};
+//#endregion
+//#region src/browser.ts
+var browser_exports = /* @__PURE__ */ __exportAll({
+	NO_BROWSER: () => NO_BROWSER,
+	browserMissing: () => browserMissing,
+	checkedRenderer: () => checkedRenderer
+});
+/** The text of the note when `playwright-core` is installed, but its browser is not. */
+const NO_BROWSER = "rendering needs a browser: npx playwright-core install chromium-headless-shell";
+/**
+* This function tells if the default renderer can start a browser. It gives no value when it can,
+* and the reason when it cannot. The command line and the MCP server use it one time at start.
+* Then a missing browser costs no wait and no error capture for each page that needs JavaScript.
+*
+* The check does not start the browser. It finds the full Chromium build or the headless shell of
+* the same revision on the disk. When the check cannot find the answer, it gives no value, and the
+* renderer then tries.
+*/
+const browserMissing = async (load = () => import("playwright-core")) => {
+	let chromium;
+	try {
+		({chromium} = await load());
+	} catch {
+		return NO_PLAYWRIGHT;
+	}
+	try {
+		const executable = chromium.executablePath();
+		if (existsSync(executable)) return void 0;
+		let folder = dirname(executable);
+		while (dirname(folder) !== folder && !/^chromium-\d+$/.test(basename(folder))) folder = dirname(folder);
+		const revision = /^chromium-(\d+)$/.exec(basename(folder))?.[1];
+		if (revision === void 0) return void 0;
+		return existsSync(join(dirname(folder), `chromium_headless_shell-${revision}`)) ? void 0 : NO_BROWSER;
+	} catch {
+		return;
+	}
+};
+/**
+* This function gives a renderer that checks one time, before its first page, that the browser is
+* on the disk (`browserMissing`). With no browser, each page gets a `RenderRefused` with the reason
+* `browser` at once, and the renderer opens nothing. `createRuntime` puts its default renderer in
+* it, so a library with no browser costs no wait and no error capture.
+*/
+const checkedRenderer = (renderer, check = () => browserMissing()) => {
+	let missing;
+	return {
+		async render(url, options) {
+			missing ??= check().catch(() => void 0);
+			const reason = await missing;
+			if (reason !== void 0) throw new RenderRefused("browser", reason);
+			return renderer.render(url, options);
+		},
+		close: () => renderer.close()
+	};
+};
+//#endregion
+export { RenderRefused as a, NO_PLAYWRIGHT as i, browser_exports as n, playwrightRenderer as o, checkedRenderer as r, __exportAll as s, browserMissing as t };
+
+//# sourceMappingURL=browser-gJ_h4AhB.mjs.map

package/build/browser-gJ_h4AhB.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-gJ_h4AhB.mjs","names":[],"sources":["../src/render.ts","../src/browser.ts"],"sourcesContent":["import type { APIResponse, Browser, BrowserContext, Page, Request, Route } from \"playwright-core\";\n\n/**\n * The renderer opens the headless browser only for a page that shows almost nothing without\n * JavaScript. The settings:\n *\n * - The default `chromium-headless-shell` of Playwright, the lighter build.\n * - One browser for many pages, and a new context for each page. Thus no cookie or storage goes\n *   from one site to the next. A context has its own HTTP cache, so routing each request costs\n *   no cache across sites.\n * - Each request of the page goes through a route, the first navigation too. The route refuses\n *   images, fonts and media by their resource type. It asks `allowRequest` about each other\n *   request, and it sends `headers` with it.\n * - Playwright routes only the first request of a redirect chain. Thus with `allowRequest`, the\n *   route sends each request itself, follows each redirect itself and asks about each step. The\n *   browser then gets each body whole, not as a stream.\n * - No download, no popup, no service worker and no dialog.\n * - A wait until the visible text stops growing, with a limit. Playwright discourages\n *   `networkidle`. The text includes open shadow roots and the frames of the same origin.\n * - One deadline for the whole render: the navigation, the wait and the reading of the page.\n * - After a crash or a disconnection, the next page launches a new browser.\n * - The browser closes after two idle minutes. An idle Chromium holds hundreds of megabytes, so\n *   it stays open only while pages need it. The open browser keeps the process alive, so a\n *   script uses `close()` when it is done.\n */\n\nexport interface Rendered {\n  readonly html: string;\n  readonly finalUrl: string;\n  readonly status: number | null;\n  readonly headers: Readonly<Record<string, string>>;\n}\n\n/** These are the options of one render. */\nexport interface RenderOptions {\n  readonly userAgent: string;\n  /** This value is the deadline of the whole render, in milliseconds. */\n  readonly timeoutMs: number;\n  /** These headers go with each request of the page, for example the `From` header. */\n  readonly headers?: Readonly<Record<string, string>> | undefined;\n  /**\n   * This function decides each request that the page makes, the first navigation included. When\n   * it gives false, the browser does not send the request. The crawler uses it to apply the\n   * egress policy and robots.txt to the requests of the page.\n   */\n  readonly allowRequest?: ((url: string) => Promise<boolean> | boolean) | undefined;\n  /** This value is the maximum number of redirects of one request of the page. The default is 5, as for a plain fetch. */\n  readonly maxRedirects?: number | undefined;\n  /** When this signal aborts, the renderer stops the page: it closes the context, and the render rejects. */\n  readonly signal?: AbortSignal | undefined;\n}\n\nexport interface Renderer {\n  render(url: string, options: RenderOptions): Promise<Rendered>;\n  close(): Promise<void>;\n}\n\n/** The reason of a `RenderRefused`. */\nexport type RefusedReason = \"refused\" | \"download\" | \"scheme\" | \"browser\";\n\n/**\n * This error tells that the renderer did not render the page, and that the page or the site is\n * not at fault. The reason is one of these:\n *\n * - `refused`: `allowRequest` refused the first navigation, a step of its redirects, or a later\n *   navigation of the page that left it on an error page. `url` names the refused URL.\n * - `download`: the URL gives a file, not a page.\n * - `scheme`: the URL is not an http or https URL.\n * - `browser`: the renderer has no browser (`playwright-core` or its browser is not installed). The\n *   crawler then renders no more pages, and gives each one the note at once.\n *\n * A crawler does not count this error as a failure of the host.\n */\nexport class RenderRefused extends Error {\n  readonly reason: RefusedReason;\n  /** This URL is the request that `allowRequest` or the limit of redirects refused, when the reason is `refused`. */\n  readonly url: string | undefined;\n  constructor(reason: RefusedReason, message: string, url?: string) {\n    super(message);\n    this.name = \"RenderRefused\";\n    this.reason = reason;\n    this.url = url;\n  }\n}\n\n/** The browser does not ask for these. The text of a page needs none of them. */\nconst BLOCKED_TYPES = new Set([\"image\", \"font\", \"media\"]);\n/** These statuses send the request on to the URL in `Location` (the Fetch standard). */\nconst REDIRECTS = new Set([301, 302, 303, 307, 308]);\nconst MAX_REDIRECTS = 5;\n\n/** The text is stable when it does not grow for this time. */\nconst SETTLED_MS = 1000;\nconst POLL_MS = 250;\nconst SETTLE_CAP_MS = 12_000;\n/** A page with no text after this time has nothing more to show. */\nconst EMPTY_SETTLE_MS = 3000;\n/** The part of the deadline that is kept to read the page after the wait. */\nconst READ_RESERVE_MS = 2000;\n/** A context that does not close in this time takes its browser with it. */\nconst CLOSE_MS = 5000;\n/** A browser that does not open a context in this time is replaced. */\nconst OPEN_MS = 5000;\n\n/** The text of the error when the optional package `playwright-core` is not installed. */\nexport const NO_PLAYWRIGHT =\n  \"rendering needs the package playwright-core and a browser: npm install playwright-core, then npx playwright-core install chromium-headless-shell. For the MCP server with npx, start it with npx -y -p polite-crawler -p playwright-core polite-crawler mcp\";\n\n// playwright-core is an optional peer: it loads only when a page first needs a browser\nconst launch = async (): Promise<Browser> => {\n  let chromium: typeof import(\"playwright-core\").chromium;\n  try {\n    ({ chromium } = await import(\"playwright-core\"));\n  } catch (cause) {\n    throw new Error(NO_PLAYWRIGHT, { cause });\n  }\n  return chromium.launch({ headless: true });\n};\n\n/** This script gives the length of the text that the page shows: its body, its open shadow roots and its frames of the same origin. */\nconst MEASURE = `(() => {\n  let total = document.body ? document.body.innerText.length : 0;\n  const visit = (root) => {\n    for (const element of root.querySelectorAll(\"*\")) {\n      if (element.shadowRoot) {\n        total += (element.shadowRoot.textContent || \"\").length;\n        visit(element.shadowRoot);\n      }\n    }\n  };\n  visit(document);\n  for (const frame of document.querySelectorAll(\"iframe, frame\")) {\n    try {\n      const inner = frame.contentDocument;\n      if (inner && inner.body) total += inner.body.innerText.length;\n    } catch (error) {}\n  }\n  return total;\n})()`;\n\n/** This script gives the HTML of the page with its open shadow roots, and the body of each frame of the same origin. */\nconst SERIALIZE = `(() => {\n  const roots = [];\n  const visit = (root) => {\n    for (const element of root.querySelectorAll(\"*\")) {\n      if (element.shadowRoot) {\n        roots.push(element.shadowRoot);\n        visit(element.shadowRoot);\n      }\n    }\n  };\n  visit(document);\n  const root = document.documentElement;\n  if (!root) return { html: \"\", frames: [] };\n  const quote = (value) => value.replace(/&/g, \"&amp;\").replace(/\"/g, \"&quot;\");\n  const open = \"<html\" + Array.from(root.attributes, (item) => \" \" + item.name + '=\"' + quote(item.value) + '\"').join(\"\") + \">\";\n  const inner =\n    roots.length > 0 && typeof root.getHTML === \"function\"\n      ? root.getHTML({ serializableShadowRoots: true, shadowRoots: roots })\n      : root.innerHTML;\n  const frames = [];\n  for (const frame of document.querySelectorAll(\"iframe, frame\")) {\n    try {\n      const doc = frame.contentDocument;\n      if (doc && doc.body) frames.push({ src: frame.src || \"\", html: doc.body.innerHTML });\n    } catch (error) {}\n  }\n  return { html: \"<!DOCTYPE html>\" + open + inner + \"</html>\", frames };\n})()`;\n\ninterface Serialized {\n  readonly html: string;\n  readonly frames: ReadonlyArray<{ readonly src: string; readonly html: string }>;\n}\n\n/** This function puts the body of each frame at the end of the page, in a section that names the frame. */\nconst withFrames = ({ html, frames }: Serialized): string => {\n  if (frames.length === 0) return html;\n  const sections = frames\n    .map(\n      ({ src, html: body }) =>\n        `<section data-polite-frame=\"${src.replace(/&/g, \"&amp;\").replace(/\"/g, \"&quot;\")}\">${body}</section>`,\n    )\n    .join(\"\");\n  const end = html.toLowerCase().lastIndexOf(\"</body>\");\n  return end === -1 ? `${html}${sections}` : `${html.slice(0, end)}${sections}${html.slice(end)}`;\n};\n\nconst messageOf = (error: unknown): string =>\n  error instanceof Error ? error.message : String(error);\n\nconst sleep = (ms: number): Promise<void> =>\n  new Promise((resolve) => {\n    setTimeout(resolve, Math.max(0, ms)).unref();\n  });\n\n/** This function gives the value of a promise, or undefined when the time ends first. */\nconst within = async <T>(promise: Promise<T>, ms: number): Promise<T | undefined> => {\n  let timer: ReturnType<typeof setTimeout> | undefined;\n  const late = new Promise<undefined>((resolve) => {\n    timer = setTimeout(() => resolve(undefined), Math.max(0, ms));\n  });\n  try {\n    return await Promise.race([promise, late]);\n  } finally {\n    clearTimeout(timer);\n  }\n};\n\nexport interface PlaywrightRenderer extends Renderer {\n  /** This function tells if a browser is open. */\n  readonly open: () => boolean;\n}\n\nexport const playwrightRenderer = (\n  options: {\n    readonly idleMs?: number;\n    /**\n     * This function starts the browser: it gives a `Browser` of `playwright-core`. The default is\n     * the headless shell of Playwright. The type names no type of `playwright-core`, so a program\n     * with no `playwright-core` can check its types.\n     */\n    readonly launch?: (() => Promise<object>) | undefined;\n  } = {},\n): PlaywrightRenderer => {\n  const idleMs = options.idleMs ?? 120_000;\n  const start = (options.launch ?? launch) as () => Promise<Browser>;\n  let browser: Promise<Browser> | undefined;\n  let inFlight = 0;\n  let idle: ReturnType<typeof setTimeout> | undefined;\n  const shutSoon = (): void => {\n    if (idle !== undefined) clearTimeout(idle);\n    idle = setTimeout(() => {\n      idle = undefined;\n      if (inFlight > 0 || browser === undefined) return;\n      const closing = browser;\n      browser = undefined;\n      void closing.then((running) => running.close()).catch(() => undefined);\n    }, idleMs);\n    // the timer alone does not keep the process alive; the open browser does\n    idle.unref();\n  };\n  const acquire = async (): Promise<Browser> => {\n    for (let attempt = 0; ; attempt += 1) {\n      if (browser === undefined) {\n        const launching: Promise<Browser> = start().then((running) => {\n          // a crash, a kill or a closed pipe: the next page launches a new browser\n          running.on(\"disconnected\", () => {\n            if (browser === launching) browser = undefined;\n          });\n          return running;\n        });\n        // a failed launch is not kept, so the next page tries again; a newer\n        // launch that replaced this one is not touched\n        launching.catch(() => {\n          if (browser === launching) browser = undefined;\n        });\n        browser = launching;\n      }\n      const held = browser;\n      const running = await held;\n      if (running.isConnected() || attempt > 0) return running;\n      if (browser === held) browser = undefined;\n    }\n  };\n  /** This function gives up on a browser that does not answer. The next page launches a new one. */\n  const forget = (running: Browser): void => {\n    const held = browser;\n    if (held !== undefined) {\n      void held.then(\n        (current) => {\n          if (current === running && browser === held) browser = undefined;\n        },\n        () => undefined,\n      );\n    }\n    void running.close().catch(() => undefined);\n  };\n\n  const renderPage = async (\n    url: string,\n    {\n      userAgent,\n      timeoutMs,\n      headers,\n      allowRequest,\n      maxRedirects = MAX_REDIRECTS,\n      signal,\n    }: RenderOptions,\n  ): Promise<Rendered> => {\n    if (signal?.aborted === true)\n      throw signal.reason instanceof Error ? signal.reason : new Error(\"the render was cancelled\");\n    const deadline = Date.now() + timeoutMs;\n    const left = (): number => Math.max(1, deadline - Date.now());\n    let running: Browser | undefined;\n    let context: BrowserContext | undefined;\n    let shownPage: Page | undefined;\n    /** The last navigation of the page that the route refused: the reason, and the refused URL. */\n    let refused: { readonly message: string; readonly url: string } | undefined;\n    /** The last navigation of the page that the route answered: the URL that the browser asked for, and the URL of the answer. */\n    let navigated: { readonly asked: string; readonly final: string } | undefined;\n    const extra = Object.fromEntries(\n      Object.entries(headers ?? {}).map(([name, value]) => [name.toLowerCase(), value]),\n    );\n    const contextOptions = {\n      userAgent,\n      serviceWorkers: \"block\" as const,\n      acceptDownloads: false,\n      viewport: { width: 1280, height: 900 },\n    };\n    const ask = async (url: string): Promise<boolean> =>\n      allowRequest === undefined\n        ? true\n        : Promise.resolve()\n            .then(() => allowRequest(url))\n            .catch(() => false);\n    /**\n     * This function sends one request of the page and follows its redirects. Each step is a new\n     * decision of `allowRequest`, and a step past `maxRedirects` is refused. It gives the answer,\n     * or the reason of a refusal.\n     *\n     * These rules are the rules of a browser. Each step gets the cookies of its own URL from the\n     * context. A credential does not go to a different origin. A 303, or a 301 or 302 after a POST,\n     * changes the request to a GET.\n     */\n    const follow = async (\n      route: Route,\n      sent: Record<string, string> | undefined,\n    ): Promise<\n      | { readonly response: APIResponse; readonly url: string }\n      | { readonly refusal: string; readonly url: string }\n    > => {\n      const request = route.request();\n      let url = request.url();\n      let method = request.method();\n      let headers: Record<string, string> = { ...request.headers(), ...extra };\n      let response = await route.fetch({\n        maxRedirects: 0,\n        timeout: left(),\n        ...(sent === undefined ? {} : { headers: sent }),\n      });\n      for (let hop = 0; ; hop += 1) {\n        const location = REDIRECTS.has(response.status())\n          ? response.headers()[\"location\"]\n          : undefined;\n        if (location === undefined) return { response, url };\n        let next: URL;\n        try {\n          next = new URL(location, url);\n        } catch {\n          return { refusal: `${url} redirects to a URL that does not parse`, url };\n        }\n        if (hop >= maxRedirects) {\n          return {\n            refusal: `${request.url()} redirects more than ${String(maxRedirects)} times`,\n            url: next.href,\n          };\n        }\n        if (!(await ask(next.href))) {\n          return {\n            refusal: `the request for ${next.href}, a redirect of ${url}, was refused`,\n            url: next.href,\n          };\n        }\n        const status = response.status();\n        const get = status === 303 || ((status === 301 || status === 302) && method === \"POST\");\n        const crossOrigin = next.origin !== new URL(url).origin;\n        headers = Object.fromEntries(\n          Object.entries(headers).filter(\n            ([name]) =>\n              name !== \"cookie\" &&\n              !(crossOrigin && name === \"authorization\") &&\n              !(get && (name === \"content-type\" || name === \"content-length\")),\n          ),\n        );\n        if (get) method = \"GET\";\n        const body = get ? undefined : (request.postDataBuffer() ?? undefined);\n        response = await context!.request.fetch(next.href, {\n          method,\n          headers,\n          maxRedirects: 0,\n          timeout: left(),\n          ...(body === undefined ? {} : { data: body }),\n        });\n        url = next.href;\n      }\n    };\n    /** This function tells if a request is a navigation of the page itself, not of a frame or a popup. */\n    const ofPage = (request: Request): boolean => {\n      if (!request.isNavigationRequest()) return false;\n      try {\n        return request.frame().parentFrame() === null && request.frame().page() === shownPage;\n      } catch {\n        // a popup asks for its first page before it has a frame\n        return false;\n      }\n    };\n    const decide = async (route: Route): Promise<void> => {\n      const request = route.request();\n      const main = ofPage(request);\n      try {\n        if (BLOCKED_TYPES.has(request.resourceType())) {\n          await route.abort(\"blockedbyclient\");\n          return;\n        }\n        if (!(await ask(request.url()))) {\n          if (main)\n            refused = {\n              message: `the request for ${request.url()} was refused`,\n              url: request.url(),\n            };\n          await route.abort(\"blockedbyclient\");\n          return;\n        }\n        const sent =\n          Object.keys(extra).length === 0\n            ? undefined\n            : {\n                ...Object.fromEntries(\n                  Object.entries(await request.allHeaders()).filter(\n                    ([name]) => !name.startsWith(\":\") && name !== \"host\",\n                  ),\n                ),\n                ...extra,\n              };\n        if (allowRequest === undefined) {\n          await route.continue(sent === undefined ? undefined : { headers: sent });\n          return;\n        }\n        const answer = await follow(route, sent);\n        if (\"refusal\" in answer) {\n          if (main) refused = { message: answer.refusal, url: answer.url };\n          await route.abort(\"blockedbyclient\");\n          return;\n        }\n        if (main) navigated = { asked: request.url(), final: answer.url };\n        await route.fulfill({ response: answer.response });\n      } catch {\n        // the page closed while the request waited: nothing to answer\n      }\n    };\n    const work = (async (): Promise<Rendered> => {\n      running = await acquire();\n      // a browser that was killed can keep its pipe open and answer nothing:\n      // it gets a few seconds, and then a new browser takes its place\n      const opened = await within(\n        running.newContext(contextOptions).catch((error: unknown) => {\n          if (!/closed|disconnected/i.test(messageOf(error))) throw error;\n          return undefined;\n        }),\n        Math.min(OPEN_MS, left()),\n      );\n      if (opened === undefined) {\n        forget(running);\n        running = await acquire();\n        context = await running.newContext(contextOptions);\n      } else {\n        context = opened;\n      }\n      await context.route(\"**/*\", decide);\n      if (allowRequest !== undefined && typeof context.routeWebSocket === \"function\") {\n        await context.routeWebSocket(/.*/, async (socket) => {\n          const allowed = await Promise.resolve()\n            .then(() => allowRequest(socket.url().replace(/^ws/i, \"http\")))\n            .catch(() => false);\n          if (allowed) socket.connectToServer();\n          else await socket.close({ code: 1008, reason: \"refused\" }).catch(() => undefined);\n        });\n      }\n      const page: Page = await context.newPage();\n      shownPage = page;\n      // a popup is not the page, and a dialog waits for no one\n      context.on(\"page\", (other) => {\n        if (other !== page) void other.close().catch(() => undefined);\n      });\n      page.on(\"dialog\", (dialog) => {\n        void dialog.dismiss().catch(() => undefined);\n      });\n      let response;\n      try {\n        response = await page.goto(url, {\n          waitUntil: \"domcontentloaded\",\n          timeout: Math.max(1, left() - READ_RESERVE_MS),\n        });\n      } catch (error) {\n        if (refused !== undefined) throw new RenderRefused(\"refused\", refused.message, refused.url);\n        if (/Download is starting/i.test(messageOf(error))) {\n          throw new RenderRefused(\"download\", `${url} gives a file, not a page`);\n        }\n        throw error;\n      }\n      const settleEnd = Math.min(deadline - READ_RESERVE_MS, Date.now() + SETTLE_CAP_MS);\n      const settleStart = Date.now();\n      let last = -1;\n      let since = Date.now();\n      while (Date.now() < settleEnd) {\n        const measured = (page.evaluate(MEASURE) as Promise<number>).catch(\n          async (error: unknown) => {\n            // a script of the page can start a navigation while the text is measured: the next poll measures the new document\n            if (!/destroyed|navigat/i.test(messageOf(error))) throw error;\n            await page\n              .waitForLoadState(\"domcontentloaded\", { timeout: left() })\n              .catch(() => undefined);\n            // no length equals NaN, so the wait goes on\n            return Number.NaN;\n          },\n        );\n        const length = await within(measured, settleEnd - Date.now());\n        // a page that keeps its thread busy does not answer: the reading below decides\n        if (length === undefined) break;\n        if (length !== last) {\n          last = length;\n          since = Date.now();\n        } else if (\n          Date.now() - since >= SETTLED_MS &&\n          (length > 0 || Date.now() - settleStart >= EMPTY_SETTLE_MS)\n        ) {\n          break;\n        }\n        await sleep(Math.min(POLL_MS, settleEnd - Date.now()));\n      }\n      // a navigation can replace the document while it is read: the read waits for the new document\n      // and tries again, until the deadline (a page can navigate many times in a row)\n      let serialized: Serialized | undefined;\n      while (serialized === undefined) {\n        try {\n          serialized = (await page.evaluate(SERIALIZE)) as Serialized;\n        } catch (error) {\n          if (!/destroyed|navigat/i.test(messageOf(error)) || Date.now() >= deadline) throw error;\n          await page\n            .waitForLoadState(\"domcontentloaded\", { timeout: left() })\n            .catch(() => undefined);\n        }\n      }\n      // a later navigation that the route refused leaves the browser on its error page, which is not the page\n      if (refused !== undefined && page.url().startsWith(\"chrome-error:\")) {\n        throw new RenderRefused(\"refused\", `the page moved: ${refused.message}`, refused.url);\n      }\n      // the route followed the redirects of the navigation, so the browser shows the first URL\n      const shown = page.url().replace(/#.*$/s, \"\");\n      return {\n        html: withFrames(serialized),\n        finalUrl:\n          navigated !== undefined && shown === navigated.asked ? navigated.final : page.url(),\n        status: response?.status() ?? null,\n        headers: response?.headers() ?? {},\n      };\n    })();\n    // the deadline can win the race; the work then ends when its context closes\n    work.catch(() => undefined);\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    let cancel: (() => void) | undefined;\n    const expired = new Promise<never>((_, reject) => {\n      timer = setTimeout(\n        () => reject(new Error(`render timed out after ${String(timeoutMs)} ms`)),\n        timeoutMs,\n      );\n      // a cancelled render ends now; the context closes below, and that stops the page\n      if (signal !== undefined) {\n        cancel = () =>\n          reject(\n            signal.reason instanceof Error ? signal.reason : new Error(\"the render was cancelled\"),\n          );\n        signal.addEventListener(\"abort\", cancel, { once: true });\n      }\n    });\n    const closeContext = async (): Promise<void> => {\n      const open = context;\n      if (open === undefined) return;\n      const closed = await within(\n        open.close().then(\n          () => true,\n          () => true,\n        ),\n        CLOSE_MS,\n      );\n      if (closed === undefined && running !== undefined) forget(running);\n    };\n    try {\n      return await Promise.race([work, expired]);\n    } finally {\n      clearTimeout(timer);\n      if (cancel !== undefined) signal?.removeEventListener(\"abort\", cancel);\n      if (context !== undefined) await closeContext();\n      // a context made after the deadline is closed when it arrives\n      else void work.then(closeContext, closeContext).catch(() => undefined);\n    }\n  };\n\n  return {\n    open: () => browser !== undefined,\n    async render(url, renderOptions) {\n      let target: URL;\n      try {\n        target = new URL(url);\n      } catch {\n        throw new RenderRefused(\"scheme\", `${url} is not a URL`);\n      }\n      if (target.protocol !== \"http:\" && target.protocol !== \"https:\") {\n        throw new RenderRefused(\n          \"scheme\",\n          `the renderer reads http and https pages, not ${target.protocol}`,\n        );\n      }\n      if (idle !== undefined) {\n        clearTimeout(idle);\n        idle = undefined;\n      }\n      inFlight += 1;\n      try {\n        return await renderPage(target.href, renderOptions);\n      } finally {\n        inFlight -= 1;\n        if (inFlight === 0) shutSoon();\n      }\n    },\n    async close() {\n      if (idle !== undefined) clearTimeout(idle);\n      idle = undefined;\n      const launched = browser;\n      browser = undefined;\n      // a launch that failed has nothing to close, and closing never throws:\n      // the idle timer calls this with no one to catch it\n      if (launched !== undefined)\n        await launched.then(\n          (open) => open.close(),\n          () => undefined,\n        );\n    },\n  };\n};\n","import { existsSync } from \"node:fs\";\nimport { basename, dirname, join } from \"node:path\";\nimport type { Renderer } from \"./render.ts\";\nimport { NO_PLAYWRIGHT, RenderRefused } from \"./render.ts\";\n\n/** The text of the note when `playwright-core` is installed, but its browser is not. */\nexport const NO_BROWSER =\n  \"rendering needs a browser: npx playwright-core install chromium-headless-shell\";\n\n/** This is the part of `playwright-core` that the check uses. */\ninterface Chromium {\n  executablePath(): string;\n}\n\n/**\n * This function tells if the default renderer can start a browser. It gives no value when it can,\n * and the reason when it cannot. The command line and the MCP server use it one time at start.\n * Then a missing browser costs no wait and no error capture for each page that needs JavaScript.\n *\n * The check does not start the browser. It finds the full Chromium build or the headless shell of\n * the same revision on the disk. When the check cannot find the answer, it gives no value, and the\n * renderer then tries.\n */\nexport const browserMissing = async (\n  load: () => Promise<{ readonly chromium: Chromium }> = () => import(\"playwright-core\"),\n): Promise<string | undefined> => {\n  let chromium: Chromium;\n  try {\n    ({ chromium } = await load());\n  } catch {\n    return NO_PLAYWRIGHT;\n  }\n  try {\n    const executable = chromium.executablePath();\n    if (existsSync(executable)) return undefined;\n    // the headless shell is in a folder next to the folder of the full build, with the same revision\n    let folder = dirname(executable);\n    while (dirname(folder) !== folder && !/^chromium-\\d+$/.test(basename(folder)))\n      folder = dirname(folder);\n    const revision = /^chromium-(\\d+)$/.exec(basename(folder))?.[1];\n    if (revision === undefined) return undefined;\n    return existsSync(join(dirname(folder), `chromium_headless_shell-${revision}`))\n      ? undefined\n      : NO_BROWSER;\n  } catch {\n    return undefined;\n  }\n};\n\n/**\n * This function gives a renderer that checks one time, before its first page, that the browser is\n * on the disk (`browserMissing`). With no browser, each page gets a `RenderRefused` with the reason\n * `browser` at once, and the renderer opens nothing. `createRuntime` puts its default renderer in\n * it, so a library with no browser costs no wait and no error capture.\n */\nexport const checkedRenderer = (\n  renderer: Renderer,\n  check: () => Promise<string | undefined> = () => browserMissing(),\n): Renderer => {\n  let missing: Promise<string | undefined> | undefined;\n  return {\n    async render(url, options) {\n      missing ??= check().catch(() => undefined);\n      const reason = await missing;\n      if (reason !== undefined) throw new RenderRefused(\"browser\", reason);\n      return renderer.render(url, options);\n    },\n    close: () => renderer.close(),\n  };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,IAAa,gBAAb,cAAmC,MAAM;CACvC;;CAEA;CACA,YAAY,QAAuB,SAAiB,KAAc;EAChE,MAAM,OAAO;EACb,KAAK,OAAO;EACZ,KAAK,SAAS;EACd,KAAK,MAAM;CACb;AACF;;AAGA,MAAM,gCAAgB,IAAI,IAAI;CAAC;CAAS;CAAQ;AAAO,CAAC;;AAExD,MAAM,4BAAY,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;CAAK;AAAG,CAAC;AACnD,MAAM,gBAAgB;;AAGtB,MAAM,aAAa;AACnB,MAAM,UAAU;AAChB,MAAM,gBAAgB;;AAEtB,MAAM,kBAAkB;;AAExB,MAAM,kBAAkB;;AAExB,MAAM,WAAW;;AAEjB,MAAM,UAAU;;AAGhB,MAAa,gBACX;AAGF,MAAM,SAAS,YAA8B;CAC3C,IAAI;CACJ,IAAI;EACF,CAAC,CAAE,YAAa,MAAM,OAAO;CAC/B,SAAS,OAAO;EACd,MAAM,IAAI,MAAM,eAAe,EAAE,MAAM,CAAC;CAC1C;CACA,OAAO,SAAS,OAAO,EAAE,UAAU,KAAK,CAAC;AAC3C;;AAGA,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;AAqBhB,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmClB,MAAM,cAAc,EAAE,MAAM,aAAiC;CAC3D,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,WAAW,OACd,KACE,EAAE,KAAK,MAAM,WACZ,+BAA+B,IAAI,QAAQ,MAAM,OAAO,CAAC,CAAC,QAAQ,MAAM,QAAQ,EAAE,IAAI,KAAK,WAC/F,CAAC,CACA,KAAK,EAAE;CACV,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,YAAY,SAAS;CACpD,OAAO,QAAQ,KAAK,GAAG,OAAO,aAAa,GAAG,KAAK,MAAM,GAAG,GAAG,IAAI,WAAW,KAAK,MAAM,GAAG;AAC9F;AAEA,MAAM,aAAa,UACjB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAEvD,MAAM,SAAS,OACb,IAAI,SAAS,YAAY;CACvB,WAAW,SAAS,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM;AAC7C,CAAC;;AAGH,MAAM,SAAS,OAAU,SAAqB,OAAuC;CACnF,IAAI;CACJ,MAAM,OAAO,IAAI,SAAoB,YAAY;EAC/C,QAAQ,iBAAiB,QAAQ,KAAA,CAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAC;CAC9D,CAAC;CACD,IAAI;EACF,OAAO,MAAM,QAAQ,KAAK,CAAC,SAAS,IAAI,CAAC;CAC3C,UAAU;EACR,aAAa,KAAK;CACpB;AACF;AAOA,MAAa,sBACX,UAQI,CAAC,MACkB;CACvB,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,QAAS,QAAQ,UAAU;CACjC,IAAI;CACJ,IAAI,WAAW;CACf,IAAI;CACJ,MAAM,iBAAuB;EAC3B,IAAI,SAAS,KAAA,GAAW,aAAa,IAAI;EACzC,OAAO,iBAAiB;GACtB,OAAO,KAAA;GACP,IAAI,WAAW,KAAK,YAAY,KAAA,GAAW;GAC3C,MAAM,UAAU;GAChB,UAAU,KAAA;GACV,QAAa,MAAM,YAAY,QAAQ,MAAM,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;EACvE,GAAG,MAAM;EAET,KAAK,MAAM;CACb;CACA,MAAM,UAAU,YAA8B;EAC5C,KAAK,IAAI,UAAU,IAAK,WAAW,GAAG;GACpC,IAAI,YAAY,KAAA,GAAW;IACzB,MAAM,YAA8B,MAAM,CAAC,CAAC,MAAM,YAAY;KAE5D,QAAQ,GAAG,sBAAsB;MAC/B,IAAI,YAAY,WAAW,UAAU,KAAA;KACvC,CAAC;KACD,OAAO;IACT,CAAC;IAGD,UAAU,YAAY;KACpB,IAAI,YAAY,WAAW,UAAU,KAAA;IACvC,CAAC;IACD,UAAU;GACZ;GACA,MAAM,OAAO;GACb,MAAM,UAAU,MAAM;GACtB,IAAI,QAAQ,YAAY,KAAK,UAAU,GAAG,OAAO;GACjD,IAAI,YAAY,MAAM,UAAU,KAAA;EAClC;CACF;;CAEA,MAAM,UAAU,YAA2B;EACzC,MAAM,OAAO;EACb,IAAI,SAAS,KAAA,GACX,KAAU,MACP,YAAY;GACX,IAAI,YAAY,WAAW,YAAY,MAAM,UAAU,KAAA;EACzD,SACM,KAAA,CACR;EAEF,QAAa,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;CAC5C;CAEA,MAAM,aAAa,OACjB,KACA,EACE,WACA,WACA,SACA,cACA,eAAe,eACf,aAEoB;EACtB,IAAI,QAAQ,YAAY,MACtB,MAAM,OAAO,kBAAkB,QAAQ,OAAO,yBAAS,IAAI,MAAM,0BAA0B;EAC7F,MAAM,WAAW,KAAK,IAAI,IAAI;EAC9B,MAAM,aAAqB,KAAK,IAAI,GAAG,WAAW,KAAK,IAAI,CAAC;EAC5D,IAAI;EACJ,IAAI;EACJ,IAAI;;EAEJ,IAAI;;EAEJ,IAAI;EACJ,MAAM,QAAQ,OAAO,YACnB,OAAO,QAAQ,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,WAAW,CAAC,KAAK,YAAY,GAAG,KAAK,CAAC,CAClF;EACA,MAAM,iBAAiB;GACrB;GACA,gBAAgB;GAChB,iBAAiB;GACjB,UAAU;IAAE,OAAO;IAAM,QAAQ;GAAI;EACvC;EACA,MAAM,MAAM,OAAO,QACjB,iBAAiB,KAAA,IACb,OACA,QAAQ,QAAQ,CAAC,CACd,WAAW,aAAa,GAAG,CAAC,CAAC,CAC7B,YAAY,KAAK;;;;;;;;;;EAU1B,MAAM,SAAS,OACb,OACA,SAIG;GACH,MAAM,UAAU,MAAM,QAAQ;GAC9B,IAAI,MAAM,QAAQ,IAAI;GACtB,IAAI,SAAS,QAAQ,OAAO;GAC5B,IAAI,UAAkC;IAAE,GAAG,QAAQ,QAAQ;IAAG,GAAG;GAAM;GACvE,IAAI,WAAW,MAAM,MAAM,MAAM;IAC/B,cAAc;IACd,SAAS,KAAK;IACd,GAAI,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,KAAK;GAChD,CAAC;GACD,KAAK,IAAI,MAAM,IAAK,OAAO,GAAG;IAC5B,MAAM,WAAW,UAAU,IAAI,SAAS,OAAO,CAAC,IAC5C,SAAS,QAAQ,CAAC,CAAC,cACnB,KAAA;IACJ,IAAI,aAAa,KAAA,GAAW,OAAO;KAAE;KAAU;IAAI;IACnD,IAAI;IACJ,IAAI;KACF,OAAO,IAAI,IAAI,UAAU,GAAG;IAC9B,QAAQ;KACN,OAAO;MAAE,SAAS,GAAG,IAAI;MAA0C;KAAI;IACzE;IACA,IAAI,OAAO,cACT,OAAO;KACL,SAAS,GAAG,QAAQ,IAAI,EAAE,uBAAuB,OAAO,YAAY,EAAE;KACtE,KAAK,KAAK;IACZ;IAEF,IAAI,CAAE,MAAM,IAAI,KAAK,IAAI,GACvB,OAAO;KACL,SAAS,mBAAmB,KAAK,KAAK,kBAAkB,IAAI;KAC5D,KAAK,KAAK;IACZ;IAEF,MAAM,SAAS,SAAS,OAAO;IAC/B,MAAM,MAAM,WAAW,QAAS,WAAW,OAAO,WAAW,QAAQ,WAAW;IAChF,MAAM,cAAc,KAAK,WAAW,IAAI,IAAI,GAAG,CAAC,CAAC;IACjD,UAAU,OAAO,YACf,OAAO,QAAQ,OAAO,CAAC,CAAC,QACrB,CAAC,UACA,SAAS,YACT,EAAE,eAAe,SAAS,oBAC1B,EAAE,QAAQ,SAAS,kBAAkB,SAAS,kBAClD,CACF;IACA,IAAI,KAAK,SAAS;IAClB,MAAM,OAAO,MAAM,KAAA,IAAa,QAAQ,eAAe,KAAK,KAAA;IAC5D,WAAW,MAAM,QAAS,QAAQ,MAAM,KAAK,MAAM;KACjD;KACA;KACA,cAAc;KACd,SAAS,KAAK;KACd,GAAI,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM,KAAK;IAC7C,CAAC;IACD,MAAM,KAAK;GACb;EACF;;EAEA,MAAM,UAAU,YAA8B;GAC5C,IAAI,CAAC,QAAQ,oBAAoB,GAAG,OAAO;GAC3C,IAAI;IACF,OAAO,QAAQ,MAAM,CAAC,CAAC,YAAY,MAAM,QAAQ,QAAQ,MAAM,CAAC,CAAC,KAAK,MAAM;GAC9E,QAAQ;IAEN,OAAO;GACT;EACF;EACA,MAAM,SAAS,OAAO,UAAgC;GACpD,MAAM,UAAU,MAAM,QAAQ;GAC9B,MAAM,OAAO,OAAO,OAAO;GAC3B,IAAI;IACF,IAAI,cAAc,IAAI,QAAQ,aAAa,CAAC,GAAG;KAC7C,MAAM,MAAM,MAAM,iBAAiB;KACnC;IACF;IACA,IAAI,CAAE,MAAM,IAAI,QAAQ,IAAI,CAAC,GAAI;KAC/B,IAAI,MACF,UAAU;MACR,SAAS,mBAAmB,QAAQ,IAAI,EAAE;MAC1C,KAAK,QAAQ,IAAI;KACnB;KACF,MAAM,MAAM,MAAM,iBAAiB;KACnC;IACF;IACA,MAAM,OACJ,OAAO,KAAK,KAAK,CAAC,CAAC,WAAW,IAC1B,KAAA,IACA;KACE,GAAG,OAAO,YACR,OAAO,QAAQ,MAAM,QAAQ,WAAW,CAAC,CAAC,CAAC,QACxC,CAAC,UAAU,CAAC,KAAK,WAAW,GAAG,KAAK,SAAS,MAChD,CACF;KACA,GAAG;IACL;IACN,IAAI,iBAAiB,KAAA,GAAW;KAC9B,MAAM,MAAM,SAAS,SAAS,KAAA,IAAY,KAAA,IAAY,EAAE,SAAS,KAAK,CAAC;KACvE;IACF;IACA,MAAM,SAAS,MAAM,OAAO,OAAO,IAAI;IACvC,IAAI,aAAa,QAAQ;KACvB,IAAI,MAAM,UAAU;MAAE,SAAS,OAAO;MAAS,KAAK,OAAO;KAAI;KAC/D,MAAM,MAAM,MAAM,iBAAiB;KACnC;IACF;IACA,IAAI,MAAM,YAAY;KAAE,OAAO,QAAQ,IAAI;KAAG,OAAO,OAAO;IAAI;IAChE,MAAM,MAAM,QAAQ,EAAE,UAAU,OAAO,SAAS,CAAC;GACnD,QAAQ,CAER;EACF;EACA,MAAM,QAAQ,YAA+B;GAC3C,UAAU,MAAM,QAAQ;GAGxB,MAAM,SAAS,MAAM,OACnB,QAAQ,WAAW,cAAc,CAAC,CAAC,OAAO,UAAmB;IAC3D,IAAI,CAAC,uBAAuB,KAAK,UAAU,KAAK,CAAC,GAAG,MAAM;GAE5D,CAAC,GACD,KAAK,IAAI,SAAS,KAAK,CAAC,CAC1B;GACA,IAAI,WAAW,KAAA,GAAW;IACxB,OAAO,OAAO;IACd,UAAU,MAAM,QAAQ;IACxB,UAAU,MAAM,QAAQ,WAAW,cAAc;GACnD,OACE,UAAU;GAEZ,MAAM,QAAQ,MAAM,QAAQ,MAAM;GAClC,IAAI,iBAAiB,KAAA,KAAa,OAAO,QAAQ,mBAAmB,YAClE,MAAM,QAAQ,eAAe,MAAM,OAAO,WAAW;IAInD,IAAI,MAHkB,QAAQ,QAAQ,CAAC,CACpC,WAAW,aAAa,OAAO,IAAI,CAAC,CAAC,QAAQ,QAAQ,MAAM,CAAC,CAAC,CAAC,CAC9D,YAAY,KAAK,GACP,OAAO,gBAAgB;SAC/B,MAAM,OAAO,MAAM;KAAE,MAAM;KAAM,QAAQ;IAAU,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAClF,CAAC;GAEH,MAAM,OAAa,MAAM,QAAQ,QAAQ;GACzC,YAAY;GAEZ,QAAQ,GAAG,SAAS,UAAU;IAC5B,IAAI,UAAU,MAAM,MAAW,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;GAC9D,CAAC;GACD,KAAK,GAAG,WAAW,WAAW;IAC5B,OAAY,QAAQ,CAAC,CAAC,YAAY,KAAA,CAAS;GAC7C,CAAC;GACD,IAAI;GACJ,IAAI;IACF,WAAW,MAAM,KAAK,KAAK,KAAK;KAC9B,WAAW;KACX,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,eAAe;IAC/C,CAAC;GACH,SAAS,OAAO;IACd,IAAI,YAAY,KAAA,GAAW,MAAM,IAAI,cAAc,WAAW,QAAQ,SAAS,QAAQ,GAAG;IAC1F,IAAI,wBAAwB,KAAK,UAAU,KAAK,CAAC,GAC/C,MAAM,IAAI,cAAc,YAAY,GAAG,IAAI,0BAA0B;IAEvE,MAAM;GACR;GACA,MAAM,YAAY,KAAK,IAAI,WAAW,iBAAiB,KAAK,IAAI,IAAI,aAAa;GACjF,MAAM,cAAc,KAAK,IAAI;GAC7B,IAAI,OAAO;GACX,IAAI,QAAQ,KAAK,IAAI;GACrB,OAAO,KAAK,IAAI,IAAI,WAAW;IAC7B,MAAM,WAAY,KAAK,SAAS,OAAO,CAAC,CAAqB,MAC3D,OAAO,UAAmB;KAExB,IAAI,CAAC,qBAAqB,KAAK,UAAU,KAAK,CAAC,GAAG,MAAM;KACxD,MAAM,KACH,iBAAiB,oBAAoB,EAAE,SAAS,KAAK,EAAE,CAAC,CAAC,CACzD,YAAY,KAAA,CAAS;KAExB,OAAO;IACT,CACF;IACA,MAAM,SAAS,MAAM,OAAO,UAAU,YAAY,KAAK,IAAI,CAAC;IAE5D,IAAI,WAAW,KAAA,GAAW;IAC1B,IAAI,WAAW,MAAM;KACnB,OAAO;KACP,QAAQ,KAAK,IAAI;IACnB,OAAO,IACL,KAAK,IAAI,IAAI,SAAS,eACrB,SAAS,KAAK,KAAK,IAAI,IAAI,eAAe,kBAE3C;IAEF,MAAM,MAAM,KAAK,IAAI,SAAS,YAAY,KAAK,IAAI,CAAC,CAAC;GACvD;GAGA,IAAI;GACJ,OAAO,eAAe,KAAA,GACpB,IAAI;IACF,aAAc,MAAM,KAAK,SAAS,SAAS;GAC7C,SAAS,OAAO;IACd,IAAI,CAAC,qBAAqB,KAAK,UAAU,KAAK,CAAC,KAAK,KAAK,IAAI,KAAK,UAAU,MAAM;IAClF,MAAM,KACH,iBAAiB,oBAAoB,EAAE,SAAS,KAAK,EAAE,CAAC,CAAC,CACzD,YAAY,KAAA,CAAS;GAC1B;GAGF,IAAI,YAAY,KAAA,KAAa,KAAK,IAAI,CAAC,CAAC,WAAW,eAAe,GAChE,MAAM,IAAI,cAAc,WAAW,mBAAmB,QAAQ,WAAW,QAAQ,GAAG;GAGtF,MAAM,QAAQ,KAAK,IAAI,CAAC,CAAC,QAAQ,SAAS,EAAE;GAC5C,OAAO;IACL,MAAM,WAAW,UAAU;IAC3B,UACE,cAAc,KAAA,KAAa,UAAU,UAAU,QAAQ,UAAU,QAAQ,KAAK,IAAI;IACpF,QAAQ,UAAU,OAAO,KAAK;IAC9B,SAAS,UAAU,QAAQ,KAAK,CAAC;GACnC;EACF,EAAA,CAAG;EAEH,KAAK,YAAY,KAAA,CAAS;EAC1B,IAAI;EACJ,IAAI;EACJ,MAAM,UAAU,IAAI,SAAgB,GAAG,WAAW;GAChD,QAAQ,iBACA,uBAAO,IAAI,MAAM,0BAA0B,OAAO,SAAS,EAAE,IAAI,CAAC,GACxE,SACF;GAEA,IAAI,WAAW,KAAA,GAAW;IACxB,eACE,OACE,OAAO,kBAAkB,QAAQ,OAAO,yBAAS,IAAI,MAAM,0BAA0B,CACvF;IACF,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;GACzD;EACF,CAAC;EACD,MAAM,eAAe,YAA2B;GAC9C,MAAM,OAAO;GACb,IAAI,SAAS,KAAA,GAAW;GAQxB,IAAI,MAPiB,OACnB,KAAK,MAAM,CAAC,CAAC,WACL,YACA,IACR,GACA,QACF,MACe,KAAA,KAAa,YAAY,KAAA,GAAW,OAAO,OAAO;EACnE;EACA,IAAI;GACF,OAAO,MAAM,QAAQ,KAAK,CAAC,MAAM,OAAO,CAAC;EAC3C,UAAU;GACR,aAAa,KAAK;GAClB,IAAI,WAAW,KAAA,GAAW,QAAQ,oBAAoB,SAAS,MAAM;GACrE,IAAI,YAAY,KAAA,GAAW,MAAM,aAAa;QAEzC,KAAU,KAAK,cAAc,YAAY,CAAC,CAAC,YAAY,KAAA,CAAS;EACvE;CACF;CAEA,OAAO;EACL,YAAY,YAAY,KAAA;EACxB,MAAM,OAAO,KAAK,eAAe;GAC/B,IAAI;GACJ,IAAI;IACF,SAAS,IAAI,IAAI,GAAG;GACtB,QAAQ;IACN,MAAM,IAAI,cAAc,UAAU,GAAG,IAAI,cAAc;GACzD;GACA,IAAI,OAAO,aAAa,WAAW,OAAO,aAAa,UACrD,MAAM,IAAI,cACR,UACA,gDAAgD,OAAO,UACzD;GAEF,IAAI,SAAS,KAAA,GAAW;IACtB,aAAa,IAAI;IACjB,OAAO,KAAA;GACT;GACA,YAAY;GACZ,IAAI;IACF,OAAO,MAAM,WAAW,OAAO,MAAM,aAAa;GACpD,UAAU;IACR,YAAY;IACZ,IAAI,aAAa,GAAG,SAAS;GAC/B;EACF;EACA,MAAM,QAAQ;GACZ,IAAI,SAAS,KAAA,GAAW,aAAa,IAAI;GACzC,OAAO,KAAA;GACP,MAAM,WAAW;GACjB,UAAU,KAAA;GAGV,IAAI,aAAa,KAAA,GACf,MAAM,SAAS,MACZ,SAAS,KAAK,MAAM,SACf,KAAA,CACR;EACJ;CACF;AACF;;;;;;;;;AChnBA,MAAa,aACX;;;;;;;;;;AAgBF,MAAa,iBAAiB,OAC5B,aAA6D,OAAO,uBACpC;CAChC,IAAI;CACJ,IAAI;EACF,CAAC,CAAE,YAAa,MAAM,KAAK;CAC7B,QAAQ;EACN,OAAO;CACT;CACA,IAAI;EACF,MAAM,aAAa,SAAS,eAAe;EAC3C,IAAI,WAAW,UAAU,GAAG,OAAO,KAAA;EAEnC,IAAI,SAAS,QAAQ,UAAU;EAC/B,OAAO,QAAQ,MAAM,MAAM,UAAU,CAAC,iBAAiB,KAAK,SAAS,MAAM,CAAC,GAC1E,SAAS,QAAQ,MAAM;EACzB,MAAM,WAAW,mBAAmB,KAAK,SAAS,MAAM,CAAC,CAAC,GAAG;EAC7D,IAAI,aAAa,KAAA,GAAW,OAAO,KAAA;EACnC,OAAO,WAAW,KAAK,QAAQ,MAAM,GAAG,2BAA2B,UAAU,CAAC,IAC1E,KAAA,IACA;CACN,QAAQ;EACN;CACF;AACF;;;;;;;AAQA,MAAa,mBACX,UACA,cAAiD,eAAe,MACnD;CACb,IAAI;CACJ,OAAO;EACL,MAAM,OAAO,KAAK,SAAS;GACzB,YAAY,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;GACzC,MAAM,SAAS,MAAM;GACrB,IAAI,WAAW,KAAA,GAAW,MAAM,IAAI,cAAc,WAAW,MAAM;GACnE,OAAO,SAAS,OAAO,KAAK,OAAO;EACrC;EACA,aAAa,SAAS,MAAM;CAC9B;AACF"}

package/build/classify-CrfyXcwo.d.mts

@@ -0,0 +1,51 @@
+//#region src/classify.d.ts
+/**
+ * The outcome tells what a response means for the crawl. The crawler decides it before it keeps
+ * the body as content. The crawler sees a refusal and stops. It does not try again with a
+ * different identity.
+ *
+ * - `challenge`: a bot challenge page. Cloudflare marks it with `cf-mitigated: challenge`. AWS WAF
+ *   answers 202 (challenge) or 405 (CAPTCHA) with an `x-amzn-waf-action` header. The host stops
+ *   for 24 hours.
+ * - `payment-required` (402): a site that wants payment or a license for a crawl. The crawler
+ *   records the price and does not pay. The host stops for 24 hours.
+ * - `blocked` (401, 403): RFC 9110 tells a client not to send the request again automatically.
+ *   The host stops for 1 hour.
+ * - `rate-limited` (429) and `unavailable` (502, 503, 504): the host waits until the time of
+ *   Retry-After, or for a backoff with jitter. A busy gateway is a reason to wait, not a failure.
+ * - `error`: a different status, or no answer. A 5xx or no answer counts toward the stop of the
+ *   host. A different 4xx (a 400 or a 422) is a problem of the request, and it does not count.
+ *
+ * The politeness policy (`PolitenessPolicy`) sets each time of a stop.
+ */
+type Outcome = "ok" | "not-modified" | "redirect" | "challenge" | "payment-required" | "blocked" | "not-found" | "rate-limited" | "unavailable" | "error";
+interface Classified {
+  readonly outcome: Outcome;
+  /** This value is the wait that the server asked for, when it asked for one. */
+  readonly retryAfterMs: number | undefined;
+  readonly note: string | undefined;
+}
+/**
+ * This function reads an HTTP date in the three forms of RFC 9110 §5.6.7: the IMF-fixdate, the
+ * RFC 850 form and the asctime form. It gives milliseconds since the epoch, or no value for a text
+ * in a different form. It does not use `Date.parse`, which reads many texts that are not dates.
+ */
+declare const parseHttpDate: (value: string | undefined) => number | undefined;
+/**
+ * Retry-After is either delta-seconds or an HTTP date (RFC 9110 §10.2.3). The function reads only
+ * these two forms. A different text, for example `1.5` or `-5`, gives no value, so the backoff
+ * applies.
+ */
+declare const parseRetryAfter: (value: string | undefined, now: number) => number | undefined;
+/** After these outcomes, the crawler does not ask the host again until the stop ends. */
+declare const STOPS_HOST: ReadonlySet<Outcome>;
+/**
+ * This function tells if a status is a client error with no meaning for the host. Such a status
+ * is a 4xx other than 401, 402, 403, 404, 408, 410 and 429. The request was wrong, and a
+ * different request can succeed. Thus the failure count of the host does not change.
+ */
+declare const isClientError: (status: number) => boolean;
+declare const classify: (status: number, header: (name: string) => string | undefined, now: number) => Classified;
+//#endregion
+export { isClientError as a, classify as i, Outcome as n, parseHttpDate as o, STOPS_HOST as r, parseRetryAfter as s, Classified as t };
+//# sourceMappingURL=classify-CrfyXcwo.d.mts.map

package/build/classify.d.mts

@@ -0,0 +1,2 @@
+import { a as isClientError, i as classify, n as Outcome, o as parseHttpDate, r as STOPS_HOST, s as parseRetryAfter, t as Classified } from "./classify-CrfyXcwo.mjs";
+export { Classified, Outcome, STOPS_HOST, classify, isClientError, parseHttpDate, parseRetryAfter };