polite-crawler 0.1.1 → 0.1.3

package/build/{open-DYZ7KL2A.mjs → runtime-JToRO63o.mjs}

@@ -1,140 +1,19 @@
-import { a as RenderRefused, o as playwrightRenderer, r as checkedRenderer, s as __exportAll } from "./browser-gJ_h4AhB.mjs";
+import { n as RenderRefused } from "./render-C1CUraWF.mjs";
+import { n as DEFAULT_TOKEN } from "./version-ntbY-6wh.mjs";
 import { classify, isClientError, parseHttpDate } from "./classify.mjs";
-import { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, MemoryHostStateStore, checkPolicy, equalJitterMs, nextValidatorMemo, policyFor, realClock, restAfter, validatorsToSend } from "./politeness.mjs";
+import { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, checkPolicy, equalJitterMs, policyFor, realClock, restAfter, validatorsToSend } from "./politeness.mjs";
 import { decide, isProductToken, robotsFromResponse, robotsFromStatus, signalsFor, unreachableRobots } from "./robots.mjs";
 import { NO_SIGNALS, metaSignals, readSignals } from "./signals.mjs";
-import { n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-no12qgos.mjs";
-import { c as canonicalUrl, o as decodeBody, r as typesOf, s as cacheKeyOf, t as extractPage } from "./html-oNpGAHwi.mjs";
-import { createHash, randomUUID } from "node:crypto";
-import { mkdir, open, readFile, readdir, rename, rm, stat } from "node:fs/promises";
-import { dirname, join } from "node:path";
-import { createReadStream, existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
+import { n as canonicalUrl, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
+import { a as signerOf } from "./webbotauth-CjJdTAfE.mjs";
+import { o as decodeBody, r as typesOf, t as extractPage } from "./html-DkQUAE_d.mjs";
+import { createHash } from "node:crypto";
 import { Transform } from "node:stream";
-import { constants, createBrotliDecompress, createGunzip, createInflate, createInflateRaw, inflateSync } from "node:zlib";
+import { brotliDecompressSync, constants, createBrotliDecompress, createGunzip, createInflate, createInflateRaw, gunzipSync, inflateRawSync, inflateSync } from "node:zlib";
 import { Agent, request } from "undici";
 import { lookup } from "node:dns";
 import { isIP } from "node:net";
-import { DatabaseSync } from "node:sqlite";
 import { getDocumentProxy, getMeta } from "unpdf";
-//#region src/blobs.ts
-const HASH = /^[0-9a-f]{64}$/;
-/** A rename can fail with these codes when a different writer puts the same blob at the same time. */
-const RACE_CODES = /* @__PURE__ */ new Set([
-	"EPERM",
-	"EEXIST",
-	"EBUSY",
-	"EACCES",
-	"ENOTEMPTY"
-]);
-const blobPath = (root, hash) => join(root, hash.slice(0, 2), hash.slice(2, 4), hash);
-const hashOf = (bytes) => createHash("sha256").update(bytes).digest("hex");
-const codeOf = (error) => typeof error === "object" && error !== null && "code" in error ? String(error.code) : "";
-const statOf = (path) => stat(path).catch(() => void 0);
-/** This function writes a folder to the disk, so a rename in it stays after a power cut. Windows cannot open a folder, so it skips this step. */
-const syncFolder = async (path) => {
-	if (process.platform === "win32") return;
-	try {
-		const handle = await open(path, "r");
-		try {
-			await handle.sync();
-		} finally {
-			await handle.close();
-		}
-	} catch {}
-};
-/**
-* This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash
-* of the bytes. These are its rules:
-*
-* - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is
-*   written again.
-* - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.
-*   A crash thus cannot leave half a blob under its name.
-* - Two writers can put the same bytes at the same time. When the rename of one fails because the
-*   other was first, the put succeeds if the stored file has the same hash.
-* - The method `get` examines the hash of the bytes. A damaged file gets a new name
-*   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.
-*/
-var FileBlobStore = class FileBlobStore {
-	dir;
-	writing = /* @__PURE__ */ new Map();
-	sync;
-	constructor(dir, options = {}) {
-		this.dir = dir;
-		this.sync = options.sync ?? true;
-	}
-	put(bytes) {
-		const hash = hashOf(bytes);
-		const running = this.writing.get(hash);
-		if (running !== void 0) return running;
-		const work = this.write(hash, bytes).finally(() => this.writing.delete(hash));
-		this.writing.set(hash, work);
-		return work;
-	}
-	/** This method tells if a file holds exactly the bytes of a hash. */
-	static async holds(path, hash) {
-		try {
-			return hashOf(await readFile(path)) === hash;
-		} catch {
-			return false;
-		}
-	}
-	async write(hash, bytes) {
-		const target = blobPath(this.dir, hash);
-		const existing = await statOf(target);
-		if (existing?.isFile() === true && existing.size === bytes.length) return {
-			hash,
-			existed: true
-		};
-		const folder = dirname(target);
-		await mkdir(folder, {
-			recursive: true,
-			mode: 448
-		});
-		const temporary = `${target}.${randomUUID()}.tmp`;
-		try {
-			const handle = await open(temporary, "wx", 384);
-			try {
-				await handle.writeFile(bytes);
-				if (this.sync) await handle.sync();
-			} finally {
-				await handle.close();
-			}
-			try {
-				await rename(temporary, target);
-			} catch (error) {
-				if (!RACE_CODES.has(codeOf(error)) || !await FileBlobStore.holds(target, hash)) throw error;
-				return {
-					hash,
-					existed: true
-				};
-			}
-			if (this.sync) await syncFolder(folder);
-			return {
-				hash,
-				existed: false
-			};
-		} finally {
-			await rm(temporary, { force: true }).catch(() => void 0);
-		}
-	}
-	async get(hash) {
-		if (!HASH.test(hash)) return void 0;
-		const target = blobPath(this.dir, hash);
-		let bytes;
-		try {
-			bytes = await readFile(target);
-		} catch {
-			return;
-		}
-		if (hashOf(bytes) !== hash) {
-			await rename(target, `${target}.${randomUUID()}.damaged`).catch(() => void 0);
-			return;
-		}
-		return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-	}
-};
-//#endregion
 //#region src/egress.ts
 /** The connection hook gives this error when the policy refuses each address of a host. */
 var EgressRefused = class extends Error {
@@ -551,6 +430,8 @@ const makeRecord = (input) => {
 };
 //#endregion
 //#region src/crawler.ts
+/** The note of a request starts with this text when its host was not free before its deadline, so the request did not start. */
+const DEADLINE_NOTE = "deadline reached";
 /**
 * This function gives the key of the cache entry of a rendered page in the form of version 0.1.
 * The index keeps the variant in a column of its own, and it reads this form too.
@@ -639,8 +520,25 @@ const CREDENTIAL_HEADERS = /* @__PURE__ */ new Set([
 const CONSTANT_HEADERS = /* @__PURE__ */ new Set([
 	"accept-encoding",
 	"user-agent",
-	"from"
+	"from",
+	"signature",
+	"signature-input",
+	"signature-agent"
+]);
+/** With a signer, the crawler sets these headers, and a caller cannot. */
+const SIGNATURE_HEADERS = /* @__PURE__ */ new Set([
+	"signature",
+	"signature-input",
+	"signature-agent"
 ]);
+/** These response headers hold usage signals. With `originalHeaderPrefix`, the archived forms of them count too. */
+const SIGNAL_HEADERS = [
+	"content-usage",
+	"content-signal",
+	"x-robots-tag",
+	"tdm-reservation",
+	"tdm-policy"
+];
 /**
 * These content codings change nothing in the body. A server that stored an object with
 * `aws-chunked` can send the name with the decoded body.
@@ -711,12 +609,21 @@ var Slots = class {
 const missingBrowser = (message) => message.startsWith("rendering needs the package playwright-core and a browser: npm install playwright-core, then npx playwright-core install chromium-headless-shell. For the MCP server with npx, start it with npx -y -p polite-crawler -p playwright-core polite-crawler mcp") || /Executable doesn't exist|Cannot find (?:module|package) 'playwright-core'/.test(message);
 /** This text tells why the crawler did not fetch a page: a rule of the site, or no robots.txt to read. */
 const robotsRefusal = (robots, by) => robots.unreachable === void 0 ? `robots.txt disallows it for ${by}` : `robots.txt could not be read (${robots.unreachable}), so nothing on the site is fetched`;
-const messageOf$2 = (error) => error instanceof Error ? error.message.split(/\r?\n/)[0] ?? "" : String(error);
+const messageOf$1 = (error) => error instanceof Error ? error.message.split(/\r?\n/)[0] ?? "" : String(error);
 const isoOf = (ms) => Number.isFinite(ms) && Math.abs(ms) < LAST_DATE_MS ? new Date(ms).toISOString() : void 0;
 const stopNote = (reason, until) => {
 	const end = isoOf(until);
 	return end === void 0 ? `host stopped: ${reason}` : `host stopped until ${end}: ${reason}`;
 };
+/** A request did not start before its deadline. The crawler gives a skip with `DEADLINE_NOTE` for it. */
+var DeadlineReached = class extends Error {
+	readyAt;
+	constructor(readyAt) {
+		super(DEADLINE_NOTE);
+		this.readyAt = readyAt;
+	}
+};
+const deadlineNote = (error, deadline) => `${DEADLINE_NOTE}: the host is free at ${isoOf(error.readyAt) ?? "a later time"}, and the request must start before ${isoOf(deadline ?? NaN) ?? "its deadline"}`;
 const secretSet = (secretParams) => new Set([...DEFAULT_SECRET_PARAMS, ...secretParams].map((name) => name.toLowerCase()));
 const nameOfParam = (raw) => {
 	try {
@@ -794,7 +701,7 @@ var Lru = class {
 var BodyError = class extends Error {
 	kind;
 	constructor(kind, cause) {
-		super(messageOf$2(cause), { cause });
+		super(messageOf$1(cause), { cause });
 		this.kind = kind;
 	}
 };
@@ -832,7 +739,7 @@ const decoderFor = (coding) => {
 	throw new BodyError("decode", /* @__PURE__ */ new Error(`unsupported content-encoding "${coding}"`));
 };
 /** This function tells if a decoder stopped because the compressed body ended early: the server cut it. */
-const endedEarly = (error) => typeof error === "object" && error !== null && "code" in error && (error.code === "Z_BUF_ERROR" || /unexpected end of file/i.test(messageOf$2(error)));
+const endedEarly = (error) => typeof error === "object" && error !== null && "code" in error && (error.code === "Z_BUF_ERROR" || /unexpected end of file/i.test(messageOf$1(error)));
 /**
 * This function reads a body as a stream and decodes each content coding in sequence. It counts
 * the decoded bytes and keeps at most `maxBytes`. Thus a decompression bomb stops at the
@@ -1027,15 +934,29 @@ var Crawler = class {
 	renders;
 	/** These tokens apply because the `User-Agent` links the bot page (`obeyDefaultToken`). */
 	pageTokens;
+	/** The deadlines of the requests that have one, by the signal that the scheduler gets for each of them. */
+	deadlines = /* @__PURE__ */ new WeakMap();
+	/** This signer signs each request. It holds the key, and the crawler writes only its key id and its agent. */
+	signer;
 	constructor(options) {
 		this.options = options;
+		this.signer = signerOf(options.signer);
 		if (!isProductToken(options.token)) throw new Error(`the token "${options.token}" is not a product token: use only letters, "_" and "-" (RFC 9309)`);
 		for (const [name, value] of [["userAgent", options.userAgent], ["from", options.from]]) if (value !== void 0 && !HEADER_TEXT.test(value)) throw new Error(`${name} "${value}" has a character that an HTTP header cannot hold: use printable ASCII`);
 		checkPolicy(options.politeness);
 		this.clock = options.clock ?? realClock;
 		this.random = options.random ?? Math.random;
 		this.egress = options.egress ?? {};
-		this.scheduler = new HostScheduler(this.clock, options.maxActive ?? 16, { store: options.hostState });
+		const clock = this.clock;
+		const deadlines = this.deadlines;
+		this.scheduler = new HostScheduler({
+			now: () => clock.now(),
+			sleep: (ms, signal) => {
+				const deadline = signal === void 0 ? void 0 : deadlines.get(signal);
+				const readyAt = clock.now() + ms;
+				return deadline !== void 0 && readyAt >= deadline ? Promise.reject(new DeadlineReached(readyAt)) : clock.sleep(ms, signal);
+			}
+		}, options.maxActive ?? 16, { store: options.hostState });
 		this.pageTokens = options.obeyDefaultToken !== false && options.token.toLowerCase() !== "PoliteCrawler".toLowerCase() && options.userAgent.includes("https://mark1russell7.github.io/crawler/bot/") ? [DEFAULT_TOKEN] : [];
 		const maxRenders = options.maxRenders ?? 2;
 		if (!Number.isInteger(maxRenders) || maxRenders < 1) throw new RangeError(`maxRenders must be an integer of 1 or more, not ${String(maxRenders)}`);
@@ -1059,6 +980,21 @@ var Crawler = class {
 	policyOf(url) {
 		return policyFor(this.options.politeness, url.hostname);
 	}
+	/**
+	* This method gives the signal of a request for the scheduler. A request with a deadline gets a
+	* signal of its own, and the clock of the scheduler reads the deadline of that signal.
+	*/
+	paced(signal, deadline) {
+		if (deadline === void 0) return signal;
+		const own = AbortSignal.any([signal]);
+		this.deadlines.set(own, deadline);
+		return own;
+	}
+	/** This method throws `DeadlineReached` when the deadline of a request came before its slot. */
+	beforeDeadline(deadline) {
+		const now = this.clock.now();
+		if (deadline !== void 0 && now >= deadline) throw new DeadlineReached(now);
+	}
 	emit(event) {
 		try {
 			this.options.onEvent?.(event);
@@ -1223,11 +1159,41 @@ var Crawler = class {
 			fetchedAt: entry.fetched_at
 		} };
 	}
+	/**
+	* This method gives the signature headers of one request, and the facts of the capture: the key id
+	* and the agent. With no signer, it gives nothing. A signer that throws gives the note of a skip,
+	* because the crawler sends no unsigned request when it has a signer.
+	*/
+	signFor(target) {
+		if (this.signer === void 0) return void 0;
+		let given;
+		try {
+			given = this.signer.sign(target, this.clock.now());
+		} catch (error) {
+			return { error: `the signer did not sign the request: ${messageOf$1(error)}` };
+		}
+		const headers = {};
+		for (const [name, value] of Object.entries(given)) {
+			const lower = name.toLowerCase();
+			if (RESERVED_HEADERS.has(lower)) continue;
+			if (typeof value !== "string" || !HEADER_TEXT.test(value)) return { error: `the signer gave a ${lower} header that HTTP cannot send` };
+			headers[lower] = value;
+		}
+		return {
+			headers,
+			facts: { signed: {
+				key_id: this.signer.keyId,
+				agent: this.signer.signatureAgent
+			} }
+		};
+	}
 	requestHeaders(request_) {
 		const headers = {};
 		for (const [name, value] of Object.entries(request_.headers ?? {})) {
 			const lower = name.toLowerCase();
-			if (!RESERVED_HEADERS.has(lower)) headers[lower] = value;
+			if (RESERVED_HEADERS.has(lower)) continue;
+			if (this.signer !== void 0 && SIGNATURE_HEADERS.has(lower)) continue;
+			headers[lower] = value;
 		}
 		headers["accept"] = request_.accept ?? headers["accept"] ?? "*/*";
 		headers["user-agent"] = this.options.userAgent;
@@ -1416,7 +1382,7 @@ var Crawler = class {
 			return {
 				...closedResult(request_.url),
 				outcome: "error",
-				note: `crawler error: ${messageOf$2(error)}`
+				note: `crawler error: ${messageOf$1(error)}`
 			};
 		}
 	}
@@ -1436,7 +1402,11 @@ var Crawler = class {
 			request_.onBehalfOf ?? null,
 			request_.job ?? null,
 			request_.scope ?? null,
-			request_.labels ?? null
+			request_.labels ?? null,
+			request_.deadline ?? null,
+			request_.range ?? null,
+			request_.signalLines ?? null,
+			request_.originalHeaderPrefix ?? null
 		]);
 		const running = this.inFlight.get(key);
 		if (running !== void 0) return running;
@@ -1479,7 +1449,7 @@ var Crawler = class {
 			good: cached?.good
 		};
 		entry.verdict = this.fetchRobots(target, policy, waiting).catch((error) => ({
-			robots: unreachableRobots(`robots.txt failed: ${messageOf$2(error)}`),
+			robots: unreachableRobots(`robots.txt failed: ${messageOf$1(error)}`),
 			capture: void 0,
 			retryAfterMs: void 0
 		})).then(async (read) => {
@@ -1514,13 +1484,63 @@ var Crawler = class {
 	* come from its memory, or from a fetch of `/robots.txt` with its redirects. A robots.txt that
 	* the crawler cannot read gives rules that permit nothing, and `unreachable` gives the reason.
 	*/
-	async readRobots(url) {
+	async readRobots(url, marks = {}) {
 		const target = new URL(url);
 		if (!WEB_PROTOCOLS.has(target.protocol)) return {
 			robots: unreachableRobots("not an http(s) URL"),
 			capture: void 0
 		};
-		return this.robotsFor(target, this.policyOf(target));
+		return this.robotsFor(target, this.policyOf(target), marks);
+	}
+	/**
+	* This method tells if robots.txt lets the crawler fetch a URL as a page. The rules of the token
+	* of the crawler and of the agents of `onBehalfOf` apply, as for a fetch. It reads robots.txt as
+	* a fetch does, so a robots.txt that the crawler cannot read permits nothing. When the signal
+	* aborts, the promise rejects, and a fetch of robots.txt in progress continues.
+	*/
+	async robotsAllow(url, options = {}) {
+		let target;
+		try {
+			target = new URL(url);
+		} catch {
+			return {
+				allowed: false,
+				note: "not a valid URL"
+			};
+		}
+		if (!WEB_PROTOCOLS.has(target.protocol)) return {
+			allowed: false,
+			note: "not an http(s) URL"
+		};
+		if (this.closed) return {
+			allowed: false,
+			note: "crawler closed"
+		};
+		const tokens = [
+			this.options.token,
+			...options.onBehalfOf ?? [],
+			...this.pageTokens
+		];
+		const badToken = tokens.find((token) => !isProductToken(token));
+		if (badToken !== void 0) return {
+			allowed: false,
+			note: `not a valid product token: "${badToken}"`
+		};
+		const work = this.robotsFor(target, this.policyOf(target), options);
+		const verdict = await (options.signal === void 0 ? work : raceAbort(work, options.signal));
+		const path = `${target.pathname}${target.search}`;
+		const decision = decide(verdict.robots, tokens, path);
+		const lines = signalsFor(verdict.robots, tokens, path);
+		const signals = lines.length > 0 ? { signals: lines } : {};
+		return decision.allowed ? {
+			allowed: true,
+			note: void 0,
+			...signals
+		} : {
+			allowed: false,
+			note: robotsRefusal(verdict.robots, decision.by ?? this.options.token),
+			...signals
+		};
 	}
 	/** This method reads the stored copy of a robots.txt, when it is younger than the limit of the policy. */
 	async storedRobots(target, policy) {
@@ -1610,21 +1630,23 @@ var Crawler = class {
 				return request_.labels;
 			}
 		};
+		const range = request_.range;
+		const badRange = range === void 0 || Number.isSafeInteger(range.offset) && range.offset >= 0 && Number.isSafeInteger(range.length) && range.length >= 1 ? void 0 : `invalid option: range is ${JSON.stringify(range)}`;
 		const admission = await this.admit({
 			transport: "http",
 			url: request_.url,
 			mode: request_.mode,
 			base,
 			secretParams,
-			invalid: invalidNumber("minIntervalMs", request_.minIntervalMs, false) ?? invalidNumber("maxBytes", request_.maxBytes, false) ?? invalidNumber("maxAgeMs", request_.maxAgeMs, true),
+			invalid: invalidNumber("minIntervalMs", request_.minIntervalMs, false) ?? invalidNumber("maxBytes", request_.maxBytes, false) ?? invalidNumber("maxAgeMs", request_.maxAgeMs, true) ?? badRange,
 			onBehalfOf: request_.onBehalfOf,
 			scope: request_.scope,
 			labels: request_.labels,
-			cached: (url) => this.fromCache(url, "plain", request_.maxAgeMs, sent, secretParams)
+			cached: (url) => range !== void 0 ? Promise.resolve(void 0) : this.fromCache(url, "plain", request_.maxAgeMs, sent, secretParams)
 		});
 		if ("answer" in admission) return admission.answer;
 		const { target, url, host, policy, check, robotsLines, crawlDelaySeconds, skip } = admission.admitted;
-		const maxBytes = request_.maxBytes ?? policy.maxBytes;
+		const maxBytes = Math.min(request_.maxBytes ?? policy.maxBytes, range?.length ?? Infinity);
 		const rest = (elapsedMs) => ({
 			kind: request_.mode,
 			elapsedMs,
@@ -1634,12 +1656,13 @@ var Crawler = class {
 		const robotsFile = request_.recipe === "robots.txt";
 		try {
 			return await this.scheduler.run(host, async () => {
-				const again = await this.fromCache(url, "plain", request_.maxAgeMs, sent, secretParams);
+				const again = range === void 0 ? await this.fromCache(url, "plain", request_.maxAgeMs, sent, secretParams) : void 0;
 				if (again !== void 0) return {
 					value: again,
 					delayMs: 0
 				};
-				const latest = this.lookup(url);
+				this.beforeDeadline(request_.deadline);
+				const latest = range === void 0 ? this.lookup(url) : void 0;
 				const usable = latest !== void 0 && latest.truncated !== true && varyMatches(latest.vary_key, sent) ? latest : void 0;
 				let stored;
 				if (usable?.blob && (usable.etag || usable.last_modified)) stored = await this.options.blobs.get(usable.blob);
@@ -1654,11 +1677,20 @@ var Crawler = class {
 				if (send.etag && usable?.etag) headers["if-none-match"] = usable.etag;
 				if (send.lastModified && usable?.last_modified) headers["if-modified-since"] = usable.last_modified;
 				const validators = ["if-none-match", "if-modified-since"].filter((name) => headers[name] !== void 0);
+				const rangeText = range === void 0 ? void 0 : `bytes=${String(range.offset)}-${String(range.offset + range.length - 1)}`;
+				if (rangeText !== void 0) headers["range"] = rangeText;
+				const signed = this.signFor(target);
+				if (signed !== void 0 && "error" in signed) return {
+					value: { fetched: await skip(signed.error) },
+					delayMs: 0
+				};
+				if (signed !== void 0) Object.assign(headers, signed.headers);
+				const signedFacts = signed?.facts ?? {};
 				const started = this.clock.now();
 				const timeout = AbortSignal.timeout(policy.requestTimeoutMs);
 				const abort = AbortSignal.any([timeout, signal]);
 				const cancelled = () => request_.signal?.aborted === true;
-				const failedNote = (prefix, error) => timeout.aborted ? `${prefix}: timed out after ${String(Math.round(policy.requestTimeoutMs / 1e3))} s` : `${prefix}: ${messageOf$2(error)}`;
+				const failedNote = (prefix, error) => timeout.aborted ? `${prefix}: timed out after ${String(Math.round(policy.requestTimeoutMs / 1e3))} s` : `${prefix}: ${messageOf$1(error)}`;
 				this.emit({
 					type: "request",
 					url,
@@ -1708,7 +1740,9 @@ var Crawler = class {
 						outcome: "error",
 						note,
 						elapsed_ms: elapsedMs,
-						...check ? { robots_check: check } : {}
+						...check ? { robots_check: check } : {},
+						...rangeText !== void 0 ? { range: rangeText } : {},
+						...signedFacts
 					}, url, marks);
 					return {
 						value: { fetched: {
@@ -1762,7 +1796,7 @@ var Crawler = class {
 						blob = (await this.options.blobs.put(body)).hash;
 					} catch (error) {
 						outcome = "error";
-						note = `blob store: ${messageOf$2(error)}`;
+						note = `blob store: ${messageOf$1(error)}`;
 						body = void 0;
 						result = "neutral";
 					}
@@ -1791,7 +1825,16 @@ var Crawler = class {
 					}
 				}
 				const revisit = outcome === "not-modified" && usable !== void 0;
-				let signals = readSignals(robotsLines, header, pageSignals(body, header("content-type")));
+				const prefix = request_.originalHeaderPrefix?.toLowerCase();
+				const archivedLines = prefix === void 0 ? [] : SIGNAL_HEADERS.flatMap((name) => {
+					const value = header(`${prefix}${name}`);
+					return value === void 0 ? [] : [`${prefix}${name}: ${value}`];
+				});
+				let signals = readSignals([
+					...robotsLines,
+					...request_.signalLines ?? [],
+					...archivedLines
+				], header, pageSignals(body, header("content-type")));
 				if (revisit) {
 					const merged = signals.ai_input === "no" || usable.ai_input === "no" ? "no" : signals.ai_input !== "unset" ? signals.ai_input : usable.ai_input;
 					signals = {
@@ -1852,10 +1895,12 @@ var Crawler = class {
 					...freshness !== null ? { fresh_until: freshness } : {},
 					...varyKey !== null ? { vary_key: varyKey } : {},
 					...location !== void 0 ? { location } : {},
-					...note !== void 0 ? { note } : {}
+					...note !== void 0 ? { note } : {},
+					...rangeText !== void 0 ? { range: rangeText } : {},
+					...signedFacts
 				}, url, marks);
 				const fetchedIso = new Date(fetchedAt).toISOString();
-				if (capture !== void 0 && (outcome === "ok" || outcome === "not-modified") && blob !== void 0 && !cut && !unasked) this.store({
+				if (capture !== void 0 && (outcome === "ok" || outcome === "not-modified") && blob !== void 0 && !cut && !unasked && range === void 0) this.store({
 					url,
 					capture,
 					blob,
@@ -1921,9 +1966,10 @@ var Crawler = class {
 					},
 					delayMs
 				};
-			}, signal);
+			}, this.paced(signal, request_.deadline));
 		} catch (error) {
 			if (error instanceof HostStopped) return { fetched: await skip(stopNote(error.reason, error.until)) };
+			if (error instanceof DeadlineReached) return { fetched: await skip(deadlineNote(error, request_.deadline)) };
 			if (this.closed) return { fetched: closedResult(url) };
 			if (request_.signal?.aborted === true) return { fetched: await skip("cancelled by the caller") };
 			throw error;
@@ -2018,7 +2064,7 @@ var Crawler = class {
 		return this.track(this.renderOnce(request_).catch((error) => this.closed ? closedResult(request_.url) : {
 			...closedResult(request_.url),
 			outcome: "error",
-			note: `crawler error: ${messageOf$2(error)}`
+			note: `crawler error: ${messageOf$1(error)}`
 		}));
 	}
 	async renderOnce(request_) {
@@ -2068,6 +2114,20 @@ var Crawler = class {
 			hosts: /* @__PURE__ */ new Set()
 		};
 		const allowRequest = this.allowRequestFor(target, verdict.robots, tokens, signal, tally);
+		const sign = this.signer === void 0 ? void 0 : (raw) => {
+			let to;
+			try {
+				to = new URL(raw);
+			} catch {
+				return;
+			}
+			const signed = this.signFor(to);
+			return signed === void 0 || "error" in signed ? void 0 : signed.headers;
+		};
+		const signedFacts = this.signer === void 0 ? {} : { signed: {
+			key_id: this.signer.keyId,
+			agent: this.signer.signatureAgent
+		} };
 		const rest = (elapsedMs) => ({
 			kind: "render",
 			elapsedMs,
@@ -2083,6 +2143,7 @@ var Crawler = class {
 		let done;
 		try {
 			done = await this.scheduler.run(host, async () => {
+				this.beforeDeadline(request_.deadline);
 				const started = this.clock.now();
 				this.emit({
 					type: "request",
@@ -2097,7 +2158,8 @@ var Crawler = class {
 						...this.options.from !== void 0 ? { headers: { from: this.options.from } } : {},
 						allowRequest,
 						maxRedirects: policy.maxRedirects,
-						signal
+						signal,
+						...sign === void 0 ? {} : { sign }
 					}), signal);
 					const header = (name) => rendered.headers[name.toLowerCase()];
 					const classified = rendered.status === null ? {
@@ -2135,7 +2197,7 @@ var Crawler = class {
 						},
 						delayMs: restAfter(policy, rest(0)).ms
 					};
-					const refused = error instanceof RenderRefused ? error : missingBrowser(messageOf$2(error)) ? new RenderRefused("browser", `rendering needs a browser: ${messageOf$2(error)}`) : void 0;
+					const refused = error instanceof RenderRefused ? error : missingBrowser(messageOf$1(error)) ? new RenderRefused("browser", `rendering needs a browser: ${messageOf$1(error)}`) : void 0;
 					if (refused !== void 0) {
 						if (refused.reason === "browser") this.renderOff = refused.message;
 						const delayMs = this.settle(host, policy, {
@@ -2167,15 +2229,16 @@ var Crawler = class {
 					return {
 						value: {
 							kind: "failed",
-							note: `render failed: ${messageOf$2(error)}`,
+							note: `render failed: ${messageOf$1(error)}`,
 							elapsedMs
 						},
 						delayMs
 					};
 				}
-			}, signal);
+			}, this.paced(signal, request_.deadline));
 		} catch (error) {
 			if (error instanceof HostStopped) return skip(stopNote(error.reason, error.until));
+			if (error instanceof DeadlineReached) return skip(deadlineNote(error, request_.deadline));
 			if (this.closed) return closedResult(url);
 			if (request_.signal?.aborted === true) return skip("cancelled by the caller");
 			throw error;
@@ -2213,7 +2276,8 @@ var Crawler = class {
 				note: done.note,
 				elapsed_ms: done.elapsedMs,
 				robots_check: check,
-				subrequests
+				subrequests,
+				...signedFacts
 			}, url, marks);
 			return {
 				...closedResult(url),
@@ -2246,7 +2310,8 @@ var Crawler = class {
 					note,
 					elapsed_ms: done.elapsedMs,
 					robots_check: check,
-					subrequests
+					subrequests,
+					...signedFacts
 				}, url, marks),
 				blob: void 0,
 				body: void 0,
@@ -2290,7 +2355,7 @@ var Crawler = class {
 				blob = (await this.options.blobs.put(body)).hash;
 			} catch (error) {
 				outcome = "error";
-				note = `blob store: ${messageOf$2(error)}`;
+				note = `blob store: ${messageOf$1(error)}`;
 				body = void 0;
 			}
 		}
@@ -2321,7 +2386,8 @@ var Crawler = class {
 			signals,
 			subrequests,
 			...freshness !== null ? { fresh_until: freshness } : {},
-			...note !== void 0 ? { note } : {}
+			...note !== void 0 ? { note } : {},
+			...signedFacts
 		}, url, marks);
 		const fetchedIso = new Date(fetchedAt).toISOString();
 		if (capture !== void 0 && blob !== void 0) this.store({
@@ -2366,6 +2432,17 @@ var Crawler = class {
 		}
 	}
 	/**
+	* This method gives the stop of the host of a URL, or undefined when the host is not stopped. A
+	* watch uses it, so a page of a stopped host is due again at the end of the stop.
+	*/
+	hostStop(url) {
+		try {
+			return this.scheduler.store.stopped(hostKey(new URL(url)), this.clock.now());
+		} catch {
+			return;
+		}
+	}
+	/**
 	* This method closes the crawler. Work in the queue ends as `skipped` with the note
 	* `crawler closed`, and the crawler writes no capture after the close. Then it closes the
 	* browser, the log and its own dispatcher. A second close waits for the first.
@@ -2661,1258 +2738,93 @@ const snippetOf = (text, terms, width = 160) => {
 	const clean = (part) => part.replace(/\s+/g, " ");
 	return `${from > 0 ? "…" : ""}${clean(text.slice(from, start))}[${clean(text.slice(start, end))}]${clean(text.slice(end, to))}${to < text.length ? "…" : ""}`;
 };
-/** The search index reads this many characters of each text, at most. */
-const MAX_INDEX_CHARS = 2e5;
-/** In the mode `fast`, a part is common when more than this share of the documents hold it. */
-const FAST_RANK_SHARE = .2;
-/** This function gives the rank mode of an option. A value that is not a rank mode throws a `RangeError`. */
-const rankModeOf = (value) => {
-	if (value === void 0 || value === "exact") return "exact";
-	if (value === "fast") return "fast";
-	throw new RangeError(`rank must be "exact" or "fast", not ${JSON.stringify(value) ?? "undefined"}`);
-};
-const namesOf = (generation) => ({
-	docs: `docs_${String(generation)}`,
-	fts: `fts_${String(generation)}`,
-	versions: `versions_${String(generation)}`,
-	texts: `texts_${String(generation)}`,
-	latest: `latest_${String(generation)}`
-});
-const GENERATION_TABLE = /^(docs|fts|versions|texts|latest)_(\d+)$/;
-/** A build that started this long ago and did not finish belongs to a process that stopped. */
-const ABANDONED_MS = 864e5;
-/** The FTS5 tokenizer: folded text, English stems, and spacing marks kept in the words. */
-const TOKENIZER = `tokenize = "porter unicode61 remove_diacritics 0 categories 'L* N* Co Mc'"`;
-const ddlOf = (n) => `
-CREATE TABLE IF NOT EXISTS ${n.texts} (
-  sha TEXT PRIMARY KEY,
-  body TEXT NOT NULL
-);
-CREATE TABLE IF NOT EXISTS ${n.versions} (
-  doc TEXT NOT NULL,
-  capture TEXT NOT NULL,
-  recipe TEXT NOT NULL,
-  rendered INTEGER NOT NULL,
-  fetched_at TEXT NOT NULL,
-  blob TEXT,
-  kind TEXT NOT NULL,
-  title TEXT NOT NULL,
-  url TEXT,
-  meta_sha TEXT NOT NULL,
-  text_sha TEXT NOT NULL,
-  text_chars INTEGER NOT NULL,
-  full_sha TEXT,
-  ai_input TEXT NOT NULL,
-  merge TEXT NOT NULL,
-  scope TEXT,
-  labels TEXT,
-  PRIMARY KEY (doc, capture)
-);
-CREATE INDEX IF NOT EXISTS ${n.versions}_time ON ${n.versions} (doc, recipe, rendered, fetched_at);
-CREATE INDEX IF NOT EXISTS ${n.versions}_capture ON ${n.versions} (capture);
-CREATE INDEX IF NOT EXISTS ${n.versions}_fetched ON ${n.versions} (fetched_at);
-CREATE INDEX IF NOT EXISTS ${n.versions}_recipe ON ${n.versions} (recipe);
-CREATE TABLE IF NOT EXISTS ${n.docs} (
-  doc TEXT PRIMARY KEY,
-  kind TEXT NOT NULL,
-  title TEXT NOT NULL,
-  url TEXT,
-  meta TEXT NOT NULL,
-  text_sha TEXT NOT NULL,
-  full_sha TEXT,
-  captures TEXT NOT NULL,
-  capture_count INTEGER NOT NULL,
-  capture TEXT NOT NULL,
-  blob TEXT,
-  fetched_at TEXT NOT NULL,
-  recipe TEXT NOT NULL,
-  rendered INTEGER NOT NULL,
-  ai_input TEXT NOT NULL,
-  scope TEXT,
-  labels TEXT,
-  updated_at TEXT NOT NULL
-);
-CREATE VIRTUAL TABLE IF NOT EXISTS ${n.fts} USING fts5(
-  title, text, content = '', contentless_delete = 1, ${TOKENIZER}
-);
-CREATE TABLE IF NOT EXISTS ${n.latest} (
-  url TEXT NOT NULL,
-  variant TEXT NOT NULL,
-  capture TEXT NOT NULL,
-  blob TEXT,
-  etag TEXT,
-  last_modified TEXT,
-  content_type TEXT,
-  fetched_at TEXT NOT NULL,
-  status INTEGER NOT NULL,
-  ai_input TEXT NOT NULL,
-  location TEXT,
-  truncated INTEGER,
-  fresh_until TEXT,
-  vary_key TEXT,
-  PRIMARY KEY (url, variant)
-);
-CREATE INDEX IF NOT EXISTS ${n.latest}_fetched ON ${n.latest} (fetched_at);
-`;
-const RENDERED_MARK$1 = "#rendered";
-/** This function splits a cache key into its URL and its variant. A key in the form of version 0.1 (`…#rendered`) is a rendered variant. */
-const keyOf = (url, variant) => {
-	if (url.endsWith(RENDERED_MARK$1)) return {
-		url: cacheKeyOf(url.slice(0, -9)),
-		variant: "rendered"
-	};
-	return {
-		url: cacheKeyOf(url),
-		variant: variant ?? "plain"
-	};
+//#endregion
+//#region src/pdf.ts
+const MAX_PAGES = 500;
+const MAX_CHARS = 2e5;
+const TIME_BUDGET_MS = 2e4;
+/**
+* The text streams of 35 stored menus and papers inflate to 0.6 MB or less. The parser of pdf.js
+* reads a page with no pause. Thus a page of 8 MB of text operators stops the process for
+* approximately one second.
+*/
+const MAX_INFLATED_BYTES$1 = 8388608;
+/** This function finds a PDF from its first bytes or its content type. Some servers put bytes before `%PDF-`. */
+const isPdf = (body, contentType) => {
+	if (/\bpdf\b/i.test(contentType ?? "")) return true;
+	const end = Math.min(body.length - 4, 1024);
+	for (let index = 0; index < end; index += 1) if (body[index] === 37 && body[index + 1] === 80 && body[index + 2] === 68 && body[index + 3] === 70 && body[index + 4] === 45) return true;
+	return false;
 };
-const sha = (text) => createHash("sha256").update(text, "utf8").digest("hex");
-const parseJson = (text, fallback) => {
-	if (text === null || text === void 0) return fallback;
-	try {
-		return JSON.parse(text);
-	} catch {
-		return fallback;
-	}
+/** This function gives one line of text for each line of the PDF. It removes the empty lines. */
+const tidy = (page) => page.split(/\r?\n/).map((line) => line.replace(/\s+/g, " ").trim()).filter((line) => line.length > 0).join("\n");
+/** A word processor writes the name of its file as the title ("Microsoft Word - menu-final-v3.docx"). */
+const JUNK_TITLE = /^(?:microsoft (?:word|powerpoint|excel) - |untitled\b|document\d*$|presentation\d*$)|\.(?:docx?|pdf|pptx?|xlsx?|indd|ai|psd|rtf|odt|pages|txt)$/i;
+const latin1 = (bytes, start, end) => {
+	let out = "";
+	for (let index = Math.max(0, start); index < end && index < bytes.length; index += 1) out += String.fromCharCode(bytes[index]);
+	return out;
 };
-const toLatest = (row) => ({
-	url: row.url,
-	capture: row.capture,
-	blob: row.blob,
-	etag: row.etag,
-	last_modified: row.last_modified,
-	content_type: row.content_type,
-	fetched_at: row.fetched_at,
-	status: row.status,
-	ai_input: row.ai_input,
-	location: row.location,
-	truncated: row.truncated === 1 ? true : void 0,
-	fresh_until: row.fresh_until,
-	vary_key: row.vary_key,
-	variant: row.variant
-});
-const toVersion = (row) => ({
-	doc: row.doc,
-	capture: row.capture,
-	recipe: row.recipe,
-	rendered: row.rendered === 1,
-	fetched_at: row.fetched_at,
-	blob: row.blob,
-	kind: row.kind,
-	title: row.title,
-	url: row.url,
-	ai_input: row.ai_input,
-	chars: row.text_chars,
-	cut: row.full_sha !== null
-});
-const messageOf$1 = (error) => error instanceof Error ? error.message : String(error);
-var DocIndex = class {
-	path;
-	db;
-	statements = /* @__PURE__ */ new Map();
-	depth = 0;
-	closed = false;
-	empty;
-	/**
-	* This value is true when the file had a different schema, or no tables, so the index opened
-	* empty. A store with captures then needs `reindex`. The commit of a rebuild makes it false.
-	*/
-	get stale() {
-		return this.empty;
-	}
-	constructor(path, options = {}) {
-		this.path = path;
-		if (path !== ":memory:") mkdirSync(dirname(path), {
-			recursive: true,
-			mode: 448
-		});
-		this.db = new DatabaseSync(path);
-		const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
-		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
-		try {
-			this.db.exec("CREATE VIRTUAL TABLE temp.polite_fts5_probe USING fts5(x, content = '', contentless_delete = 1); DROP TABLE temp.polite_fts5_probe;");
-		} catch (cause) {
-			this.db.close();
-			throw new Error(`polite-crawler needs SQLite FTS5 with contentless_delete, and the SQLite of Node.js ${process.version} does not have it: use Node.js 22.19 or later`, { cause });
-		}
-		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
-		this.db.exec("PRAGMA synchronous = NORMAL;");
-		this.empty = this.write(() => this.prepareSchema());
-	}
-	/** This method makes the tables of the schema. It gives true when the index opened empty. */
-	prepareSchema() {
-		const version = this.db.prepare("PRAGMA user_version").get().user_version;
-		const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
-		if (version === 2 && tables.some((table) => table.name === "index_meta")) {
-			this.dropAbandoned();
-			return false;
-		}
-		const virtual = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'").all();
-		for (const { name } of virtual) this.db.exec(`DROP TABLE IF EXISTS "${name}"`);
-		const rest = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
-		for (const { name } of rest) this.db.exec(`DROP TABLE IF EXISTS "${name}"`);
-		this.db.exec("CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)");
-		this.db.exec(ddlOf(namesOf(1)));
-		this.statements.clear();
-		this.setMeta("generation", "1");
-		this.setMeta("next_generation", "2");
-		this.db.exec(`PRAGMA user_version = ${String(2)}`);
-		return true;
-	}
-	setMeta(key, value) {
-		this.stmt("INSERT INTO index_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
-	}
-	getMeta(key) {
-		return this.stmt("SELECT value FROM index_meta WHERE key = ?").get(key)?.value;
-	}
-	stmt(sql) {
-		let statement = this.statements.get(sql);
-		if (statement === void 0) {
-			statement = this.db.prepare(sql);
-			this.statements.set(sql, statement);
-		}
-		return statement;
-	}
-	/**
-	* This method does a write in one transaction. A transaction in progress takes the write in,
-	* also a transaction that the caller opened on the connection.
-	*/
-	write(work) {
-		if (this.closed) throw new Error("the index is closed");
-		if (this.depth > 0 || this.db.isTransaction) {
-			this.depth += 1;
-			try {
-				return work();
-			} finally {
-				this.depth -= 1;
-			}
-		}
-		this.db.exec("BEGIN IMMEDIATE");
-		this.depth = 1;
-		try {
-			const result = work();
-			this.db.exec("COMMIT");
-			return result;
-		} catch (error) {
-			if (this.db.isTransaction) this.db.exec("ROLLBACK");
-			throw error;
-		} finally {
-			this.depth = 0;
+const indexOf = (bytes, needle, from) => {
+	const first = needle.charCodeAt(0);
+	for (let index = bytes.indexOf(first, from); index !== -1; index = bytes.indexOf(first, index + 1)) {
+		let match = true;
+		for (let at = 1; at < needle.length; at += 1) if (bytes[index + at] !== needle.charCodeAt(at)) {
+			match = false;
+			break;
 		}
+		if (match) return index;
 	}
-	generation() {
-		return Number(this.getMeta("generation") ?? "1");
-	}
-	/**
-	* This method does a step on the tables of the current generation. A rebuild of a different
-	* process can remove those tables in the meantime. Then the method reads the generation again
-	* and does the step once more.
-	*/
-	live(step) {
-		if (this.closed) throw new Error("the index is closed");
+	return -1;
+};
+/**
+* This function adds up the bytes that the compressed streams of a file inflate to, and stops at
+* the limit. It tries each stream: a stream in another format fails at its first bytes. An image
+* stream does not count, because the text reader does not decode it. The function gives true when
+* the file stays inside the limit.
+*/
+const inflatesWithin = (bytes, limit) => {
+	let total = 0;
+	let from = 0;
+	for (;;) {
+		const keyword = indexOf(bytes, "stream", from);
+		if (keyword === -1) return true;
+		from = keyword + 6;
+		if (latin1(bytes, keyword - 3, keyword) === "end") continue;
+		const next = bytes[keyword + 6];
+		if (next !== 10 && next !== 13) continue;
+		const dictionary = latin1(bytes, keyword - 1024, keyword);
+		const head = dictionary.slice(Math.max(0, dictionary.lastIndexOf("<<")));
+		if (/\/Subtype\s*\/Image\b/.test(head)) continue;
+		let start = keyword + 7;
+		if (next === 13 && bytes[start] === 10) start += 1;
+		const declared = /\/Length\s+(\d+)(?!\s+\d+\s+R)/.exec(head);
+		const stop = indexOf(bytes, "endstream", start);
+		const end = declared !== null && start + Number(declared[1]) <= bytes.length ? start + Number(declared[1]) : stop === -1 ? bytes.length : stop;
 		try {
-			return step(namesOf(this.generation()));
+			total += inflateSync(bytes.subarray(start, end), {
+				maxOutputLength: Math.max(1, limit - total + 1),
+				finishFlush: constants.Z_SYNC_FLUSH
+			}).length;
 		} catch (error) {
-			if (!/no such table/i.test(messageOf$1(error)) || this.depth > 0) throw error;
-			this.statements.clear();
-			return step(namesOf(this.generation()));
-		}
-	}
-	/** This method removes the tables of a generation. */
-	dropGeneration(generation) {
-		const n = namesOf(generation);
-		for (const table of [
-			n.fts,
-			n.docs,
-			n.versions,
-			n.texts,
-			n.latest
-		]) this.db.exec(`DROP TABLE IF EXISTS ${table}`);
-		this.db.prepare("DELETE FROM index_meta WHERE key = ?").run(`building:${String(generation)}`);
-		this.statements.clear();
-	}
-	/** This method removes the generations of builds that a stopped process left. */
-	dropAbandoned() {
-		const current = this.generation();
-		const generations = /* @__PURE__ */ new Set();
-		for (const { name } of this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all()) {
-			const match = GENERATION_TABLE.exec(name);
-			if (match !== null) generations.add(Number(match[2]));
-		}
-		for (const generation of generations) {
-			if (generation === current) continue;
-			const started = this.getMeta(`building:${String(generation)}`);
-			if (!((started === void 0 ? Number.POSITIVE_INFINITY : Date.now() - Date.parse(started)) < ABANDONED_MS)) this.dropGeneration(generation);
-		}
-	}
-	putText(n, body) {
-		const key = sha(body);
-		this.stmt(`INSERT OR IGNORE INTO ${n.texts} (sha, body) VALUES (?, ?)`).run(key, body);
-		return key;
-	}
-	textOf(n, key) {
-		if (key === null) return void 0;
-		return this.stmt(`SELECT body FROM ${n.texts} WHERE sha = ?`).get(key)?.body;
-	}
-	putVersion(n, input, now) {
-		const full = input.full_text !== void 0 && input.full_text !== input.text ? this.putText(n, input.full_text) : null;
-		this.stmt(`INSERT INTO ${n.versions} (doc, capture, recipe, rendered, fetched_at, blob, kind, title, url,
-         meta_sha, text_sha, text_chars, full_sha, ai_input, merge, scope, labels)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-       ON CONFLICT(doc, capture) DO UPDATE SET recipe = excluded.recipe, rendered = excluded.rendered,
-         fetched_at = excluded.fetched_at, blob = excluded.blob, kind = excluded.kind, title = excluded.title,
-         url = excluded.url, meta_sha = excluded.meta_sha, text_sha = excluded.text_sha,
-         text_chars = excluded.text_chars, full_sha = excluded.full_sha, ai_input = excluded.ai_input,
-         merge = excluded.merge, scope = excluded.scope, labels = excluded.labels`).run(input.doc, input.capture ?? input.captures[0] ?? "", input.recipe ?? "", input.rendered === true ? 1 : 0, input.fetched_at ?? now, input.blob ?? null, input.kind, input.title, input.url, this.putText(n, JSON.stringify(input.meta ?? {})), this.putText(n, input.text), input.text.length, full, input.ai_input ?? "unset", input.merge ?? "richest", input.scope ?? null, input.labels === void 0 ? null : JSON.stringify(input.labels));
-	}
-	newest(n, doc, recipe, rendered) {
-		return this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = ?
-        ORDER BY fetched_at DESC, capture DESC LIMIT 1`).get(doc, recipe, rendered);
-	}
-	/** This method chooses the current version of a document from its versions, and writes it with its search entry. */
-	recompute(n, doc, now) {
-		const groups = this.stmt(`SELECT DISTINCT recipe, rendered FROM ${n.versions} WHERE doc = ?`).all(doc);
-		const before = this.stmt(`SELECT rowid, text_sha, title FROM ${n.docs} WHERE doc = ?`).get(doc);
-		if (groups.length === 0) {
-			if (before !== void 0) {
-				this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);
-				this.stmt(`DELETE FROM ${n.docs} WHERE doc = ?`).run(doc);
-			}
-			return;
-		}
-		const byRecipe = /* @__PURE__ */ new Map();
-		for (const group of groups) {
-			const row = this.newest(n, doc, group.recipe, group.rendered);
-			if (row === void 0) continue;
-			const entry = byRecipe.get(group.recipe) ?? {};
-			if (group.rendered === 1) entry.rendered = row;
-			else entry.plain = row;
-			byRecipe.set(group.recipe, entry);
-		}
-		const candidates = [];
-		for (const [recipe, { plain, rendered }] of byRecipe) {
-			let pick;
-			if (rendered === void 0) pick = plain;
-			else if (plain === void 0 || rendered.fetched_at >= plain.fetched_at) pick = rendered;
-			else {
-				const base = this.stmt(`SELECT blob FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = 0 AND fetched_at <= ?
-            ORDER BY fetched_at DESC, capture DESC LIMIT 1`).get(doc, recipe, rendered.fetched_at);
-				pick = base !== void 0 && base.blob !== null && base.blob === plain.blob ? rendered : plain;
-			}
-			const newest = plain === void 0 ? rendered : rendered === void 0 || plain.fetched_at > rendered.fetched_at ? plain : rendered;
-			const others = [plain, rendered].filter((row) => row !== void 0 && row !== pick);
-			candidates.push({
-				row: pick,
-				ai: newest.ai_input,
-				others
-			});
-		}
-		let winner = candidates[0];
-		if (candidates.length > 1) {
-			const latest = candidates.every((candidate) => candidate.row.merge === "latest");
-			for (const candidate of candidates.slice(1)) {
-				const a = candidate.row;
-				const b = winner.row;
-				if (latest ? a.fetched_at > b.fetched_at : a.text_chars > b.text_chars || a.text_chars === b.text_chars && a.fetched_at > b.fetched_at) winner = candidate;
-			}
-		}
-		const row = winner.row;
-		const aiInput = row.ai_input === "no" || candidates.some((candidate) => candidate.ai === "no") ? "no" : winner.ai !== "unset" ? winner.ai : candidates.some((candidate) => candidate.ai === "yes") ? "yes" : "unset";
-		let meta = parseJson(this.textOf(n, row.meta_sha), {});
-		for (const candidate of candidates) if (candidate !== winner) meta = {
-			...parseJson(this.textOf(n, candidate.row.meta_sha), {}),
-			...meta
-		};
-		const same = this.stmt(`SELECT capture FROM ${n.versions} WHERE doc = ? AND text_sha = ?
-        ORDER BY fetched_at DESC, capture DESC LIMIT ?`).all(doc, row.text_sha, 10);
-		const captures = [.../* @__PURE__ */ new Set([
-			row.capture,
-			...same.map((entry) => entry.capture),
-			...winner.others.map((other) => other.capture),
-			...candidates.filter((candidate) => candidate !== winner).map((candidate) => candidate.row.capture)
-		])].slice(0, 10);
-		const count = this.stmt(`SELECT count(*) AS n FROM ${n.versions} WHERE doc = ?`).get(doc).n;
-		const url = row.url ?? candidates.map((candidate) => candidate.row.url).find((value) => value !== null) ?? null;
-		const saved = this.stmt(`INSERT INTO ${n.docs} (doc, kind, title, url, meta, text_sha, full_sha, captures, capture_count, capture,
-         blob, fetched_at, recipe, rendered, ai_input, scope, labels, updated_at)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-       ON CONFLICT(doc) DO UPDATE SET kind = excluded.kind, title = excluded.title, url = excluded.url,
-         meta = excluded.meta, text_sha = excluded.text_sha, full_sha = excluded.full_sha,
-         captures = excluded.captures, capture_count = excluded.capture_count, capture = excluded.capture,
-         blob = excluded.blob, fetched_at = excluded.fetched_at, recipe = excluded.recipe,
-         rendered = excluded.rendered, ai_input = excluded.ai_input, scope = excluded.scope,
-         labels = excluded.labels, updated_at = excluded.updated_at
-       RETURNING rowid`).get(doc, row.kind, row.title, url, JSON.stringify(meta), row.text_sha, row.full_sha, JSON.stringify(captures), count, row.capture, row.blob, row.fetched_at, row.recipe, row.rendered, aiInput, row.scope, row.labels, now);
-		const text = this.textOf(n, row.text_sha) ?? "";
-		if (before === void 0 || before.text_sha !== row.text_sha || before.title !== row.title) {
-			if (before !== void 0) this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);
-			this.stmt(`INSERT INTO ${n.fts} (rowid, title, text) VALUES (?, ?, ?)`).run(saved.rowid, fold(row.title, "search"), fold(text.length > MAX_INDEX_CHARS ? text.slice(0, MAX_INDEX_CHARS) : text, "search"));
+			if (error instanceof RangeError) return false;
 		}
-		return this.docOf(n, doc, text);
-	}
-	docOf(n, doc, knownText) {
-		const row = this.stmt(`SELECT rowid, * FROM ${n.docs} WHERE doc = ?`).get(doc);
-		if (row === void 0) return void 0;
-		return this.toDoc(n, row, knownText);
-	}
-	toDoc(n, row, knownText) {
-		const labels = parseJson(row.labels, void 0);
-		return {
-			doc: row.doc,
-			kind: row.kind,
-			title: row.title,
-			text: knownText ?? this.textOf(n, row.text_sha) ?? "",
-			url: row.url,
-			meta: parseJson(row.meta, {}),
-			captures: parseJson(row.captures, []),
-			capture_count: row.capture_count,
-			capture: row.capture,
-			blob: row.blob,
-			fetched_at: row.fetched_at,
-			recipe: row.recipe,
-			rendered: row.rendered === 1,
-			ai_input: row.ai_input,
-			cut: row.full_sha !== null,
-			...row.scope !== null ? { scope: row.scope } : {},
-			...labels !== void 0 ? { labels } : {},
-			updated_at: row.updated_at
-		};
-	}
-	versionTextOf(n, row) {
-		return {
-			...toVersion(row),
-			text: this.textOf(n, row.text_sha) ?? "",
-			full_text: this.textOf(n, row.full_sha),
-			meta: parseJson(this.textOf(n, row.meta_sha), {})
-		};
-	}
-	get(doc) {
-		return this.live((n) => this.docOf(n, doc));
-	}
-	/**
-	* This method adds one version of a document, and it gives the current document. The rules of
-	* the class choose the current version. An upsert of a known capture replaces that version.
-	*/
-	upsert(input, now) {
-		return this.live((n) => this.write(() => {
-			this.putVersion(n, input, now);
-			return this.recompute(n, input.doc, now);
-		}));
+		if (total > limit) return false;
+		from = Math.max(from, end);
 	}
-	/** This method gives the versions of a document, the newest first. */
-	versions(doc, options = {}) {
-		const limit = Math.max(1, Math.floor(options.limit ?? 100));
-		return this.live((n) => this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? ORDER BY fetched_at DESC, capture DESC LIMIT ?`).all(doc, limit).map(toVersion));
-	}
-	/**
-	* This method gives the text of one version of a document. With no capture, it gives the current
-	* version. It also gives the text of a held-back version: the caller applies `isHeldBack` to the
-	* `ai_input` of the version.
-	*/
-	versionText(doc, capture) {
-		return this.live((n) => {
-			const id = capture ?? this.docOf(n, doc)?.capture;
-			if (id === void 0) return void 0;
-			const row = this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND capture = ?`).get(doc, id);
-			return row === void 0 ? void 0 : this.versionTextOf(n, row);
-		});
-	}
-	/**
-	* This method gives the texts of the earlier versions of a document, the newest first. Each
-	* text comes one time, with the newest version that has it. It skips the current text. It also
-	* gives held-back texts: the caller applies `isHeldBack` to the `ai_input` of each version.
-	*/
-	earlierTexts(doc, options = {}) {
-		const limit = Math.max(1, Math.floor(options.limit ?? 50));
-		return this.live((n) => {
-			const head = this.stmt(`SELECT text_sha FROM ${n.docs} WHERE doc = ?`).get(doc);
-			if (head === void 0) return [];
-			const rows = this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND text_sha != ? ORDER BY fetched_at DESC, capture DESC`).all(doc, head.text_sha);
-			const seen = /* @__PURE__ */ new Set();
-			const out = [];
-			for (const row of rows) {
-				if (seen.has(row.text_sha)) continue;
-				seen.add(row.text_sha);
-				out.push(this.versionTextOf(n, row));
-				if (out.length >= limit) break;
-			}
-			return out;
-		});
-	}
-	/** This method tells if a capture gave a version of a document. */
-	hasVersion(capture) {
-		return this.live((n) => this.stmt(`SELECT 1 AS found FROM ${n.versions} WHERE capture = ? LIMIT 1`).get(capture) !== void 0);
-	}
-	/** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
-	newestFetch() {
-		return this.live((n) => this.stmt(`SELECT max(fetched_at) AS newest FROM ${n.latest}`).get()?.newest ?? void 0);
-	}
-	/** This method gives the current documents that a capture gave. */
-	docsOfCapture(capture) {
-		return this.live((n) => this.stmt(`SELECT DISTINCT doc FROM ${n.versions} WHERE capture = ?`).all(capture).map((row) => this.docOf(n, row.doc)).filter((doc) => doc !== void 0));
-	}
-	/**
-	* This method searches the documents with BM25, the best first. The syntax of the query is the
-	* syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it
-	* adds the documents that match one part. A document that asked not to be AI input is not in
-	* the results, so a search cannot show its text. The option `rank` chooses the ranking (`RankMode`).
-	*/
-	find(q, options) {
-		const rank = rankModeOf(options.rank);
-		const query = parseQuery(q);
-		if (query === void 0) return [];
-		const limit = Math.max(1, Math.min(1e3, Math.floor(options.limit) || 10));
-		return this.live((n) => {
-			const kindClause = options.kind === void 0 || options.kind === null ? "" : "AND d.kind = ?";
-			const sql = `SELECT d.rowid AS rowid, d.*, bm25(${n.fts}, 3.0, 1.0) AS rank
-          FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid
-         WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' ${kindClause}
-         ORDER BY rank LIMIT ?`;
-			const run = (match) => {
-				const params = [match];
-				if (kindClause.length > 0) params.push(options.kind);
-				params.push(limit);
-				return this.stmt(sql).all(...params);
-			};
-			const rows = run(query.all);
-			if (rows.length < limit && query.parts > 1) {
-				const seen = new Set(rows.map((row) => row.doc));
-				for (const row of run(rank === "fast" ? this.rareParts(n, query.anyParts) : query.any)) {
-					if (rows.length >= limit) break;
-					if (!seen.has(row.doc)) rows.push(row);
-				}
-			}
-			return rows.map((row) => ({
-				doc: row.doc,
-				kind: row.kind,
-				title: row.title,
-				url: row.url,
-				snippet: snippetOf(this.textOf(n, row.text_sha) ?? "", query.terms),
-				score: Math.round(-row.rank * 1e3) / 1e3,
-				ai_input: row.ai_input,
-				fetched_at: row.fetched_at,
-				capture: row.capture
-			}));
-		});
-	}
-	/**
-	* This method scores a set of documents against a query, with the best first. It leaves out a
-	* document that matches no term, and a document that asked not to be AI input. It ranks the
-	* results of a source again, because the order of the source is only as good as its search engine.
-	* In the mode `fast`, it also leaves out a document that holds only common parts (`RankMode`).
-	*/
-	rankWithin(q, docs, options = {}) {
-		const rank = rankModeOf(options.rank);
-		const query = parseQuery(q);
-		if (query === void 0 || docs.length === 0) return [];
-		return this.live((n) => this.stmt(`SELECT d.doc AS doc, bm25(${n.fts}, 3.0, 1.0) AS rank
-             FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid
-            WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' AND d.doc IN (SELECT value FROM json_each(?))
-            ORDER BY rank`).all(rank === "fast" ? this.rareParts(n, query.anyParts) : query.any, JSON.stringify(docs)).map((row) => ({
-			doc: row.doc,
-			score: Math.round(-row.rank * 1e3) / 1e3
-		})));
-	}
-	/**
-	* This method gives the form of a query that needs one part, without the common parts (the mode
-	* `fast` of `RankMode`). It counts the documents that hold each part. When each part is common,
-	* it keeps the rarest part.
-	*/
-	rareParts(n, parts) {
-		if (parts.length < 2) return parts.join(" OR ");
-		const total = this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get().n;
-		const counted = parts.map((part) => ({
-			part,
-			count: this.stmt(`SELECT count(*) AS n FROM ${n.fts} WHERE ${n.fts} MATCH ?`).get(part).n
-		}));
-		const rare = counted.filter((one) => one.count <= FAST_RANK_SHARE * total);
-		if (rare.length > 0) return rare.map((one) => one.part).join(" OR ");
-		return counted.reduce((best, one) => one.count < best.count ? one : best).part;
-	}
-	/** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */
-	latest(url, variant) {
-		const key = keyOf(url, variant);
-		return this.live((n) => {
-			const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(key.url, key.variant);
-			return row === void 0 ? void 0 : toLatest(row);
-		});
-	}
-	putLatest(n, entry) {
-		const key = keyOf(entry.url, entry.variant);
-		this.stmt(`INSERT INTO ${n.latest} (url, variant, capture, blob, etag, last_modified, content_type, fetched_at,
-         status, ai_input, location, truncated, fresh_until, vary_key)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-       ON CONFLICT(url, variant) DO UPDATE SET capture = excluded.capture, blob = excluded.blob,
-         etag = excluded.etag, last_modified = excluded.last_modified, content_type = excluded.content_type,
-         fetched_at = excluded.fetched_at, status = excluded.status, ai_input = excluded.ai_input,
-         location = excluded.location, truncated = excluded.truncated, fresh_until = excluded.fresh_until,
-         vary_key = excluded.vary_key
-       WHERE excluded.fetched_at >= ${n.latest}.fetched_at`).run(key.url, key.variant, entry.capture, entry.blob, entry.etag, entry.last_modified, entry.content_type, entry.fetched_at, entry.status, entry.ai_input, entry.location ?? null, entry.truncated === true ? 1 : null, entry.fresh_until ?? null, entry.vary_key ?? null);
-	}
-	/** This method keeps the last answer for a URL. An entry that is older than the stored entry changes nothing. */
-	setLatest(entry) {
-		this.live((n) => this.write(() => this.putLatest(n, entry)));
-	}
-	counts() {
-		return this.live((n) => ({
-			docs: this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get().n,
-			urls: this.stmt(`SELECT count(*) AS n FROM ${n.latest}`).get().n
-		}));
-	}
-	/** This method empties the current generation. A rebuild uses `beginRebuild`, which keeps the index until its commit. */
-	reset() {
-		this.live((n) => this.write(() => {
-			for (const table of [
-				n.fts,
-				n.docs,
-				n.versions,
-				n.texts,
-				n.latest
-			]) this.db.exec(`DELETE FROM ${table}`);
-		}));
-	}
-	/**
-	* This method starts a rebuild into a new generation of the tables. The current generation
-	* answers each read and takes each write until the commit. A failed rebuild thus leaves the
-	* index as it was.
-	*/
-	beginRebuild(options = {}) {
-		const batchSize = Math.max(1, Math.floor(options.batch ?? 500));
-		const generation = this.write(() => {
-			const next = Number(this.getMeta("next_generation") ?? "2");
-			this.setMeta("next_generation", String(next + 1));
-			this.setMeta(`building:${String(next)}`, (/* @__PURE__ */ new Date()).toISOString());
-			this.db.exec(ddlOf(namesOf(next)));
-			return next;
-		});
-		const n = namesOf(generation);
-		const versions = [];
-		const latest = /* @__PURE__ */ new Map();
-		let open = true;
-		const keyText = (url, variant) => {
-			const key = keyOf(url, variant);
-			return `${key.variant} ${key.url}`;
-		};
-		const flush = () => {
-			if (!open) throw new Error("the rebuild is over");
-			if (versions.length === 0 && latest.size === 0) return;
-			this.write(() => {
-				const touched = /* @__PURE__ */ new Map();
-				for (const { input, now } of versions) {
-					this.putVersion(n, input, now);
-					touched.set(input.doc, now);
-				}
-				for (const entry of latest.values()) this.putLatest(n, entry);
-				for (const [doc, now] of touched) this.recompute(n, doc, now);
-			});
-			versions.length = 0;
-			latest.clear();
-		};
-		/** This function copies the texts of the versions of the current generation that agree with `where`. */
-		const copyTexts = (current, where, params) => {
-			for (const column of [
-				"text_sha",
-				"meta_sha",
-				"full_sha"
-			]) this.db.prepare(`INSERT OR IGNORE INTO ${n.texts} (sha, body)
-             SELECT t.sha, t.body FROM ${current.texts} t
-              WHERE t.sha IN (SELECT ${column} FROM ${current.versions} WHERE ${where})`).run(...params);
-		};
-		return {
-			upsert: (input, now) => {
-				if (!open) throw new Error("the rebuild is over");
-				versions.push({
-					input,
-					now
-				});
-				if (versions.length >= batchSize) flush();
-			},
-			setLatest: (entry) => {
-				if (!open) throw new Error("the rebuild is over");
-				const key = keyText(entry.url, entry.variant);
-				const before = latest.get(key);
-				if (before === void 0 || entry.fetched_at >= before.fetched_at) latest.set(key, entry);
-				if (latest.size >= batchSize) flush();
-			},
-			latest: (url, variant) => {
-				const waiting = latest.get(keyText(url, variant));
-				if (waiting !== void 0) return waiting;
-				const key = keyOf(url, variant);
-				const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(key.url, key.variant);
-				return row === void 0 ? void 0 : toLatest(row);
-			},
-			flush,
-			carry: (recipes) => {
-				flush();
-				const known = JSON.stringify([...recipes]);
-				const current = namesOf(this.generation());
-				const docs = this.db.prepare(`SELECT DISTINCT doc FROM ${current.versions} WHERE recipe NOT IN (SELECT value FROM json_each(?))`).all(known);
-				const where = "recipe NOT IN (SELECT value FROM json_each(?)) AND doc IN (SELECT value FROM json_each(?))";
-				for (let at = 0; at < docs.length; at += batchSize) {
-					const part = JSON.stringify(docs.slice(at, at + batchSize).map((row) => row.doc));
-					this.write(() => {
-						copyTexts(current, where, [known, part]);
-						this.db.prepare(`INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${current.versions} WHERE ${where}`).run(known, part);
-						const now = (/* @__PURE__ */ new Date()).toISOString();
-						for (const doc of JSON.parse(part)) this.recompute(n, doc, now);
-					});
-				}
-				return docs.length;
-			},
-			commit: (since) => {
-				flush();
-				open = false;
-				return this.write(() => {
-					const current = this.generation();
-					const old = namesOf(current);
-					copyTexts(old, "fetched_at >= ?", [since]);
-					const added = this.db.prepare(`SELECT DISTINCT doc FROM ${old.versions} WHERE fetched_at >= ?`).all(since);
-					this.db.prepare(`INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${old.versions} WHERE fetched_at >= ?`).run(since);
-					const now = (/* @__PURE__ */ new Date()).toISOString();
-					for (const { doc } of added) this.recompute(n, doc, now);
-					for (const row of this.db.prepare(`SELECT * FROM ${old.latest} WHERE fetched_at >= ?`).all(since)) this.putLatest(n, toLatest(row));
-					this.setMeta("generation", String(generation));
-					this.db.prepare("DELETE FROM index_meta WHERE key = ?").run(`building:${String(generation)}`);
-					this.dropGeneration(current);
-					this.empty = false;
-					return {
-						docs: this.db.prepare(`SELECT count(*) AS n FROM ${n.docs}`).get().n,
-						urls: this.db.prepare(`SELECT count(*) AS n FROM ${n.latest}`).get().n
-					};
-				});
-			},
-			abort: () => {
-				open = false;
-				versions.length = 0;
-				latest.clear();
-				if (this.closed) return;
-				this.write(() => this.dropGeneration(generation));
-			}
-		};
-	}
-	/** This method closes the file. A second close does nothing. */
-	close() {
-		if (this.closed) return;
-		this.closed = true;
-		this.statements.clear();
-		this.db.close();
-	}
-};
-//#endregion
-//#region src/hoststate.ts
-/** A stop with no end goes into the table as this value, which is past each date that JavaScript can hold. */
-const NO_END = 9e15;
-/** A process that shares the store asks again after this wait, at most, during the lease of a different process. */
-const SHARED_WAIT_MS = 1e3;
-/** The store removes the rows of idle hosts after this many reservations. */
-const SWEEP_EVERY = 1e3;
-const verdictOf = (value) => value === "works" || value === "fails" ? value : "unknown";
-const memoOf = (row) => ({
-	skipEtag: row.skip_etag === 1,
-	etag: verdictOf(row.etag),
-	lastModified: verdictOf(row.last_modified),
-	etagMisses: row.etag_misses,
-	dateMisses: row.date_misses,
-	sinceProbe: row.since_probe,
-	changedAt: row.changed_at
-});
-/**
-* This store keeps the state of each host in a SQLite file. Two or more processes on one store
-* share it, so they share the pace and the stops of each host. Thus a host gets one request at a
-* time from all of them. Each method is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock
-* of a different process (`busy_timeout`).
-*
-* The lease of a request belongs to the store that gave it. A different process that asks during
-* the lease gets a wait of one second or less, and asks again. A process that stops during a
-* request holds its lease until the lease ends (5 minutes by default).
-*
-* The table `validators` keeps the validator memo of each host (`ValidatorMemo`), so the processes
-* share it and it stays after a restart. A file of version 0.1.0 gets the table when it opens.
-*
-* The state is not a projection of the capture log. A deletion of the file loses only the pace,
-* the counts, the stops and the memos, and the crawler starts them again.
-*/
-var SqliteHostStateStore = class {
-	path;
-	db;
-	statements = /* @__PURE__ */ new Map();
-	/** This id names the leases of this store. */
-	owner = randomUUID();
-	reservations = 0;
-	closed = false;
-	pollMs;
-	constructor(path, options = {}) {
-		this.path = path;
-		if (path !== ":memory:") mkdirSync(dirname(path), {
-			recursive: true,
-			mode: 448
-		});
-		this.pollMs = Math.max(1, Math.min(SHARED_WAIT_MS, options.pollMs ?? SHARED_WAIT_MS));
-		this.db = new DatabaseSync(path);
-		const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
-		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
-		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
-		this.db.exec("PRAGMA synchronous = NORMAL;");
-		this.migrate();
-	}
-	/**
-	* This method makes the tables, or adds the tables of a later version to an older file. It is one
-	* transaction, so two processes that open one old file at the same time do not collide.
-	*/
-	migrate() {
-		this.step(() => {
-			this.db.exec(`CREATE TABLE IF NOT EXISTS hosts (
-        host TEXT PRIMARY KEY,
-        next_at REAL NOT NULL DEFAULT 0,
-        leased_until REAL NOT NULL DEFAULT 0,
-        owner TEXT,
-        failures INTEGER NOT NULL DEFAULT 0,
-        waits INTEGER NOT NULL DEFAULT 0,
-        stop_until REAL,
-        stop_reason TEXT
-      )`);
-			this.db.exec(`CREATE TABLE IF NOT EXISTS validators (
-        host TEXT PRIMARY KEY,
-        skip_etag INTEGER NOT NULL DEFAULT 0,
-        etag TEXT NOT NULL DEFAULT 'unknown',
-        last_modified TEXT NOT NULL DEFAULT 'unknown',
-        etag_misses INTEGER NOT NULL DEFAULT 0,
-        date_misses INTEGER NOT NULL DEFAULT 0,
-        since_probe INTEGER NOT NULL DEFAULT 0,
-        changed_at REAL NOT NULL DEFAULT 0
-      )`);
-			const { user_version: version } = this.db.prepare("PRAGMA user_version").get();
-			if (version < 1) this.db.exec(`PRAGMA user_version = ${String(1)}`);
-		});
-	}
-	stmt(sql) {
-		let statement = this.statements.get(sql);
-		if (statement === void 0) {
-			statement = this.db.prepare(sql);
-			this.statements.set(sql, statement);
-		}
-		return statement;
-	}
-	/** This method does one step in one transaction. */
-	step(work) {
-		if (this.closed) throw new Error("the host state store is closed");
-		this.db.exec("BEGIN IMMEDIATE");
-		try {
-			const result = work();
-			this.db.exec("COMMIT");
-			return result;
-		} catch (error) {
-			if (this.db.isTransaction) this.db.exec("ROLLBACK");
-			throw error;
-		}
-	}
-	row(host) {
-		this.stmt("INSERT OR IGNORE INTO hosts (host) VALUES (?)").run(host);
-		return this.stmt("SELECT next_at, leased_until, owner, failures, waits, stop_until, stop_reason FROM hosts WHERE host = ?").get(host);
-	}
-	/** This method removes the hosts that have nothing to remember: no rest, no lease, no counts and no stop. */
-	sweep(now) {
-		this.stmt(`DELETE FROM hosts WHERE next_at <= ? AND leased_until <= ? AND failures = 0 AND waits = 0
-         AND (stop_until IS NULL OR stop_until <= ?)`).run(now, now, now);
-	}
-	reserve(host, now, leaseMs) {
-		return this.step(() => {
-			this.reservations += 1;
-			if (this.reservations % SWEEP_EVERY === 0) this.sweep(now);
-			const row = this.row(host);
-			if (row.leased_until > now) return row.owner === this.owner ? row.leased_until - now : Math.min(this.pollMs, row.leased_until - now);
-			if (row.next_at > now) return row.next_at - now;
-			this.stmt("UPDATE hosts SET leased_until = ?, owner = ? WHERE host = ?").run(now + leaseMs, this.owner, host);
-			return 0;
-		});
-	}
-	record(host, result) {
-		return this.step(() => {
-			const row = this.row(host);
-			let { failures, waits } = row;
-			if (result === "success") {
-				failures = 0;
-				waits = 0;
-			} else if (result === "failure") failures += 1;
-			else if (result === "wait") waits += 1;
-			if (failures !== row.failures || waits !== row.waits) this.stmt("UPDATE hosts SET failures = ?, waits = ? WHERE host = ?").run(failures, waits, host);
-			return {
-				failures,
-				waits
-			};
-		});
-	}
-	release(host, nextAt) {
-		this.step(() => {
-			this.row(host);
-			this.stmt(`UPDATE hosts SET next_at = max(next_at, ?),
-           leased_until = CASE WHEN owner = ? THEN 0 ELSE leased_until END,
-           owner = CASE WHEN owner = ? THEN NULL ELSE owner END
-         WHERE host = ?`).run(nextAt, this.owner, this.owner, host);
-		});
-	}
-	defer(host, until) {
-		this.step(() => {
-			this.row(host);
-			this.stmt("UPDATE hosts SET next_at = max(next_at, ?) WHERE host = ?").run(until, host);
-		});
-	}
-	stop(host, until, reason) {
-		const end = Number.isFinite(until) ? until : NO_END;
-		this.step(() => {
-			this.row(host);
-			this.stmt(`UPDATE hosts SET stop_until = ?, stop_reason = ?
-         WHERE host = ? AND (stop_until IS NULL OR stop_until < ?)`).run(end, reason, host, end);
-		});
-	}
-	stopped(host, now) {
-		if (this.closed) throw new Error("the host state store is closed");
-		const row = this.stmt("SELECT stop_until, stop_reason FROM hosts WHERE host = ?").get(host);
-		if (row === void 0 || row.stop_until === null || row.stop_until <= now) return void 0;
-		return {
-			until: row.stop_until >= NO_END ? Number.POSITIVE_INFINITY : row.stop_until,
-			reason: row.stop_reason ?? "stopped"
-		};
-	}
-	reset(host) {
-		this.step(() => {
-			if (host === void 0) this.stmt("UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL").run();
-			else this.stmt("UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL WHERE host = ?").run(host);
-		});
-	}
-	validators(host) {
-		if (this.closed) throw new Error("the host state store is closed");
-		const row = this.stmt(`SELECT skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at
-         FROM validators WHERE host = ?`).get(host);
-		return row === void 0 ? void 0 : memoOf(row);
-	}
-	noteValidators(host, report, now) {
-		return this.step(() => {
-			const memo = nextValidatorMemo(this.validators(host), report, now);
-			this.stmt(`INSERT INTO validators (host, skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at)
-           VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-         ON CONFLICT(host) DO UPDATE SET skip_etag = excluded.skip_etag, etag = excluded.etag,
-           last_modified = excluded.last_modified, etag_misses = excluded.etag_misses,
-           date_misses = excluded.date_misses, since_probe = excluded.since_probe,
-           changed_at = excluded.changed_at`).run(host, memo.skipEtag ? 1 : 0, memo.etag, memo.lastModified, memo.etagMisses, memo.dateMisses, memo.sinceProbe, memo.changedAt);
-			return memo;
-		});
-	}
-	/** This value is the number of hosts that the store holds. */
-	get size() {
-		return this.stmt("SELECT count(*) AS n FROM hosts").get().n;
-	}
-	/** This method closes the file. A second close does nothing. */
-	close() {
-		if (this.closed) return;
-		this.closed = true;
-		this.statements.clear();
-		this.db.close();
-	}
-};
-//#endregion
-//#region src/log.ts
-const MONTH_FILE = /^(\d{2})(?:\.[^.]+)?\.ndjson$/;
-const YEAR_DIR = /^\d{4}$/;
-const NEWLINE = 10;
-/** This function writes the whole buffer. A write to a file can write less than the buffer, and the loop writes the rest. */
-const writeAll = async (handle, bytes) => {
-	let at = 0;
-	while (at < bytes.length) {
-		const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);
-		at += bytesWritten;
-	}
-};
-/**
-* This function reads a file line by line, as bytes, so a long file does not become one string. It
-* gives each line and tells if a line break ends it.
-*/
-async function* linesOf(path) {
-	const decoder = new TextDecoder("utf-8");
-	let rest;
-	for await (const chunk of createReadStream(path)) {
-		const data = rest === void 0 ? chunk : Buffer.concat([rest, chunk]);
-		rest = void 0;
-		let start = 0;
-		for (;;) {
-			const end = data.indexOf(NEWLINE, start);
-			if (end === -1) break;
-			yield {
-				text: decoder.decode(data.subarray(start, end)),
-				complete: true
-			};
-			start = end + 1;
-		}
-		if (start < data.length) rest = Buffer.from(data.subarray(start));
-	}
-	if (rest !== void 0 && rest.length > 0) yield {
-		text: decoder.decode(rest),
-		complete: false
-	};
-}
-const monthKey = (year, file) => `${year}-${MONTH_FILE.exec(file)?.[1] ?? "00"}`;
-/**
-* The capture log keeps each fetch decision as one line of JSON in
-* `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.
-*
-* These are the rules of the log:
-*
-* - The log appends a batch of records with one write, and it writes the batch to the disk before
-*   the appends finish.
-* - An append always starts on a new line. After a crash leaves a partial line, the next append
-*   starts after a line break. The partial line stays as a damaged line, and the record is good.
-* - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.
-* - The log does not remove or change a line.
-*
-* Two processes can append to one log, because each batch is one write to a file in append mode.
-*/
-var CaptureLog = class CaptureLog {
-	dir;
-	handles = /* @__PURE__ */ new Map();
-	pending = [];
-	draining;
-	sync;
-	constructor(dir, options = {}) {
-		this.dir = dir;
-		this.sync = options.sync ?? true;
-	}
-	pathOf(record) {
-		return join(this.dir, record.source, record.received_at.slice(0, 4), `${record.received_at.slice(5, 7)}.ndjson`);
-	}
-	async handleFor(path) {
-		const known = this.handles.get(path);
-		if (known !== void 0) return known;
-		await mkdir(dirname(path), {
-			recursive: true,
-			mode: 448
-		});
-		const handle = await open(path, "a+", 384);
-		this.handles.set(path, handle);
-		return handle;
-	}
-	/** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */
-	static async needsBreak(handle) {
-		const { size } = await handle.stat();
-		if (size === 0) return false;
-		const last = Buffer.alloc(1);
-		await handle.read(last, 0, 1, size - 1);
-		return last[0] !== NEWLINE;
-	}
-	append(record) {
-		let line;
-		try {
-			line = `${JSON.stringify(record)}\n`;
-		} catch (error) {
-			return Promise.reject(error instanceof Error ? error : new Error(String(error)));
-		}
-		return new Promise((resolve, reject) => {
-			this.pending.push({
-				path: this.pathOf(record),
-				line,
-				resolve,
-				reject
-			});
-			this.kick();
-		});
-	}
-	/** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */
-	kick() {
-		if (this.draining !== void 0) return;
-		this.draining = this.drain().finally(() => {
-			this.draining = void 0;
-			if (this.pending.length > 0) this.kick();
-		});
-	}
-	/** This method writes the waiting records in batches: one write and one sync for each file. */
-	async drain() {
-		while (this.pending.length > 0) {
-			const batch = this.pending.splice(0);
-			const byPath = /* @__PURE__ */ new Map();
-			for (const item of batch) {
-				const list = byPath.get(item.path);
-				if (list === void 0) byPath.set(item.path, [item]);
-				else list.push(item);
-			}
-			for (const [path, items] of byPath) try {
-				const handle = await this.handleFor(path);
-				const lead = await CaptureLog.needsBreak(handle) ? "\n" : "";
-				await writeAll(handle, Buffer.from(`${lead}${items.map((item) => item.line).join("")}`, "utf8"));
-				if (this.sync) await handle.datasync();
-				for (const item of items) item.resolve();
-			} catch (error) {
-				const handle = this.handles.get(path);
-				this.handles.delete(path);
-				await handle?.close().catch(() => void 0);
-				for (const item of items) item.reject(error);
-			}
-		}
-	}
-	/** This method waits until each append in progress is on the disk. */
-	async flush() {
-		while (this.draining !== void 0) await this.draining;
-	}
-	/** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */
-	async filesOf(source, since) {
-		const sourceDir = join(this.dir, source);
-		if (!existsSync(sourceDir)) return [];
-		const from = since?.slice(0, 7);
-		const years = (await readdir(sourceDir, { withFileTypes: true })).filter((entry) => entry.isDirectory() && YEAR_DIR.test(entry.name)).map((entry) => entry.name).sort();
-		const files = [];
-		for (const year of years) {
-			if (from !== void 0 && year < from.slice(0, 4)) continue;
-			const months = (await readdir(join(sourceDir, year))).filter((name) => MONTH_FILE.test(name)).sort();
-			for (const month of months) {
-				if (from !== void 0 && monthKey(year, month) < from) continue;
-				files.push(join(sourceDir, year, month));
-			}
-		}
-		return files;
-	}
-	/** This method reads the records of one source in the sequence of the files. */
-	async *readSource(source, filter, options) {
-		for (const path of await this.filesOf(source, filter?.since)) {
-			let number = 0;
-			for await (const { text, complete } of linesOf(path)) {
-				number += 1;
-				const trimmed = text.trim();
-				if (trimmed.length === 0) continue;
-				let row;
-				try {
-					row = JSON.parse(trimmed);
-					if (typeof row !== "object" || row === null || typeof row.received_at !== "string") throw new Error("not a record");
-				} catch {
-					if (complete) options?.onDamaged?.({
-						path,
-						line: number
-					});
-					continue;
-				}
-				if (filter?.kind !== void 0 && row.kind !== filter.kind) continue;
-				if (filter?.since !== void 0 && row.received_at < filter.since) continue;
-				yield row;
-			}
-		}
-	}
-	/**
-	* This method reads the records that agree with the filter. It first waits for the appends that
-	* are in progress. It reads each file as a stream, so its memory does not grow with the log.
-	*
-	* The records of one source come in the sequence of the files. The records of two or more sources
-	* merge in the sequence of `received_at`.
-	*/
-	async *scan(filter, options) {
-		await this.flush();
-		const sources = filter?.source !== void 0 ? [filter.source] : existsSync(this.dir) ? (await readdir(this.dir, { withFileTypes: true })).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort() : [];
-		if (sources.length === 1) {
-			yield* this.readSource(sources[0], filter, options);
-			return;
-		}
-		const streams = sources.map((source) => this.readSource(source, filter, options)[Symbol.asyncIterator]());
-		const heads = [];
-		for (const stream of streams) heads.push((await stream.next()).value);
-		for (;;) {
-			let pick = -1;
-			for (const [index, head] of heads.entries()) if (head !== void 0 && (pick === -1 || head.received_at < heads[pick].received_at)) pick = index;
-			if (pick === -1) return;
-			yield heads[pick];
-			heads[pick] = (await streams[pick].next()).value;
-		}
-	}
-	/** This method writes the appends in progress, then closes the files. A later append opens them again. */
-	async close() {
-		await this.flush();
-		const handles = [...this.handles.values()];
-		this.handles.clear();
-		for (const handle of handles) await handle.close();
-	}
-};
-//#endregion
-//#region src/pdf.ts
-const MAX_PAGES = 500;
-const MAX_CHARS = 2e5;
-const TIME_BUDGET_MS = 2e4;
-/**
-* The text streams of 35 stored menus and papers inflate to 0.6 MB or less. The parser of pdf.js
-* reads a page with no pause. Thus a page of 8 MB of text operators stops the process for
-* approximately one second.
-*/
-const MAX_INFLATED_BYTES = 8388608;
-/** This function finds a PDF from its first bytes or its content type. Some servers put bytes before `%PDF-`. */
-const isPdf = (body, contentType) => {
-	if (/\bpdf\b/i.test(contentType ?? "")) return true;
-	const end = Math.min(body.length - 4, 1024);
-	for (let index = 0; index < end; index += 1) if (body[index] === 37 && body[index + 1] === 80 && body[index + 2] === 68 && body[index + 3] === 70 && body[index + 4] === 45) return true;
-	return false;
-};
-/** This function gives one line of text for each line of the PDF. It removes the empty lines. */
-const tidy = (page) => page.split(/\r?\n/).map((line) => line.replace(/\s+/g, " ").trim()).filter((line) => line.length > 0).join("\n");
-/** A word processor writes the name of its file as the title ("Microsoft Word - menu-final-v3.docx"). */
-const JUNK_TITLE = /^(?:microsoft (?:word|powerpoint|excel) - |untitled\b|document\d*$|presentation\d*$)|\.(?:docx?|pdf|pptx?|xlsx?|indd|ai|psd|rtf|odt|pages|txt)$/i;
-const latin1 = (bytes, start, end) => {
-	let out = "";
-	for (let index = Math.max(0, start); index < end && index < bytes.length; index += 1) out += String.fromCharCode(bytes[index]);
-	return out;
-};
-const indexOf = (bytes, needle, from) => {
-	const first = needle.charCodeAt(0);
-	for (let index = bytes.indexOf(first, from); index !== -1; index = bytes.indexOf(first, index + 1)) {
-		let match = true;
-		for (let at = 1; at < needle.length; at += 1) if (bytes[index + at] !== needle.charCodeAt(at)) {
-			match = false;
-			break;
-		}
-		if (match) return index;
-	}
-	return -1;
-};
-/**
-* This function adds up the bytes that the compressed streams of a file inflate to, and stops at
-* the limit. It tries each stream: a stream in another format fails at its first bytes. An image
-* stream does not count, because the text reader does not decode it. The function gives true when
-* the file stays inside the limit.
-*/
-const inflatesWithin = (bytes, limit) => {
-	let total = 0;
-	let from = 0;
-	for (;;) {
-		const keyword = indexOf(bytes, "stream", from);
-		if (keyword === -1) return true;
-		from = keyword + 6;
-		if (latin1(bytes, keyword - 3, keyword) === "end") continue;
-		const next = bytes[keyword + 6];
-		if (next !== 10 && next !== 13) continue;
-		const dictionary = latin1(bytes, keyword - 1024, keyword);
-		const head = dictionary.slice(Math.max(0, dictionary.lastIndexOf("<<")));
-		if (/\/Subtype\s*\/Image\b/.test(head)) continue;
-		let start = keyword + 7;
-		if (next === 13 && bytes[start] === 10) start += 1;
-		const declared = /\/Length\s+(\d+)(?!\s+\d+\s+R)/.exec(head);
-		const stop = indexOf(bytes, "endstream", start);
-		const end = declared !== null && start + Number(declared[1]) <= bytes.length ? start + Number(declared[1]) : stop === -1 ? bytes.length : stop;
-		try {
-			total += inflateSync(bytes.subarray(start, end), {
-				maxOutputLength: Math.max(1, limit - total + 1),
-				finishFlush: constants.Z_SYNC_FLUSH
-			}).length;
-		} catch (error) {
-			if (error instanceof RangeError) return false;
-		}
-		if (total > limit) return false;
-		from = Math.max(from, end);
-	}
-};
-/** This function lets the other work of the process run, for example another request. */
-const pause = () => new Promise((resolve) => setImmediate(resolve));
-/** This function gives the value of a promise, or undefined when the time ends first. */
-const within = async (promise, ms) => {
-	let timer;
-	const late = new Promise((resolve) => {
-		timer = setTimeout(() => resolve(void 0), Math.max(0, ms));
-	});
-	try {
-		return await Promise.race([promise, late]);
-	} finally {
-		clearTimeout(timer);
+};
+/** This function lets the other work of the process run, for example another request. */
+const pause = () => new Promise((resolve) => setImmediate(resolve));
+/** This function gives the value of a promise, or undefined when the time ends first. */
+const within = async (promise, ms) => {
+	let timer;
+	const late = new Promise((resolve) => {
+		timer = setTimeout(() => resolve(void 0), Math.max(0, ms));
+	});
+	try {
+		return await Promise.race([promise, late]);
+	} finally {
+		clearTimeout(timer);
 	}
 };
 /** This function keeps the first characters of a text, and does not cut a surrogate pair. */
@@ -3926,7 +2838,7 @@ const readPdf = async (bytes, options = {}) => {
 	const maxPages = options.maxPages ?? MAX_PAGES;
 	const maxChars = options.maxChars ?? MAX_CHARS;
 	const budgetMs = options.timeBudgetMs ?? TIME_BUDGET_MS;
-	const maxInflated = options.maxInflatedBytes ?? MAX_INFLATED_BYTES;
+	const maxInflated = options.maxInflatedBytes ?? MAX_INFLATED_BYTES$1;
 	if (!inflatesWithin(bytes, maxInflated)) return {
 		title: null,
 		text: "",
@@ -3987,6 +2899,511 @@ const readPdf = async (bytes, options = {}) => {
 	}
 };
 //#endregion
+//#region src/fallback.ts
+/** The recipe of the request for an archived copy. Its extractor reads a Wayback Machine copy or a Common Crawl WARC record. */
+const ARCHIVE_COPY_RECIPE = "archive-copy";
+/** The recipe of the requests to the indexes of the archives. They give no document. */
+const ARCHIVE_INDEX_RECIPE = "archive-index";
+/** The kind of the document of an archived copy. */
+const ARCHIVED_KIND = "archived";
+/** This function gives the document id of the archived copy of a page. */
+const archivedDocId = (url) => `archive:${canonicalUrl(url)}`;
+/** The addresses of the archives. */
+const WAYBACK_AVAILABLE = "https://archive.org/wayback/available";
+const WAYBACK = "https://web.archive.org/web/";
+const CC_INDEX = "https://index.commoncrawl.org/";
+const CC_DATA = "https://data.commoncrawl.org/";
+/** The Common Crawl indexes that the fallback asks, the newest first. */
+const CC_INDEXES = 3;
+/** The largest compressed WARC record that the fallback asks for. */
+const MAX_RECORD_BYTES = 5242880;
+/** The largest decompressed WARC record or body that the fallback reads. */
+const MAX_INFLATED_BYTES = 20971520;
+/** The Wayback Machine path of a raw copy: the 14 digits of the time, `id_`, and the URL of the page. */
+const WAYBACK_RAW = /^\/web\/(\d{14})id_\/(https?:\/\/.+)$/s;
+const HTML_ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
+const JSON_ACCEPT = "application/json";
+const ROBOTS_DISALLOW = "robots.txt disallows it";
+const ROBOTS_UNREACHABLE = "robots.txt could not be read";
+const HOST_STOPPED = "host stopped";
+/**
+* This function tells if the result of a live fetch can get an archived copy, and why. These
+* refusals can: robots.txt, a stop of the host, a 401, a 402, a 403 and a challenge. A 502, 503 or
+* 504, a 5xx error and a dead host can too. A 404, a 429, a cancel and a refusal of the egress
+* policy cannot.
+*/
+const fallbackReason = (indexed) => {
+	const { fetched } = indexed;
+	if (indexed.blocked !== void 0) return "challenge";
+	const note = fetched.note ?? "";
+	switch (fetched.outcome) {
+		case "skipped":
+			if (note.startsWith(ROBOTS_DISALLOW)) return "robots";
+			if (note.startsWith(ROBOTS_UNREACHABLE)) return "unreachable";
+			if (note.startsWith(HOST_STOPPED)) return "stopped";
+			return;
+		case "blocked":
+		case "challenge":
+		case "payment-required":
+		case "unavailable": return fetched.outcome;
+		case "error":
+			if (fetched.status === 0 && note.startsWith("no response")) return "unreachable";
+			return fetched.status >= 500 ? "error" : void 0;
+		default: return;
+	}
+};
+/** This function gives the `archived` of the metadata of a document, when the document is an archived copy. */
+const archivedOf = (meta) => {
+	const value = meta["archived"];
+	if (typeof value !== "object" || value === null) return void 0;
+	const { source, timestamp, url, original, record } = value;
+	if (source !== "wayback" && source !== "commoncrawl" || typeof timestamp !== "string" || typeof url !== "string" || typeof original !== "string") return void 0;
+	return {
+		source,
+		timestamp,
+		url,
+		original,
+		...typeof record === "string" ? { record } : {}
+	};
+};
+const ARCHIVE_NAMES = {
+	wayback: "the Wayback Machine",
+	commoncrawl: "Common Crawl"
+};
+/** This function gives the note of an archived copy: the archive and the date, and that it is not a fresh fetch. */
+const archivedNote = (archived) => `an archived copy from ${ARCHIVE_NAMES[archived.source]}, captured ${archived.timestamp}, not a fresh fetch of the page`;
+/** This function gives an ISO 8601 time with no milliseconds, or undefined for a time that is not valid. */
+const isoSeconds = (ms) => Number.isFinite(ms) ? new Date(ms).toISOString().replace(/\.\d{3}Z$/, "Z") : void 0;
+/** This function reads the 14 digits of a Wayback Machine time (`20240131235959`) as an ISO 8601 time. */
+const waybackTime = (digits) => {
+	const match = /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$/.exec(digits);
+	if (match === null) return void 0;
+	const [, year, month, day, hour, minute, second] = match;
+	const ms = Date.UTC(Number(year), Number(month) - 1, Number(day), Number(hour), Number(minute), Number(second));
+	const iso = isoSeconds(ms);
+	return iso !== void 0 && iso.replace(/\D/g, "").startsWith(digits) ? iso : void 0;
+};
+const httpUrlOf = (text) => {
+	try {
+		const url = new URL(text);
+		return url.protocol === "http:" || url.protocol === "https:" ? url.href : void 0;
+	} catch {
+		return;
+	}
+};
+const CRLF2 = Buffer.from("\r\n\r\n");
+/** This function reads the header lines of a block as a map. A name can be in any case, and two lines of one name join with ", ". */
+const headerMap = (text) => {
+	const out = /* @__PURE__ */ new Map();
+	for (const line of text.split(/\r?\n/)) {
+		const colon = line.indexOf(":");
+		if (colon <= 0) continue;
+		const name = line.slice(0, colon).trim().toLowerCase();
+		const value = line.slice(colon + 1).trim();
+		const known = out.get(name);
+		out.set(name, known === void 0 ? value : `${known}, ${value}`);
+	}
+	return out;
+};
+/** This function joins the chunks of a body with `Transfer-Encoding: chunked`. A body that is not in that form gives undefined. */
+const dechunk = (body) => {
+	const parts = [];
+	let at = 0;
+	let total = 0;
+	for (;;) {
+		const end = body.indexOf("\r\n", at);
+		if (end === -1) return void 0;
+		const size = Number.parseInt(body.subarray(at, end).toString("latin1").split(";")[0].trim(), 16);
+		if (!Number.isFinite(size) || size < 0) return void 0;
+		if (size === 0) return Buffer.concat(parts);
+		const start = end + 2;
+		if (start + size > body.length) return void 0;
+		total += size;
+		if (total > MAX_INFLATED_BYTES) return void 0;
+		parts.push(body.subarray(start, start + size));
+		at = start + size + 2;
+	}
+};
+/** This function decodes the content coding of an archived body, with a limit on the output. */
+const decodeCoding = (body, coding) => {
+	const options = { maxOutputLength: MAX_INFLATED_BYTES };
+	let out = body;
+	for (const name of (coding ?? "").split(",").map((part) => part.trim().toLowerCase()).filter((part) => part.length > 0 && part !== "identity").reverse()) if (name === "gzip" || name === "x-gzip") out = gunzipSync(out, options);
+	else if (name === "br") out = brotliDecompressSync(out, options);
+	else if (name === "deflate") try {
+		out = inflateSync(out, options);
+	} catch {
+		out = inflateRawSync(out, options);
+	}
+	else return void 0;
+	return out;
+};
+/**
+* This function reads one WARC record of the type `response`, as Common Crawl gives it. The
+* record is a gzip member with the WARC headers, the HTTP headers and the body. It gives undefined for a different
+* record, or for bytes that do not decode. Common Crawl keeps the decoded body and renames
+* `Content-Encoding` to `X-Crawler-Content-Encoding`. A record with the original codings is decoded too.
+*/
+const readWarcResponse = (bytes) => {
+	if (bytes.length < 2 || bytes[0] !== 31 || bytes[1] !== 139) return void 0;
+	let record;
+	try {
+		record = gunzipSync(bytes, { maxOutputLength: MAX_INFLATED_BYTES });
+	} catch {
+		return;
+	}
+	if (!record.subarray(0, 8).toString("latin1").startsWith("WARC/1.")) return void 0;
+	const warcEnd = record.indexOf(CRLF2);
+	if (warcEnd === -1) return void 0;
+	const warc = headerMap(record.subarray(0, warcEnd).toString("utf8"));
+	if (warc.get("warc-type")?.toLowerCase() !== "response") return void 0;
+	const target = httpUrlOf(warc.get("warc-target-uri")?.replace(/^<|>$/g, "") ?? "");
+	const date = isoSeconds(Date.parse(warc.get("warc-date") ?? ""));
+	if (target === void 0 || date === void 0) return void 0;
+	const length = Number(warc.get("content-length") ?? NaN);
+	const start = warcEnd + 4;
+	const block = record.subarray(start, Number.isInteger(length) && length >= 0 ? Math.min(record.length, start + length) : record.length);
+	const httpEnd = block.indexOf(CRLF2);
+	if (httpEnd === -1) return void 0;
+	const head = block.subarray(0, httpEnd).toString("latin1");
+	const statusLine = /^HTTP\/\d(?:\.\d)? (\d{3})/.exec(head);
+	if (statusLine === null) return void 0;
+	const headers = headerMap(head.slice(head.indexOf("\n") + 1));
+	let body = block.subarray(httpEnd + 4);
+	try {
+		if (/\bchunked\b/i.test(headers.get("transfer-encoding") ?? "")) body = dechunk(body);
+		if (body !== void 0) body = decodeCoding(body, headers.get("content-encoding"));
+	} catch {
+		body = void 0;
+	}
+	if (body === void 0) return void 0;
+	return {
+		target,
+		date,
+		record: warc.get("warc-record-id")?.replace(/^<|>$/g, ""),
+		status: Number(statusLine[1]),
+		header: (name) => headers.get(name.toLowerCase()),
+		body: new Uint8Array(body)
+	};
+};
+/** This function changes the documents of an archived body into the documents of the copy. A bot check is not the content, so it gives no document. */
+const asArchived = (docs, archived, aiInput) => docs.filter((doc) => typeof doc.meta["blocked"] !== "string" || doc.meta["blocked"] === "").map((doc) => ({
+	...doc,
+	doc: archivedDocId(archived.original),
+	kind: ARCHIVED_KIND,
+	url: archived.url,
+	meta: {
+		...doc.meta,
+		archived_kind: doc.kind,
+		archived
+	},
+	...aiInput === "no" ? { ai_input: "no" } : {}
+}));
+/**
+* This extractor reads the captures of the recipe `archive-copy`. It has no side effects, so
+* `reindex` replays them. It reads two forms:
+*
+* - A Wayback Machine copy in the raw form (`/web/<time>id_/<url>`). The body is the body of the page.
+* - A Common Crawl WARC record (a gzip member). It reads the HTTP headers and the body from the record.
+*
+* It reads the body with the page reader, and gives a document of the kind `archived` with
+* `archived` in its metadata. The usage signals of the HTTP headers in a WARC record apply too.
+*/
+const archiveExtractor = async (body, capture) => {
+	let url;
+	try {
+		url = new URL(capture.url);
+	} catch {
+		return [];
+	}
+	const raw = WAYBACK_RAW.exec(`${url.pathname}${url.search}`);
+	if (raw !== null) {
+		const timestamp = waybackTime(raw[1]);
+		const original = httpUrlOf(raw[2]);
+		if (timestamp === void 0 || original === void 0) return [];
+		const archived = {
+			source: "wayback",
+			timestamp,
+			url: `${url.origin}/web/${raw[1]}/${original}`,
+			original
+		};
+		return asArchived(await pageExtractor(body, {
+			url: original,
+			contentType: capture.contentType
+		}), archived, void 0);
+	}
+	const response = readWarcResponse(body);
+	if (response === void 0 || response.status < 200 || response.status >= 300) return [];
+	const archived = {
+		source: "commoncrawl",
+		timestamp: response.date,
+		url: capture.url,
+		original: response.target,
+		...response.record === void 0 ? {} : { record: response.record }
+	};
+	const signals = readSignals([], response.header);
+	return asArchived(await pageExtractor(response.body, {
+		url: response.target,
+		contentType: response.header("content-type") ?? null
+	}), archived, signals.ai_input === "no" ? "no" : void 0);
+};
+/** The parts of each request to an archive: the identity of the request, its budget and its labels. */
+const common = (options) => ({
+	maxAgeMs: options.maxAgeMs,
+	onBehalfOf: options.onBehalfOf,
+	job: options.job,
+	signal: options.signal,
+	scope: options.scope,
+	labels: options.labels,
+	deadline: options.deadline
+});
+const got = (fetched) => (fetched.outcome === "ok" || fetched.outcome === "not-modified") && fetched.body !== void 0;
+const said = (fetched) => `${fetched.outcome}${fetched.status > 0 ? ` ${String(fetched.status)}` : ""}${fetched.note === void 0 ? "" : `: ${fetched.note}`}`;
+const jsonOf = (fetched) => {
+	try {
+		return JSON.parse(new TextDecoder().decode(fetched.body));
+	} catch {
+		return;
+	}
+};
+/** This function gives the archived document of a fetch of a copy, or a note that tells why there is none. */
+const copyOf = (indexed, name) => {
+	const doc = indexed.docs.find((one) => one.kind === ARCHIVED_KIND);
+	const archived = doc === void 0 ? void 0 : archivedOf(doc.meta);
+	if (doc !== void 0 && archived !== void 0) return {
+		doc,
+		archived
+	};
+	if (!got(indexed.fetched)) return { note: `${name}: the copy did not come (${said(indexed.fetched)})` };
+	return { note: `${name}: the copy gave no text${indexed.error === void 0 ? " (a bot check or an empty page)" : ` (${indexed.error})`}` };
+};
+/** This function asks the Wayback Machine for its newest copy of a page, and fetches the raw copy. */
+const fromWayback = async (runtime, url, signalLines, options) => {
+	const name = ARCHIVE_NAMES.wayback;
+	const lookup = await runtime.crawler.fetch({
+		url: `${WAYBACK_AVAILABLE}?url=${encodeURIComponent(url)}`,
+		mode: "page",
+		recipe: ARCHIVE_INDEX_RECIPE,
+		accept: JSON_ACCEPT,
+		...common(options)
+	});
+	if (!got(lookup)) return { note: `${name}: the index did not answer (${said(lookup)})` };
+	const closest = jsonOf(lookup)?.archived_snapshots?.closest;
+	if (closest === void 0 || closest["available"] === false) return { note: `${name}: no copy` };
+	const status = closest["status"];
+	if (status !== void 0 && status !== "200") return { note: `${name}: the newest copy has the status ${JSON.stringify(status) ?? "unknown"}, not 200` };
+	const digits = typeof closest["timestamp"] === "string" ? closest["timestamp"] : "";
+	const shown = typeof closest["url"] === "string" ? closest["url"] : "";
+	const original = httpUrlOf(/\/web\/\d{14}[a-z_]*\/(.+)$/s.exec(shown)?.[1] ?? "");
+	if (waybackTime(digits) === void 0 || original === void 0) return { note: `${name}: the index gave no usable time or URL` };
+	const copy = await fetchAndIndex(runtime, {
+		url: `${WAYBACK}${digits}id_/${original}`,
+		mode: "page",
+		recipe: ARCHIVE_COPY_RECIPE,
+		accept: HTML_ACCEPT,
+		originalHeaderPrefix: "x-archive-orig-",
+		signalLines,
+		...common(options)
+	});
+	const found = copyOf(copy, name);
+	return "note" in found ? found : {
+		...found,
+		fromCache: lookup.fromCache && copy.fetched.fromCache
+	};
+};
+/** This function reads the JSON lines of a Common Crawl index answer, and gives the newest copy with the status 200. */
+const newestOf = (fetched) => {
+	const text = new TextDecoder().decode(fetched.body);
+	let best;
+	for (const line of text.split("\n")) {
+		if (line.trim() === "") continue;
+		let row;
+		try {
+			row = JSON.parse(line);
+		} catch {
+			continue;
+		}
+		const { timestamp, filename, status } = row;
+		const offset = Number(row["offset"]);
+		const length = Number(row["length"]);
+		if (status !== "200" || typeof timestamp !== "string" || !/^\d{14}$/.test(timestamp) || typeof filename !== "string" || !/^crawl-data\/[\w\-./]+\.warc\.gz$/.test(filename) || filename.includes("..") || !Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(length) || length < 1 || length > MAX_RECORD_BYTES) continue;
+		if (best === void 0 || timestamp > best.timestamp) best = {
+			timestamp,
+			filename,
+			offset,
+			length
+		};
+	}
+	return best;
+};
+/** This function asks the newest Common Crawl indexes for a page, and fetches its WARC record with a range request. */
+const fromCommonCrawl = async (runtime, url, signalLines, options) => {
+	const name = ARCHIVE_NAMES.commoncrawl;
+	const info = await runtime.crawler.fetch({
+		url: `${CC_INDEX}collinfo.json`,
+		mode: "page",
+		recipe: ARCHIVE_INDEX_RECIPE,
+		accept: JSON_ACCEPT,
+		...common(options)
+	});
+	if (!got(info)) return { note: `${name}: the list of indexes did not come (${said(info)})` };
+	const listed = jsonOf(info);
+	const apis = (Array.isArray(listed) ? listed : []).map((entry) => typeof entry === "object" && entry !== null ? entry["cdx-api"] : void 0).filter((api) => typeof api === "string" && api.startsWith(CC_INDEX)).slice(0, CC_INDEXES);
+	if (apis.length === 0) return { note: `${name}: the list of indexes is empty` };
+	let entry;
+	let fromCache = info.fromCache;
+	const misses = [];
+	for (const api of apis) {
+		const query = await runtime.crawler.fetch({
+			url: `${api}?url=${encodeURIComponent(url)}&output=json`,
+			mode: "page",
+			recipe: ARCHIVE_INDEX_RECIPE,
+			accept: JSON_ACCEPT,
+			...common(options)
+		});
+		fromCache &&= query.fromCache;
+		if (got(query)) entry = newestOf(query);
+		else if (query.outcome !== "not-found") misses.push(said(query));
+		if (entry !== void 0) break;
+		if (query.outcome === "skipped") break;
+	}
+	if (entry === void 0) return { note: `${name}: no copy in the ${String(apis.length)} newest indexes${misses.length > 0 ? ` (${misses.join("; ")})` : ""}` };
+	const copy = await fetchAndIndex(runtime, {
+		url: `${CC_DATA}${entry.filename}`,
+		mode: "page",
+		recipe: ARCHIVE_COPY_RECIPE,
+		accept: "*/*",
+		range: {
+			offset: entry.offset,
+			length: entry.length
+		},
+		signalLines,
+		...common(options)
+	});
+	if (got(copy.fetched) && copy.fetched.status !== 206) return { note: `${name}: the server sent the status ${String(copy.fetched.status)}, not the part of the file` };
+	const found = copyOf(copy, name);
+	return "note" in found ? found : {
+		...found,
+		fromCache: false
+	};
+};
+/**
+* This function gives an archived copy of a page after a refusal. It applies the rule for
+* robots.txt first: with no `archiveDespiteRobots`, a page that the current robots.txt of its site
+* disallows gets no copy. A stored copy younger than `maxAgeMs` answers with no request. Then it asks
+* the Wayback Machine, then Common Crawl, each one with polite fetches of the crawler.
+*/
+const archivedCopy = async (runtime, url, reason, options) => {
+	let target;
+	try {
+		target = new URL(url);
+	} catch {
+		return {
+			fromCache: false,
+			notes: []
+		};
+	}
+	target.hash = "";
+	if (redact(target) !== target.href) return {
+		fromCache: false,
+		notes: ["the URL holds a secret, so the crawler does not send it to a web archive"]
+	};
+	const despite = options.archiveDespiteRobots === true;
+	const refused = "the current robots.txt of the site disallows the page for this agent, so the crawler gives no archived copy (archiveDespiteRobots permits one)";
+	if (reason === "robots" && !despite) return {
+		fromCache: false,
+		notes: [refused]
+	};
+	let rules;
+	try {
+		rules = await runtime.crawler.robotsAllow(target.href, {
+			onBehalfOf: options.onBehalfOf,
+			scope: options.scope,
+			labels: options.labels,
+			signal: options.signal
+		});
+	} catch {
+		return {
+			fromCache: false,
+			notes: ["the fallback stopped before it read robots.txt"]
+		};
+	}
+	if (!rules.allowed && (rules.note ?? "").startsWith(ROBOTS_DISALLOW) && !despite) return {
+		fromCache: false,
+		notes: [refused]
+	};
+	const signalLines = rules.signals ?? [];
+	const known = runtime.index.get(archivedDocId(target.href));
+	const archivedKnown = known === void 0 ? void 0 : archivedOf(known.meta);
+	if (known !== void 0 && archivedKnown !== void 0 && typeof options.maxAgeMs === "number" && Date.parse(runtime.now()) - Date.parse(known.fetched_at) < options.maxAgeMs) return {
+		doc: known,
+		archived: archivedKnown,
+		fromCache: true,
+		notes: []
+	};
+	const notes = [];
+	for (const ask of [fromWayback, fromCommonCrawl]) {
+		if (options.signal?.aborted === true) break;
+		const answer = await ask(runtime, target.href, signalLines, options);
+		if ("note" in answer) notes.push(answer.note);
+		else return {
+			doc: answer.doc,
+			archived: answer.archived,
+			fromCache: answer.fromCache,
+			notes
+		};
+	}
+	return {
+		fromCache: false,
+		notes: [`no archived copy: ${notes.join("; ")}`]
+	};
+};
+/**
+* This function gives the result of a page after a refusal, with an archived copy when an archive
+* had one. The result keeps the outcome and the status of the live fetch. It gives undefined when
+* the refusal gets no copy, and `notes` tell why.
+*/
+const archivedPage = async (runtime, url, live, options) => {
+	const reason = fallbackReason(live);
+	if (reason === void 0) return { notes: [] };
+	const answer = await archivedCopy(runtime, url, reason, options);
+	if (answer.doc === void 0 || answer.archived === void 0) return { notes: answer.notes };
+	const doc = runtime.index.get(answer.doc.doc) ?? answer.doc;
+	const archived = archivedOf(doc.meta) ?? answer.archived;
+	const held = isHeldBack(doc);
+	const types = doc.meta["jsonld_types"];
+	const links = doc.meta["links"];
+	const note = [
+		live.fetched.note,
+		live.error,
+		...answer.notes,
+		archivedNote(archived),
+		...held ? [HELD_BACK] : []
+	].filter((part) => part !== void 0 && part.length > 0);
+	return {
+		page: {
+			url,
+			doc: doc.doc,
+			outcome: live.fetched.outcome,
+			status: live.fetched.status,
+			title: held ? null : doc.title,
+			chars: doc.text.length,
+			rendered: false,
+			structured: Array.isArray(types) ? types : [],
+			links: Array.isArray(links) ? links.length : 0,
+			captures: [...doc.captures],
+			capture: doc.capture,
+			capture_count: doc.capture_count,
+			fetched_at: doc.fetched_at,
+			from_cache: answer.fromCache,
+			held_back: held,
+			archived,
+			note: note.join("; ")
+		},
+		notes: []
+	};
+};
+//#endregion
 //#region src/runtime.ts
 /** This function gives the document id of a web page. Each page has one id. */
 const pageDocId = (url) => `url:${canonicalUrl(url)}`;
@@ -4088,6 +3505,7 @@ const pageExtractor = (body, capture) => {
 			needs_render: page.needsRender,
 			render_reason: page.renderReason,
 			canonical: page.canonical,
+			...page.feeds !== void 0 && page.feeds.length > 0 ? { feeds: page.feeds } : {},
 			redirect: page.redirect,
 			blocked: page.blocked,
 			...page.notes.length > 0 ? { notes: page.notes } : {}
@@ -4147,9 +3565,10 @@ const pdfDocs = async (body, url) => {
 	}];
 };
 var Extractors = class {
-	byRecipe = /* @__PURE__ */ new Map([["page", pageExtractor]]);
+	byRecipe = /* @__PURE__ */ new Map([["page", pageExtractor], ["archive-copy", archiveExtractor]]);
+	replaceable = /* @__PURE__ */ new Set(["archive-copy"]);
 	register(recipe, extractor) {
-		if (this.byRecipe.has(recipe)) throw new Error(`recipe ${recipe} already has an extractor`);
+		if (this.byRecipe.has(recipe) && !this.replaceable.delete(recipe)) throw new Error(`recipe ${recipe} already has an extractor`);
 		this.byRecipe.set(recipe, extractor);
 	}
 	get(recipe) {
@@ -4175,7 +3594,9 @@ const STRUCTURAL_META = [
 	"format",
 	"pages",
 	"ocr_needed",
-	"truncated"
+	"truncated",
+	"archived",
+	"archived_kind"
 ];
 /** The note of a document that is held back. */
 const HELD_BACK = "This page asked not to be used as AI input, so its text is held back. Open the URL to read it, or read again with userProvided when a person gave this URL directly (the crawler logs that).";
@@ -4232,7 +3653,7 @@ const indexCapture = async (runtime, fetched, recipe, rendered) => {
 		blob: fetched.blob ?? null,
 		recipe,
 		rendered,
-		ai_input: fetched.signals.ai_input,
+		ai_input: doc.ai_input === "no" ? "no" : fetched.signals.ai_input,
 		...fetched.scope !== void 0 ? { scope: fetched.scope } : {},
 		...fetched.labels !== void 0 ? { labels: fetched.labels } : {}
 	}, now)) };
@@ -4284,7 +3705,8 @@ const fetchPage = async (runtime, url, options) => {
 		job: options.job,
 		signal: options.signal,
 		scope: options.scope,
-		labels: options.labels
+		labels: options.labels,
+		deadline: options.deadline
 	});
 	let plain = await fetchAndIndex(runtime, plainRequest(url));
 	const notes = [];
@@ -4317,7 +3739,8 @@ const fetchPage = async (runtime, url, options) => {
 				job: options.job,
 				signal: options.signal,
 				scope: options.scope,
-				labels: options.labels
+				labels: options.labels,
+				deadline: options.deadline
 			});
 			if (shown.docs.length > 0) {
 				docs = shown.docs;
@@ -4326,6 +3749,11 @@ const fetchPage = async (runtime, url, options) => {
 			} else notes.push(shown.error ?? shown.fetched.note ?? "the render gave no document");
 		}
 	}
+	if (options.fallback === "archive" && docs.length === 0) {
+		const fallback = await archivedPage(runtime, url, plain, options);
+		if (fallback.page !== void 0) return fallback.page;
+		notes.push(...fallback.notes);
+	}
 	const doc = docs[0] === void 0 ? void 0 : runtime.index.get(docs[0].doc) ?? docs[0];
 	const held = doc !== void 0 && isHeldBack(doc);
 	const types = doc?.meta["jsonld_types"];
@@ -4356,6 +3784,38 @@ const knownDoc = (runtime, id) => {
 	if (doc === void 0) throw new Error(`no stored document ${id}: find one with find(), or fetch its page`);
 	return doc;
 };
+/**
+* This function gives the ISO 8601 form of a time. A text that is not a time throws a `RangeError`.
+* A date with no time is the start of that day in UTC.
+*/
+const isoTime = (name, value) => {
+	const ms = value instanceof Date ? value.getTime() : typeof value === "string" ? Date.parse(value) : NaN;
+	if (!Number.isFinite(ms)) throw new RangeError(`${name} must be a time, such as 2026-10-01T12:00:00Z, not ${JSON.stringify(String(value))}`);
+	return new Date(ms).toISOString();
+};
+/**
+* This function gives the version of a document that was current at a time (ISO 8601). It uses
+* `ReadingIndex.versionAt` when the index has it. Else it takes the newest version at or before
+* the time.
+*/
+const versionAtTime = (runtime, id, at) => {
+	const index = runtime.index;
+	if (index.versionAt !== void 0) return index.versionAt(id, at);
+	const all = index.versions(id, { limit: 1e6 });
+	const pick = all.find((version) => version.fetched_at <= at);
+	return {
+		version: pick === void 0 ? void 0 : index.versionText(id, pick.capture),
+		first: all.at(-1)?.fetched_at
+	};
+};
+/** This function gives the note of a time before the first version of a document. */
+const beforeFirstNote = (id, at, first) => `the document ${id} has no version at ${at} or before${first === void 0 ? "" : `: its first capture is from ${first}`}`;
+/** This function gives the time of `at`, or undefined. It refuses `capture` and `at` together. */
+const atOf = (options) => {
+	if (options.at === void 0) return void 0;
+	if (options.capture !== void 0) throw new RangeError("give capture or at, not the two: each one chooses a version");
+	return isoTime("at", options.at);
+};
 const wholeNumber = (name, value, fallback) => {
 	if (value === void 0) return fallback;
 	if (!Number.isFinite(value) || value < 0) throw new RangeError(`${name} must be a number of 0 or more, not ${String(value)}`);
@@ -4399,9 +3859,37 @@ const recordOverride = async (runtime, doc, action, marks = {}) => {
 const readDocument = async (runtime, id, options = {}) => {
 	const from = wholeNumber("from", options.from, 0);
 	const length = wholeNumber("length", options.length, 4e3);
+	const at = atOf(options);
 	const doc = knownDoc(runtime, id);
-	const version = options.capture === void 0 ? void 0 : runtime.index.versionText(id, options.capture);
+	let version = options.capture === void 0 ? void 0 : runtime.index.versionText(id, options.capture);
 	if (options.capture !== void 0 && version === void 0) throw new Error(`the document ${id} has no version from the capture ${options.capture}`);
+	const timed = at === void 0 ? {} : {
+		at,
+		before_first: false
+	};
+	if (at !== void 0) {
+		const found = versionAtTime(runtime, id, at);
+		if (found.version === void 0) return {
+			doc: doc.doc,
+			kind: doc.kind,
+			title: "",
+			url: doc.url,
+			meta: {},
+			from: 0,
+			to: 0,
+			total: 0,
+			text: "",
+			captures: doc.captures,
+			capture: "",
+			fetched_at: "",
+			ai_input: doc.ai_input,
+			held_back: false,
+			note: beforeFirstNote(id, at, found.first),
+			at,
+			before_first: true
+		};
+		version = found.version;
+	}
 	const aiInput = doc.ai_input === "no" || version?.ai_input === "no" ? "no" : version?.ai_input ?? doc.ai_input;
 	const userProvided = options.userProvided ?? false;
 	const held = isHeldBack({ ai_input: aiInput }, userProvided);
@@ -4410,6 +3898,8 @@ const readDocument = async (runtime, id, options = {}) => {
 	const start = onBoundary(text, Math.min(from, text.length));
 	const end = held ? start : onBoundary(text, Math.min(text.length, start + length));
 	const meta = version?.meta ?? doc.meta;
+	const archived = archivedOf(meta);
+	const notes = [...held ? [HELD_BACK] : [], ...archived ? [archivedNote(archived)] : []];
 	return {
 		doc: doc.doc,
 		kind: version?.kind ?? doc.kind,
@@ -4425,7 +3915,24 @@ const readDocument = async (runtime, id, options = {}) => {
 		fetched_at: version?.fetched_at ?? doc.fetched_at,
 		ai_input: aiInput,
 		held_back: held,
-		note: held ? HELD_BACK : null
+		note: notes.length > 0 ? notes.join(" ") : null,
+		...timed,
+		...archived !== void 0 ? { archived } : {}
+	};
+};
+/** This function gives the `archived` field of a result, for the metadata of an archived copy. */
+const archivedField = (meta) => {
+	const archived = archivedOf(meta);
+	return archived === void 0 ? {} : { archived };
+};
+/** This function gives the `archived` field and the note of a result: the note of an archived copy goes after the other note. */
+const withArchived = (meta, note) => {
+	const archived = archivedOf(meta);
+	if (archived === void 0) return { note };
+	const said = archivedNote(archived);
+	return {
+		note: note === null ? said : `${note}; ${said}`,
+		archived
 	};
 };
 /**
@@ -4433,8 +3940,9 @@ const readDocument = async (runtime, id, options = {}) => {
 * if `userProvided` opened a held-back version, so that `verifyQuote` logs the override.
 */
 const examineQuote = (runtime, id, quote, options) => {
+	const at = atOf(options);
 	const doc = knownDoc(runtime, id);
-	const pinned = options.capture === void 0 ? void 0 : runtime.index.versionText(id, options.capture);
+	let pinned = options.capture === void 0 ? void 0 : runtime.index.versionText(id, options.capture);
 	if (options.capture !== void 0 && pinned === void 0) throw new Error(`the document ${id} has no version from the capture ${options.capture}`);
 	const userProvided = options.userProvided ?? false;
 	const base = {
@@ -4442,23 +3950,48 @@ const examineQuote = (runtime, id, quote, options) => {
 		url: doc.url,
 		captures: doc.captures
 	};
+	const timed = at === void 0 ? {} : {
+		at,
+		before_first: false
+	};
+	const nothing = {
+		found: false,
+		start: null,
+		end: null,
+		excerpt: null,
+		capture: null,
+		blob: null,
+		fetched_at: null,
+		stale: false,
+		in_full_text: false,
+		count: 0
+	};
+	if (at !== void 0) {
+		const found = versionAtTime(runtime, id, at);
+		if (found.version === void 0) return {
+			result: {
+				...base,
+				...nothing,
+				complete: true,
+				held_back: false,
+				note: beforeFirstNote(id, at, found.first),
+				at,
+				before_first: true
+			},
+			override: false
+		};
+		pinned = found.version;
+	}
 	const aiInput = doc.ai_input === "no" || pinned?.ai_input === "no" ? "no" : doc.ai_input;
 	if (isHeldBack({ ai_input: aiInput }, userProvided)) return {
 		result: {
 			...base,
-			found: false,
-			start: null,
-			end: null,
-			excerpt: null,
-			capture: null,
-			blob: null,
-			fetched_at: null,
-			stale: false,
-			in_full_text: false,
+			...nothing,
 			complete: false,
-			count: 0,
 			held_back: true,
-			note: HELD_BACK
+			note: HELD_BACK,
+			...timed,
+			...archivedField(pinned?.meta ?? doc.meta)
 		},
 		override: false
 	};
@@ -4491,7 +4024,8 @@ const examineQuote = (runtime, id, quote, options) => {
 				complete: true,
 				count: hit.count,
 				held_back: false,
-				note: stale ? "only an earlier capture of the document holds the quote" : null
+				...withArchived(version.meta, stale ? "only an earlier capture of the document holds the quote" : null),
+				...timed
 			},
 			override
 		};
@@ -4513,7 +4047,8 @@ const examineQuote = (runtime, id, quote, options) => {
 			complete: !incomplete,
 			count: 0,
 			held_back: false,
-			note: notes.length > 0 ? notes.join("; ") : null
+			...withArchived(current?.meta ?? doc.meta, notes.length > 0 ? notes.join("; ") : null),
+			...timed
 		},
 		override
 	};
@@ -4548,10 +4083,27 @@ const findDocuments = async (runtime, q, options = {}) => {
 		limit: options.limit ?? 10,
 		kind: options.kind ?? null,
 		...options.rank !== void 0 ? { rank: options.rank } : {}
+	}).map((hit) => {
+		if (hit.kind !== "archived") return hit;
+		const meta = runtime.index.get(hit.doc)?.meta;
+		return meta === void 0 ? hit : {
+			...hit,
+			...archivedField(meta)
+		};
 	});
 };
 const RENDERED_MARK = "#rendered";
 const MAX_ERRORS = 20;
+/**
+* The crawler reads the captures of these recipes for itself. They are robots.txt, the sitemaps
+* and feeds of `discover`, and the indexes of the web archives. They give no document.
+*/
+const CRAWLER_RECIPES = /* @__PURE__ */ new Set([
+	"robots.txt",
+	"sitemap",
+	"feed",
+	"archive-index"
+]);
 /** This function gives the URL of a capture and its variant. A URL of version 0.1 can end in `#rendered`. */
 const captureTarget = (payload) => {
 	if (payload.url.endsWith(RENDERED_MARK)) return {
@@ -4594,7 +4146,7 @@ const replayLatest = (build, envelope, payload) => {
 		});
 		return;
 	}
-	if (payload.cut === true || payload.unasked === true) return;
+	if (payload.cut === true || payload.unasked === true || payload.range !== void 0) return;
 	const previous = build.latest(url, variant);
 	const revisit = envelope.kind === "revisit" || payload.outcome === "not-modified";
 	const truncated = payload.truncated === true;
@@ -4634,7 +4186,7 @@ const replayRecord = async (runtime, target, envelope, tally) => {
 	replayLatest(target, envelope, payload);
 	const extractor = runtime.extractors.get(payload.recipe);
 	if (extractor === void 0) {
-		if (payload.recipe !== "robots.txt") tally.unknown.add(payload.recipe);
+		if (!CRAWLER_RECIPES.has(payload.recipe)) tally.unknown.add(payload.recipe);
 		return;
 	}
 	const body = await runtime.blobs.get(payload.blob);
@@ -4667,7 +4219,7 @@ const replayRecord = async (runtime, target, envelope, tally) => {
 		blob: payload.blob,
 		recipe: payload.recipe,
 		rendered: variant === "rendered",
-		ai_input: payload.signals?.ai_input ?? "unset",
+		ai_input: doc.ai_input === "no" ? "no" : payload.signals?.ai_input ?? "unset",
 		...envelope.scope !== void 0 ? { scope: envelope.scope } : {},
 		...envelope.labels !== void 0 ? { labels: envelope.labels } : {}
 	}, envelope.received_at);
@@ -4722,6 +4274,7 @@ const missed = (runtime, envelope) => {
 	const entry = runtime.index.latest(url, variant);
 	if (payload.outcome === "redirect") return entry === void 0 || entry.fetched_at < fetchedAt;
 	if (typeof payload.blob !== "string" || payload.outcome !== "ok" && payload.outcome !== "not-modified") return false;
+	if (payload.range !== void 0) return runtime.extractors.get(payload.recipe) !== void 0 && !runtime.index.hasVersion(envelope.id);
 	if (entry === void 0 || entry.fetched_at < fetchedAt) return true;
 	return runtime.extractors.get(payload.recipe) !== void 0 && !runtime.index.hasVersion(envelope.id);
 };
@@ -4766,230 +4319,6 @@ const catchUpIndex = async (runtime) => {
 	};
 };
 //#endregion
-//#region src/open.ts
-var open_exports = /* @__PURE__ */ __exportAll({
-	BOT_PAGE: () => BOT_PAGE,
-	DEFAULT_TOKEN: () => DEFAULT_TOKEN,
-	createRuntime: () => createRuntime,
-	defaultUserAgent: () => defaultUserAgent,
-	durationMs: () => durationMs,
-	isMailbox: () => isMailbox,
-	openCrawler: () => openCrawler
-});
-/**
-* This function gives the default `User-Agent` header. The header names the crawler, its version
-* and the page about it, so a site owner can find out what made each request.
-*/
-const defaultUserAgent = (contact) => `${DEFAULT_TOKEN}/${VERSION} (+${BOT_PAGE}${contact === void 0 ? "" : `; ${contact}`})`;
-/** A mailbox is `name@host`, or a name and `<name@host>`. A URL is not a mailbox, also when it holds an `@`. */
-const MAILBOX = /^(?:[^\s<>@:/]+@[^\s<>@/]+|[^<>]*<[^\s<>@:/]+@[^\s<>@/]+>)$/;
-/**
-* This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,
-* section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in
-* the `User-Agent` only.
-*/
-const isMailbox = (contact) => MAILBOX.test(contact.trim());
-const UNITS = {
-	ms: 1,
-	s: 1e3,
-	m: 6e4,
-	h: 36e5,
-	d: 864e5,
-	w: 6048e5
-};
-/**
-* This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,
-* `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`
-* is zero. The function throws an error for a text that it cannot read.
-*/
-const durationMs = (duration) => {
-	if (typeof duration === "number") {
-		if (!Number.isFinite(duration) || duration < 0) throw new Error(`not a duration: ${String(duration)}`);
-		return duration;
-	}
-	if (/^\s*0+(\.0+)?\s*$/.test(duration)) return 0;
-	const match = /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h|d|w)\s*$/i.exec(duration);
-	const unit = match?.[2];
-	const factor = unit === void 0 || unit === "M" ? void 0 : UNITS[unit.toLowerCase()];
-	if (match === null || factor === void 0) throw new Error(`not a duration: "${duration}" (write a number and ms, s, m, h, d or w; m is minutes)`);
-	return Number(match[1]) * factor;
-};
-/** The parts of a store that can be a symbolic link to a different place only by mistake or by attack. */
-const STORE_PARTS = [
-	"captures",
-	"blobs",
-	"index.sqlite",
-	"hosts.sqlite"
-];
-/**
-* This function makes the folder of a store for the owner only, where the system has modes. It
-* refuses a store folder of a different user, and a part of the store that is a symbolic link.
-* The store folder itself can be a link, for example to a larger disk.
-*/
-const guardStore = (dir) => {
-	mkdirSync(dir, {
-		recursive: true,
-		mode: 448
-	});
-	const real = lstatSync(dir).isSymbolicLink() ? realpathSync(dir) : dir;
-	const info = statSync(real);
-	if (!info.isDirectory()) throw new Error(`the store ${dir} is not a folder`);
-	if (process.platform !== "win32" && typeof process.getuid === "function" && info.uid !== process.getuid()) throw new Error(`the store ${dir} belongs to a different user: use a folder of your own`);
-	for (const part of STORE_PARTS) {
-		let link = false;
-		try {
-			link = lstatSync(join(dir, part)).isSymbolicLink();
-		} catch {}
-		if (link) throw new Error(`${part} in the store ${dir} is a symbolic link: the crawler does not follow it`);
-	}
-};
-/**
-* This function makes a runtime. It is the one place that puts the log, the blob store, the index,
-* the host state, the crawler and the extractors together. The function `openCrawler` uses it.
-* An application that needs the parts uses it too. The runtime and the crawler share one clock.
-*/
-const createRuntime = (options) => {
-	const dir = options.dir;
-	if (dir === void 0 && (options.log === void 0 || options.blobs === void 0 || options.index === void 0)) throw new Error("a runtime needs dir, or a log, a blob store and an index");
-	const contact = options.contact;
-	if (contact !== void 0 && !/^[\t -~]*$/.test(contact)) throw new Error(`contact "${contact}" has a character that an HTTP header cannot hold: use printable ASCII`);
-	if (dir !== void 0) guardStore(dir);
-	const owned = [];
-	const log = options.log ?? new CaptureLog(join(dir, "captures"));
-	const blobs = options.blobs ?? new FileBlobStore(join(dir, "blobs"));
-	let index = options.index;
-	if (index === void 0) {
-		const made = new DocIndex(join(dir, "index.sqlite"));
-		owned.push(() => made.close());
-		index = made;
-	}
-	let hostState = options.hostState;
-	if (hostState === void 0) {
-		if (dir === void 0) hostState = new MemoryHostStateStore();
-		else {
-			const made = new SqliteHostStateStore(join(dir, "hosts.sqlite"));
-			owned.push(() => made.close());
-			hostState = made;
-		}
-	}
-	const clock = options.clock ?? realClock;
-	const renderer = options.renderer === false ? void 0 : options.renderer ?? checkedRenderer(playwrightRenderer());
-	const crawler = new Crawler({
-		token: options.token ?? "PoliteCrawler",
-		userAgent: options.userAgent ?? defaultUserAgent(contact),
-		from: contact !== void 0 && isMailbox(contact) ? contact.trim() : void 0,
-		log,
-		blobs,
-		index,
-		renderer,
-		maxActive: options.maxActive,
-		maxRenders: options.maxRenders,
-		dispatcher: options.dispatcher,
-		egress: options.egress,
-		politeness: options.politeness,
-		hostState,
-		clock,
-		random: options.random,
-		scope: options.scope,
-		scopeRank: options.scopeRank,
-		obeyDefaultToken: options.obeyDefaultToken,
-		onEvent: options.onEvent
-	});
-	const extractors = new Extractors();
-	for (const [recipe, extractor] of Object.entries(options.extractors ?? {})) extractors.register(recipe, extractor);
-	let closing;
-	return {
-		crawler,
-		index,
-		blobs,
-		captures: log,
-		log,
-		extractors,
-		hostState,
-		now: () => new Date(clock.now()).toISOString(),
-		close() {
-			closing ??= (async () => {
-				await crawler.close();
-				for (const close of owned) await close();
-			})();
-			return closing;
-		}
-	};
-};
-/**
-* This function opens a crawler on a folder. The folder keeps the capture log (`captures/`), the
-* response bodies (`blobs/`), the search index (`index.sqlite`) and the state of each host
-* (`hosts.sqlite`). The index is a projection: `reindex` makes it again from the log and the bodies.
-* At open, `catchUpIndex` rebuilds an index of an earlier version, and gives an index that is
-* behind the log the captures that it missed.
-*
-* Each method gives a promise, and none throws before its promise. After `close`, each method
-* rejects.
-*/
-const openCrawler = (options) => {
-	const runtime = createRuntime(options);
-	let closed = false;
-	let rebuilt;
-	const ready = existsSync(join(options.dir, "captures")) ? catchUpIndex(runtime).then((done) => {
-		if (done?.mode === "rebuild") {
-			const { mode, ...counts } = done;
-			rebuilt = mode === "rebuild" ? counts : void 0;
-		}
-	}, () => void 0) : Promise.resolve();
-	const live = async () => {
-		if (closed) throw new Error("the crawler is closed");
-		await ready;
-		if (closed) throw new Error("the crawler is closed");
-	};
-	return {
-		runtime,
-		async fetch(url, fetchOptions = {}) {
-			await live();
-			rebuilt = void 0;
-			const maxAge = fetchOptions.maxAge ?? "24h";
-			return fetchPage(runtime, url, {
-				render: fetchOptions.render ?? "auto",
-				maxAgeMs: maxAge === "auto" ? "auto" : durationMs(maxAge),
-				onBehalfOf: [...options.onBehalfOf ?? [], ...fetchOptions.onBehalfOf ?? []],
-				signal: fetchOptions.signal,
-				scope: fetchOptions.scope,
-				labels: fetchOptions.labels
-			});
-		},
-		async find(q, findOptions = {}) {
-			await live();
-			return findDocuments(runtime, q, findOptions);
-		},
-		async read(id, readOptions = {}) {
-			await live();
-			return readDocument(runtime, id, readOptions);
-		},
-		async verify(id, quote, verifyOptions = {}) {
-			await live();
-			return verifyQuote(runtime, id, quote, verifyOptions);
-		},
-		async versions(id, versionOptions = {}) {
-			await live();
-			const held = runtime.index.get(id)?.ai_input === "no";
-			return runtime.index.versions(id, versionOptions).map((version) => held || version.ai_input === "no" ? {
-				...version,
-				title: ""
-			} : version);
-		},
-		async reindex() {
-			await live();
-			const done = rebuilt;
-			rebuilt = void 0;
-			return done ?? reindex(runtime);
-		},
-		async close() {
-			closed = true;
-			await ready;
-			await runtime.close();
-		}
-	};
-};
-//#endregion
-export { Crawler as A, readPdf as C, FAST_RANK_SHARE as D, DocIndex as E, egressLookup as F, FileBlobStore as I, renderedKey as M, makeRecord as N, locate as O, EgressRefused as P, isPdf as S, SqliteHostStateStore as T, readDocument as _, openCrawler as a, renderAndIndex as b, HELD_BACK as c, fetchPage as d, findDocuments as f, pageExtractor as g, pageDocId as h, isMailbox as i, DEFAULT_SECRET_PARAMS as j, CAPTURE_SCHEMA as k, catchUpIndex as l, isHeldBack as m, defaultUserAgent as n, open_exports as o, heldBackMeta as p, durationMs as r, Extractors as s, createRuntime as t, fetchAndIndex as u, recordOverride as v, CaptureLog as w, verifyQuote as x, reindex as y };
+export { snippetOf as A, egressLookup as B, fallbackReason as C, foldText as D, fold as E, renderedKey as F, makeRecord as I, sha256 as L, Crawler as M, DEADLINE_NOTE as N, locate as O, DEFAULT_SECRET_PARAMS as P, stableStringify as R, archivedOf as S, readPdf as T, versionAtTime as _, fetchPage as a, archivedDocId as b, isHeldBack as c, pageExtractor as d, readDocument as f, verifyQuote as g, renderAndIndex as h, fetchAndIndex as i, CAPTURE_SCHEMA as j, parseQuery as k, isoTime as l, reindex as m, HELD_BACK as n, findDocuments as o, recordOverride as p, catchUpIndex as r, heldBackMeta as s, Extractors as t, pageDocId as u, archiveExtractor as v, isPdf as w, archivedNote as x, archivedCopy as y, EgressRefused as z };
 
-//# sourceMappingURL=open-DYZ7KL2A.mjs.map
+//# sourceMappingURL=runtime-JToRO63o.mjs.map