polite-crawler 0.1.1 → 0.1.2

package/build/open-DnrWgkC0.mjs

@@ -0,0 +1,3161 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { n as FileBlobStore, t as CaptureLog } from "./log-Dx-wXjJl.mjs";
+import { r as playwrightRenderer } from "./render-C1CUraWF.mjs";
+import { r as checkedRenderer } from "./browser-CKRNbIDT.mjs";
+import { A as Crawler, D as parseQuery, F as sha256, I as stableStringify, O as snippetOf, a as fetchPage, c as isHeldBack, f as readDocument, g as verifyQuote, i as fetchAndIndex, j as DEADLINE_NOTE, m as reindex, o as findDocuments, r as catchUpIndex, t as Extractors, w as fold } from "./runtime-k8mo1_7U.mjs";
+import { MemoryHostStateStore, nextValidatorMemo, realClock } from "./politeness.mjs";
+import { matchesPattern } from "./robots.mjs";
+import { i as VERSION, n as BOT_PAGE, r as DEFAULT_TOKEN, t as durationMs } from "./duration-iIuMMofi.mjs";
+import { n as canonicalUrl, r as scopeTest, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
+import { o as decodeBody, t as extractPage } from "./html-DkQUAE_d.mjs";
+import { n as diffDocument, r as diffTexts } from "./diff-Dk2eSvi-.mjs";
+import { r as extractDocument } from "./extract-DcwNrTK7.mjs";
+import { createHash, randomUUID } from "node:crypto";
+import { dirname, join } from "node:path";
+import { existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
+import { createGunzip } from "node:zlib";
+import { DatabaseSync } from "node:sqlite";
+/** The search index reads this many characters of each text, at most. */
+const MAX_INDEX_CHARS = 2e5;
+/** In the mode `fast`, a part is common when more than this share of the documents hold it. */
+const FAST_RANK_SHARE = .2;
+/** This function gives the rank mode of an option. A value that is not a rank mode throws a `RangeError`. */
+const rankModeOf = (value) => {
+	if (value === void 0 || value === "exact") return "exact";
+	if (value === "fast") return "fast";
+	throw new RangeError(`rank must be "exact" or "fast", not ${JSON.stringify(value) ?? "undefined"}`);
+};
+const namesOf = (generation) => ({
+	docs: `docs_${String(generation)}`,
+	fts: `fts_${String(generation)}`,
+	versions: `versions_${String(generation)}`,
+	texts: `texts_${String(generation)}`,
+	latest: `latest_${String(generation)}`
+});
+const GENERATION_TABLE = /^(docs|fts|versions|texts|latest)_(\d+)$/;
+/** A build that started this long ago and did not finish belongs to a process that stopped. */
+const ABANDONED_MS = 864e5;
+/** The FTS5 tokenizer: folded text, English stems, and spacing marks kept in the words. */
+const TOKENIZER = `tokenize = "porter unicode61 remove_diacritics 0 categories 'L* N* Co Mc'"`;
+const ddlOf = (n) => `
+CREATE TABLE IF NOT EXISTS ${n.texts} (
+  sha TEXT PRIMARY KEY,
+  body TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS ${n.versions} (
+  doc TEXT NOT NULL,
+  capture TEXT NOT NULL,
+  recipe TEXT NOT NULL,
+  rendered INTEGER NOT NULL,
+  fetched_at TEXT NOT NULL,
+  blob TEXT,
+  kind TEXT NOT NULL,
+  title TEXT NOT NULL,
+  url TEXT,
+  meta_sha TEXT NOT NULL,
+  text_sha TEXT NOT NULL,
+  text_chars INTEGER NOT NULL,
+  full_sha TEXT,
+  ai_input TEXT NOT NULL,
+  merge TEXT NOT NULL,
+  scope TEXT,
+  labels TEXT,
+  PRIMARY KEY (doc, capture)
+);
+CREATE INDEX IF NOT EXISTS ${n.versions}_time ON ${n.versions} (doc, recipe, rendered, fetched_at);
+CREATE INDEX IF NOT EXISTS ${n.versions}_capture ON ${n.versions} (capture);
+CREATE INDEX IF NOT EXISTS ${n.versions}_fetched ON ${n.versions} (fetched_at);
+CREATE INDEX IF NOT EXISTS ${n.versions}_recipe ON ${n.versions} (recipe);
+CREATE TABLE IF NOT EXISTS ${n.docs} (
+  doc TEXT PRIMARY KEY,
+  kind TEXT NOT NULL,
+  title TEXT NOT NULL,
+  url TEXT,
+  meta TEXT NOT NULL,
+  text_sha TEXT NOT NULL,
+  full_sha TEXT,
+  captures TEXT NOT NULL,
+  capture_count INTEGER NOT NULL,
+  capture TEXT NOT NULL,
+  blob TEXT,
+  fetched_at TEXT NOT NULL,
+  recipe TEXT NOT NULL,
+  rendered INTEGER NOT NULL,
+  ai_input TEXT NOT NULL,
+  scope TEXT,
+  labels TEXT,
+  updated_at TEXT NOT NULL
+);
+CREATE VIRTUAL TABLE IF NOT EXISTS ${n.fts} USING fts5(
+  title, text, content = '', contentless_delete = 1, ${TOKENIZER}
+);
+CREATE TABLE IF NOT EXISTS ${n.latest} (
+  url TEXT NOT NULL,
+  variant TEXT NOT NULL,
+  capture TEXT NOT NULL,
+  blob TEXT,
+  etag TEXT,
+  last_modified TEXT,
+  content_type TEXT,
+  fetched_at TEXT NOT NULL,
+  status INTEGER NOT NULL,
+  ai_input TEXT NOT NULL,
+  location TEXT,
+  truncated INTEGER,
+  fresh_until TEXT,
+  vary_key TEXT,
+  PRIMARY KEY (url, variant)
+);
+CREATE INDEX IF NOT EXISTS ${n.latest}_fetched ON ${n.latest} (fetched_at);
+`;
+const RENDERED_MARK = "#rendered";
+/** This function splits a cache key into its URL and its variant. A key in the form of version 0.1 (`…#rendered`) is a rendered variant. */
+const keyOf = (url, variant) => {
+	if (url.endsWith(RENDERED_MARK)) return {
+		url: cacheKeyOf(url.slice(0, -9)),
+		variant: "rendered"
+	};
+	return {
+		url: cacheKeyOf(url),
+		variant: variant ?? "plain"
+	};
+};
+const sha = (text) => createHash("sha256").update(text, "utf8").digest("hex");
+const parseJson$1 = (text, fallback) => {
+	if (text === null || text === void 0) return fallback;
+	try {
+		return JSON.parse(text);
+	} catch {
+		return fallback;
+	}
+};
+const toLatest = (row) => ({
+	url: row.url,
+	capture: row.capture,
+	blob: row.blob,
+	etag: row.etag,
+	last_modified: row.last_modified,
+	content_type: row.content_type,
+	fetched_at: row.fetched_at,
+	status: row.status,
+	ai_input: row.ai_input,
+	location: row.location,
+	truncated: row.truncated === 1 ? true : void 0,
+	fresh_until: row.fresh_until,
+	vary_key: row.vary_key,
+	variant: row.variant
+});
+const toVersion = (row) => ({
+	doc: row.doc,
+	capture: row.capture,
+	recipe: row.recipe,
+	rendered: row.rendered === 1,
+	fetched_at: row.fetched_at,
+	blob: row.blob,
+	kind: row.kind,
+	title: row.title,
+	url: row.url,
+	ai_input: row.ai_input,
+	chars: row.text_chars,
+	cut: row.full_sha !== null
+});
+const messageOf = (error) => error instanceof Error ? error.message : String(error);
+var DocIndex = class {
+	path;
+	db;
+	statements = /* @__PURE__ */ new Map();
+	depth = 0;
+	closed = false;
+	empty;
+	/**
+	* This value is true when the file had a different schema, or no tables, so the index opened
+	* empty. A store with captures then needs `reindex`. The commit of a rebuild makes it false.
+	*/
+	get stale() {
+		return this.empty;
+	}
+	constructor(path, options = {}) {
+		this.path = path;
+		if (path !== ":memory:") mkdirSync(dirname(path), {
+			recursive: true,
+			mode: 448
+		});
+		this.db = new DatabaseSync(path);
+		const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
+		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
+		try {
+			this.db.exec("CREATE VIRTUAL TABLE temp.polite_fts5_probe USING fts5(x, content = '', contentless_delete = 1); DROP TABLE temp.polite_fts5_probe;");
+		} catch (cause) {
+			this.db.close();
+			throw new Error(`polite-crawler needs SQLite FTS5 with contentless_delete, and the SQLite of Node.js ${process.version} does not have it: use Node.js 22.19 or later`, { cause });
+		}
+		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
+		this.db.exec("PRAGMA synchronous = NORMAL;");
+		this.empty = this.write(() => this.prepareSchema());
+	}
+	/** This method makes the tables of the schema. It gives true when the index opened empty. */
+	prepareSchema() {
+		const version = this.db.prepare("PRAGMA user_version").get().user_version;
+		const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
+		if (version === 2 && tables.some((table) => table.name === "index_meta")) {
+			this.dropAbandoned();
+			return false;
+		}
+		const virtual = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'").all();
+		for (const { name } of virtual) this.db.exec(`DROP TABLE IF EXISTS "${name}"`);
+		const rest = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
+		for (const { name } of rest) this.db.exec(`DROP TABLE IF EXISTS "${name}"`);
+		this.db.exec("CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)");
+		this.db.exec(ddlOf(namesOf(1)));
+		this.statements.clear();
+		this.setMeta("generation", "1");
+		this.setMeta("next_generation", "2");
+		this.db.exec(`PRAGMA user_version = ${String(2)}`);
+		return true;
+	}
+	setMeta(key, value) {
+		this.stmt("INSERT INTO index_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
+	}
+	getMeta(key) {
+		return this.stmt("SELECT value FROM index_meta WHERE key = ?").get(key)?.value;
+	}
+	stmt(sql) {
+		let statement = this.statements.get(sql);
+		if (statement === void 0) {
+			statement = this.db.prepare(sql);
+			this.statements.set(sql, statement);
+		}
+		return statement;
+	}
+	/**
+	* This method does a write in one transaction. A transaction in progress takes the write in,
+	* also a transaction that the caller opened on the connection.
+	*/
+	write(work) {
+		if (this.closed) throw new Error("the index is closed");
+		if (this.depth > 0 || this.db.isTransaction) {
+			this.depth += 1;
+			try {
+				return work();
+			} finally {
+				this.depth -= 1;
+			}
+		}
+		this.db.exec("BEGIN IMMEDIATE");
+		this.depth = 1;
+		try {
+			const result = work();
+			this.db.exec("COMMIT");
+			return result;
+		} catch (error) {
+			if (this.db.isTransaction) this.db.exec("ROLLBACK");
+			throw error;
+		} finally {
+			this.depth = 0;
+		}
+	}
+	generation() {
+		return Number(this.getMeta("generation") ?? "1");
+	}
+	/**
+	* This method does a step on the tables of the current generation. A rebuild of a different
+	* process can remove those tables in the meantime. Then the method reads the generation again
+	* and does the step once more.
+	*/
+	live(step) {
+		if (this.closed) throw new Error("the index is closed");
+		try {
+			return step(namesOf(this.generation()));
+		} catch (error) {
+			if (!/no such table/i.test(messageOf(error)) || this.depth > 0) throw error;
+			this.statements.clear();
+			return step(namesOf(this.generation()));
+		}
+	}
+	/** This method removes the tables of a generation. */
+	dropGeneration(generation) {
+		const n = namesOf(generation);
+		for (const table of [
+			n.fts,
+			n.docs,
+			n.versions,
+			n.texts,
+			n.latest
+		]) this.db.exec(`DROP TABLE IF EXISTS ${table}`);
+		this.db.prepare("DELETE FROM index_meta WHERE key = ?").run(`building:${String(generation)}`);
+		this.statements.clear();
+	}
+	/** This method removes the generations of builds that a stopped process left. */
+	dropAbandoned() {
+		const current = this.generation();
+		const generations = /* @__PURE__ */ new Set();
+		for (const { name } of this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all()) {
+			const match = GENERATION_TABLE.exec(name);
+			if (match !== null) generations.add(Number(match[2]));
+		}
+		for (const generation of generations) {
+			if (generation === current) continue;
+			const started = this.getMeta(`building:${String(generation)}`);
+			if (!((started === void 0 ? Number.POSITIVE_INFINITY : Date.now() - Date.parse(started)) < ABANDONED_MS)) this.dropGeneration(generation);
+		}
+	}
+	putText(n, body) {
+		const key = sha(body);
+		this.stmt(`INSERT OR IGNORE INTO ${n.texts} (sha, body) VALUES (?, ?)`).run(key, body);
+		return key;
+	}
+	textOf(n, key) {
+		if (key === null) return void 0;
+		return this.stmt(`SELECT body FROM ${n.texts} WHERE sha = ?`).get(key)?.body;
+	}
+	putVersion(n, input, now) {
+		const full = input.full_text !== void 0 && input.full_text !== input.text ? this.putText(n, input.full_text) : null;
+		this.stmt(`INSERT INTO ${n.versions} (doc, capture, recipe, rendered, fetched_at, blob, kind, title, url,
+         meta_sha, text_sha, text_chars, full_sha, ai_input, merge, scope, labels)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(doc, capture) DO UPDATE SET recipe = excluded.recipe, rendered = excluded.rendered,
+         fetched_at = excluded.fetched_at, blob = excluded.blob, kind = excluded.kind, title = excluded.title,
+         url = excluded.url, meta_sha = excluded.meta_sha, text_sha = excluded.text_sha,
+         text_chars = excluded.text_chars, full_sha = excluded.full_sha, ai_input = excluded.ai_input,
+         merge = excluded.merge, scope = excluded.scope, labels = excluded.labels`).run(input.doc, input.capture ?? input.captures[0] ?? "", input.recipe ?? "", input.rendered === true ? 1 : 0, input.fetched_at ?? now, input.blob ?? null, input.kind, input.title, input.url, this.putText(n, JSON.stringify(input.meta ?? {})), this.putText(n, input.text), input.text.length, full, input.ai_input ?? "unset", input.merge ?? "richest", input.scope ?? null, input.labels === void 0 ? null : JSON.stringify(input.labels));
+	}
+	/** This method gives the newest version of a document for one recipe and one variant, at `until` or before when it is given. */
+	newest(n, doc, recipe, rendered, until) {
+		if (until === void 0) return this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = ?
+          ORDER BY fetched_at DESC, capture DESC LIMIT 1`).get(doc, recipe, rendered);
+		return this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = ? AND fetched_at <= ?
+        ORDER BY fetched_at DESC, capture DESC LIMIT 1`).get(doc, recipe, rendered, until);
+	}
+	/**
+	* This method chooses the current version of a document with the rules of the class. With
+	* `until`, it uses only the versions that the crawler fetched at that time or before. Thus it
+	* gives the version that was current at that time. It gives undefined when no version agrees.
+	*/
+	choose(n, doc, until) {
+		const groups = until === void 0 ? this.stmt(`SELECT DISTINCT recipe, rendered FROM ${n.versions} WHERE doc = ?`).all(doc) : this.stmt(`SELECT DISTINCT recipe, rendered FROM ${n.versions} WHERE doc = ? AND fetched_at <= ?`).all(doc, until);
+		if (groups.length === 0) return void 0;
+		const byRecipe = /* @__PURE__ */ new Map();
+		for (const group of groups) {
+			const row = this.newest(n, doc, group.recipe, group.rendered, until);
+			if (row === void 0) continue;
+			const entry = byRecipe.get(group.recipe) ?? {};
+			if (group.rendered === 1) entry.rendered = row;
+			else entry.plain = row;
+			byRecipe.set(group.recipe, entry);
+		}
+		const candidates = [];
+		for (const [recipe, { plain, rendered }] of byRecipe) {
+			let pick;
+			if (rendered === void 0) pick = plain;
+			else if (plain === void 0 || rendered.fetched_at >= plain.fetched_at) pick = rendered;
+			else {
+				const base = this.stmt(`SELECT blob FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = 0 AND fetched_at <= ?
+            ORDER BY fetched_at DESC, capture DESC LIMIT 1`).get(doc, recipe, rendered.fetched_at);
+				pick = base !== void 0 && base.blob !== null && base.blob === plain.blob ? rendered : plain;
+			}
+			const newest = plain === void 0 ? rendered : rendered === void 0 || plain.fetched_at > rendered.fetched_at ? plain : rendered;
+			const others = [plain, rendered].filter((row) => row !== void 0 && row !== pick);
+			candidates.push({
+				row: pick,
+				ai: newest.ai_input,
+				others
+			});
+		}
+		let winner = candidates[0];
+		if (candidates.length > 1) {
+			const latest = candidates.every((candidate) => candidate.row.merge === "latest");
+			for (const candidate of candidates.slice(1)) {
+				const a = candidate.row;
+				const b = winner.row;
+				if (latest ? a.fetched_at > b.fetched_at : a.text_chars > b.text_chars || a.text_chars === b.text_chars && a.fetched_at > b.fetched_at) winner = candidate;
+			}
+		}
+		const aiInput = winner.row.ai_input === "no" || candidates.some((candidate) => candidate.ai === "no") ? "no" : winner.ai !== "unset" ? winner.ai : candidates.some((candidate) => candidate.ai === "yes") ? "yes" : "unset";
+		return {
+			winner,
+			candidates,
+			aiInput
+		};
+	}
+	/** This method chooses the current version of a document from its versions, and writes it with its search entry. */
+	recompute(n, doc, now) {
+		const before = this.stmt(`SELECT rowid, text_sha, title FROM ${n.docs} WHERE doc = ?`).get(doc);
+		const choice = this.choose(n, doc);
+		if (choice === void 0) {
+			if (before !== void 0) {
+				this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);
+				this.stmt(`DELETE FROM ${n.docs} WHERE doc = ?`).run(doc);
+			}
+			return;
+		}
+		const { winner, candidates, aiInput } = choice;
+		const row = winner.row;
+		let meta = parseJson$1(this.textOf(n, row.meta_sha), {});
+		for (const candidate of candidates) if (candidate !== winner) meta = {
+			...parseJson$1(this.textOf(n, candidate.row.meta_sha), {}),
+			...meta
+		};
+		const same = this.stmt(`SELECT capture FROM ${n.versions} WHERE doc = ? AND text_sha = ?
+        ORDER BY fetched_at DESC, capture DESC LIMIT ?`).all(doc, row.text_sha, 10);
+		const captures = [.../* @__PURE__ */ new Set([
+			row.capture,
+			...same.map((entry) => entry.capture),
+			...winner.others.map((other) => other.capture),
+			...candidates.filter((candidate) => candidate !== winner).map((candidate) => candidate.row.capture)
+		])].slice(0, 10);
+		const count = this.stmt(`SELECT count(*) AS n FROM ${n.versions} WHERE doc = ?`).get(doc).n;
+		const url = row.url ?? candidates.map((candidate) => candidate.row.url).find((value) => value !== null) ?? null;
+		const saved = this.stmt(`INSERT INTO ${n.docs} (doc, kind, title, url, meta, text_sha, full_sha, captures, capture_count, capture,
+         blob, fetched_at, recipe, rendered, ai_input, scope, labels, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(doc) DO UPDATE SET kind = excluded.kind, title = excluded.title, url = excluded.url,
+         meta = excluded.meta, text_sha = excluded.text_sha, full_sha = excluded.full_sha,
+         captures = excluded.captures, capture_count = excluded.capture_count, capture = excluded.capture,
+         blob = excluded.blob, fetched_at = excluded.fetched_at, recipe = excluded.recipe,
+         rendered = excluded.rendered, ai_input = excluded.ai_input, scope = excluded.scope,
+         labels = excluded.labels, updated_at = excluded.updated_at
+       RETURNING rowid`).get(doc, row.kind, row.title, url, JSON.stringify(meta), row.text_sha, row.full_sha, JSON.stringify(captures), count, row.capture, row.blob, row.fetched_at, row.recipe, row.rendered, aiInput, row.scope, row.labels, now);
+		const text = this.textOf(n, row.text_sha) ?? "";
+		if (before === void 0 || before.text_sha !== row.text_sha || before.title !== row.title) {
+			if (before !== void 0) this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);
+			this.stmt(`INSERT INTO ${n.fts} (rowid, title, text) VALUES (?, ?, ?)`).run(saved.rowid, fold(row.title, "search"), fold(text.length > MAX_INDEX_CHARS ? text.slice(0, MAX_INDEX_CHARS) : text, "search"));
+		}
+		return this.docOf(n, doc, text);
+	}
+	docOf(n, doc, knownText) {
+		const row = this.stmt(`SELECT rowid, * FROM ${n.docs} WHERE doc = ?`).get(doc);
+		if (row === void 0) return void 0;
+		return this.toDoc(n, row, knownText);
+	}
+	toDoc(n, row, knownText) {
+		const labels = parseJson$1(row.labels, void 0);
+		return {
+			doc: row.doc,
+			kind: row.kind,
+			title: row.title,
+			text: knownText ?? this.textOf(n, row.text_sha) ?? "",
+			url: row.url,
+			meta: parseJson$1(row.meta, {}),
+			captures: parseJson$1(row.captures, []),
+			capture_count: row.capture_count,
+			capture: row.capture,
+			blob: row.blob,
+			fetched_at: row.fetched_at,
+			recipe: row.recipe,
+			rendered: row.rendered === 1,
+			ai_input: row.ai_input,
+			cut: row.full_sha !== null,
+			...row.scope !== null ? { scope: row.scope } : {},
+			...labels !== void 0 ? { labels } : {},
+			updated_at: row.updated_at
+		};
+	}
+	versionTextOf(n, row) {
+		return {
+			...toVersion(row),
+			text: this.textOf(n, row.text_sha) ?? "",
+			full_text: this.textOf(n, row.full_sha),
+			meta: parseJson$1(this.textOf(n, row.meta_sha), {})
+		};
+	}
+	get(doc) {
+		return this.live((n) => this.docOf(n, doc));
+	}
+	/**
+	* This method adds one version of a document, and it gives the current document. The rules of
+	* the class choose the current version. An upsert of a known capture replaces that version.
+	*/
+	upsert(input, now) {
+		return this.live((n) => this.write(() => {
+			this.putVersion(n, input, now);
+			return this.recompute(n, input.doc, now);
+		}));
+	}
+	/** This method gives the versions of a document, the newest first. */
+	versions(doc, options = {}) {
+		const limit = Math.max(1, Math.floor(options.limit ?? 100));
+		return this.live((n) => this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? ORDER BY fetched_at DESC, capture DESC LIMIT ?`).all(doc, limit).map(toVersion));
+	}
+	/**
+	* This method gives the text of one version of a document. With no capture, it gives the current
+	* version. It also gives the text of a held-back version: the caller applies `isHeldBack` to the
+	* `ai_input` of the version.
+	*/
+	versionText(doc, capture) {
+		return this.live((n) => {
+			const id = capture ?? this.docOf(n, doc)?.capture;
+			if (id === void 0) return void 0;
+			const row = this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND capture = ?`).get(doc, id);
+			return row === void 0 ? void 0 : this.versionTextOf(n, row);
+		});
+	}
+	/**
+	* This method gives the version of a document that was current at a time (ISO 8601). It applies
+	* the rules of the class to the versions that the crawler fetched at that time or before. Thus
+	* a later capture does not change the answer. It also gives a held-back version: its `ai_input`
+	* is `no` when the document was held back at that time, and the caller applies `isHeldBack`.
+	*/
+	versionAt(doc, at) {
+		return this.live((n) => {
+			const first = this.stmt(`SELECT min(fetched_at) AS first FROM ${n.versions} WHERE doc = ?`).get(doc)?.first ?? void 0;
+			const choice = this.choose(n, doc, at);
+			if (choice === void 0) return {
+				version: void 0,
+				first
+			};
+			const version = this.versionTextOf(n, choice.winner.row);
+			return {
+				version: choice.aiInput === "no" ? {
+					...version,
+					ai_input: "no"
+				} : version,
+				first
+			};
+		});
+	}
+	/**
+	* This method gives the texts of the earlier versions of a document, the newest first. Each
+	* text comes one time, with the newest version that has it. It skips the current text. It also
+	* gives held-back texts: the caller applies `isHeldBack` to the `ai_input` of each version.
+	*/
+	earlierTexts(doc, options = {}) {
+		const limit = Math.max(1, Math.floor(options.limit ?? 50));
+		return this.live((n) => {
+			const head = this.stmt(`SELECT text_sha FROM ${n.docs} WHERE doc = ?`).get(doc);
+			if (head === void 0) return [];
+			const rows = this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND text_sha != ? ORDER BY fetched_at DESC, capture DESC`).all(doc, head.text_sha);
+			const seen = /* @__PURE__ */ new Set();
+			const out = [];
+			for (const row of rows) {
+				if (seen.has(row.text_sha)) continue;
+				seen.add(row.text_sha);
+				out.push(this.versionTextOf(n, row));
+				if (out.length >= limit) break;
+			}
+			return out;
+		});
+	}
+	/** This method tells if a capture gave a version of a document. */
+	hasVersion(capture) {
+		return this.live((n) => this.stmt(`SELECT 1 AS found FROM ${n.versions} WHERE capture = ? LIMIT 1`).get(capture) !== void 0);
+	}
+	/** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
+	newestFetch() {
+		return this.live((n) => this.stmt(`SELECT max(fetched_at) AS newest FROM ${n.latest}`).get()?.newest ?? void 0);
+	}
+	/** This method gives the current documents that a capture gave. */
+	docsOfCapture(capture) {
+		return this.live((n) => this.stmt(`SELECT DISTINCT doc FROM ${n.versions} WHERE capture = ?`).all(capture).map((row) => this.docOf(n, row.doc)).filter((doc) => doc !== void 0));
+	}
+	/**
+	* This method searches the documents with BM25, the best first. The syntax of the query is the
+	* syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it
+	* adds the documents that match one part. A document that asked not to be AI input is not in
+	* the results, so a search cannot show its text. The option `rank` chooses the ranking (`RankMode`).
+	*/
+	find(q, options) {
+		const rank = rankModeOf(options.rank);
+		const query = parseQuery(q);
+		if (query === void 0) return [];
+		const limit = Math.max(1, Math.min(1e3, Math.floor(options.limit) || 10));
+		return this.live((n) => {
+			const kindClause = options.kind === void 0 || options.kind === null ? "" : "AND d.kind = ?";
+			const sql = `SELECT d.rowid AS rowid, d.*, bm25(${n.fts}, 3.0, 1.0) AS rank
+          FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid
+         WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' ${kindClause}
+         ORDER BY rank LIMIT ?`;
+			const run = (match) => {
+				const params = [match];
+				if (kindClause.length > 0) params.push(options.kind);
+				params.push(limit);
+				return this.stmt(sql).all(...params);
+			};
+			const rows = run(query.all);
+			if (rows.length < limit && query.parts > 1) {
+				const seen = new Set(rows.map((row) => row.doc));
+				for (const row of run(rank === "fast" ? this.rareParts(n, query.anyParts) : query.any)) {
+					if (rows.length >= limit) break;
+					if (!seen.has(row.doc)) rows.push(row);
+				}
+			}
+			return rows.map((row) => ({
+				doc: row.doc,
+				kind: row.kind,
+				title: row.title,
+				url: row.url,
+				snippet: snippetOf(this.textOf(n, row.text_sha) ?? "", query.terms),
+				score: Math.round(-row.rank * 1e3) / 1e3,
+				ai_input: row.ai_input,
+				fetched_at: row.fetched_at,
+				capture: row.capture
+			}));
+		});
+	}
+	/**
+	* This method scores a set of documents against a query, with the best first. It leaves out a
+	* document that matches no term, and a document that asked not to be AI input. It ranks the
+	* results of a source again, because the order of the source is only as good as its search engine.
+	* In the mode `fast`, it also leaves out a document that holds only common parts (`RankMode`).
+	*/
+	rankWithin(q, docs, options = {}) {
+		const rank = rankModeOf(options.rank);
+		const query = parseQuery(q);
+		if (query === void 0 || docs.length === 0) return [];
+		return this.live((n) => this.stmt(`SELECT d.doc AS doc, bm25(${n.fts}, 3.0, 1.0) AS rank
+             FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid
+            WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' AND d.doc IN (SELECT value FROM json_each(?))
+            ORDER BY rank`).all(rank === "fast" ? this.rareParts(n, query.anyParts) : query.any, JSON.stringify(docs)).map((row) => ({
+			doc: row.doc,
+			score: Math.round(-row.rank * 1e3) / 1e3
+		})));
+	}
+	/**
+	* This method gives the form of a query that needs one part, without the common parts (the mode
+	* `fast` of `RankMode`). It counts the documents that hold each part. When each part is common,
+	* it keeps the rarest part.
+	*/
+	rareParts(n, parts) {
+		if (parts.length < 2) return parts.join(" OR ");
+		const total = this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get().n;
+		const counted = parts.map((part) => ({
+			part,
+			count: this.stmt(`SELECT count(*) AS n FROM ${n.fts} WHERE ${n.fts} MATCH ?`).get(part).n
+		}));
+		const rare = counted.filter((one) => one.count <= FAST_RANK_SHARE * total);
+		if (rare.length > 0) return rare.map((one) => one.part).join(" OR ");
+		return counted.reduce((best, one) => one.count < best.count ? one : best).part;
+	}
+	/** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */
+	latest(url, variant) {
+		const key = keyOf(url, variant);
+		return this.live((n) => {
+			const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(key.url, key.variant);
+			return row === void 0 ? void 0 : toLatest(row);
+		});
+	}
+	putLatest(n, entry) {
+		const key = keyOf(entry.url, entry.variant);
+		this.stmt(`INSERT INTO ${n.latest} (url, variant, capture, blob, etag, last_modified, content_type, fetched_at,
+         status, ai_input, location, truncated, fresh_until, vary_key)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(url, variant) DO UPDATE SET capture = excluded.capture, blob = excluded.blob,
+         etag = excluded.etag, last_modified = excluded.last_modified, content_type = excluded.content_type,
+         fetched_at = excluded.fetched_at, status = excluded.status, ai_input = excluded.ai_input,
+         location = excluded.location, truncated = excluded.truncated, fresh_until = excluded.fresh_until,
+         vary_key = excluded.vary_key
+       WHERE excluded.fetched_at >= ${n.latest}.fetched_at`).run(key.url, key.variant, entry.capture, entry.blob, entry.etag, entry.last_modified, entry.content_type, entry.fetched_at, entry.status, entry.ai_input, entry.location ?? null, entry.truncated === true ? 1 : null, entry.fresh_until ?? null, entry.vary_key ?? null);
+	}
+	/** This method keeps the last answer for a URL. An entry that is older than the stored entry changes nothing. */
+	setLatest(entry) {
+		this.live((n) => this.write(() => this.putLatest(n, entry)));
+	}
+	counts() {
+		return this.live((n) => ({
+			docs: this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get().n,
+			urls: this.stmt(`SELECT count(*) AS n FROM ${n.latest}`).get().n
+		}));
+	}
+	/** This method empties the current generation. A rebuild uses `beginRebuild`, which keeps the index until its commit. */
+	reset() {
+		this.live((n) => this.write(() => {
+			for (const table of [
+				n.fts,
+				n.docs,
+				n.versions,
+				n.texts,
+				n.latest
+			]) this.db.exec(`DELETE FROM ${table}`);
+		}));
+	}
+	/**
+	* This method starts a rebuild into a new generation of the tables. The current generation
+	* answers each read and takes each write until the commit. A failed rebuild thus leaves the
+	* index as it was.
+	*/
+	beginRebuild(options = {}) {
+		const batchSize = Math.max(1, Math.floor(options.batch ?? 500));
+		const generation = this.write(() => {
+			const next = Number(this.getMeta("next_generation") ?? "2");
+			this.setMeta("next_generation", String(next + 1));
+			this.setMeta(`building:${String(next)}`, (/* @__PURE__ */ new Date()).toISOString());
+			this.db.exec(ddlOf(namesOf(next)));
+			return next;
+		});
+		const n = namesOf(generation);
+		const versions = [];
+		const latest = /* @__PURE__ */ new Map();
+		let open = true;
+		const keyText = (url, variant) => {
+			const key = keyOf(url, variant);
+			return `${key.variant} ${key.url}`;
+		};
+		const flush = () => {
+			if (!open) throw new Error("the rebuild is over");
+			if (versions.length === 0 && latest.size === 0) return;
+			this.write(() => {
+				const touched = /* @__PURE__ */ new Map();
+				for (const { input, now } of versions) {
+					this.putVersion(n, input, now);
+					touched.set(input.doc, now);
+				}
+				for (const entry of latest.values()) this.putLatest(n, entry);
+				for (const [doc, now] of touched) this.recompute(n, doc, now);
+			});
+			versions.length = 0;
+			latest.clear();
+		};
+		/** This function copies the texts of the versions of the current generation that agree with `where`. */
+		const copyTexts = (current, where, params) => {
+			for (const column of [
+				"text_sha",
+				"meta_sha",
+				"full_sha"
+			]) this.db.prepare(`INSERT OR IGNORE INTO ${n.texts} (sha, body)
+             SELECT t.sha, t.body FROM ${current.texts} t
+              WHERE t.sha IN (SELECT ${column} FROM ${current.versions} WHERE ${where})`).run(...params);
+		};
+		return {
+			upsert: (input, now) => {
+				if (!open) throw new Error("the rebuild is over");
+				versions.push({
+					input,
+					now
+				});
+				if (versions.length >= batchSize) flush();
+			},
+			setLatest: (entry) => {
+				if (!open) throw new Error("the rebuild is over");
+				const key = keyText(entry.url, entry.variant);
+				const before = latest.get(key);
+				if (before === void 0 || entry.fetched_at >= before.fetched_at) latest.set(key, entry);
+				if (latest.size >= batchSize) flush();
+			},
+			latest: (url, variant) => {
+				const waiting = latest.get(keyText(url, variant));
+				if (waiting !== void 0) return waiting;
+				const key = keyOf(url, variant);
+				const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(key.url, key.variant);
+				return row === void 0 ? void 0 : toLatest(row);
+			},
+			flush,
+			carry: (recipes) => {
+				flush();
+				const known = JSON.stringify([...recipes]);
+				const current = namesOf(this.generation());
+				const docs = this.db.prepare(`SELECT DISTINCT doc FROM ${current.versions} WHERE recipe NOT IN (SELECT value FROM json_each(?))`).all(known);
+				const where = "recipe NOT IN (SELECT value FROM json_each(?)) AND doc IN (SELECT value FROM json_each(?))";
+				for (let at = 0; at < docs.length; at += batchSize) {
+					const part = JSON.stringify(docs.slice(at, at + batchSize).map((row) => row.doc));
+					this.write(() => {
+						copyTexts(current, where, [known, part]);
+						this.db.prepare(`INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${current.versions} WHERE ${where}`).run(known, part);
+						const now = (/* @__PURE__ */ new Date()).toISOString();
+						for (const doc of JSON.parse(part)) this.recompute(n, doc, now);
+					});
+				}
+				return docs.length;
+			},
+			commit: (since) => {
+				flush();
+				open = false;
+				return this.write(() => {
+					const current = this.generation();
+					const old = namesOf(current);
+					copyTexts(old, "fetched_at >= ?", [since]);
+					const added = this.db.prepare(`SELECT DISTINCT doc FROM ${old.versions} WHERE fetched_at >= ?`).all(since);
+					this.db.prepare(`INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${old.versions} WHERE fetched_at >= ?`).run(since);
+					const now = (/* @__PURE__ */ new Date()).toISOString();
+					for (const { doc } of added) this.recompute(n, doc, now);
+					for (const row of this.db.prepare(`SELECT * FROM ${old.latest} WHERE fetched_at >= ?`).all(since)) this.putLatest(n, toLatest(row));
+					this.setMeta("generation", String(generation));
+					this.db.prepare("DELETE FROM index_meta WHERE key = ?").run(`building:${String(generation)}`);
+					this.dropGeneration(current);
+					this.empty = false;
+					return {
+						docs: this.db.prepare(`SELECT count(*) AS n FROM ${n.docs}`).get().n,
+						urls: this.db.prepare(`SELECT count(*) AS n FROM ${n.latest}`).get().n
+					};
+				});
+			},
+			abort: () => {
+				open = false;
+				versions.length = 0;
+				latest.clear();
+				if (this.closed) return;
+				this.write(() => this.dropGeneration(generation));
+			}
+		};
+	}
+	/** This method closes the file. A second close does nothing. */
+	close() {
+		if (this.closed) return;
+		this.closed = true;
+		this.statements.clear();
+		this.db.close();
+	}
+};
+//#endregion
+//#region src/sitemaps.ts
+/** These are the default limits of the reader, and the maximum length of one value. */
+const SITEMAP_LIMITS = {
+	maxEntries: 5e4,
+	maxDepth: 32,
+	/** A value (a URL or a date) longer than this number of characters is not valid. */
+	maxValue: 4096
+};
+const CHANGEFREQ = /* @__PURE__ */ new Set([
+	"always",
+	"hourly",
+	"daily",
+	"weekly",
+	"monthly",
+	"yearly",
+	"never"
+]);
+const SITEMAP_NS = [
+	"http://www.sitemaps.org/schemas/sitemap/0.9",
+	"http://www.google.com/schemas/sitemap/0.84",
+	"http://www.google.com/schemas/sitemap/0.9"
+];
+const ATOM_NS = ["http://www.w3.org/2005/Atom"];
+const RSS1_NS = ["http://purl.org/rss/1.0/"];
+const DC_NS = ["http://purl.org/dc/elements/1.1/"];
+const RDF_NS = ["http://www.w3.org/1999/02/22-rdf-syntax-ns#"];
+const XML_ENTITIES = {
+	amp: "&",
+	lt: "<",
+	gt: ">",
+	quot: "\"",
+	apos: "'"
+};
+/** A tag keeps this number of attributes at most. The reader reads the others and keeps nothing of them. */
+const MAX_ATTRIBUTES = 64;
+const NAME = /^[A-Za-z_][\w.:-]*$/;
+const W3C_DATE = /^\d{4}(?:-\d{2}(?:-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?)?)?$/;
+const MAIL_DATE = /^(?:[A-Za-z]{3},?\s+)?\d{1,2}\s+[A-Za-z]{3,9}\s+\d{2,4}\s+\d{1,2}:\d{2}(?::\d{2})?(?:\s*(?:[A-Za-z]{1,5}|[+-]\d{4}))?$/;
+const short = (name) => name === void 0 ? "nothing" : name.length > 40 ? `${name.slice(0, 40)}...` : name;
+/**
+* This function changes a date of a sitemap or a feed into ISO 8601. It reads the W3C form of the
+* sitemap protocol and Atom, and the mail form of RSS. A time with no zone is UTC. It gives null
+* for a text that it cannot read.
+*/
+const feedDate = (value) => {
+	const text = value.trim();
+	if (text.length === 0 || text.length > 64) return null;
+	let parse;
+	const w3c = W3C_DATE.exec(text);
+	if (w3c !== null) {
+		const iso = text.replace(" ", "T");
+		parse = iso.includes("T") && w3c[1] === void 0 ? `${iso}Z` : iso;
+	} else if (MAIL_DATE.test(text)) parse = text;
+	if (parse === void 0) return null;
+	const ms = Date.parse(parse);
+	return Number.isFinite(ms) && Math.abs(ms) < 864e13 ? new Date(ms).toISOString() : null;
+};
+/** This function gives the canonical form of an http(s) URL, or undefined. A relative URL needs a base. */
+const webUrl = (value, base) => {
+	const text = value.trim();
+	if (text.length === 0 || text.length > SITEMAP_LIMITS.maxValue) return void 0;
+	try {
+		const url = base === void 0 ? new URL(text) : new URL(text, base);
+		if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
+		return canonicalUrl(url.href);
+	} catch {
+		return;
+	}
+};
+/** This function decodes the five entities of XML and the character references. It gives undefined for each other entity. */
+const decodeText = (raw, meter) => {
+	meter.steps += raw.length;
+	if (!raw.includes("&")) return raw;
+	let out = "";
+	let at = 0;
+	for (;;) {
+		const amp = raw.indexOf("&", at);
+		if (amp === -1) return out + raw.slice(at);
+		out += raw.slice(at, amp);
+		const semi = raw.indexOf(";", amp + 1);
+		if (semi === -1 || semi - amp > 10) return void 0;
+		const name = raw.slice(amp + 1, semi);
+		let char = XML_ENTITIES[name];
+		if (char === void 0 && /^#(?:\d{1,7}|x[0-9a-fA-F]{1,6})$/.test(name)) {
+			const code = name[1] === "x" ? Number.parseInt(name.slice(2), 16) : Number(name.slice(1));
+			if (code > 0 && code <= 1114111 && (code < 55296 || code > 57343)) char = String.fromCodePoint(code);
+		}
+		if (char === void 0) return void 0;
+		out += char;
+		at = semi + 1;
+	}
+};
+const isSpace = (code) => code === 32 || code === 9 || code === 10 || code === 13;
+/** This function reads a start tag from the position after its `<`. It gives undefined for a tag that is not well formed. */
+const readTag = (text, from, meter) => {
+	const tag = tagAt(text, from);
+	meter.steps += (tag?.end ?? text.length) - from;
+	return tag;
+};
+const tagAt = (text, from) => {
+	const n = text.length;
+	let at = from;
+	while (at < n) {
+		const code = text.charCodeAt(at);
+		if (isSpace(code) || code === 47 || code === 62) break;
+		at += 1;
+	}
+	const name = text.slice(from, Math.min(at, from + 256));
+	if (at - from > 256 || !NAME.test(name)) return void 0;
+	const attributes = [];
+	for (;;) {
+		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
+		if (at >= n) return void 0;
+		const code = text.charCodeAt(at);
+		if (code === 62) return {
+			name,
+			attributes,
+			selfClosing: false,
+			end: at + 1
+		};
+		if (code === 47) return text.charCodeAt(at + 1) === 62 ? {
+			name,
+			attributes,
+			selfClosing: true,
+			end: at + 2
+		} : void 0;
+		const start = at;
+		while (at < n) {
+			const next = text.charCodeAt(at);
+			if (isSpace(next) || next === 61 || next === 62 || next === 47) break;
+			at += 1;
+		}
+		const key = text.slice(start, Math.min(at, start + 256));
+		if (at - start > 256 || !NAME.test(key)) return void 0;
+		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
+		if (text.charCodeAt(at) !== 61) return void 0;
+		at += 1;
+		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
+		const quote = text[at];
+		if (quote !== "\"" && quote !== "'") return void 0;
+		const close = text.indexOf(quote, at + 1);
+		if (close === -1) return void 0;
+		if (attributes.length < MAX_ATTRIBUTES && close - at <= SITEMAP_LIMITS.maxValue) attributes.push([key, text.slice(at + 1, close)]);
+		at = close + 1;
+	}
+};
+/**
+* This function finds the end of a declaration (`<!DOCTYPE ...>`) with its internal subset. It
+* gives the position after the `>`, or -1, and it tells if the declaration has an internal subset.
+*/
+const declarationEnd = (text, from, meter) => {
+	let depth = 0;
+	let quote = 0;
+	let subset = false;
+	for (let at = from; at < text.length; at += 1) {
+		meter.steps += 1;
+		const code = text.charCodeAt(at);
+		if (quote !== 0) {
+			if (code === quote) quote = 0;
+		} else if (code === 34 || code === 39) quote = code;
+		else if (code === 91) {
+			depth += 1;
+			subset = true;
+		} else if (code === 93) depth = Math.max(0, depth - 1);
+		else if (code === 62 && depth === 0) return {
+			end: at + 1,
+			subset
+		};
+	}
+	return {
+		end: -1,
+		subset
+	};
+};
+/** This function reads a text sitemap: each line that is an absolute http(s) URL is an entry. */
+const readLines = (text, maxEntries, meter) => {
+	const entries = [];
+	let invalid = 0;
+	let at = 0;
+	while (at < text.length) {
+		let end = text.indexOf("\n", at);
+		if (end === -1) end = text.length;
+		meter.steps += end - at + 1;
+		const line = end - at > SITEMAP_LIMITS.maxValue + 2 ? "-" : text.slice(at, end).trim();
+		at = end + 1;
+		if (line.length === 0) continue;
+		const url = webUrl(line, void 0);
+		if (url === void 0) invalid += 1;
+		else entries.push({
+			url,
+			lastmod: null,
+			changefreq: null
+		});
+		if (entries.length >= maxEntries) {
+			const complete = at >= text.length;
+			return {
+				kind: "text",
+				entries,
+				invalid,
+				complete,
+				notes: complete ? [] : [`the reader stopped at ${String(maxEntries)} entries`]
+			};
+		}
+	}
+	return {
+		kind: "text",
+		entries,
+		invalid,
+		complete: true,
+		notes: []
+	};
+};
+/**
+* This function reads a sitemap, a sitemap index, a feed or a text sitemap, and it counts its
+* steps in `meter`. The function `readSitemap` uses it. A test uses it to show that the time is linear.
+*/
+const scanSitemap = (text, base, limits = {}, meter = { steps: 0 }) => {
+	const maxEntries = Math.max(1, Math.floor(limits.maxEntries ?? SITEMAP_LIMITS.maxEntries));
+	const maxDepth = Math.max(1, Math.floor(limits.maxDepth ?? SITEMAP_LIMITS.maxDepth));
+	let at = text.charCodeAt(0) === 65279 ? 1 : 0;
+	while (at < text.length && isSpace(text.charCodeAt(at))) at += 1;
+	if (at < text.length && text[at] !== "<") return readLines(text.slice(at), maxEntries, meter);
+	const n = text.length;
+	const notes = [];
+	const entries = [];
+	const stack = [];
+	const prefixes = /* @__PURE__ */ new Map();
+	let kind = "unknown";
+	let invalid = 0;
+	let complete = true;
+	let done = false;
+	let feedBase = base;
+	let entry;
+	let field;
+	const stop = (note) => {
+		complete = false;
+		done = true;
+		notes.push(note);
+	};
+	/** This function tells if a name is a local name of one of the namespaces, or the local name with no prefix. */
+	const named = (qname, local, spaces) => {
+		const colon = qname.indexOf(":");
+		if (colon === -1) return qname === local;
+		return qname.slice(colon + 1) === local && spaces.includes(prefixes.get(qname.slice(0, colon)) ?? "");
+	};
+	const attribute = (tag, local, spaces) => {
+		for (const [key, raw] of tag.attributes) if (spaces === void 0 ? key === local : named(key, local, spaces) && key.includes(":")) return decodeText(raw, meter);
+	};
+	const fieldOf = (qname) => {
+		switch (kind) {
+			case "urlset":
+			case "sitemapindex":
+				if (named(qname, "loc", SITEMAP_NS)) return "loc";
+				if (named(qname, "lastmod", SITEMAP_NS)) return "lastmod";
+				return kind === "urlset" && named(qname, "changefreq", SITEMAP_NS) ? "changefreq" : void 0;
+			case "rss":
+				if (qname === "link") return "link";
+				if (qname === "guid") return "guid";
+				return qname === "pubDate" ? "date" : void 0;
+			case "atom": return named(qname, "updated", ATOM_NS) || named(qname, "published", ATOM_NS) ? "date" : void 0;
+			case "rdf":
+				if (named(qname, "link", RSS1_NS)) return "link";
+				return named(qname, "date", DC_NS) ? "date" : void 0;
+			default: return;
+		}
+	};
+	const isEntry = (qname, depth) => {
+		switch (kind) {
+			case "urlset": return depth === 2 && named(qname, "url", SITEMAP_NS);
+			case "sitemapindex": return depth === 2 && named(qname, "sitemap", SITEMAP_NS);
+			case "rss": return depth === 3 && qname === "item" && stack[1] === "channel";
+			case "atom": return depth === 2 && named(qname, "entry", ATOM_NS);
+			case "rdf": return depth === 2 && named(qname, "item", RSS1_NS);
+			default: return false;
+		}
+	};
+	const resolveBase = (tag, against) => {
+		const declared = attribute(tag, "xml:base");
+		if (declared === void 0) return against;
+		try {
+			return new URL(declared.trim(), against).href;
+		} catch {
+			return against;
+		}
+	};
+	const open = (tag, depth) => {
+		for (const [key, raw] of tag.attributes) if (key.startsWith("xmlns:")) {
+			const uri = decodeText(raw, meter);
+			if (uri !== void 0) prefixes.set(key.slice(6), uri.trim());
+		}
+		if (depth === 1) {
+			const local = tag.name.slice(tag.name.indexOf(":") + 1);
+			if (local === "urlset" || local === "sitemapindex" || local === "rss") kind = local;
+			else if (local === "feed") kind = "atom";
+			else if (local === "RDF") kind = "rdf";
+			else {
+				stop(`not a sitemap or a feed: the root element is <${short(tag.name)}>`);
+				return;
+			}
+			feedBase = resolveBase(tag, base);
+			return;
+		}
+		if (entry === void 0) {
+			if (isEntry(tag.name, depth)) entry = {
+				depth,
+				base: kind === "atom" ? resolveBase(tag, feedBase) : feedBase,
+				loc: void 0,
+				link: void 0,
+				guid: void 0,
+				lastmod: void 0,
+				changefreq: void 0,
+				about: kind === "rdf" ? attribute(tag, "about", RDF_NS) : void 0,
+				linkPage: false
+			};
+			return;
+		}
+		if (field !== void 0) {
+			field.bad = true;
+			return;
+		}
+		if (depth !== entry.depth + 1) return;
+		if (kind === "atom" && named(tag.name, "link", ATOM_NS)) {
+			const rel = (attribute(tag, "rel") ?? "alternate").trim().toLowerCase();
+			const href = attribute(tag, "href");
+			const type = (attribute(tag, "type") ?? "text/html").trim().toLowerCase();
+			const page = type.startsWith("text/html") || type.startsWith("application/xhtml");
+			if (rel === "alternate" && href !== void 0 && (entry.link === void 0 || page && !entry.linkPage)) {
+				entry.link = href;
+				entry.linkPage = page;
+			}
+			return;
+		}
+		const which = fieldOf(tag.name);
+		if (which === void 0) return;
+		if (which === "guid") {
+			if ((attribute(tag, "isPermaLink") ?? "true").trim().toLowerCase() === "false") return;
+		}
+		field = {
+			field: which,
+			depth,
+			parts: [],
+			length: 0,
+			bad: false
+		};
+	};
+	const finishEntry = (current) => {
+		const raw = kind === "urlset" || kind === "sitemapindex" ? current.loc : current.link ?? (kind === "rdf" ? current.about : current.guid);
+		const url = raw === void 0 ? void 0 : webUrl(raw, kind === "urlset" || kind === "sitemapindex" ? void 0 : current.base);
+		if (url === void 0) {
+			invalid += 1;
+			return;
+		}
+		const freq = current.changefreq?.trim().toLowerCase();
+		entries.push({
+			url,
+			lastmod: current.lastmod === void 0 ? null : feedDate(current.lastmod),
+			changefreq: freq !== void 0 && CHANGEFREQ.has(freq) && kind === "urlset" ? freq : null
+		});
+		if (entries.length >= maxEntries) stop(`the reader stopped at ${String(maxEntries)} entries`);
+	};
+	const close = (depth) => {
+		if (field !== void 0 && depth === field.depth) {
+			const finished = field;
+			field = void 0;
+			if (finished.bad || entry === void 0) return;
+			const value = finished.parts.join("").trim();
+			if (value.length === 0) return;
+			if (finished.field === "loc") entry.loc ??= value;
+			else if (finished.field === "link") entry.link ??= value;
+			else if (finished.field === "guid") entry.guid ??= value;
+			else if (finished.field === "lastmod" || finished.field === "date") entry.lastmod ??= value;
+			else entry.changefreq ??= value;
+			return;
+		}
+		if (entry !== void 0 && depth === entry.depth) {
+			const finished = entry;
+			entry = void 0;
+			finishEntry(finished);
+		}
+	};
+	const addText = (value) => {
+		if (field === void 0 || field.bad) return;
+		if (value === void 0 || field.length + value.length > SITEMAP_LIMITS.maxValue) {
+			field.bad = true;
+			return;
+		}
+		field.parts.push(value);
+		field.length += value.length;
+	};
+	while (at < n && !done) {
+		meter.steps += 1;
+		const lt = text.indexOf("<", at);
+		const end = lt === -1 ? n : lt;
+		if (end > at) {
+			meter.steps += end - at;
+			if (field !== void 0 && !field.bad) addText(end - at > SITEMAP_LIMITS.maxValue ? void 0 : decodeText(text.slice(at, end), meter));
+		}
+		if (lt === -1) break;
+		at = lt;
+		if (text.startsWith("<!--", at)) {
+			const endOf = text.indexOf("-->", at + 4);
+			if (endOf === -1) {
+				stop("a comment does not end");
+				break;
+			}
+			meter.steps += endOf + 3 - at;
+			at = endOf + 3;
+			continue;
+		}
+		if (text.startsWith("<![CDATA[", at)) {
+			const endOf = text.indexOf("]]>", at + 9);
+			if (endOf === -1) {
+				stop("a CDATA section does not end");
+				break;
+			}
+			meter.steps += endOf + 3 - at;
+			if (field !== void 0 && !field.bad) addText(endOf - at - 9 > SITEMAP_LIMITS.maxValue ? void 0 : text.slice(at + 9, endOf));
+			at = endOf + 3;
+			continue;
+		}
+		if (text.startsWith("<?", at)) {
+			const endOf = text.indexOf("?>", at + 2);
+			if (endOf === -1) {
+				stop("a processing instruction does not end");
+				break;
+			}
+			meter.steps += endOf + 2 - at;
+			at = endOf + 2;
+			continue;
+		}
+		if (text.startsWith("<!", at)) {
+			const declaration = declarationEnd(text, at + 2, meter);
+			if (declaration.end === -1) {
+				stop("a declaration does not end");
+				break;
+			}
+			if (declaration.subset && /^<!DOCTYPE/i.test(text.slice(at, at + 9))) notes.push("the file declares a DTD: the reader ignores it and expands no entity");
+			at = declaration.end;
+			continue;
+		}
+		if (text.startsWith("</", at)) {
+			const endOf = text.indexOf(">", at + 2);
+			if (endOf === -1) {
+				stop("an end tag does not end");
+				break;
+			}
+			meter.steps += endOf + 1 - at;
+			const name = text.slice(at + 2, Math.min(endOf, at + 300)).trim();
+			const top = stack.pop();
+			if (top !== name) {
+				stop(`the end tag </${short(name)}> does not match <${short(top)}>`);
+				break;
+			}
+			close(stack.length + 1);
+			at = endOf + 1;
+			if (stack.length === 0) break;
+			continue;
+		}
+		const tag = readTag(text, at + 1, meter);
+		if (tag === void 0) {
+			stop("a start tag is not well formed");
+			break;
+		}
+		at = tag.end;
+		const depth = stack.length + 1;
+		if (depth > maxDepth) {
+			stop(`the elements nest deeper than ${String(maxDepth)}`);
+			break;
+		}
+		open(tag, depth);
+		if (done) break;
+		if (tag.selfClosing) close(depth);
+		else stack.push(tag.name);
+		if (depth === 1 && tag.selfClosing) break;
+	}
+	if (!done && stack.length > 0) stop("the file ends inside an element");
+	if (kind === "unknown" && notes.length === 0) {
+		complete = false;
+		notes.push("not a sitemap or a feed: the file has no root element");
+	}
+	return {
+		kind,
+		entries,
+		invalid,
+		complete,
+		notes
+	};
+};
+/**
+* This function reads a sitemap, a sitemap index, an RSS or Atom feed, or a text sitemap. The
+* root element of the file tells its kind. The entries of a feed can have URLs relative to `base`
+* (the URL of the file). The entries of a sitemap must have absolute URLs.
+*/
+const readSitemap = (text, base, limits) => scanSitemap(text, base, limits);
+//#endregion
+//#region src/discover.ts
+/**
+* Discovery finds the pages of a site from its sitemaps and its feeds. Each file is a fetch of the
+* crawler, so each polite rule applies to it. These rules are robots.txt, one request at a time for
+* each host, the pace, the stops, the egress policy and the capture log. These are the sources, in sequence:
+*
+* 1. The page at the URL of the site, for the RSS and Atom feeds that it links.
+* 2. The feeds that the caller gives.
+* 3. The sitemaps that the caller gives, and the `Sitemap:` lines of robots.txt. When there are
+*    none, `/sitemap.xml`.
+* 4. The sitemaps of each sitemap index, the newest first, to a maximum depth.
+*
+* Discovery keeps a URL only when it is in scope (the same site by default) and robots.txt permits
+* it. It stops at a maximum of files, a maximum of URLs and a maximum time.
+*/
+/** These are the default limits of `discover`. */
+const DISCOVER_LIMITS = {
+	/** The most sitemap and feed files that one discovery reads. */
+	maxFiles: 20,
+	/** The most URLs that one discovery gives. */
+	maxUrls: 1e4,
+	/** The most bytes of one file, after decompression: 10 MiB. */
+	maxBytes: 10485760,
+	/** The most levels of sitemap index under a file that discovery found first. */
+	maxNesting: 3,
+	/** The longest time of one discovery: 1 minute. */
+	budgetMs: 6e4,
+	/** The most origins whose robots.txt one discovery reads for the URLs that it finds. */
+	maxOrigins: 10,
+	/** The most feeds that discovery reads from the links of the page. */
+	maxLinkedFeeds: 5
+};
+const PAGE_ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
+const SITEMAP_ACCEPT = "application/xml,text/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5";
+const FEED_ACCEPT = "application/rss+xml,application/atom+xml,application/xml;q=0.9,text/xml;q=0.8,*/*;q=0.5";
+const FEED_KINDS = /* @__PURE__ */ new Set([
+	"rss",
+	"atom",
+	"rdf"
+]);
+const whole$1 = (name, value, fallback, min) => {
+	if (value === void 0) return fallback;
+	if (!Number.isInteger(value) || value < min) throw new RangeError(`${name} must be an integer of ${String(min)} or more, not ${String(value)}`);
+	return value;
+};
+/**
+* This function reads a body that is gzip data (a `.xml.gz` sitemap). It keeps at most `maxBytes`
+* of the decompressed data, so a decompression bomb stops at the limit.
+*/
+const gunzipBounded = async (body, maxBytes) => {
+	const gunzip = createGunzip();
+	const chunks = [];
+	let size = 0;
+	let truncated = false;
+	let error;
+	gunzip.end(body);
+	try {
+		for await (const chunk of gunzip) {
+			const room = maxBytes - size;
+			if (chunk.length > room) {
+				chunks.push(chunk.subarray(0, room));
+				size += room;
+				truncated = true;
+				break;
+			}
+			chunks.push(chunk);
+			size += chunk.length;
+		}
+	} catch (caught) {
+		error = caught instanceof Error ? caught.message : String(caught);
+		truncated = true;
+	} finally {
+		gunzip.destroy();
+	}
+	return {
+		bytes: Buffer.concat(chunks, size),
+		truncated,
+		...error === void 0 ? {} : { error }
+	};
+};
+const isGzip = (body) => body[0] === 31 && body[1] === 139;
+const parsedUrl = (value, base) => {
+	try {
+		const url = base === void 0 ? new URL(value.trim()) : new URL(value.trim(), base);
+		return url.protocol === "http:" || url.protocol === "https:" ? url : void 0;
+	} catch {
+		return;
+	}
+};
+/** This function gives the value of a promise, or rejects when the signal aborts first. */
+const untilAbort = (work, signal) => {
+	if (signal === void 0) return work;
+	return new Promise((resolve, reject) => {
+		const cancel = () => reject(signal.reason);
+		if (signal.aborted) cancel();
+		signal.addEventListener("abort", cancel, { once: true });
+		work.then(resolve, reject).finally(() => signal.removeEventListener("abort", cancel));
+	});
+};
+/** This function tells if the note of a fetch says that its deadline came first. */
+const pastDeadline = (note) => note?.startsWith(DEADLINE_NOTE) === true;
+/**
+* This function finds the pages of a site from its sitemaps and its feeds. `PoliteCrawler.discover`
+* uses it. Each file is a fetch of the crawler with the recipe `sitemap` or `feed`, so the
+* capture log records it, and robots.txt applies to it.
+*/
+const discoverSite = async (runtime, site, options = {}) => {
+	const start = parsedUrl(site);
+	if (start === void 0) throw new Error(`discover wants an http(s) URL, not "${site}"`);
+	const maxFiles = whole$1("maxFiles", options.maxFiles, DISCOVER_LIMITS.maxFiles, 1);
+	const maxUrls = whole$1("maxUrls", options.maxUrls, DISCOVER_LIMITS.maxUrls, 1);
+	const maxBytes = whole$1("maxBytes", options.maxBytes, DISCOVER_LIMITS.maxBytes, 1);
+	const maxNesting = whole$1("maxNesting", options.maxNesting, DISCOVER_LIMITS.maxNesting, 0);
+	const maxAge = options.maxAge ?? "24h";
+	const maxAgeMs = maxAge === "auto" ? "auto" : durationMs(maxAge);
+	const budgetMs = options.budgetMs ?? DISCOVER_LIMITS.budgetMs;
+	if (!(budgetMs >= 0)) throw new RangeError(`budgetMs must be 0 or more, not ${String(budgetMs)}`);
+	const now = () => Date.parse(runtime.now());
+	const deadline = Math.min(now() + budgetMs, options.deadline ?? Number.POSITIVE_INFINITY);
+	const crawler = runtime.crawler;
+	const signal = options.signal;
+	const cancelled = () => signal?.aborted === true;
+	const marks = {
+		scope: options.scope,
+		labels: options.labels
+	};
+	const inScope = scopeTest([start], options);
+	const urls = /* @__PURE__ */ new Map();
+	const sources = [];
+	const dropped = {
+		scope: 0,
+		robots: 0,
+		duplicate: 0,
+		invalid: 0,
+		limit: 0
+	};
+	const origins = /* @__PURE__ */ new Set();
+	const queued = /* @__PURE__ */ new Set();
+	const pending = [];
+	let stopped;
+	let note = null;
+	let files = 0;
+	let next = 0;
+	const stopNote = () => {
+		switch (stopped) {
+			case "max-files": return `discovery read ${String(maxFiles)} files, its limit, and ${String(pending.length - next)} more files wait`;
+			case "max-urls": return `discovery found ${String(maxUrls)} URLs, its limit`;
+			case "budget": return `the budget of ${String(budgetMs)} ms ended`;
+			case "cancelled": return "the caller cancelled the discovery";
+			case "closed": return "the crawler closed";
+			default: return null;
+		}
+	};
+	const result = () => ({
+		site: start.origin,
+		urls: [...urls.values()].map((url, order) => ({
+			url,
+			order
+		})).sort((a, b) => (b.url.lastmod === null ? -Infinity : Date.parse(b.url.lastmod)) - (a.url.lastmod === null ? -Infinity : Date.parse(a.url.lastmod)) || a.order - b.order).map(({ url }) => url),
+		sources,
+		dropped,
+		stopped: stopped ?? "done",
+		note: note ?? stopNote()
+	});
+	/** This function sets the reason of the stop from the note of a fetch, when the note gives one. */
+	const stoppedBy = (fetched) => {
+		if (cancelled()) stopped = "cancelled";
+		else if (pastDeadline(fetched.note)) stopped = "budget";
+		else if (fetched.note === "crawler closed") stopped = "closed";
+		else return false;
+		return true;
+	};
+	const enqueue = (url, via, from, nesting = 0) => {
+		const parsed = parsedUrl(url, start.href);
+		if (parsed === void 0) {
+			dropped.invalid += 1;
+			return;
+		}
+		const key = cacheKeyOf(parsed.href);
+		if (queued.has(key)) return;
+		queued.add(key);
+		pending.push({
+			url: parsed.href,
+			via,
+			from,
+			nesting
+		});
+	};
+	if (cancelled()) {
+		stopped = "cancelled";
+		return result();
+	}
+	let sitemapLines = [];
+	try {
+		const robots = await untilAbort(crawler.readRobots(start.href, marks), signal);
+		if (robots.robots.unreachable === "crawler closed") {
+			stopped = "closed";
+			return result();
+		}
+		if (robots.robots.unreachable !== void 0) {
+			stopped = "robots";
+			note = `robots.txt could not be read (${robots.robots.unreachable}), so nothing on the site is fetched`;
+			return result();
+		}
+		sitemapLines = robots.robots.sitemaps;
+	} catch {
+		stopped = cancelled() ? "cancelled" : "closed";
+		return result();
+	}
+	const fetchFile = (url, recipe) => crawler.fetch({
+		url,
+		mode: "page",
+		recipe,
+		accept: recipe === "feed" ? FEED_ACCEPT : SITEMAP_ACCEPT,
+		maxBytes,
+		maxAgeMs,
+		onBehalfOf: options.onBehalfOf,
+		signal,
+		scope: options.scope,
+		labels: options.labels,
+		deadline
+	});
+	if (options.linkedFeeds !== false) {
+		const page = await fetchAndIndex(runtime, {
+			url: start.href,
+			mode: "page",
+			recipe: "page",
+			accept: PAGE_ACCEPT,
+			maxAgeMs,
+			onBehalfOf: options.onBehalfOf,
+			signal,
+			scope: options.scope,
+			labels: options.labels,
+			deadline
+		});
+		if (stoppedBy(page.fetched)) return result();
+		const doc = page.docs[0];
+		const body = page.fetched.body;
+		const html = /html/i.test(page.fetched.contentType ?? "text/html");
+		if (body !== void 0 && html && doc !== void 0 && !isHeldBack(doc)) {
+			let linked = [];
+			try {
+				linked = extractPage(decodeBody(body, page.fetched.contentType ?? null), page.fetched.url).feeds ?? [];
+			} catch {}
+			let taken = 0;
+			for (const feed of linked) {
+				const parsed = parsedUrl(feed);
+				if (parsed === void 0 || !inScope(parsed)) {
+					dropped.scope += 1;
+					continue;
+				}
+				if (taken >= DISCOVER_LIMITS.maxLinkedFeeds) {
+					dropped.limit += 1;
+					continue;
+				}
+				taken += 1;
+				enqueue(parsed.href, "feed", "page");
+			}
+		}
+	}
+	for (const feed of options.feeds ?? []) enqueue(feed, "feed", "given");
+	for (const sitemap of options.sitemaps ?? []) enqueue(sitemap, "sitemap", "given");
+	for (const sitemap of sitemapLines) enqueue(sitemap, "sitemap", "robots.txt");
+	if ((options.sitemaps ?? []).length === 0 && sitemapLines.length === 0) enqueue(new URL("/sitemap.xml", start.origin).href, "sitemap", "default");
+	/** This function keeps the page URLs of one file that are new, in scope and permitted. */
+	const keep = async (read, source, via) => {
+		let kept = 0;
+		dropped.invalid += read.invalid;
+		for (const entry of read.entries) {
+			if (urls.size >= maxUrls) {
+				stopped = "max-urls";
+				return kept;
+			}
+			const known = urls.get(entry.url);
+			if (known !== void 0) {
+				dropped.duplicate += 1;
+				if (entry.lastmod !== null && (known.lastmod === null || Date.parse(entry.lastmod) > Date.parse(known.lastmod))) urls.set(entry.url, {
+					...known,
+					lastmod: entry.lastmod
+				});
+				continue;
+			}
+			const url = new URL(entry.url);
+			if (!inScope(url)) {
+				dropped.scope += 1;
+				continue;
+			}
+			if (!origins.has(url.origin)) {
+				if (origins.size >= DISCOVER_LIMITS.maxOrigins) {
+					dropped.limit += 1;
+					continue;
+				}
+				origins.add(url.origin);
+			}
+			let allowed;
+			try {
+				allowed = (await crawler.robotsAllow(entry.url, {
+					onBehalfOf: options.onBehalfOf,
+					signal,
+					...marks
+				})).allowed;
+			} catch {
+				stopped = "cancelled";
+				return kept;
+			}
+			if (!allowed) {
+				dropped.robots += 1;
+				continue;
+			}
+			urls.set(entry.url, {
+				url: entry.url,
+				lastmod: entry.lastmod,
+				changefreq: entry.changefreq,
+				via,
+				source
+			});
+			kept += 1;
+		}
+		return kept;
+	};
+	for (; next < pending.length && stopped === void 0; next += 1) {
+		if (cancelled()) {
+			stopped = "cancelled";
+			break;
+		}
+		if (now() >= deadline) {
+			stopped = "budget";
+			break;
+		}
+		if (files >= maxFiles) {
+			stopped = "max-files";
+			break;
+		}
+		const file = pending[next];
+		files += 1;
+		const fetched = await fetchFile(file.url, file.via);
+		const base = {
+			url: fetched.url,
+			via: file.via,
+			from: file.from,
+			outcome: fetched.outcome,
+			status: fetched.status,
+			capture: fetched.capture ?? null,
+			fetched_at: fetched.fetchedAt ?? null,
+			from_cache: fetched.fromCache
+		};
+		if (stoppedBy(fetched)) {
+			sources.push({
+				...base,
+				kind: null,
+				entries: 0,
+				urls: 0,
+				truncated: false,
+				note: fetched.note ?? null
+			});
+			break;
+		}
+		const body = fetched.body;
+		if (fetched.outcome !== "ok" && fetched.outcome !== "not-modified" || body === void 0) {
+			sources.push({
+				...base,
+				kind: null,
+				entries: 0,
+				urls: 0,
+				truncated: false,
+				note: fetched.note ?? `${fetched.outcome} ${String(fetched.status)}`
+			});
+			continue;
+		}
+		let bytes = body;
+		let truncated = fetched.truncated === true;
+		const notes = [];
+		if (isGzip(body)) {
+			const unzipped = await gunzipBounded(body, maxBytes);
+			bytes = unzipped.bytes;
+			truncated ||= unzipped.truncated;
+			if (unzipped.error !== void 0) notes.push(`the gzip data ended early (${unzipped.error})`);
+		}
+		const read = readSitemap(decodeBody(bytes, isGzip(body) ? null : fetched.contentType ?? null), fetched.url);
+		if (truncated) notes.push(`the file is longer than ${String(maxBytes)} bytes, so discovery read its first part`);
+		notes.push(...read.notes);
+		let kept = 0;
+		if (read.kind === "sitemapindex") {
+			const children = [...read.entries].sort((a, b) => (b.lastmod === null ? -Infinity : Date.parse(b.lastmod)) - (a.lastmod === null ? -Infinity : Date.parse(a.lastmod)));
+			for (const child of children) if (!inScope(new URL(child.url))) dropped.scope += 1;
+			else if (file.nesting + 1 > maxNesting) dropped.limit += 1;
+			else enqueue(child.url, "sitemap", "index", file.nesting + 1);
+			if (file.nesting + 1 > maxNesting && children.length > 0) notes.push(`the files of the index are deeper than maxNesting (${String(maxNesting)}), so discovery reads none of them`);
+		} else if (read.kind !== "unknown") kept = await keep(read, fetched.url, FEED_KINDS.has(read.kind) ? "feed" : "sitemap");
+		sources.push({
+			...base,
+			kind: read.kind,
+			entries: read.entries.length,
+			urls: kept,
+			truncated,
+			note: notes.length > 0 ? notes.join("; ") : null
+		});
+	}
+	if (stopped === void 0 && urls.size >= maxUrls) stopped = "max-urls";
+	return result();
+};
+//#endregion
+//#region src/frontier.ts
+/**
+* The frontier is a small crawl of a site. It starts at the seeds. It follows the links of each page
+* that it fetched, to a maximum of pages, a maximum depth and a maximum time.
+*
+* Each page is a fetch of the crawler, so each polite rule applies. These rules are robots.txt, one
+* request at a time for each host, the pace, the stops, the egress policy and the capture log. The
+* frontier does not start a second request to a host with a request in progress. The crawler
+* limits the hosts in progress (`maxActive`).
+*
+* The state of the frontier is in memory, for one call. A later version can make it resumable.
+*/
+/** These are the default limits of `crawl`, and the limits of the memory of the frontier. */
+const CRAWL_LIMITS = {
+	/** The most pages that one crawl gives. */
+	maxPages: 100,
+	/** The most links from a seed to a page. A seed has the depth 0. */
+	maxDepth: 3,
+	/** The longest time of one crawl: 10 minutes. */
+	budgetMs: 6e5,
+	/** The most hosts that get a request of the crawl at one time. */
+	hostsAtOnce: 16,
+	/** The most URLs that wait in the frontier. */
+	maxQueue: 1e4,
+	/** The most URLs that the frontier remembers, so it fetches none of them two times. */
+	maxSeen: 1e5
+};
+/** The lower depth first, then the lower tier, then the newer `lastmod`, then the first found. */
+const before = (a, b) => a.depth - b.depth || a.tier - b.tier || b.lastmod - a.lastmod || a.seq - b.seq;
+/** A binary heap of the URLs of one host, the first URL to fetch at the top. */
+var Heap = class {
+	items = [];
+	get size() {
+		return this.items.length;
+	}
+	peek() {
+		return this.items[0];
+	}
+	push(item) {
+		const items = this.items;
+		items.push(item);
+		let at = items.length - 1;
+		while (at > 0) {
+			const parent = at - 1 >> 1;
+			if (before(items[at], items[parent]) >= 0) break;
+			[items[at], items[parent]] = [items[parent], items[at]];
+			at = parent;
+		}
+	}
+	pop() {
+		const items = this.items;
+		const top = items[0];
+		const last = items.pop();
+		if (top === void 0 || last === void 0 || items.length === 0) return top;
+		items[0] = last;
+		let at = 0;
+		for (;;) {
+			const left = at * 2 + 1;
+			const right = left + 1;
+			let first = at;
+			if (left < items.length && before(items[left], items[first]) < 0) first = left;
+			if (right < items.length && before(items[right], items[first]) < 0) first = right;
+			if (first === at) return top;
+			[items[at], items[first]] = [items[first], items[at]];
+			at = first;
+		}
+	}
+};
+const hostOf = (url) => `${url.hostname.toLowerCase().replace(/\.$/, "")}${url.port === "" ? "" : `:${url.port}`}`;
+const matcher = (pattern) => {
+	if (pattern instanceof RegExp) return (url) => {
+		pattern.lastIndex = 0;
+		return pattern.test(url.href);
+	};
+	if (typeof pattern !== "string" || pattern.length === 0) throw new RangeError("a URL pattern is a regular expression or a text that is not empty");
+	return pattern.startsWith("/") ? (url) => matchesPattern(pattern, `${url.pathname}${url.search}`) : (url) => matchesPattern(pattern, url.href);
+};
+const whole = (name, value, fallback, min) => {
+	if (value === void 0) return fallback;
+	if (!Number.isInteger(value) || value < min) throw new RangeError(`${name} must be an integer of ${String(min)} or more, not ${String(value)}`);
+	return value;
+};
+const failedPage = (url, note) => ({
+	url,
+	doc: null,
+	outcome: "error",
+	status: 0,
+	title: null,
+	chars: 0,
+	rendered: false,
+	structured: [],
+	links: 0,
+	captures: [],
+	capture: null,
+	capture_count: 0,
+	fetched_at: null,
+	from_cache: false,
+	held_back: false,
+	note
+});
+const REFUSALS$1 = /* @__PURE__ */ new Set([
+	"skipped",
+	"blocked",
+	"challenge",
+	"payment-required"
+]);
+/**
+* This function crawls a site from its seeds. `PoliteCrawler.crawl` uses it. It gives each page
+* when its fetch ends, and a summary at the end. The seeds come first, then the pages of the
+* sitemaps (the newest first), then the links, the lower depth first.
+*/
+const crawlSite = (runtime, seeds, options = {}) => {
+	let resolveSummary = () => void 0;
+	const summary = new Promise((resolve) => {
+		resolveSummary = resolve;
+	});
+	let used = false;
+	async function* pages() {
+		const now = () => Date.parse(runtime.now());
+		const startedAt = now();
+		const abort = new AbortController();
+		const outer = options.signal;
+		const cancel = () => abort.abort(outer?.reason);
+		if (outer?.aborted === true) cancel();
+		outer?.addEventListener("abort", cancel, { once: true });
+		const inflight = /* @__PURE__ */ new Map();
+		const heaps = /* @__PURE__ */ new Map();
+		const seen = /* @__PURE__ */ new Set();
+		const counts = {
+			pages: 0,
+			fetched: 0,
+			from_cache: 0,
+			refused: 0,
+			failed: 0,
+			sitemap_urls: 0
+		};
+		const skips = {
+			scope: 0,
+			pattern: 0,
+			depth: 0,
+			held_back: 0,
+			limit: 0,
+			duplicate: 0
+		};
+		const got = /* @__PURE__ */ new Set();
+		let queued = 0;
+		let returned = 0;
+		let seq = 0;
+		let started = 0;
+		let stopping;
+		let reason;
+		let finished = false;
+		let maxPages = CRAWL_LIMITS.maxPages;
+		let budgetMs = CRAWL_LIMITS.budgetMs;
+		const crawler = runtime.crawler;
+		const marks = {
+			scope: options.scope,
+			labels: options.labels
+		};
+		try {
+			await options.ready?.();
+			maxPages = whole("maxPages", options.maxPages, CRAWL_LIMITS.maxPages, 1);
+			const maxDepth = whole("maxDepth", options.maxDepth, CRAWL_LIMITS.maxDepth, 0);
+			budgetMs = options.budgetMs ?? CRAWL_LIMITS.budgetMs;
+			if (!(budgetMs >= 0)) throw new RangeError(`budgetMs must be 0 or more, not ${String(budgetMs)}`);
+			const maxAge = options.maxAge ?? "24h";
+			const maxAgeMs = maxAge === "auto" ? "auto" : durationMs(maxAge);
+			const includes = (options.include ?? []).map(matcher);
+			const excludes = (options.exclude ?? []).map(matcher);
+			const deadline = startedAt + budgetMs;
+			const list = typeof seeds === "string" ? [seeds] : [...seeds];
+			if (list.length === 0) throw new RangeError("a crawl needs one seed URL or more");
+			const starts = list.flatMap((seed) => {
+				try {
+					const url = new URL(seed);
+					return url.protocol === "http:" || url.protocol === "https:" ? [url] : [];
+				} catch {
+					return [];
+				}
+			});
+			const inScope = scopeTest(starts, options);
+			const followed = (url) => (includes.length === 0 || includes.some((test) => test(url))) && !excludes.some((test) => test(url));
+			/** This function adds a URL to the frontier, when it is new, in scope and not too deep. */
+			const offer = (raw, depth, tier, lastmod, source, referrer) => {
+				let url;
+				try {
+					url = new URL(raw);
+				} catch {
+					url = void 0;
+				}
+				const web = url !== void 0 && (url.protocol === "http:" || url.protocol === "https:");
+				if (!web && source !== "seed") return;
+				const key = cacheKeyOf(url?.href ?? raw);
+				if (seen.has(key)) return;
+				if (seen.size >= CRAWL_LIMITS.maxSeen) {
+					skips.limit += 1;
+					return;
+				}
+				seen.add(key);
+				if (source !== "seed" && url !== void 0) {
+					if (!inScope(url)) {
+						skips.scope += 1;
+						return;
+					}
+					if (!followed(url)) {
+						skips.pattern += 1;
+						return;
+					}
+					if (depth > maxDepth) {
+						skips.depth += 1;
+						return;
+					}
+				}
+				if (queued >= CRAWL_LIMITS.maxQueue) {
+					skips.limit += 1;
+					return;
+				}
+				const host = url === void 0 || !web ? `not-web:${raw}` : hostOf(url);
+				let heap = heaps.get(host);
+				if (heap === void 0) {
+					heap = new Heap();
+					heaps.set(host, heap);
+				}
+				seq += 1;
+				heap.push({
+					url: raw,
+					key,
+					host,
+					depth,
+					tier,
+					lastmod,
+					seq,
+					source,
+					referrer
+				});
+				queued += 1;
+			};
+			/** This function gives the first URL of a host with no request of the crawl in progress. */
+			const pick = () => {
+				for (;;) {
+					let best;
+					for (const [host, heap] of heaps) {
+						if (inflight.has(host)) continue;
+						const top = heap.peek();
+						if (top === void 0) continue;
+						if (best === void 0 || before(top, best.heap.peek()) < 0) best = {
+							host,
+							heap
+						};
+					}
+					if (best === void 0) return void 0;
+					const item = best.heap.pop();
+					queued -= 1;
+					if (best.heap.size === 0) heaps.delete(best.host);
+					if (!got.has(item.key)) return item;
+					skips.duplicate += 1;
+				}
+			};
+			const fetchItem = async (item) => {
+				try {
+					if (item.source !== "seed") {
+						if (!(await crawler.robotsAllow(item.url, {
+							onBehalfOf: options.onBehalfOf,
+							signal: abort.signal,
+							...marks
+						})).allowed) return {
+							kind: "refused",
+							item
+						};
+					}
+					return {
+						kind: "page",
+						item,
+						page: await fetchPage(runtime, item.url, {
+							render: options.render ?? "auto",
+							maxAgeMs,
+							onBehalfOf: options.onBehalfOf,
+							signal: abort.signal,
+							scope: options.scope,
+							labels: options.labels,
+							deadline
+						})
+					};
+				} catch (error) {
+					if (abort.signal.aborted) return {
+						kind: "aborted",
+						item
+					};
+					const message = error instanceof Error ? error.message : String(error);
+					return {
+						kind: "page",
+						item,
+						page: failedPage(item.url, `crawl error: ${message}`)
+					};
+				}
+			};
+			/** This function adds the links of a page to the frontier. The canonical URL of the page is known after its fetch. */
+			const follow = (page, item) => {
+				if (page.doc === null) return;
+				const doc = runtime.index.get(page.doc);
+				if (doc === void 0) return;
+				for (const same of [doc.url, doc.meta["canonical"]]) {
+					if (typeof same !== "string") continue;
+					seen.add(cacheKeyOf(same));
+					got.add(cacheKeyOf(same));
+				}
+				const links = doc.meta["links"];
+				if (!Array.isArray(links) || links.length === 0) return;
+				if (isHeldBack(doc)) {
+					skips.held_back += 1;
+					return;
+				}
+				for (const link of links) if (typeof link?.href === "string") offer(link.href, item.depth + 1, 2, Number.NEGATIVE_INFINITY, "link", page.url);
+			};
+			for (const seed of list) offer(seed, 0, 0, Number.NEGATIVE_INFINITY, "seed", null);
+			if (options.useSitemaps === true) {
+				const sites = [...new Set(starts.map((start) => start.origin))];
+				for (const site of sites) {
+					if (abort.signal.aborted || now() >= deadline) break;
+					try {
+						const found = await discoverSite(runtime, site, {
+							sameSite: options.sameSite,
+							allowHosts: options.allowHosts,
+							maxUrls: Math.min(1e4, Math.max(50, maxPages * 4)),
+							linkedFeeds: false,
+							maxAge,
+							onBehalfOf: options.onBehalfOf,
+							signal: abort.signal,
+							scope: options.scope,
+							labels: options.labels,
+							deadline
+						});
+						counts.sitemap_urls += found.urls.length;
+						for (const url of found.urls) offer(url.url, 0, 1, url.lastmod === null ? Number.NEGATIVE_INFINITY : Date.parse(url.lastmod), "sitemap", null);
+					} catch {}
+				}
+			}
+			for (;;) {
+				if (stopping === void 0 && abort.signal.aborted) stopping = "cancelled";
+				if (stopping === void 0 && now() >= deadline) stopping = "budget";
+				while (stopping === void 0 && inflight.size < CRAWL_LIMITS.hostsAtOnce && started < maxPages) {
+					const item = pick();
+					if (item === void 0) break;
+					started += 1;
+					inflight.set(item.host, fetchItem(item));
+				}
+				if (inflight.size === 0) break;
+				const done = await Promise.race(inflight.values());
+				inflight.delete(done.item.host);
+				if (done.kind === "aborted") {
+					started -= 1;
+					stopping ??= "cancelled";
+					continue;
+				}
+				if (done.kind === "refused") {
+					started -= 1;
+					counts.refused += 1;
+					continue;
+				}
+				const page = done.page;
+				if (pastDeadline(page.note)) {
+					started -= 1;
+					returned += 1;
+					stopping ??= "budget";
+					continue;
+				}
+				if (abort.signal.aborted) {
+					started -= 1;
+					stopping ??= "cancelled";
+					continue;
+				}
+				if (page.outcome === "skipped" && page.note === "crawler closed") {
+					started -= 1;
+					stopping ??= "closed";
+					continue;
+				}
+				counts.pages += 1;
+				if (page.outcome === "ok" || page.outcome === "not-modified") {
+					counts.fetched += 1;
+					if (page.from_cache) counts.from_cache += 1;
+				} else if (REFUSALS$1.has(page.outcome)) counts.refused += 1;
+				else counts.failed += 1;
+				follow(page, done.item);
+				yield {
+					...page,
+					depth: done.item.depth,
+					source: done.item.source,
+					referrer: done.item.referrer
+				};
+			}
+			stopping ??= queued > 0 && started >= maxPages ? "max-pages" : "done";
+			finished = true;
+		} catch (error) {
+			stopping = "error";
+			reason = error instanceof Error ? error.message : String(error);
+			finished = true;
+			throw error;
+		} finally {
+			if (!finished) {
+				stopping ??= "cancelled";
+				reason ??= "the caller stopped the loop";
+			}
+			outer?.removeEventListener("abort", cancel);
+			abort.abort(/* @__PURE__ */ new Error("the crawl ended"));
+			await Promise.allSettled(inflight.values());
+			const stop = stopping ?? "done";
+			reason ??= stop === "done" ? "the crawl fetched each page in scope that it found" : stop === "max-pages" ? `the crawl gave ${String(maxPages)} pages, its limit (maxPages)` : stop === "budget" ? `the budget of ${String(budgetMs)} ms ended (budgetMs)` : stop === "cancelled" ? "the caller cancelled the crawl" : "the crawler closed";
+			resolveSummary({
+				...counts,
+				skipped: skips.scope + skips.pattern + skips.depth + skips.held_back + skips.limit + skips.duplicate,
+				skips,
+				queued: queued + returned,
+				stopped: stop,
+				reason,
+				elapsed_ms: Math.max(0, now() - startedAt)
+			});
+		}
+	}
+	return {
+		summary,
+		[Symbol.asyncIterator]() {
+			if (used) throw new Error("a crawl runs one time: call crawl again for a new crawl");
+			used = true;
+			return pages();
+		}
+	};
+};
+//#endregion
+//#region src/hoststate.ts
+/** A stop with no end goes into the table as this value, which is past each date that JavaScript can hold. */
+const NO_END = 9e15;
+/** A process that shares the store asks again after this wait, at most, during the lease of a different process. */
+const SHARED_WAIT_MS = 1e3;
+/** The store removes the rows of idle hosts after this many reservations. */
+const SWEEP_EVERY = 1e3;
+const verdictOf = (value) => value === "works" || value === "fails" ? value : "unknown";
+const memoOf = (row) => ({
+	skipEtag: row.skip_etag === 1,
+	etag: verdictOf(row.etag),
+	lastModified: verdictOf(row.last_modified),
+	etagMisses: row.etag_misses,
+	dateMisses: row.date_misses,
+	sinceProbe: row.since_probe,
+	changedAt: row.changed_at
+});
+/**
+* This store keeps the state of each host in a SQLite file. Two or more processes on one store
+* share it, so they share the pace and the stops of each host. Thus a host gets one request at a
+* time from all of them. Each method is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock
+* of a different process (`busy_timeout`).
+*
+* The lease of a request belongs to the store that gave it. A different process that asks during
+* the lease gets a wait of one second or less, and asks again. A process that stops during a
+* request holds its lease until the lease ends (5 minutes by default).
+*
+* The table `validators` keeps the validator memo of each host (`ValidatorMemo`), so the processes
+* share it and it stays after a restart. A file of version 0.1.0 gets the table when it opens.
+*
+* The state is not a projection of the capture log. A deletion of the file loses only the pace,
+* the counts, the stops and the memos, and the crawler starts them again.
+*/
+var SqliteHostStateStore = class {
+	path;
+	db;
+	statements = /* @__PURE__ */ new Map();
+	/** This id names the leases of this store. */
+	owner = randomUUID();
+	reservations = 0;
+	closed = false;
+	pollMs;
+	constructor(path, options = {}) {
+		this.path = path;
+		if (path !== ":memory:") mkdirSync(dirname(path), {
+			recursive: true,
+			mode: 448
+		});
+		this.pollMs = Math.max(1, Math.min(SHARED_WAIT_MS, options.pollMs ?? SHARED_WAIT_MS));
+		this.db = new DatabaseSync(path);
+		const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
+		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
+		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
+		this.db.exec("PRAGMA synchronous = NORMAL;");
+		this.migrate();
+	}
+	/**
+	* This method makes the tables, or adds the tables of a later version to an older file. It is one
+	* transaction, so two processes that open one old file at the same time do not collide.
+	*/
+	migrate() {
+		this.step(() => {
+			this.db.exec(`CREATE TABLE IF NOT EXISTS hosts (
+        host TEXT PRIMARY KEY,
+        next_at REAL NOT NULL DEFAULT 0,
+        leased_until REAL NOT NULL DEFAULT 0,
+        owner TEXT,
+        failures INTEGER NOT NULL DEFAULT 0,
+        waits INTEGER NOT NULL DEFAULT 0,
+        stop_until REAL,
+        stop_reason TEXT
+      )`);
+			this.db.exec(`CREATE TABLE IF NOT EXISTS validators (
+        host TEXT PRIMARY KEY,
+        skip_etag INTEGER NOT NULL DEFAULT 0,
+        etag TEXT NOT NULL DEFAULT 'unknown',
+        last_modified TEXT NOT NULL DEFAULT 'unknown',
+        etag_misses INTEGER NOT NULL DEFAULT 0,
+        date_misses INTEGER NOT NULL DEFAULT 0,
+        since_probe INTEGER NOT NULL DEFAULT 0,
+        changed_at REAL NOT NULL DEFAULT 0
+      )`);
+			const { user_version: version } = this.db.prepare("PRAGMA user_version").get();
+			if (version < 1) this.db.exec(`PRAGMA user_version = ${String(1)}`);
+		});
+	}
+	stmt(sql) {
+		let statement = this.statements.get(sql);
+		if (statement === void 0) {
+			statement = this.db.prepare(sql);
+			this.statements.set(sql, statement);
+		}
+		return statement;
+	}
+	/** This method does one step in one transaction. */
+	step(work) {
+		if (this.closed) throw new Error("the host state store is closed");
+		this.db.exec("BEGIN IMMEDIATE");
+		try {
+			const result = work();
+			this.db.exec("COMMIT");
+			return result;
+		} catch (error) {
+			if (this.db.isTransaction) this.db.exec("ROLLBACK");
+			throw error;
+		}
+	}
+	row(host) {
+		this.stmt("INSERT OR IGNORE INTO hosts (host) VALUES (?)").run(host);
+		return this.stmt("SELECT next_at, leased_until, owner, failures, waits, stop_until, stop_reason FROM hosts WHERE host = ?").get(host);
+	}
+	/** This method removes the hosts that have nothing to remember: no rest, no lease, no counts and no stop. */
+	sweep(now) {
+		this.stmt(`DELETE FROM hosts WHERE next_at <= ? AND leased_until <= ? AND failures = 0 AND waits = 0
+         AND (stop_until IS NULL OR stop_until <= ?)`).run(now, now, now);
+	}
+	reserve(host, now, leaseMs) {
+		return this.step(() => {
+			this.reservations += 1;
+			if (this.reservations % SWEEP_EVERY === 0) this.sweep(now);
+			const row = this.row(host);
+			if (row.leased_until > now) return row.owner === this.owner ? row.leased_until - now : Math.min(this.pollMs, row.leased_until - now);
+			if (row.next_at > now) return row.next_at - now;
+			this.stmt("UPDATE hosts SET leased_until = ?, owner = ? WHERE host = ?").run(now + leaseMs, this.owner, host);
+			return 0;
+		});
+	}
+	record(host, result) {
+		return this.step(() => {
+			const row = this.row(host);
+			let { failures, waits } = row;
+			if (result === "success") {
+				failures = 0;
+				waits = 0;
+			} else if (result === "failure") failures += 1;
+			else if (result === "wait") waits += 1;
+			if (failures !== row.failures || waits !== row.waits) this.stmt("UPDATE hosts SET failures = ?, waits = ? WHERE host = ?").run(failures, waits, host);
+			return {
+				failures,
+				waits
+			};
+		});
+	}
+	release(host, nextAt) {
+		this.step(() => {
+			this.row(host);
+			this.stmt(`UPDATE hosts SET next_at = max(next_at, ?),
+           leased_until = CASE WHEN owner = ? THEN 0 ELSE leased_until END,
+           owner = CASE WHEN owner = ? THEN NULL ELSE owner END
+         WHERE host = ?`).run(nextAt, this.owner, this.owner, host);
+		});
+	}
+	defer(host, until) {
+		this.step(() => {
+			this.row(host);
+			this.stmt("UPDATE hosts SET next_at = max(next_at, ?) WHERE host = ?").run(until, host);
+		});
+	}
+	stop(host, until, reason) {
+		const end = Number.isFinite(until) ? until : NO_END;
+		this.step(() => {
+			this.row(host);
+			this.stmt(`UPDATE hosts SET stop_until = ?, stop_reason = ?
+         WHERE host = ? AND (stop_until IS NULL OR stop_until < ?)`).run(end, reason, host, end);
+		});
+	}
+	stopped(host, now) {
+		if (this.closed) throw new Error("the host state store is closed");
+		const row = this.stmt("SELECT stop_until, stop_reason FROM hosts WHERE host = ?").get(host);
+		if (row === void 0 || row.stop_until === null || row.stop_until <= now) return void 0;
+		return {
+			until: row.stop_until >= NO_END ? Number.POSITIVE_INFINITY : row.stop_until,
+			reason: row.stop_reason ?? "stopped"
+		};
+	}
+	reset(host) {
+		this.step(() => {
+			if (host === void 0) this.stmt("UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL").run();
+			else this.stmt("UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL WHERE host = ?").run(host);
+		});
+	}
+	validators(host) {
+		if (this.closed) throw new Error("the host state store is closed");
+		const row = this.stmt(`SELECT skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at
+         FROM validators WHERE host = ?`).get(host);
+		return row === void 0 ? void 0 : memoOf(row);
+	}
+	noteValidators(host, report, now) {
+		return this.step(() => {
+			const memo = nextValidatorMemo(this.validators(host), report, now);
+			this.stmt(`INSERT INTO validators (host, skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(host) DO UPDATE SET skip_etag = excluded.skip_etag, etag = excluded.etag,
+           last_modified = excluded.last_modified, etag_misses = excluded.etag_misses,
+           date_misses = excluded.date_misses, since_probe = excluded.since_probe,
+           changed_at = excluded.changed_at`).run(host, memo.skipEtag ? 1 : 0, memo.etag, memo.lastModified, memo.etagMisses, memo.dateMisses, memo.sinceProbe, memo.changedAt);
+			return memo;
+		});
+	}
+	/** This value is the number of hosts that the store holds. */
+	get size() {
+		return this.stmt("SELECT count(*) AS n FROM hosts").get().n;
+	}
+	/** This method closes the file. A second close does nothing. */
+	close() {
+		if (this.closed) return;
+		this.closed = true;
+		this.statements.clear();
+		this.db.close();
+	}
+};
+//#endregion
+//#region src/watch.ts
+/**
+* A watch list keeps the pages that the crawler checks again on a schedule. `watch.run` fetches the
+* pages that are due through the normal fetch, so each polite rule applies. It gives an event for
+* each page: new, changed, unchanged, gone, refused or error.
+*
+* These are the rules of a watch:
+*
+* - A watch compares the title, the text and the JSON-LD of the page with the last confirmed version.
+* - With `confirm: 2` (the default), a change is suspected when a check first sees it. The next
+*   check confirms it when it sees the same new version. A check that sees the earlier version
+*   again clears the suspicion, because one read can be a glitch. With `confirm: 1`, each check confirms.
+* - `every` has a floor of 1 hour by default. A check that a stop of the host refused is due again
+*   at the end of the stop, or later.
+* - The list is in `watch.sqlite` in the store, so it stays after a restart. A run claims each entry
+*   with a lease in that file, so two processes do not check one entry two times.
+*
+* The package has no daemon. A scheduler of the operator starts `watch.run`.
+*/
+/** This value is the version of the schema of `watch.sqlite`. */
+const WATCH_SCHEMA = 1;
+/** These are the defaults and the limits of the watch list. */
+const WATCH_LIMITS = {
+	/** The shortest `every` of a watch by default: 1 hour, so a watch cannot hammer a site. */
+	floorMs: 36e5,
+	/** The default `every`: 24 hours. */
+	everyMs: 864e5,
+	/** The default time of one run: 10 minutes. No request starts after it. */
+	budgetMs: 6e5,
+	/** The default number of entries that one run checks. */
+	maxPages: 100,
+	/** The most hosts that get a check of one run at one time. */
+	hostsAtOnce: 8,
+	/** A run holds the lease of an entry until the end of its budget and this time more: 2 minutes. */
+	leaseMs: 12e4
+};
+const iso = (ms) => new Date(ms).toISOString();
+const parseJson = (text, fallback) => {
+	if (text === null) return fallback;
+	try {
+		return JSON.parse(text);
+	} catch {
+		return fallback;
+	}
+};
+const RENDERS = /* @__PURE__ */ new Set([
+	"auto",
+	"never",
+	"always"
+]);
+const entryOf = (row, now) => ({
+	url: row.target,
+	every_ms: row.every_ms,
+	render: row.render,
+	confirm: row.confirm === 1 ? 1 : 2,
+	scope: row.scope,
+	labels: parseJson(row.labels, null),
+	on_behalf_of: parseJson(row.on_behalf_of, []),
+	added_at: iso(row.added_at),
+	next_at: iso(row.next_at),
+	checked_at: row.last_at === null ? null : iso(row.last_at),
+	last_kind: row.last_kind,
+	last_note: row.last_note,
+	checks: row.checks,
+	changes: row.changes,
+	doc: row.doc,
+	capture: row.base_capture,
+	fetched_at: row.base_fetched_at,
+	gone: row.base_state === "gone",
+	pending: row.pend_state === null ? null : {
+		kind: row.pend_state === "gone" ? "gone" : row.base_state === "gone" ? "new" : "changed",
+		capture: row.pend_capture,
+		since: iso(row.pend_since ?? row.last_at ?? row.added_at)
+	},
+	leased_until: row.lease_until > now ? iso(row.lease_until) : null
+});
+/** This function gives the canonical key, the URL and the host of a page to watch. A URL that is not http(s) throws a `RangeError`. */
+const targetOf = (url) => {
+	let parsed;
+	try {
+		parsed = new URL(url);
+	} catch {
+		throw new RangeError(`not a valid URL: ${url}`);
+	}
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new RangeError(`a watch needs an http or https URL, not ${url}`);
+	parsed.hash = "";
+	return {
+		key: cacheKeyOf(parsed.href),
+		target: parsed.href,
+		host: parsed.host.toLowerCase()
+	};
+};
+/**
+* This store keeps the watch list in a SQLite file (`watch.sqlite`). Two or more processes on one
+* store share it. Each step is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock of a
+* different process (`busy_timeout`).
+*
+* The list is the decision of the operator, not a projection of the capture log. Thus it has its
+* own file: a deletion of `hosts.sqlite` or of the index does not lose it.
+*/
+var WatchStore = class {
+	path;
+	db;
+	statements = /* @__PURE__ */ new Map();
+	closed = false;
+	/** This value is the shortest `every` in milliseconds. */
+	floorMs;
+	constructor(path, options = {}) {
+		this.path = path;
+		if (path !== ":memory:") mkdirSync(dirname(path), {
+			recursive: true,
+			mode: 448
+		});
+		const floor = options.floorMs ?? WATCH_LIMITS.floorMs;
+		if (!Number.isFinite(floor) || floor < 0) throw new RangeError(`the floor of every must be 0 or more milliseconds, not ${String(floor)}`);
+		this.floorMs = floor;
+		this.db = new DatabaseSync(path);
+		const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
+		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
+		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
+		this.db.exec("PRAGMA synchronous = NORMAL;");
+		this.step(() => {
+			this.db.exec(`CREATE TABLE IF NOT EXISTS watches (
+        url TEXT PRIMARY KEY,
+        target TEXT NOT NULL,
+        host TEXT NOT NULL,
+        every_ms REAL NOT NULL,
+        render TEXT NOT NULL,
+        confirm INTEGER NOT NULL,
+        scope TEXT,
+        labels TEXT,
+        on_behalf_of TEXT,
+        added_at REAL NOT NULL,
+        next_at REAL NOT NULL,
+        lease_until REAL NOT NULL DEFAULT 0,
+        lease_owner TEXT,
+        checks INTEGER NOT NULL DEFAULT 0,
+        changes INTEGER NOT NULL DEFAULT 0,
+        last_at REAL,
+        last_kind TEXT,
+        last_note TEXT,
+        doc TEXT,
+        base_state TEXT,
+        base_sha TEXT,
+        base_capture TEXT,
+        base_fetched_at TEXT,
+        pend_state TEXT,
+        pend_sha TEXT,
+        pend_capture TEXT,
+        pend_fetched_at TEXT,
+        pend_since REAL
+      )`);
+			this.db.exec("CREATE INDEX IF NOT EXISTS watches_due ON watches (next_at)");
+			const { user_version: version } = this.db.prepare("PRAGMA user_version").get();
+			if (version < 1) this.db.exec(`PRAGMA user_version = ${String(1)}`);
+		});
+	}
+	stmt(sql) {
+		let statement = this.statements.get(sql);
+		if (statement === void 0) {
+			statement = this.db.prepare(sql);
+			this.statements.set(sql, statement);
+		}
+		return statement;
+	}
+	/** This method does one step in one transaction. */
+	step(work) {
+		if (this.closed) throw new Error("the watch store is closed");
+		this.db.exec("BEGIN IMMEDIATE");
+		try {
+			const result = work();
+			this.db.exec("COMMIT");
+			return result;
+		} catch (error) {
+			if (this.db.isTransaction) this.db.exec("ROLLBACK");
+			throw error;
+		}
+	}
+	row(key) {
+		return this.stmt("SELECT * FROM watches WHERE url = ?").get(key);
+	}
+	/**
+	* This method adds a page, or changes the options of its watch and keeps its state. A new entry
+	* is due at once. A changed entry is due one `every` after its last check. A bad option throws
+	* a `RangeError`.
+	*/
+	add(url, options, now) {
+		const { key, target, host } = targetOf(url);
+		const every = options.every === void 0 ? WATCH_LIMITS.everyMs : durationMs(options.every);
+		if (every < this.floorMs) throw new RangeError(`every must be ${String(this.floorMs / 6e4)} minutes or more, so a watch cannot hammer a site, not ${String(every / 6e4)} minutes`);
+		const render = options.render ?? "auto";
+		if (!RENDERS.has(render)) throw new RangeError(`render is auto, never or always, not ${JSON.stringify(render)}`);
+		const confirm = options.confirm ?? 2;
+		if (confirm !== 1 && confirm !== 2) throw new RangeError(`confirm is 1 or 2, not ${JSON.stringify(confirm)}`);
+		const labels = options.labels === void 0 || Object.keys(options.labels).length === 0 ? null : JSON.stringify(options.labels);
+		const agents = options.onBehalfOf === void 0 || options.onBehalfOf.length === 0 ? null : JSON.stringify([...options.onBehalfOf]);
+		return this.step(() => {
+			this.stmt(`INSERT INTO watches (url, target, host, every_ms, render, confirm, scope, labels, on_behalf_of, added_at, next_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(url) DO UPDATE SET target = excluded.target, host = excluded.host,
+           every_ms = excluded.every_ms, render = excluded.render, confirm = excluded.confirm,
+           scope = excluded.scope, labels = excluded.labels, on_behalf_of = excluded.on_behalf_of,
+           next_at = CASE WHEN watches.last_at IS NULL THEN watches.next_at
+                          ELSE watches.last_at + excluded.every_ms END`).run(key, target, host, every, render, confirm, options.scope ?? null, labels, agents, now, now);
+			return entryOf(this.row(key), now);
+		});
+	}
+	/** This method removes a page. It gives false when the list does not have the page. */
+	remove(url) {
+		const { key } = targetOf(url);
+		return this.step(() => Number(this.stmt("DELETE FROM watches WHERE url = ?").run(key).changes) > 0);
+	}
+	/** This method gives the entry of a page, or undefined. */
+	get(url, now) {
+		if (this.closed) throw new Error("the watch store is closed");
+		const row = this.row(targetOf(url).key);
+		return row === void 0 ? void 0 : entryOf(row, now);
+	}
+	/** This method gives the entries, the first due first. */
+	list(now) {
+		if (this.closed) throw new Error("the watch store is closed");
+		return this.stmt("SELECT * FROM watches ORDER BY next_at, url").all().map((row) => entryOf(row, now));
+	}
+	/** This method counts the entries that are due at a time. */
+	due(now) {
+		if (this.closed) throw new Error("the watch store is closed");
+		return this.stmt("SELECT count(*) AS n FROM watches WHERE next_at <= ?").get(now).n;
+	}
+	/**
+	* This method claims the first due entry with no lease, on a host that is not in `busy`. The
+	* lease of `owner` holds until `until`. Thus two processes do not get one entry at one time.
+	*/
+	claim(now, owner, until, busy) {
+		return this.step(() => {
+			const row = this.stmt(`SELECT * FROM watches WHERE next_at <= ? AND lease_until <= ?
+           AND host NOT IN (SELECT value FROM json_each(?)) ORDER BY next_at, url LIMIT 1`).get(now, now, JSON.stringify(busy));
+			if (row === void 0) return void 0;
+			this.stmt("UPDATE watches SET lease_until = ?, lease_owner = ? WHERE url = ?").run(until, owner, row.url);
+			return {
+				...row,
+				lease_until: until,
+				lease_owner: owner
+			};
+		});
+	}
+	/** This method ends the lease of `owner` and changes nothing else, so the entry stays due. A closed store does nothing. */
+	release(key, owner) {
+		if (this.closed) return;
+		this.step(() => {
+			this.stmt("UPDATE watches SET lease_until = 0, lease_owner = NULL WHERE url = ? AND lease_owner = ?").run(key, owner);
+		});
+	}
+	/**
+	* This method writes the result of a check and ends the lease. It gives false when `owner` does
+	* not hold the lease, or when the store is closed.
+	*/
+	settle(key, owner, update) {
+		if (this.closed) return false;
+		return this.step(() => {
+			const row = this.row(key);
+			if (row === void 0 || row.lease_owner !== owner) return false;
+			const base = update.base;
+			const pending = update.pending;
+			this.stmt(`UPDATE watches SET lease_until = 0, lease_owner = NULL, next_at = ?, last_at = ?, last_kind = ?,
+           last_note = ?, checks = checks + 1, changes = changes + ?, doc = ?,
+           base_state = ?, base_sha = ?, base_capture = ?, base_fetched_at = ?,
+           pend_state = ?, pend_sha = ?, pend_capture = ?, pend_fetched_at = ?, pend_since = ?
+         WHERE url = ?`).run(update.next_at, update.last_at, update.last_kind, update.last_note, update.changed ? 1 : 0, update.doc ?? row.doc, base === void 0 ? row.base_state : base.state, base === void 0 ? row.base_sha : base.sha, base === void 0 ? row.base_capture : base.capture, base === void 0 ? row.base_fetched_at : base.fetched_at, pending === void 0 ? row.pend_state : pending?.state ?? null, pending === void 0 ? row.pend_sha : pending?.sha ?? null, pending === void 0 ? row.pend_capture : pending?.capture ?? null, pending === void 0 ? row.pend_fetched_at : pending?.fetched_at ?? null, pending === void 0 ? row.pend_since : pending?.since ?? null, key);
+			return true;
+		});
+	}
+	/** This method closes the file. A second close does nothing. */
+	close() {
+		if (this.closed) return;
+		this.closed = true;
+		this.statements.clear();
+		this.db.close();
+	}
+};
+/** The outcomes of a check that the site or the crawler refused. */
+const REFUSALS = /* @__PURE__ */ new Set([
+	"skipped",
+	"blocked",
+	"challenge",
+	"payment-required"
+]);
+/** This function gives the fingerprint of a version: a hash of its title, its text and its JSON-LD. */
+const watchFingerprint = (version) => sha256(stableStringify([
+	version.title,
+	version.text,
+	version.meta["jsonld"] ?? null
+]));
+const sameState = (a, b) => a !== void 0 && a.state === b.state && (a.state === "gone" || a.sha === b.sha);
+/**
+* This function checks the entries of a watch store that are due. `PoliteCrawler.watch.run` uses
+* it. Each check is a fetch of the page with `fetchPage`, with no stored copy (`maxAgeMs: 0`), so
+* each polite rule applies. A check that the budget stops before its request stays due. The store
+* can be a function that opens it: the run uses it before the first check.
+*/
+const runWatch = (runtime, source, options = {}) => {
+	let resolveSummary = () => void 0;
+	const summary = new Promise((resolve) => {
+		resolveSummary = resolve;
+	});
+	let used = false;
+	async function* events() {
+		const now = () => Date.parse(runtime.now());
+		const startedAt = now();
+		const owner = randomUUID();
+		const abort = new AbortController();
+		const outer = options.signal;
+		const cancel = () => abort.abort(outer?.reason);
+		if (outer?.aborted === true) cancel();
+		outer?.addEventListener("abort", cancel, { once: true });
+		const inflight = /* @__PURE__ */ new Map();
+		const counts = {
+			checked: 0,
+			new: 0,
+			changed: 0,
+			unchanged: 0,
+			gone: 0,
+			refused: 0,
+			error: 0,
+			suspected: 0,
+			confirmed: 0
+		};
+		let started = 0;
+		let stopping;
+		let reason;
+		let finished = false;
+		let budgetMs = WATCH_LIMITS.budgetMs;
+		let maxPages = WATCH_LIMITS.maxPages;
+		let opened;
+		/** This function fetches one entry, compares it with its state, and writes the new state. */
+		const check = async (store, row, deadline) => {
+			const agents = [...options.onBehalfOf ?? [], ...parseJson(row.on_behalf_of, [])];
+			const labels = parseJson(row.labels, void 0);
+			let page;
+			let failure;
+			try {
+				page = await fetchPage(runtime, row.target, {
+					render: row.render,
+					maxAgeMs: 0,
+					onBehalfOf: agents,
+					signal: abort.signal,
+					scope: row.scope ?? void 0,
+					labels,
+					deadline
+				});
+			} catch (error) {
+				failure = `watch error: ${error instanceof Error ? error.message : String(error)}`;
+			}
+			const deferred = (stop) => {
+				store.release(row.url, owner);
+				return {
+					kind: "deferred",
+					host: row.host,
+					stop
+				};
+			};
+			if (abort.signal.aborted) return deferred("cancelled");
+			if (page !== void 0 && pastDeadline(page.note)) return deferred("budget");
+			if (page?.outcome === "skipped" && page.note === "crawler closed") return deferred("closed");
+			const at = now();
+			const base = row.base_state === null ? void 0 : {
+				state: row.base_state,
+				sha: row.base_sha,
+				capture: row.base_capture,
+				fetched_at: row.base_fetched_at
+			};
+			const pending = row.pend_state === null ? void 0 : {
+				state: row.pend_state,
+				sha: row.pend_sha,
+				capture: row.pend_capture,
+				fetched_at: row.pend_fetched_at
+			};
+			let seen;
+			let version;
+			let held = false;
+			let kind;
+			let note = page?.note ?? failure ?? null;
+			const ok = page !== void 0 && (page.outcome === "ok" || page.outcome === "not-modified");
+			if (page !== void 0 && ok && page.doc !== null && page.capture !== null) {
+				version = runtime.index.versionText(page.doc, page.capture);
+				const doc = runtime.index.get(page.doc);
+				held = doc !== void 0 && isHeldBack(doc) || version !== void 0 && isHeldBack(version);
+				if (version !== void 0) seen = {
+					state: "present",
+					sha: watchFingerprint(version),
+					capture: version.capture,
+					fetched_at: version.fetched_at
+				};
+			} else if (page?.outcome === "not-found") seen = {
+				state: "gone",
+				sha: null,
+				capture: null,
+				fetched_at: page.fetched_at
+			};
+			let update;
+			let suspected = false;
+			let confirmed = false;
+			if (seen === void 0) {
+				kind = page !== void 0 && REFUSALS.has(page.outcome) ? "refused" : "error";
+				if (ok) note = note ?? "the page gave no document, so the watch cannot compare it";
+				update = { changed: false };
+			} else if (base === void 0) {
+				kind = seen.state === "gone" ? "gone" : "new";
+				update = {
+					changed: false,
+					base: seen,
+					pending: null
+				};
+			} else if (sameState(base, seen)) {
+				kind = "unchanged";
+				if (pending !== void 0) note = "the change that the last check saw did not repeat, so the watch takes it for a glitch";
+				update = {
+					changed: false,
+					pending: null
+				};
+			} else {
+				kind = seen.state === "gone" ? "gone" : base.state === "gone" ? "new" : "changed";
+				const repeated = sameState(pending, seen) && (seen.state === "gone" || pending?.capture !== seen.capture);
+				if (row.confirm === 1 || repeated) {
+					confirmed = true;
+					update = {
+						changed: true,
+						base: seen,
+						pending: null
+					};
+				} else {
+					suspected = true;
+					note = pending === void 0 ? "the watch saw this change one time: the next check confirms it when it sees the same version" : "the watch saw a different change than the last check: the next check confirms it when it sees the same version";
+					update = {
+						changed: false,
+						pending: {
+							...seen,
+							since: at
+						}
+					};
+				}
+			}
+			let diff = null;
+			if (kind === "changed" && base?.state === "present" && base.capture !== null && version !== void 0 && !held) {
+				const docId = row.doc ?? version.doc;
+				const before = runtime.index.versionText(docId, base.capture);
+				if (before !== void 0 && !isHeldBack(before)) diff = diffTexts({
+					text: before.text,
+					fields: {
+						title: before.title,
+						meta: before.meta
+					}
+				}, {
+					text: version.text,
+					fields: {
+						title: version.title,
+						meta: version.meta
+					}
+				}).summary;
+			}
+			let next = at + row.every_ms;
+			const stop = runtime.crawler.hostStop(row.target);
+			if (stop !== void 0 && Number.isFinite(stop.until) && stop.until > next) next = stop.until;
+			store.settle(row.url, owner, {
+				...update,
+				next_at: next,
+				last_at: at,
+				last_kind: kind,
+				last_note: note,
+				doc: page?.doc ?? null
+			});
+			const event = {
+				url: row.target,
+				doc: page?.doc ?? row.doc,
+				kind,
+				suspected,
+				confirmed,
+				capture: seen?.capture ?? null,
+				fetched_at: seen?.fetched_at ?? null,
+				previous: base?.capture ?? null,
+				previous_fetched_at: base?.fetched_at ?? null,
+				outcome: page?.outcome ?? "error",
+				status: page?.status ?? 0,
+				diff,
+				held_back: held,
+				next_at: iso(next),
+				note
+			};
+			return {
+				kind: "event",
+				host: row.host,
+				event
+			};
+		};
+		try {
+			const store = typeof source === "function" ? await source() : source;
+			opened = store;
+			await options.ready?.();
+			budgetMs = options.budgetMs ?? WATCH_LIMITS.budgetMs;
+			if (!(budgetMs >= 0)) throw new RangeError(`budgetMs must be 0 or more, not ${String(budgetMs)}`);
+			maxPages = options.maxPages ?? WATCH_LIMITS.maxPages;
+			if (!Number.isInteger(maxPages) || maxPages < 1) throw new RangeError(`maxPages must be an integer of 1 or more, not ${String(maxPages)}`);
+			const deadline = startedAt + budgetMs;
+			for (;;) {
+				if (stopping === void 0 && abort.signal.aborted) stopping = "cancelled";
+				if (stopping === void 0 && now() >= deadline) stopping = "budget";
+				while (stopping === void 0 && inflight.size < WATCH_LIMITS.hostsAtOnce && started < maxPages) {
+					const claimed = store.claim(now(), owner, Math.max(now(), deadline) + WATCH_LIMITS.leaseMs, [...inflight.keys()]);
+					if (claimed === void 0) break;
+					started += 1;
+					inflight.set(claimed.host, check(store, claimed, deadline));
+				}
+				if (inflight.size === 0) break;
+				const done = await Promise.race(inflight.values());
+				inflight.delete(done.host);
+				if (done.kind === "deferred") {
+					started -= 1;
+					stopping ??= done.stop;
+					continue;
+				}
+				const event = done.event;
+				counts.checked += 1;
+				counts[event.kind] += 1;
+				if (event.suspected) counts.suspected += 1;
+				if (event.confirmed) counts.confirmed += 1;
+				yield event;
+			}
+			stopping ??= started >= maxPages && store.due(now()) > 0 ? "max-pages" : "done";
+			finished = true;
+		} catch (error) {
+			stopping = "error";
+			reason = error instanceof Error ? error.message : String(error);
+			finished = true;
+			throw error;
+		} finally {
+			if (!finished) {
+				stopping ??= "cancelled";
+				reason ??= "the caller stopped the loop";
+			}
+			outer?.removeEventListener("abort", cancel);
+			abort.abort(/* @__PURE__ */ new Error("the watch run ended"));
+			await Promise.allSettled(inflight.values());
+			const stop = stopping ?? "done";
+			reason ??= stop === "done" ? "the run checked each entry that was due" : stop === "max-pages" ? `the run checked ${String(maxPages)} entries, its limit (maxPages)` : stop === "budget" ? `the budget of ${String(budgetMs)} ms ended (budgetMs)` : stop === "cancelled" ? "the caller cancelled the run" : "the crawler closed";
+			let due = 0;
+			try {
+				due = opened?.due(now()) ?? 0;
+			} catch {}
+			resolveSummary({
+				...counts,
+				due,
+				stopped: stop,
+				reason,
+				elapsed_ms: Math.max(0, now() - startedAt)
+			});
+		}
+	}
+	return {
+		summary,
+		[Symbol.asyncIterator]() {
+			if (used) throw new Error("a watch run operates one time: call run again for a new run");
+			used = true;
+			return events();
+		}
+	};
+};
+//#endregion
+//#region src/open.ts
+var open_exports = /* @__PURE__ */ __exportAll({
+	BOT_PAGE: () => BOT_PAGE,
+	DEFAULT_TOKEN: () => DEFAULT_TOKEN,
+	createRuntime: () => createRuntime,
+	defaultUserAgent: () => defaultUserAgent,
+	isMailbox: () => isMailbox,
+	openCrawler: () => openCrawler
+});
+/**
+* This function gives the default `User-Agent` header. The header names the crawler, its version
+* and the page about it, so a site owner can find out what made each request.
+*/
+const defaultUserAgent = (contact) => `${DEFAULT_TOKEN}/${VERSION} (+${BOT_PAGE}${contact === void 0 ? "" : `; ${contact}`})`;
+/** A mailbox is `name@host`, or a name and `<name@host>`. A URL is not a mailbox, also when it holds an `@`. */
+const MAILBOX = /^(?:[^\s<>@:/]+@[^\s<>@/]+|[^<>]*<[^\s<>@:/]+@[^\s<>@/]+>)$/;
+/**
+* This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,
+* section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in
+* the `User-Agent` only.
+*/
+const isMailbox = (contact) => MAILBOX.test(contact.trim());
+/** The parts of a store that can be a symbolic link to a different place only by mistake or by attack. */
+const STORE_PARTS = [
+	"captures",
+	"blobs",
+	"index.sqlite",
+	"hosts.sqlite",
+	"watch.sqlite"
+];
+/**
+* This function makes the folder of a store for the owner only, where the system has modes. It
+* refuses a store folder of a different user, and a part of the store that is a symbolic link.
+* The store folder itself can be a link, for example to a larger disk.
+*/
+const guardStore = (dir) => {
+	mkdirSync(dir, {
+		recursive: true,
+		mode: 448
+	});
+	const real = lstatSync(dir).isSymbolicLink() ? realpathSync(dir) : dir;
+	const info = statSync(real);
+	if (!info.isDirectory()) throw new Error(`the store ${dir} is not a folder`);
+	if (process.platform !== "win32" && typeof process.getuid === "function" && info.uid !== process.getuid()) throw new Error(`the store ${dir} belongs to a different user: use a folder of your own`);
+	for (const part of STORE_PARTS) {
+		let link = false;
+		try {
+			link = lstatSync(join(dir, part)).isSymbolicLink();
+		} catch {}
+		if (link) throw new Error(`${part} in the store ${dir} is a symbolic link: the crawler does not follow it`);
+	}
+};
+/**
+* This function makes a runtime. It is the one place that puts the log, the blob store, the index,
+* the host state, the crawler and the extractors together. The function `openCrawler` uses it.
+* An application that needs the parts uses it too. The runtime and the crawler share one clock.
+*/
+const createRuntime = (options) => {
+	const dir = options.dir;
+	if (dir === void 0 && (options.log === void 0 || options.blobs === void 0 || options.index === void 0)) throw new Error("a runtime needs dir, or a log, a blob store and an index");
+	const contact = options.contact;
+	if (contact !== void 0 && !/^[\t -~]*$/.test(contact)) throw new Error(`contact "${contact}" has a character that an HTTP header cannot hold: use printable ASCII`);
+	if (dir !== void 0) guardStore(dir);
+	const owned = [];
+	const log = options.log ?? new CaptureLog(join(dir, "captures"));
+	const blobs = options.blobs ?? new FileBlobStore(join(dir, "blobs"));
+	let index = options.index;
+	if (index === void 0) {
+		const made = new DocIndex(join(dir, "index.sqlite"));
+		owned.push(() => made.close());
+		index = made;
+	}
+	let hostState = options.hostState;
+	if (hostState === void 0) {
+		if (dir === void 0) hostState = new MemoryHostStateStore();
+		else {
+			const made = new SqliteHostStateStore(join(dir, "hosts.sqlite"));
+			owned.push(() => made.close());
+			hostState = made;
+		}
+	}
+	const clock = options.clock ?? realClock;
+	const renderer = options.renderer === false ? void 0 : options.renderer ?? checkedRenderer(playwrightRenderer());
+	const crawler = new Crawler({
+		token: options.token ?? "PoliteCrawler",
+		userAgent: options.userAgent ?? defaultUserAgent(contact),
+		from: contact !== void 0 && isMailbox(contact) ? contact.trim() : void 0,
+		log,
+		blobs,
+		index,
+		renderer,
+		maxActive: options.maxActive,
+		maxRenders: options.maxRenders,
+		dispatcher: options.dispatcher,
+		egress: options.egress,
+		politeness: options.politeness,
+		hostState,
+		clock,
+		random: options.random,
+		scope: options.scope,
+		scopeRank: options.scopeRank,
+		obeyDefaultToken: options.obeyDefaultToken,
+		onEvent: options.onEvent,
+		signer: options.signer
+	});
+	const extractors = new Extractors();
+	for (const [recipe, extractor] of Object.entries(options.extractors ?? {})) extractors.register(recipe, extractor);
+	let closing;
+	return {
+		crawler,
+		index,
+		blobs,
+		captures: log,
+		log,
+		extractors,
+		hostState,
+		now: () => new Date(clock.now()).toISOString(),
+		close() {
+			closing ??= (async () => {
+				await crawler.close();
+				for (const close of owned) await close();
+			})();
+			return closing;
+		}
+	};
+};
+/**
+* This function opens a crawler on a folder. The folder keeps the capture log (`captures/`), the
+* response bodies (`blobs/`), the search index (`index.sqlite`) and the state of each host
+* (`hosts.sqlite`). The index is a projection: `reindex` makes it again from the log and the bodies.
+* At open, `catchUpIndex` rebuilds an index of an earlier version, and gives an index that is
+* behind the log the captures that it missed.
+*
+* Each method gives a promise, and none throws before its promise. After `close`, each method
+* rejects.
+*/
+const openCrawler = (options) => {
+	const floorMs = options.watchFloor === void 0 ? WATCH_LIMITS.floorMs : durationMs(options.watchFloor);
+	const runtime = createRuntime(options);
+	let closed = false;
+	let rebuilt;
+	const ready = existsSync(join(options.dir, "captures")) ? catchUpIndex(runtime).then((done) => {
+		if (done?.mode === "rebuild") {
+			const { mode, ...counts } = done;
+			rebuilt = mode === "rebuild" ? counts : void 0;
+		}
+	}, () => void 0) : Promise.resolve();
+	const live = async () => {
+		if (closed) throw new Error("the crawler is closed");
+		await ready;
+		if (closed) throw new Error("the crawler is closed");
+	};
+	let watches;
+	const watchStore = async () => {
+		await live();
+		watches ??= new WatchStore(join(options.dir, "watch.sqlite"), { floorMs });
+		return watches;
+	};
+	const clockNow = () => Date.parse(runtime.now());
+	return {
+		runtime,
+		async fetch(url, fetchOptions = {}) {
+			await live();
+			rebuilt = void 0;
+			const maxAge = fetchOptions.maxAge ?? "24h";
+			return fetchPage(runtime, url, {
+				render: fetchOptions.render ?? "auto",
+				maxAgeMs: maxAge === "auto" ? "auto" : durationMs(maxAge),
+				onBehalfOf: [...options.onBehalfOf ?? [], ...fetchOptions.onBehalfOf ?? []],
+				signal: fetchOptions.signal,
+				scope: fetchOptions.scope,
+				labels: fetchOptions.labels,
+				fallback: fetchOptions.fallback,
+				archiveDespiteRobots: fetchOptions.archiveDespiteRobots
+			});
+		},
+		async find(q, findOptions = {}) {
+			await live();
+			return findDocuments(runtime, q, findOptions);
+		},
+		async read(id, readOptions = {}) {
+			await live();
+			return readDocument(runtime, id, readOptions);
+		},
+		async verify(id, quote, verifyOptions = {}) {
+			await live();
+			return verifyQuote(runtime, id, quote, verifyOptions);
+		},
+		async extract(id, fields, extractOptions = {}) {
+			await live();
+			return extractDocument(runtime, id, fields, extractOptions);
+		},
+		async versions(id, versionOptions = {}) {
+			await live();
+			const held = runtime.index.get(id)?.ai_input === "no";
+			return runtime.index.versions(id, versionOptions).map((version) => held || version.ai_input === "no" ? {
+				...version,
+				title: ""
+			} : version);
+		},
+		async discover(site, discoverOptions = {}) {
+			await live();
+			rebuilt = void 0;
+			return discoverSite(runtime, site, {
+				...discoverOptions,
+				onBehalfOf: [...options.onBehalfOf ?? [], ...discoverOptions.onBehalfOf ?? []]
+			});
+		},
+		crawl(seeds, crawlOptions = {}) {
+			return crawlSite(runtime, seeds, {
+				...crawlOptions,
+				onBehalfOf: [...options.onBehalfOf ?? [], ...crawlOptions.onBehalfOf ?? []],
+				ready: async () => {
+					await live();
+					rebuilt = void 0;
+				}
+			});
+		},
+		async reindex() {
+			await live();
+			const done = rebuilt;
+			rebuilt = void 0;
+			return done ?? reindex(runtime);
+		},
+		async diff(id, diffOptions = {}) {
+			await live();
+			return diffDocument(runtime, id, diffOptions);
+		},
+		watch: {
+			async add(url, watchOptions = {}) {
+				return (await watchStore()).add(url, watchOptions, clockNow());
+			},
+			async remove(url) {
+				return (await watchStore()).remove(url);
+			},
+			async list() {
+				return (await watchStore()).list(clockNow());
+			},
+			run(runOptions = {}) {
+				return runWatch(runtime, watchStore, {
+					...runOptions,
+					onBehalfOf: options.onBehalfOf,
+					ready: () => {
+						rebuilt = void 0;
+						return Promise.resolve();
+					}
+				});
+			}
+		},
+		async close() {
+			closed = true;
+			await ready;
+			await runtime.close();
+			watches?.close();
+		}
+	};
+};
+//#endregion
+export { readSitemap as _, open_exports as a, WatchStore as c, CRAWL_LIMITS as d, crawlSite as f, feedDate as g, SITEMAP_LIMITS as h, openCrawler as i, runWatch as l, discoverSite as m, defaultUserAgent as n, WATCH_LIMITS as o, DISCOVER_LIMITS as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, SqliteHostStateStore as u, DocIndex as v, FAST_RANK_SHARE as y };
+
+//# sourceMappingURL=open-DnrWgkC0.mjs.map