polite-crawler 0.1.1 → 0.1.3

package/build/open-C4fhXJgD.mjs

@@ -0,0 +1,4544 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { n as FileBlobStore, t as CaptureLog } from "./log-Dx-wXjJl.mjs";
+import { r as playwrightRenderer } from "./render-C1CUraWF.mjs";
+import { r as checkedRenderer } from "./browser-CKRNbIDT.mjs";
+import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-ntbY-6wh.mjs";
+import { A as snippetOf, C as fallbackReason, E as fold, L as sha256$1, M as Crawler, N as DEADLINE_NOTE, R as stableStringify, S as archivedOf, a as fetchPage, c as isHeldBack, f as readDocument, g as verifyQuote, i as fetchAndIndex, k as parseQuery, m as reindex, n as HELD_BACK, o as findDocuments, p as recordOverride, r as catchUpIndex, s as heldBackMeta, t as Extractors, y as archivedCopy } from "./runtime-JToRO63o.mjs";
+import { MemoryHostStateStore, nextValidatorMemo, realClock } from "./politeness.mjs";
+import { matchesPattern } from "./robots.mjs";
+import { n as canonicalUrl, r as scopeTest, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
+import { o as decodeBody, t as extractPage } from "./html-DkQUAE_d.mjs";
+import { n as diffDocument, r as diffTexts } from "./diff-NQcQqKe9.mjs";
+import { r as extractDocument } from "./extract-ByFfc8Zk.mjs";
+import { createHash, randomUUID } from "node:crypto";
+import { dirname, join } from "node:path";
+import { existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
+import { createGunzip } from "node:zlib";
+import { DatabaseSync } from "node:sqlite";
+/** The search index reads this many characters of each text, at most. */
+const MAX_INDEX_CHARS = 2e5;
+/** In the mode `fast`, a part is common when more than this share of the documents hold it. */
+const FAST_RANK_SHARE = .2;
+/** The text of the error of hybrid search with no embedder. */
+const NO_EMBEDDER = "hybrid search needs an embedder: open the crawler with the option embedder (transformersEmbedder or ollamaEmbedder), or give --embedder transformers or ollama to the command line";
+/** This function gives the rank mode of an option. A value that is not a rank mode throws a `RangeError`. */
+const rankModeOf = (value) => {
+	if (value === void 0 || value === "exact") return "exact";
+	if (value === "fast") return "fast";
+	if (value === "hybrid") throw new RangeError("the index ranks with \"exact\" and \"fast\" only: rank \"hybrid\" needs the vectors of an embedder, so use find or rankWithin of a crawler with the option embedder");
+	throw new RangeError(`rank must be "exact" or "fast", not ${JSON.stringify(value) ?? "undefined"}`);
+};
+const namesOf = (generation) => ({
+	docs: `docs_${String(generation)}`,
+	fts: `fts_${String(generation)}`,
+	versions: `versions_${String(generation)}`,
+	texts: `texts_${String(generation)}`,
+	latest: `latest_${String(generation)}`
+});
+const GENERATION_TABLE = /^(docs|fts|versions|texts|latest)_(\d+)$/;
+/** A build that started this long ago and did not finish belongs to a process that stopped. */
+const ABANDONED_MS = 864e5;
+/** The FTS5 tokenizer: folded text, English stems, and spacing marks kept in the words. */
+const TOKENIZER = `tokenize = "porter unicode61 remove_diacritics 0 categories 'L* N* Co Mc'"`;
+const ddlOf = (n) => `
+CREATE TABLE IF NOT EXISTS ${n.texts} (
+  sha TEXT PRIMARY KEY,
+  body TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS ${n.versions} (
+  doc TEXT NOT NULL,
+  capture TEXT NOT NULL,
+  recipe TEXT NOT NULL,
+  rendered INTEGER NOT NULL,
+  fetched_at TEXT NOT NULL,
+  blob TEXT,
+  kind TEXT NOT NULL,
+  title TEXT NOT NULL,
+  url TEXT,
+  meta_sha TEXT NOT NULL,
+  text_sha TEXT NOT NULL,
+  text_chars INTEGER NOT NULL,
+  full_sha TEXT,
+  ai_input TEXT NOT NULL,
+  merge TEXT NOT NULL,
+  scope TEXT,
+  labels TEXT,
+  PRIMARY KEY (doc, capture)
+);
+CREATE INDEX IF NOT EXISTS ${n.versions}_time ON ${n.versions} (doc, recipe, rendered, fetched_at);
+CREATE INDEX IF NOT EXISTS ${n.versions}_capture ON ${n.versions} (capture);
+CREATE INDEX IF NOT EXISTS ${n.versions}_fetched ON ${n.versions} (fetched_at);
+CREATE INDEX IF NOT EXISTS ${n.versions}_recipe ON ${n.versions} (recipe);
+CREATE TABLE IF NOT EXISTS ${n.docs} (
+  doc TEXT PRIMARY KEY,
+  kind TEXT NOT NULL,
+  title TEXT NOT NULL,
+  url TEXT,
+  meta TEXT NOT NULL,
+  text_sha TEXT NOT NULL,
+  full_sha TEXT,
+  captures TEXT NOT NULL,
+  capture_count INTEGER NOT NULL,
+  capture TEXT NOT NULL,
+  blob TEXT,
+  fetched_at TEXT NOT NULL,
+  recipe TEXT NOT NULL,
+  rendered INTEGER NOT NULL,
+  ai_input TEXT NOT NULL,
+  scope TEXT,
+  labels TEXT,
+  updated_at TEXT NOT NULL
+);
+CREATE VIRTUAL TABLE IF NOT EXISTS ${n.fts} USING fts5(
+  title, text, content = '', contentless_delete = 1, ${TOKENIZER}
+);
+CREATE TABLE IF NOT EXISTS ${n.latest} (
+  url TEXT NOT NULL,
+  variant TEXT NOT NULL,
+  capture TEXT NOT NULL,
+  blob TEXT,
+  etag TEXT,
+  last_modified TEXT,
+  content_type TEXT,
+  fetched_at TEXT NOT NULL,
+  status INTEGER NOT NULL,
+  ai_input TEXT NOT NULL,
+  location TEXT,
+  truncated INTEGER,
+  fresh_until TEXT,
+  vary_key TEXT,
+  PRIMARY KEY (url, variant)
+);
+CREATE INDEX IF NOT EXISTS ${n.latest}_fetched ON ${n.latest} (fetched_at);
+`;
+const RENDERED_MARK = "#rendered";
+/** This function splits a cache key into its URL and its variant. A key in the form of version 0.1 (`…#rendered`) is a rendered variant. */
+const keyOf = (url, variant) => {
+	if (url.endsWith(RENDERED_MARK)) return {
+		url: cacheKeyOf(url.slice(0, -9)),
+		variant: "rendered"
+	};
+	return {
+		url: cacheKeyOf(url),
+		variant: variant ?? "plain"
+	};
+};
+const sha = (text) => createHash("sha256").update(text, "utf8").digest("hex");
+const parseJson$1 = (text, fallback) => {
+	if (text === null || text === void 0) return fallback;
+	try {
+		return JSON.parse(text);
+	} catch {
+		return fallback;
+	}
+};
+const toLatest = (row) => ({
+	url: row.url,
+	capture: row.capture,
+	blob: row.blob,
+	etag: row.etag,
+	last_modified: row.last_modified,
+	content_type: row.content_type,
+	fetched_at: row.fetched_at,
+	status: row.status,
+	ai_input: row.ai_input,
+	location: row.location,
+	truncated: row.truncated === 1 ? true : void 0,
+	fresh_until: row.fresh_until,
+	vary_key: row.vary_key,
+	variant: row.variant
+});
+const toVersion = (row) => ({
+	doc: row.doc,
+	capture: row.capture,
+	recipe: row.recipe,
+	rendered: row.rendered === 1,
+	fetched_at: row.fetched_at,
+	blob: row.blob,
+	kind: row.kind,
+	title: row.title,
+	url: row.url,
+	ai_input: row.ai_input,
+	chars: row.text_chars,
+	cut: row.full_sha !== null
+});
+const messageOf = (error) => error instanceof Error ? error.message : String(error);
+var DocIndex = class {
+	path;
+	db;
+	statements = /* @__PURE__ */ new Map();
+	depth = 0;
+	closed = false;
+	empty;
+	/**
+	* This value is true when the file had a different schema, or no tables, so the index opened
+	* empty. A store with captures then needs `reindex`. The commit of a rebuild makes it false.
+	*/
+	get stale() {
+		return this.empty;
+	}
+	constructor(path, options = {}) {
+		this.path = path;
+		if (path !== ":memory:") mkdirSync(dirname(path), {
+			recursive: true,
+			mode: 448
+		});
+		this.db = new DatabaseSync(path);
+		const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
+		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
+		try {
+			this.db.exec("CREATE VIRTUAL TABLE temp.polite_fts5_probe USING fts5(x, content = '', contentless_delete = 1); DROP TABLE temp.polite_fts5_probe;");
+		} catch (cause) {
+			this.db.close();
+			throw new Error(`polite-crawler needs SQLite FTS5 with contentless_delete, and the SQLite of Node.js ${process.version} does not have it: use Node.js 22.19 or later`, { cause });
+		}
+		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
+		this.db.exec("PRAGMA synchronous = NORMAL;");
+		this.empty = this.write(() => this.prepareSchema());
+	}
+	/** This method makes the tables of the schema. It gives true when the index opened empty. */
+	prepareSchema() {
+		const version = this.db.prepare("PRAGMA user_version").get().user_version;
+		const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
+		if (version === 2 && tables.some((table) => table.name === "index_meta")) {
+			this.dropAbandoned();
+			return false;
+		}
+		const virtual = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'").all();
+		for (const { name } of virtual) this.db.exec(`DROP TABLE IF EXISTS "${name}"`);
+		const rest = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
+		for (const { name } of rest) this.db.exec(`DROP TABLE IF EXISTS "${name}"`);
+		this.db.exec("CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)");
+		this.db.exec(ddlOf(namesOf(1)));
+		this.statements.clear();
+		this.setMeta("generation", "1");
+		this.setMeta("next_generation", "2");
+		this.db.exec(`PRAGMA user_version = ${String(2)}`);
+		return true;
+	}
+	setMeta(key, value) {
+		this.stmt("INSERT INTO index_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
+	}
+	getMeta(key) {
+		return this.stmt("SELECT value FROM index_meta WHERE key = ?").get(key)?.value;
+	}
+	stmt(sql) {
+		let statement = this.statements.get(sql);
+		if (statement === void 0) {
+			statement = this.db.prepare(sql);
+			this.statements.set(sql, statement);
+		}
+		return statement;
+	}
+	/**
+	* This method does a write in one transaction. A transaction in progress takes the write in,
+	* also a transaction that the caller opened on the connection.
+	*/
+	write(work) {
+		if (this.closed) throw new Error("the index is closed");
+		if (this.depth > 0 || this.db.isTransaction) {
+			this.depth += 1;
+			try {
+				return work();
+			} finally {
+				this.depth -= 1;
+			}
+		}
+		this.db.exec("BEGIN IMMEDIATE");
+		this.depth = 1;
+		try {
+			const result = work();
+			this.db.exec("COMMIT");
+			return result;
+		} catch (error) {
+			if (this.db.isTransaction) this.db.exec("ROLLBACK");
+			throw error;
+		} finally {
+			this.depth = 0;
+		}
+	}
+	generation() {
+		return Number(this.getMeta("generation") ?? "1");
+	}
+	/**
+	* This method does a step on the tables of the current generation. A rebuild of a different
+	* process can remove those tables in the meantime. Then the method reads the generation again
+	* and does the step once more.
+	*/
+	live(step) {
+		if (this.closed) throw new Error("the index is closed");
+		try {
+			return step(namesOf(this.generation()));
+		} catch (error) {
+			if (!/no such table/i.test(messageOf(error)) || this.depth > 0) throw error;
+			this.statements.clear();
+			return step(namesOf(this.generation()));
+		}
+	}
+	/** This method removes the tables of a generation. */
+	dropGeneration(generation) {
+		const n = namesOf(generation);
+		for (const table of [
+			n.fts,
+			n.docs,
+			n.versions,
+			n.texts,
+			n.latest
+		]) this.db.exec(`DROP TABLE IF EXISTS ${table}`);
+		this.db.prepare("DELETE FROM index_meta WHERE key = ?").run(`building:${String(generation)}`);
+		this.statements.clear();
+	}
+	/** This method removes the generations of builds that a stopped process left. */
+	dropAbandoned() {
+		const current = this.generation();
+		const generations = /* @__PURE__ */ new Set();
+		for (const { name } of this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all()) {
+			const match = GENERATION_TABLE.exec(name);
+			if (match !== null) generations.add(Number(match[2]));
+		}
+		for (const generation of generations) {
+			if (generation === current) continue;
+			const started = this.getMeta(`building:${String(generation)}`);
+			if (!((started === void 0 ? Number.POSITIVE_INFINITY : Date.now() - Date.parse(started)) < ABANDONED_MS)) this.dropGeneration(generation);
+		}
+	}
+	putText(n, body) {
+		const key = sha(body);
+		this.stmt(`INSERT OR IGNORE INTO ${n.texts} (sha, body) VALUES (?, ?)`).run(key, body);
+		return key;
+	}
+	textOf(n, key) {
+		if (key === null) return void 0;
+		return this.stmt(`SELECT body FROM ${n.texts} WHERE sha = ?`).get(key)?.body;
+	}
+	putVersion(n, input, now) {
+		const full = input.full_text !== void 0 && input.full_text !== input.text ? this.putText(n, input.full_text) : null;
+		this.stmt(`INSERT INTO ${n.versions} (doc, capture, recipe, rendered, fetched_at, blob, kind, title, url,
+         meta_sha, text_sha, text_chars, full_sha, ai_input, merge, scope, labels)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(doc, capture) DO UPDATE SET recipe = excluded.recipe, rendered = excluded.rendered,
+         fetched_at = excluded.fetched_at, blob = excluded.blob, kind = excluded.kind, title = excluded.title,
+         url = excluded.url, meta_sha = excluded.meta_sha, text_sha = excluded.text_sha,
+         text_chars = excluded.text_chars, full_sha = excluded.full_sha, ai_input = excluded.ai_input,
+         merge = excluded.merge, scope = excluded.scope, labels = excluded.labels`).run(input.doc, input.capture ?? input.captures[0] ?? "", input.recipe ?? "", input.rendered === true ? 1 : 0, input.fetched_at ?? now, input.blob ?? null, input.kind, input.title, input.url, this.putText(n, JSON.stringify(input.meta ?? {})), this.putText(n, input.text), input.text.length, full, input.ai_input ?? "unset", input.merge ?? "richest", input.scope ?? null, input.labels === void 0 ? null : JSON.stringify(input.labels));
+	}
+	/** This method gives the newest version of a document for one recipe and one variant, at `until` or before when it is given. */
+	newest(n, doc, recipe, rendered, until) {
+		if (until === void 0) return this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = ?
+          ORDER BY fetched_at DESC, capture DESC LIMIT 1`).get(doc, recipe, rendered);
+		return this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = ? AND fetched_at <= ?
+        ORDER BY fetched_at DESC, capture DESC LIMIT 1`).get(doc, recipe, rendered, until);
+	}
+	/**
+	* This method chooses the current version of a document with the rules of the class. With
+	* `until`, it uses only the versions that the crawler fetched at that time or before. Thus it
+	* gives the version that was current at that time. It gives undefined when no version agrees.
+	*/
+	choose(n, doc, until) {
+		const groups = until === void 0 ? this.stmt(`SELECT DISTINCT recipe, rendered FROM ${n.versions} WHERE doc = ?`).all(doc) : this.stmt(`SELECT DISTINCT recipe, rendered FROM ${n.versions} WHERE doc = ? AND fetched_at <= ?`).all(doc, until);
+		if (groups.length === 0) return void 0;
+		const byRecipe = /* @__PURE__ */ new Map();
+		for (const group of groups) {
+			const row = this.newest(n, doc, group.recipe, group.rendered, until);
+			if (row === void 0) continue;
+			const entry = byRecipe.get(group.recipe) ?? {};
+			if (group.rendered === 1) entry.rendered = row;
+			else entry.plain = row;
+			byRecipe.set(group.recipe, entry);
+		}
+		const candidates = [];
+		for (const [recipe, { plain, rendered }] of byRecipe) {
+			let pick;
+			if (rendered === void 0) pick = plain;
+			else if (plain === void 0 || rendered.fetched_at >= plain.fetched_at) pick = rendered;
+			else {
+				const base = this.stmt(`SELECT blob FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = 0 AND fetched_at <= ?
+            ORDER BY fetched_at DESC, capture DESC LIMIT 1`).get(doc, recipe, rendered.fetched_at);
+				pick = base !== void 0 && base.blob !== null && base.blob === plain.blob ? rendered : plain;
+			}
+			const newest = plain === void 0 ? rendered : rendered === void 0 || plain.fetched_at > rendered.fetched_at ? plain : rendered;
+			const others = [plain, rendered].filter((row) => row !== void 0 && row !== pick);
+			candidates.push({
+				row: pick,
+				ai: newest.ai_input,
+				others
+			});
+		}
+		let winner = candidates[0];
+		if (candidates.length > 1) {
+			const latest = candidates.every((candidate) => candidate.row.merge === "latest");
+			for (const candidate of candidates.slice(1)) {
+				const a = candidate.row;
+				const b = winner.row;
+				if (latest ? a.fetched_at > b.fetched_at : a.text_chars > b.text_chars || a.text_chars === b.text_chars && a.fetched_at > b.fetched_at) winner = candidate;
+			}
+		}
+		const aiInput = winner.row.ai_input === "no" || candidates.some((candidate) => candidate.ai === "no") ? "no" : winner.ai !== "unset" ? winner.ai : candidates.some((candidate) => candidate.ai === "yes") ? "yes" : "unset";
+		return {
+			winner,
+			candidates,
+			aiInput
+		};
+	}
+	/** This method chooses the current version of a document from its versions, and writes it with its search entry. */
+	recompute(n, doc, now) {
+		const before = this.stmt(`SELECT rowid, text_sha, title FROM ${n.docs} WHERE doc = ?`).get(doc);
+		const choice = this.choose(n, doc);
+		if (choice === void 0) {
+			if (before !== void 0) {
+				this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);
+				this.stmt(`DELETE FROM ${n.docs} WHERE doc = ?`).run(doc);
+			}
+			return;
+		}
+		const { winner, candidates, aiInput } = choice;
+		const row = winner.row;
+		let meta = parseJson$1(this.textOf(n, row.meta_sha), {});
+		for (const candidate of candidates) if (candidate !== winner) meta = {
+			...parseJson$1(this.textOf(n, candidate.row.meta_sha), {}),
+			...meta
+		};
+		const same = this.stmt(`SELECT capture FROM ${n.versions} WHERE doc = ? AND text_sha = ?
+        ORDER BY fetched_at DESC, capture DESC LIMIT ?`).all(doc, row.text_sha, 10);
+		const captures = [.../* @__PURE__ */ new Set([
+			row.capture,
+			...same.map((entry) => entry.capture),
+			...winner.others.map((other) => other.capture),
+			...candidates.filter((candidate) => candidate !== winner).map((candidate) => candidate.row.capture)
+		])].slice(0, 10);
+		const count = this.stmt(`SELECT count(*) AS n FROM ${n.versions} WHERE doc = ?`).get(doc).n;
+		const url = row.url ?? candidates.map((candidate) => candidate.row.url).find((value) => value !== null) ?? null;
+		const saved = this.stmt(`INSERT INTO ${n.docs} (doc, kind, title, url, meta, text_sha, full_sha, captures, capture_count, capture,
+         blob, fetched_at, recipe, rendered, ai_input, scope, labels, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(doc) DO UPDATE SET kind = excluded.kind, title = excluded.title, url = excluded.url,
+         meta = excluded.meta, text_sha = excluded.text_sha, full_sha = excluded.full_sha,
+         captures = excluded.captures, capture_count = excluded.capture_count, capture = excluded.capture,
+         blob = excluded.blob, fetched_at = excluded.fetched_at, recipe = excluded.recipe,
+         rendered = excluded.rendered, ai_input = excluded.ai_input, scope = excluded.scope,
+         labels = excluded.labels, updated_at = excluded.updated_at
+       RETURNING rowid`).get(doc, row.kind, row.title, url, JSON.stringify(meta), row.text_sha, row.full_sha, JSON.stringify(captures), count, row.capture, row.blob, row.fetched_at, row.recipe, row.rendered, aiInput, row.scope, row.labels, now);
+		const text = this.textOf(n, row.text_sha) ?? "";
+		if (before === void 0 || before.text_sha !== row.text_sha || before.title !== row.title) {
+			if (before !== void 0) this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);
+			this.stmt(`INSERT INTO ${n.fts} (rowid, title, text) VALUES (?, ?, ?)`).run(saved.rowid, fold(row.title, "search"), fold(text.length > MAX_INDEX_CHARS ? text.slice(0, MAX_INDEX_CHARS) : text, "search"));
+		}
+		return this.docOf(n, doc, text);
+	}
+	docOf(n, doc, knownText) {
+		const row = this.stmt(`SELECT rowid, * FROM ${n.docs} WHERE doc = ?`).get(doc);
+		if (row === void 0) return void 0;
+		return this.toDoc(n, row, knownText);
+	}
+	toDoc(n, row, knownText) {
+		const labels = parseJson$1(row.labels, void 0);
+		return {
+			doc: row.doc,
+			kind: row.kind,
+			title: row.title,
+			text: knownText ?? this.textOf(n, row.text_sha) ?? "",
+			url: row.url,
+			meta: parseJson$1(row.meta, {}),
+			captures: parseJson$1(row.captures, []),
+			capture_count: row.capture_count,
+			capture: row.capture,
+			blob: row.blob,
+			fetched_at: row.fetched_at,
+			recipe: row.recipe,
+			rendered: row.rendered === 1,
+			ai_input: row.ai_input,
+			cut: row.full_sha !== null,
+			...row.scope !== null ? { scope: row.scope } : {},
+			...labels !== void 0 ? { labels } : {},
+			updated_at: row.updated_at
+		};
+	}
+	versionTextOf(n, row) {
+		return {
+			...toVersion(row),
+			text: this.textOf(n, row.text_sha) ?? "",
+			full_text: this.textOf(n, row.full_sha),
+			meta: parseJson$1(this.textOf(n, row.meta_sha), {})
+		};
+	}
+	get(doc) {
+		return this.live((n) => this.docOf(n, doc));
+	}
+	/**
+	* This method adds one version of a document, and it gives the current document. The rules of
+	* the class choose the current version. An upsert of a known capture replaces that version.
+	*/
+	upsert(input, now) {
+		return this.live((n) => this.write(() => {
+			this.putVersion(n, input, now);
+			return this.recompute(n, input.doc, now);
+		}));
+	}
+	/** This method gives the versions of a document, the newest first. */
+	versions(doc, options = {}) {
+		const limit = Math.max(1, Math.floor(options.limit ?? 100));
+		return this.live((n) => this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? ORDER BY fetched_at DESC, capture DESC LIMIT ?`).all(doc, limit).map(toVersion));
+	}
+	/**
+	* This method gives the text of one version of a document. With no capture, it gives the current
+	* version. It also gives the text of a held-back version: the caller applies `isHeldBack` to the
+	* `ai_input` of the version.
+	*/
+	versionText(doc, capture) {
+		return this.live((n) => {
+			const id = capture ?? this.docOf(n, doc)?.capture;
+			if (id === void 0) return void 0;
+			const row = this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND capture = ?`).get(doc, id);
+			return row === void 0 ? void 0 : this.versionTextOf(n, row);
+		});
+	}
+	/**
+	* This method gives the version of a document that was current at a time (ISO 8601). It applies
+	* the rules of the class to the versions that the crawler fetched at that time or before. Thus
+	* a later capture does not change the answer. It also gives a held-back version: its `ai_input`
+	* is `no` when the document was held back at that time, and the caller applies `isHeldBack`.
+	*/
+	versionAt(doc, at) {
+		return this.live((n) => {
+			const first = this.stmt(`SELECT min(fetched_at) AS first FROM ${n.versions} WHERE doc = ?`).get(doc)?.first ?? void 0;
+			const choice = this.choose(n, doc, at);
+			if (choice === void 0) return {
+				version: void 0,
+				first
+			};
+			const version = this.versionTextOf(n, choice.winner.row);
+			return {
+				version: choice.aiInput === "no" ? {
+					...version,
+					ai_input: "no"
+				} : version,
+				first
+			};
+		});
+	}
+	/**
+	* This method gives the texts of the earlier versions of a document, the newest first. Each
+	* text comes one time, with the newest version that has it. It skips the current text. It also
+	* gives held-back texts: the caller applies `isHeldBack` to the `ai_input` of each version.
+	*/
+	earlierTexts(doc, options = {}) {
+		const limit = Math.max(1, Math.floor(options.limit ?? 50));
+		return this.live((n) => {
+			const head = this.stmt(`SELECT text_sha FROM ${n.docs} WHERE doc = ?`).get(doc);
+			if (head === void 0) return [];
+			const rows = this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND text_sha != ? ORDER BY fetched_at DESC, capture DESC`).all(doc, head.text_sha);
+			const seen = /* @__PURE__ */ new Set();
+			const out = [];
+			for (const row of rows) {
+				if (seen.has(row.text_sha)) continue;
+				seen.add(row.text_sha);
+				out.push(this.versionTextOf(n, row));
+				if (out.length >= limit) break;
+			}
+			return out;
+		});
+	}
+	/** This method tells if a capture gave a version of a document. */
+	hasVersion(capture) {
+		return this.live((n) => this.stmt(`SELECT 1 AS found FROM ${n.versions} WHERE capture = ? LIMIT 1`).get(capture) !== void 0);
+	}
+	/**
+	* This method gives the hash of the current text of documents, with no text. It gives the
+	* documents of `docs`, or each document after `after` in the sequence of the ids, to `limit`.
+	*/
+	digests(options = {}) {
+		return this.live((n) => {
+			if (options.docs !== void 0) {
+				if (options.docs.length === 0) return [];
+				return this.stmt(`SELECT doc, kind, ai_input, text_sha FROM ${n.docs} WHERE doc IN (SELECT value FROM json_each(?))`).all(JSON.stringify(options.docs));
+			}
+			const limit = Math.max(1, Math.floor(options.limit ?? 500));
+			return this.stmt(`SELECT doc, kind, ai_input, text_sha FROM ${n.docs} WHERE doc > ? ORDER BY doc LIMIT ?`).all(options.after ?? "", limit);
+		});
+	}
+	/** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
+	newestFetch() {
+		return this.live((n) => this.stmt(`SELECT max(fetched_at) AS newest FROM ${n.latest}`).get()?.newest ?? void 0);
+	}
+	/** This method gives the current documents that a capture gave. */
+	docsOfCapture(capture) {
+		return this.live((n) => this.stmt(`SELECT DISTINCT doc FROM ${n.versions} WHERE capture = ?`).all(capture).map((row) => this.docOf(n, row.doc)).filter((doc) => doc !== void 0));
+	}
+	/**
+	* This method searches the documents with BM25, the best first. The syntax of the query is the
+	* syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it
+	* adds the documents that match one part. A document that asked not to be AI input is not in
+	* the results, so a search cannot show its text. The option `rank` chooses the ranking (`RankMode`).
+	*/
+	find(q, options) {
+		const rank = rankModeOf(options.rank);
+		const query = parseQuery(q);
+		if (query === void 0) return [];
+		const limit = Math.max(1, Math.min(1e3, Math.floor(options.limit) || 10));
+		return this.live((n) => {
+			const kindClause = options.kind === void 0 || options.kind === null ? "" : "AND d.kind = ?";
+			const sql = `SELECT d.rowid AS rowid, d.*, bm25(${n.fts}, 3.0, 1.0) AS rank
+          FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid
+         WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' ${kindClause}
+         ORDER BY rank LIMIT ?`;
+			const run = (match) => {
+				const params = [match];
+				if (kindClause.length > 0) params.push(options.kind);
+				params.push(limit);
+				return this.stmt(sql).all(...params);
+			};
+			const rows = run(query.all);
+			if (rows.length < limit && query.parts > 1) {
+				const seen = new Set(rows.map((row) => row.doc));
+				for (const row of run(rank === "fast" ? this.rareParts(n, query.anyParts) : query.any)) {
+					if (rows.length >= limit) break;
+					if (!seen.has(row.doc)) rows.push(row);
+				}
+			}
+			return rows.map((row) => ({
+				doc: row.doc,
+				kind: row.kind,
+				title: row.title,
+				url: row.url,
+				snippet: snippetOf(this.textOf(n, row.text_sha) ?? "", query.terms),
+				score: Math.round(-row.rank * 1e3) / 1e3,
+				ai_input: row.ai_input,
+				fetched_at: row.fetched_at,
+				capture: row.capture
+			}));
+		});
+	}
+	/**
+	* This method scores a set of documents against a query, with the best first. It leaves out a
+	* document that matches no term, and a document that asked not to be AI input. It ranks the
+	* results of a source again, because the order of the source is only as good as its search engine.
+	* In the mode `fast`, it also leaves out a document that holds only common parts (`RankMode`).
+	*/
+	rankWithin(q, docs, options = {}) {
+		const rank = rankModeOf(options.rank);
+		const query = parseQuery(q);
+		if (query === void 0 || docs.length === 0) return [];
+		return this.live((n) => this.stmt(`SELECT d.doc AS doc, bm25(${n.fts}, 3.0, 1.0) AS rank
+             FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid
+            WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' AND d.doc IN (SELECT value FROM json_each(?))
+            ORDER BY rank`).all(rank === "fast" ? this.rareParts(n, query.anyParts) : query.any, JSON.stringify(docs)).map((row) => ({
+			doc: row.doc,
+			score: Math.round(-row.rank * 1e3) / 1e3
+		})));
+	}
+	/**
+	* This method gives the form of a query that needs one part, without the common parts (the mode
+	* `fast` of `RankMode`). It counts the documents that hold each part. When each part is common,
+	* it keeps the rarest part.
+	*/
+	rareParts(n, parts) {
+		if (parts.length < 2) return parts.join(" OR ");
+		const total = this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get().n;
+		const counted = parts.map((part) => ({
+			part,
+			count: this.stmt(`SELECT count(*) AS n FROM ${n.fts} WHERE ${n.fts} MATCH ?`).get(part).n
+		}));
+		const rare = counted.filter((one) => one.count <= FAST_RANK_SHARE * total);
+		if (rare.length > 0) return rare.map((one) => one.part).join(" OR ");
+		return counted.reduce((best, one) => one.count < best.count ? one : best).part;
+	}
+	/** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */
+	latest(url, variant) {
+		const key = keyOf(url, variant);
+		return this.live((n) => {
+			const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(key.url, key.variant);
+			return row === void 0 ? void 0 : toLatest(row);
+		});
+	}
+	putLatest(n, entry) {
+		const key = keyOf(entry.url, entry.variant);
+		this.stmt(`INSERT INTO ${n.latest} (url, variant, capture, blob, etag, last_modified, content_type, fetched_at,
+         status, ai_input, location, truncated, fresh_until, vary_key)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(url, variant) DO UPDATE SET capture = excluded.capture, blob = excluded.blob,
+         etag = excluded.etag, last_modified = excluded.last_modified, content_type = excluded.content_type,
+         fetched_at = excluded.fetched_at, status = excluded.status, ai_input = excluded.ai_input,
+         location = excluded.location, truncated = excluded.truncated, fresh_until = excluded.fresh_until,
+         vary_key = excluded.vary_key
+       WHERE excluded.fetched_at >= ${n.latest}.fetched_at`).run(key.url, key.variant, entry.capture, entry.blob, entry.etag, entry.last_modified, entry.content_type, entry.fetched_at, entry.status, entry.ai_input, entry.location ?? null, entry.truncated === true ? 1 : null, entry.fresh_until ?? null, entry.vary_key ?? null);
+	}
+	/** This method keeps the last answer for a URL. An entry that is older than the stored entry changes nothing. */
+	setLatest(entry) {
+		this.live((n) => this.write(() => this.putLatest(n, entry)));
+	}
+	counts() {
+		return this.live((n) => ({
+			docs: this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get().n,
+			urls: this.stmt(`SELECT count(*) AS n FROM ${n.latest}`).get().n
+		}));
+	}
+	/** This method empties the current generation. A rebuild uses `beginRebuild`, which keeps the index until its commit. */
+	reset() {
+		this.live((n) => this.write(() => {
+			for (const table of [
+				n.fts,
+				n.docs,
+				n.versions,
+				n.texts,
+				n.latest
+			]) this.db.exec(`DELETE FROM ${table}`);
+		}));
+	}
+	/**
+	* This method starts a rebuild into a new generation of the tables. The current generation
+	* answers each read and takes each write until the commit. A failed rebuild thus leaves the
+	* index as it was.
+	*/
+	beginRebuild(options = {}) {
+		const batchSize = Math.max(1, Math.floor(options.batch ?? 500));
+		const generation = this.write(() => {
+			const next = Number(this.getMeta("next_generation") ?? "2");
+			this.setMeta("next_generation", String(next + 1));
+			this.setMeta(`building:${String(next)}`, (/* @__PURE__ */ new Date()).toISOString());
+			this.db.exec(ddlOf(namesOf(next)));
+			return next;
+		});
+		const n = namesOf(generation);
+		const versions = [];
+		const latest = /* @__PURE__ */ new Map();
+		let open = true;
+		const keyText = (url, variant) => {
+			const key = keyOf(url, variant);
+			return `${key.variant} ${key.url}`;
+		};
+		const flush = () => {
+			if (!open) throw new Error("the rebuild is over");
+			if (versions.length === 0 && latest.size === 0) return;
+			this.write(() => {
+				const touched = /* @__PURE__ */ new Map();
+				for (const { input, now } of versions) {
+					this.putVersion(n, input, now);
+					touched.set(input.doc, now);
+				}
+				for (const entry of latest.values()) this.putLatest(n, entry);
+				for (const [doc, now] of touched) this.recompute(n, doc, now);
+			});
+			versions.length = 0;
+			latest.clear();
+		};
+		/** This function copies the texts of the versions of the current generation that agree with `where`. */
+		const copyTexts = (current, where, params) => {
+			for (const column of [
+				"text_sha",
+				"meta_sha",
+				"full_sha"
+			]) this.db.prepare(`INSERT OR IGNORE INTO ${n.texts} (sha, body)
+             SELECT t.sha, t.body FROM ${current.texts} t
+              WHERE t.sha IN (SELECT ${column} FROM ${current.versions} WHERE ${where})`).run(...params);
+		};
+		return {
+			upsert: (input, now) => {
+				if (!open) throw new Error("the rebuild is over");
+				versions.push({
+					input,
+					now
+				});
+				if (versions.length >= batchSize) flush();
+			},
+			setLatest: (entry) => {
+				if (!open) throw new Error("the rebuild is over");
+				const key = keyText(entry.url, entry.variant);
+				const before = latest.get(key);
+				if (before === void 0 || entry.fetched_at >= before.fetched_at) latest.set(key, entry);
+				if (latest.size >= batchSize) flush();
+			},
+			latest: (url, variant) => {
+				const waiting = latest.get(keyText(url, variant));
+				if (waiting !== void 0) return waiting;
+				const key = keyOf(url, variant);
+				const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(key.url, key.variant);
+				return row === void 0 ? void 0 : toLatest(row);
+			},
+			flush,
+			carry: (recipes) => {
+				flush();
+				const known = JSON.stringify([...recipes]);
+				const current = namesOf(this.generation());
+				const docs = this.db.prepare(`SELECT DISTINCT doc FROM ${current.versions} WHERE recipe NOT IN (SELECT value FROM json_each(?))`).all(known);
+				const where = "recipe NOT IN (SELECT value FROM json_each(?)) AND doc IN (SELECT value FROM json_each(?))";
+				for (let at = 0; at < docs.length; at += batchSize) {
+					const part = JSON.stringify(docs.slice(at, at + batchSize).map((row) => row.doc));
+					this.write(() => {
+						copyTexts(current, where, [known, part]);
+						this.db.prepare(`INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${current.versions} WHERE ${where}`).run(known, part);
+						const now = (/* @__PURE__ */ new Date()).toISOString();
+						for (const doc of JSON.parse(part)) this.recompute(n, doc, now);
+					});
+				}
+				return docs.length;
+			},
+			commit: (since) => {
+				flush();
+				open = false;
+				return this.write(() => {
+					const current = this.generation();
+					const old = namesOf(current);
+					copyTexts(old, "fetched_at >= ?", [since]);
+					const added = this.db.prepare(`SELECT DISTINCT doc FROM ${old.versions} WHERE fetched_at >= ?`).all(since);
+					this.db.prepare(`INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${old.versions} WHERE fetched_at >= ?`).run(since);
+					const now = (/* @__PURE__ */ new Date()).toISOString();
+					for (const { doc } of added) this.recompute(n, doc, now);
+					for (const row of this.db.prepare(`SELECT * FROM ${old.latest} WHERE fetched_at >= ?`).all(since)) this.putLatest(n, toLatest(row));
+					this.setMeta("generation", String(generation));
+					this.db.prepare("DELETE FROM index_meta WHERE key = ?").run(`building:${String(generation)}`);
+					this.dropGeneration(current);
+					this.empty = false;
+					return {
+						docs: this.db.prepare(`SELECT count(*) AS n FROM ${n.docs}`).get().n,
+						urls: this.db.prepare(`SELECT count(*) AS n FROM ${n.latest}`).get().n
+					};
+				});
+			},
+			abort: () => {
+				open = false;
+				versions.length = 0;
+				latest.clear();
+				if (this.closed) return;
+				this.write(() => this.dropGeneration(generation));
+			}
+		};
+	}
+	/** This method closes the file. A second close does nothing. */
+	close() {
+		if (this.closed) return;
+		this.closed = true;
+		this.statements.clear();
+		this.db.close();
+	}
+};
+//#endregion
+//#region src/sitemaps.ts
+/** These are the default limits of the reader, and the maximum length of one value. */
+const SITEMAP_LIMITS = {
+	maxEntries: 5e4,
+	maxDepth: 32,
+	/** A value (a URL or a date) longer than this number of characters is not valid. */
+	maxValue: 4096
+};
+const CHANGEFREQ = /* @__PURE__ */ new Set([
+	"always",
+	"hourly",
+	"daily",
+	"weekly",
+	"monthly",
+	"yearly",
+	"never"
+]);
+const SITEMAP_NS = [
+	"http://www.sitemaps.org/schemas/sitemap/0.9",
+	"http://www.google.com/schemas/sitemap/0.84",
+	"http://www.google.com/schemas/sitemap/0.9"
+];
+const ATOM_NS = ["http://www.w3.org/2005/Atom"];
+const RSS1_NS = ["http://purl.org/rss/1.0/"];
+const DC_NS = ["http://purl.org/dc/elements/1.1/"];
+const RDF_NS = ["http://www.w3.org/1999/02/22-rdf-syntax-ns#"];
+const XML_ENTITIES = {
+	amp: "&",
+	lt: "<",
+	gt: ">",
+	quot: "\"",
+	apos: "'"
+};
+/** A tag keeps this number of attributes at most. The reader reads the others and keeps nothing of them. */
+const MAX_ATTRIBUTES = 64;
+const NAME$1 = /^[A-Za-z_][\w.:-]*$/;
+const W3C_DATE = /^\d{4}(?:-\d{2}(?:-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?)?)?$/;
+const MAIL_DATE = /^(?:[A-Za-z]{3},?\s+)?\d{1,2}\s+[A-Za-z]{3,9}\s+\d{2,4}\s+\d{1,2}:\d{2}(?::\d{2})?(?:\s*(?:[A-Za-z]{1,5}|[+-]\d{4}))?$/;
+const short = (name) => name === void 0 ? "nothing" : name.length > 40 ? `${name.slice(0, 40)}...` : name;
+/**
+* This function changes a date of a sitemap or a feed into ISO 8601. It reads the W3C form of the
+* sitemap protocol and Atom, and the mail form of RSS. A time with no zone is UTC. It gives null
+* for a text that it cannot read.
+*/
+const feedDate = (value) => {
+	const text = value.trim();
+	if (text.length === 0 || text.length > 64) return null;
+	let parse;
+	const w3c = W3C_DATE.exec(text);
+	if (w3c !== null) {
+		const iso = text.replace(" ", "T");
+		parse = iso.includes("T") && w3c[1] === void 0 ? `${iso}Z` : iso;
+	} else if (MAIL_DATE.test(text)) parse = text;
+	if (parse === void 0) return null;
+	const ms = Date.parse(parse);
+	return Number.isFinite(ms) && Math.abs(ms) < 864e13 ? new Date(ms).toISOString() : null;
+};
+/** This function gives the canonical form of an http(s) URL, or undefined. A relative URL needs a base. */
+const webUrl = (value, base) => {
+	const text = value.trim();
+	if (text.length === 0 || text.length > SITEMAP_LIMITS.maxValue) return void 0;
+	try {
+		const url = base === void 0 ? new URL(text) : new URL(text, base);
+		if (url.protocol !== "http:" && url.protocol !== "https:") return void 0;
+		return canonicalUrl(url.href);
+	} catch {
+		return;
+	}
+};
+/** This function decodes the five entities of XML and the character references. It gives undefined for each other entity. */
+const decodeText = (raw, meter) => {
+	meter.steps += raw.length;
+	if (!raw.includes("&")) return raw;
+	let out = "";
+	let at = 0;
+	for (;;) {
+		const amp = raw.indexOf("&", at);
+		if (amp === -1) return out + raw.slice(at);
+		out += raw.slice(at, amp);
+		const semi = raw.indexOf(";", amp + 1);
+		if (semi === -1 || semi - amp > 10) return void 0;
+		const name = raw.slice(amp + 1, semi);
+		let char = XML_ENTITIES[name];
+		if (char === void 0 && /^#(?:\d{1,7}|x[0-9a-fA-F]{1,6})$/.test(name)) {
+			const code = name[1] === "x" ? Number.parseInt(name.slice(2), 16) : Number(name.slice(1));
+			if (code > 0 && code <= 1114111 && (code < 55296 || code > 57343)) char = String.fromCodePoint(code);
+		}
+		if (char === void 0) return void 0;
+		out += char;
+		at = semi + 1;
+	}
+};
+const isSpace = (code) => code === 32 || code === 9 || code === 10 || code === 13;
+/** This function reads a start tag from the position after its `<`. It gives undefined for a tag that is not well formed. */
+const readTag = (text, from, meter) => {
+	const tag = tagAt(text, from);
+	meter.steps += (tag?.end ?? text.length) - from;
+	return tag;
+};
+const tagAt = (text, from) => {
+	const n = text.length;
+	let at = from;
+	while (at < n) {
+		const code = text.charCodeAt(at);
+		if (isSpace(code) || code === 47 || code === 62) break;
+		at += 1;
+	}
+	const name = text.slice(from, Math.min(at, from + 256));
+	if (at - from > 256 || !NAME$1.test(name)) return void 0;
+	const attributes = [];
+	for (;;) {
+		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
+		if (at >= n) return void 0;
+		const code = text.charCodeAt(at);
+		if (code === 62) return {
+			name,
+			attributes,
+			selfClosing: false,
+			end: at + 1
+		};
+		if (code === 47) return text.charCodeAt(at + 1) === 62 ? {
+			name,
+			attributes,
+			selfClosing: true,
+			end: at + 2
+		} : void 0;
+		const start = at;
+		while (at < n) {
+			const next = text.charCodeAt(at);
+			if (isSpace(next) || next === 61 || next === 62 || next === 47) break;
+			at += 1;
+		}
+		const key = text.slice(start, Math.min(at, start + 256));
+		if (at - start > 256 || !NAME$1.test(key)) return void 0;
+		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
+		if (text.charCodeAt(at) !== 61) return void 0;
+		at += 1;
+		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
+		const quote = text[at];
+		if (quote !== "\"" && quote !== "'") return void 0;
+		const close = text.indexOf(quote, at + 1);
+		if (close === -1) return void 0;
+		if (attributes.length < MAX_ATTRIBUTES && close - at <= SITEMAP_LIMITS.maxValue) attributes.push([key, text.slice(at + 1, close)]);
+		at = close + 1;
+	}
+};
+/**
+* This function finds the end of a declaration (`<!DOCTYPE ...>`) with its internal subset. It
+* gives the position after the `>`, or -1, and it tells if the declaration has an internal subset.
+*/
+const declarationEnd = (text, from, meter) => {
+	let depth = 0;
+	let quote = 0;
+	let subset = false;
+	for (let at = from; at < text.length; at += 1) {
+		meter.steps += 1;
+		const code = text.charCodeAt(at);
+		if (quote !== 0) {
+			if (code === quote) quote = 0;
+		} else if (code === 34 || code === 39) quote = code;
+		else if (code === 91) {
+			depth += 1;
+			subset = true;
+		} else if (code === 93) depth = Math.max(0, depth - 1);
+		else if (code === 62 && depth === 0) return {
+			end: at + 1,
+			subset
+		};
+	}
+	return {
+		end: -1,
+		subset
+	};
+};
+/** This function reads a text sitemap: each line that is an absolute http(s) URL is an entry. */
+const readLines = (text, maxEntries, meter) => {
+	const entries = [];
+	let invalid = 0;
+	let at = 0;
+	while (at < text.length) {
+		let end = text.indexOf("\n", at);
+		if (end === -1) end = text.length;
+		meter.steps += end - at + 1;
+		const line = end - at > SITEMAP_LIMITS.maxValue + 2 ? "-" : text.slice(at, end).trim();
+		at = end + 1;
+		if (line.length === 0) continue;
+		const url = webUrl(line, void 0);
+		if (url === void 0) invalid += 1;
+		else entries.push({
+			url,
+			lastmod: null,
+			changefreq: null
+		});
+		if (entries.length >= maxEntries) {
+			const complete = at >= text.length;
+			return {
+				kind: "text",
+				entries,
+				invalid,
+				complete,
+				notes: complete ? [] : [`the reader stopped at ${String(maxEntries)} entries`]
+			};
+		}
+	}
+	return {
+		kind: "text",
+		entries,
+		invalid,
+		complete: true,
+		notes: []
+	};
+};
+/**
+* This function reads a sitemap, a sitemap index, a feed or a text sitemap, and it counts its
+* steps in `meter`. The function `readSitemap` uses it. A test uses it to show that the time is linear.
+*/
+const scanSitemap = (text, base, limits = {}, meter = { steps: 0 }) => {
+	const maxEntries = Math.max(1, Math.floor(limits.maxEntries ?? SITEMAP_LIMITS.maxEntries));
+	const maxDepth = Math.max(1, Math.floor(limits.maxDepth ?? SITEMAP_LIMITS.maxDepth));
+	let at = text.charCodeAt(0) === 65279 ? 1 : 0;
+	while (at < text.length && isSpace(text.charCodeAt(at))) at += 1;
+	if (at < text.length && text[at] !== "<") return readLines(text.slice(at), maxEntries, meter);
+	const n = text.length;
+	const notes = [];
+	const entries = [];
+	const stack = [];
+	const prefixes = /* @__PURE__ */ new Map();
+	let kind = "unknown";
+	let invalid = 0;
+	let complete = true;
+	let done = false;
+	let feedBase = base;
+	let entry;
+	let field;
+	const stop = (note) => {
+		complete = false;
+		done = true;
+		notes.push(note);
+	};
+	/** This function tells if a name is a local name of one of the namespaces, or the local name with no prefix. */
+	const named = (qname, local, spaces) => {
+		const colon = qname.indexOf(":");
+		if (colon === -1) return qname === local;
+		return qname.slice(colon + 1) === local && spaces.includes(prefixes.get(qname.slice(0, colon)) ?? "");
+	};
+	const attribute = (tag, local, spaces) => {
+		for (const [key, raw] of tag.attributes) if (spaces === void 0 ? key === local : named(key, local, spaces) && key.includes(":")) return decodeText(raw, meter);
+	};
+	const fieldOf = (qname) => {
+		switch (kind) {
+			case "urlset":
+			case "sitemapindex":
+				if (named(qname, "loc", SITEMAP_NS)) return "loc";
+				if (named(qname, "lastmod", SITEMAP_NS)) return "lastmod";
+				return kind === "urlset" && named(qname, "changefreq", SITEMAP_NS) ? "changefreq" : void 0;
+			case "rss":
+				if (qname === "link") return "link";
+				if (qname === "guid") return "guid";
+				return qname === "pubDate" ? "date" : void 0;
+			case "atom": return named(qname, "updated", ATOM_NS) || named(qname, "published", ATOM_NS) ? "date" : void 0;
+			case "rdf":
+				if (named(qname, "link", RSS1_NS)) return "link";
+				return named(qname, "date", DC_NS) ? "date" : void 0;
+			default: return;
+		}
+	};
+	const isEntry = (qname, depth) => {
+		switch (kind) {
+			case "urlset": return depth === 2 && named(qname, "url", SITEMAP_NS);
+			case "sitemapindex": return depth === 2 && named(qname, "sitemap", SITEMAP_NS);
+			case "rss": return depth === 3 && qname === "item" && stack[1] === "channel";
+			case "atom": return depth === 2 && named(qname, "entry", ATOM_NS);
+			case "rdf": return depth === 2 && named(qname, "item", RSS1_NS);
+			default: return false;
+		}
+	};
+	const resolveBase = (tag, against) => {
+		const declared = attribute(tag, "xml:base");
+		if (declared === void 0) return against;
+		try {
+			return new URL(declared.trim(), against).href;
+		} catch {
+			return against;
+		}
+	};
+	const open = (tag, depth) => {
+		for (const [key, raw] of tag.attributes) if (key.startsWith("xmlns:")) {
+			const uri = decodeText(raw, meter);
+			if (uri !== void 0) prefixes.set(key.slice(6), uri.trim());
+		}
+		if (depth === 1) {
+			const local = tag.name.slice(tag.name.indexOf(":") + 1);
+			if (local === "urlset" || local === "sitemapindex" || local === "rss") kind = local;
+			else if (local === "feed") kind = "atom";
+			else if (local === "RDF") kind = "rdf";
+			else {
+				stop(`not a sitemap or a feed: the root element is <${short(tag.name)}>`);
+				return;
+			}
+			feedBase = resolveBase(tag, base);
+			return;
+		}
+		if (entry === void 0) {
+			if (isEntry(tag.name, depth)) entry = {
+				depth,
+				base: kind === "atom" ? resolveBase(tag, feedBase) : feedBase,
+				loc: void 0,
+				link: void 0,
+				guid: void 0,
+				lastmod: void 0,
+				changefreq: void 0,
+				about: kind === "rdf" ? attribute(tag, "about", RDF_NS) : void 0,
+				linkPage: false
+			};
+			return;
+		}
+		if (field !== void 0) {
+			field.bad = true;
+			return;
+		}
+		if (depth !== entry.depth + 1) return;
+		if (kind === "atom" && named(tag.name, "link", ATOM_NS)) {
+			const rel = (attribute(tag, "rel") ?? "alternate").trim().toLowerCase();
+			const href = attribute(tag, "href");
+			const type = (attribute(tag, "type") ?? "text/html").trim().toLowerCase();
+			const page = type.startsWith("text/html") || type.startsWith("application/xhtml");
+			if (rel === "alternate" && href !== void 0 && (entry.link === void 0 || page && !entry.linkPage)) {
+				entry.link = href;
+				entry.linkPage = page;
+			}
+			return;
+		}
+		const which = fieldOf(tag.name);
+		if (which === void 0) return;
+		if (which === "guid") {
+			if ((attribute(tag, "isPermaLink") ?? "true").trim().toLowerCase() === "false") return;
+		}
+		field = {
+			field: which,
+			depth,
+			parts: [],
+			length: 0,
+			bad: false
+		};
+	};
+	const finishEntry = (current) => {
+		const raw = kind === "urlset" || kind === "sitemapindex" ? current.loc : current.link ?? (kind === "rdf" ? current.about : current.guid);
+		const url = raw === void 0 ? void 0 : webUrl(raw, kind === "urlset" || kind === "sitemapindex" ? void 0 : current.base);
+		if (url === void 0) {
+			invalid += 1;
+			return;
+		}
+		const freq = current.changefreq?.trim().toLowerCase();
+		entries.push({
+			url,
+			lastmod: current.lastmod === void 0 ? null : feedDate(current.lastmod),
+			changefreq: freq !== void 0 && CHANGEFREQ.has(freq) && kind === "urlset" ? freq : null
+		});
+		if (entries.length >= maxEntries) stop(`the reader stopped at ${String(maxEntries)} entries`);
+	};
+	const close = (depth) => {
+		if (field !== void 0 && depth === field.depth) {
+			const finished = field;
+			field = void 0;
+			if (finished.bad || entry === void 0) return;
+			const value = finished.parts.join("").trim();
+			if (value.length === 0) return;
+			if (finished.field === "loc") entry.loc ??= value;
+			else if (finished.field === "link") entry.link ??= value;
+			else if (finished.field === "guid") entry.guid ??= value;
+			else if (finished.field === "lastmod" || finished.field === "date") entry.lastmod ??= value;
+			else entry.changefreq ??= value;
+			return;
+		}
+		if (entry !== void 0 && depth === entry.depth) {
+			const finished = entry;
+			entry = void 0;
+			finishEntry(finished);
+		}
+	};
+	const addText = (value) => {
+		if (field === void 0 || field.bad) return;
+		if (value === void 0 || field.length + value.length > SITEMAP_LIMITS.maxValue) {
+			field.bad = true;
+			return;
+		}
+		field.parts.push(value);
+		field.length += value.length;
+	};
+	while (at < n && !done) {
+		meter.steps += 1;
+		const lt = text.indexOf("<", at);
+		const end = lt === -1 ? n : lt;
+		if (end > at) {
+			meter.steps += end - at;
+			if (field !== void 0 && !field.bad) addText(end - at > SITEMAP_LIMITS.maxValue ? void 0 : decodeText(text.slice(at, end), meter));
+		}
+		if (lt === -1) break;
+		at = lt;
+		if (text.startsWith("<!--", at)) {
+			const endOf = text.indexOf("-->", at + 4);
+			if (endOf === -1) {
+				stop("a comment does not end");
+				break;
+			}
+			meter.steps += endOf + 3 - at;
+			at = endOf + 3;
+			continue;
+		}
+		if (text.startsWith("<![CDATA[", at)) {
+			const endOf = text.indexOf("]]>", at + 9);
+			if (endOf === -1) {
+				stop("a CDATA section does not end");
+				break;
+			}
+			meter.steps += endOf + 3 - at;
+			if (field !== void 0 && !field.bad) addText(endOf - at - 9 > SITEMAP_LIMITS.maxValue ? void 0 : text.slice(at + 9, endOf));
+			at = endOf + 3;
+			continue;
+		}
+		if (text.startsWith("<?", at)) {
+			const endOf = text.indexOf("?>", at + 2);
+			if (endOf === -1) {
+				stop("a processing instruction does not end");
+				break;
+			}
+			meter.steps += endOf + 2 - at;
+			at = endOf + 2;
+			continue;
+		}
+		if (text.startsWith("<!", at)) {
+			const declaration = declarationEnd(text, at + 2, meter);
+			if (declaration.end === -1) {
+				stop("a declaration does not end");
+				break;
+			}
+			if (declaration.subset && /^<!DOCTYPE/i.test(text.slice(at, at + 9))) notes.push("the file declares a DTD: the reader ignores it and expands no entity");
+			at = declaration.end;
+			continue;
+		}
+		if (text.startsWith("</", at)) {
+			const endOf = text.indexOf(">", at + 2);
+			if (endOf === -1) {
+				stop("an end tag does not end");
+				break;
+			}
+			meter.steps += endOf + 1 - at;
+			const name = text.slice(at + 2, Math.min(endOf, at + 300)).trim();
+			const top = stack.pop();
+			if (top !== name) {
+				stop(`the end tag </${short(name)}> does not match <${short(top)}>`);
+				break;
+			}
+			close(stack.length + 1);
+			at = endOf + 1;
+			if (stack.length === 0) break;
+			continue;
+		}
+		const tag = readTag(text, at + 1, meter);
+		if (tag === void 0) {
+			stop("a start tag is not well formed");
+			break;
+		}
+		at = tag.end;
+		const depth = stack.length + 1;
+		if (depth > maxDepth) {
+			stop(`the elements nest deeper than ${String(maxDepth)}`);
+			break;
+		}
+		open(tag, depth);
+		if (done) break;
+		if (tag.selfClosing) close(depth);
+		else stack.push(tag.name);
+		if (depth === 1 && tag.selfClosing) break;
+	}
+	if (!done && stack.length > 0) stop("the file ends inside an element");
+	if (kind === "unknown" && notes.length === 0) {
+		complete = false;
+		notes.push("not a sitemap or a feed: the file has no root element");
+	}
+	return {
+		kind,
+		entries,
+		invalid,
+		complete,
+		notes
+	};
+};
+/**
+* This function reads a sitemap, a sitemap index, an RSS or Atom feed, or a text sitemap. The
+* root element of the file tells its kind. The entries of a feed can have URLs relative to `base`
+* (the URL of the file). The entries of a sitemap must have absolute URLs.
+*/
+const readSitemap = (text, base, limits) => scanSitemap(text, base, limits);
+//#endregion
+//#region src/discover.ts
+/**
+* Discovery finds the pages of a site from its sitemaps and its feeds. Each file is a fetch of the
+* crawler, so each polite rule applies to it. These rules are robots.txt, one request at a time for
+* each host, the pace, the stops, the egress policy and the capture log. These are the sources, in sequence:
+*
+* 1. The page at the URL of the site, for the RSS and Atom feeds that it links.
+* 2. The feeds that the caller gives.
+* 3. The sitemaps that the caller gives, and the `Sitemap:` lines of robots.txt. When there are
+*    none, `/sitemap.xml`.
+* 4. The sitemaps of each sitemap index, the newest first, to a maximum depth.
+*
+* Discovery keeps a URL only when it is in scope (the same site by default) and robots.txt permits
+* it. It stops at a maximum of files, a maximum of URLs and a maximum time.
+*/
+/** These are the default limits of `discover`. */
+const DISCOVER_LIMITS = {
+	/** The most sitemap and feed files that one discovery reads. */
+	maxFiles: 20,
+	/** The most URLs that one discovery gives. */
+	maxUrls: 1e4,
+	/** The most bytes of one file, after decompression: 10 MiB. */
+	maxBytes: 10485760,
+	/** The most levels of sitemap index under a file that discovery found first. */
+	maxNesting: 3,
+	/** The longest time of one discovery: 1 minute. */
+	budgetMs: 6e4,
+	/** The most origins whose robots.txt one discovery reads for the URLs that it finds. */
+	maxOrigins: 10,
+	/** The most feeds that discovery reads from the links of the page. */
+	maxLinkedFeeds: 5
+};
+const PAGE_ACCEPT$1 = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
+const SITEMAP_ACCEPT = "application/xml,text/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5";
+const FEED_ACCEPT = "application/rss+xml,application/atom+xml,application/xml;q=0.9,text/xml;q=0.8,*/*;q=0.5";
+const FEED_KINDS = /* @__PURE__ */ new Set([
+	"rss",
+	"atom",
+	"rdf"
+]);
+const whole$1 = (name, value, fallback, min) => {
+	if (value === void 0) return fallback;
+	if (!Number.isInteger(value) || value < min) throw new RangeError(`${name} must be an integer of ${String(min)} or more, not ${String(value)}`);
+	return value;
+};
+/**
+* This function reads a body that is gzip data (a `.xml.gz` sitemap). It keeps at most `maxBytes`
+* of the decompressed data, so a decompression bomb stops at the limit.
+*/
+const gunzipBounded = async (body, maxBytes) => {
+	const gunzip = createGunzip();
+	const chunks = [];
+	let size = 0;
+	let truncated = false;
+	let error;
+	gunzip.end(body);
+	try {
+		for await (const chunk of gunzip) {
+			const room = maxBytes - size;
+			if (chunk.length > room) {
+				chunks.push(chunk.subarray(0, room));
+				size += room;
+				truncated = true;
+				break;
+			}
+			chunks.push(chunk);
+			size += chunk.length;
+		}
+	} catch (caught) {
+		error = caught instanceof Error ? caught.message : String(caught);
+		truncated = true;
+	} finally {
+		gunzip.destroy();
+	}
+	return {
+		bytes: Buffer.concat(chunks, size),
+		truncated,
+		...error === void 0 ? {} : { error }
+	};
+};
+const isGzip = (body) => body[0] === 31 && body[1] === 139;
+const parsedUrl = (value, base) => {
+	try {
+		const url = base === void 0 ? new URL(value.trim()) : new URL(value.trim(), base);
+		return url.protocol === "http:" || url.protocol === "https:" ? url : void 0;
+	} catch {
+		return;
+	}
+};
+/** This function gives the value of a promise, or rejects when the signal aborts first. */
+const untilAbort = (work, signal) => {
+	if (signal === void 0) return work;
+	return new Promise((resolve, reject) => {
+		const cancel = () => reject(signal.reason);
+		if (signal.aborted) cancel();
+		signal.addEventListener("abort", cancel, { once: true });
+		work.then(resolve, reject).finally(() => signal.removeEventListener("abort", cancel));
+	});
+};
+/** This function tells if the note of a fetch says that its deadline came first. */
+const pastDeadline = (note) => note?.startsWith(DEADLINE_NOTE) === true;
+/**
+* This function finds the pages of a site from its sitemaps and its feeds. `PoliteCrawler.discover`
+* uses it. Each file is a fetch of the crawler with the recipe `sitemap` or `feed`, so the
+* capture log records it, and robots.txt applies to it.
+*/
+const discoverSite = async (runtime, site, options = {}) => {
+	const start = parsedUrl(site);
+	if (start === void 0) throw new Error(`discover wants an http(s) URL, not "${site}"`);
+	const maxFiles = whole$1("maxFiles", options.maxFiles, DISCOVER_LIMITS.maxFiles, 1);
+	const maxUrls = whole$1("maxUrls", options.maxUrls, DISCOVER_LIMITS.maxUrls, 1);
+	const maxBytes = whole$1("maxBytes", options.maxBytes, DISCOVER_LIMITS.maxBytes, 1);
+	const maxNesting = whole$1("maxNesting", options.maxNesting, DISCOVER_LIMITS.maxNesting, 0);
+	const maxAge = options.maxAge ?? "24h";
+	const maxAgeMs = maxAge === "auto" ? "auto" : durationMs(maxAge);
+	const budgetMs = options.budgetMs ?? DISCOVER_LIMITS.budgetMs;
+	if (!(budgetMs >= 0)) throw new RangeError(`budgetMs must be 0 or more, not ${String(budgetMs)}`);
+	const now = () => Date.parse(runtime.now());
+	const deadline = Math.min(now() + budgetMs, options.deadline ?? Number.POSITIVE_INFINITY);
+	const crawler = runtime.crawler;
+	const signal = options.signal;
+	const cancelled = () => signal?.aborted === true;
+	const marks = {
+		scope: options.scope,
+		labels: options.labels
+	};
+	const inScope = scopeTest([start], options);
+	const urls = /* @__PURE__ */ new Map();
+	const sources = [];
+	const dropped = {
+		scope: 0,
+		robots: 0,
+		duplicate: 0,
+		invalid: 0,
+		limit: 0
+	};
+	const origins = /* @__PURE__ */ new Set();
+	const queued = /* @__PURE__ */ new Set();
+	const pending = [];
+	let stopped;
+	let note = null;
+	let files = 0;
+	let next = 0;
+	const stopNote = () => {
+		switch (stopped) {
+			case "max-files": return `discovery read ${String(maxFiles)} files, its limit, and ${String(pending.length - next)} more files wait`;
+			case "max-urls": return `discovery found ${String(maxUrls)} URLs, its limit`;
+			case "budget": return `the budget of ${String(budgetMs)} ms ended`;
+			case "cancelled": return "the caller cancelled the discovery";
+			case "closed": return "the crawler closed";
+			default: return null;
+		}
+	};
+	const result = () => ({
+		site: start.origin,
+		urls: [...urls.values()].map((url, order) => ({
+			url,
+			order
+		})).sort((a, b) => (b.url.lastmod === null ? -Infinity : Date.parse(b.url.lastmod)) - (a.url.lastmod === null ? -Infinity : Date.parse(a.url.lastmod)) || a.order - b.order).map(({ url }) => url),
+		sources,
+		dropped,
+		stopped: stopped ?? "done",
+		note: note ?? stopNote()
+	});
+	/** This function sets the reason of the stop from the note of a fetch, when the note gives one. */
+	const stoppedBy = (fetched) => {
+		if (cancelled()) stopped = "cancelled";
+		else if (pastDeadline(fetched.note)) stopped = "budget";
+		else if (fetched.note === "crawler closed") stopped = "closed";
+		else return false;
+		return true;
+	};
+	const enqueue = (url, via, from, nesting = 0) => {
+		const parsed = parsedUrl(url, start.href);
+		if (parsed === void 0) {
+			dropped.invalid += 1;
+			return;
+		}
+		const key = cacheKeyOf(parsed.href);
+		if (queued.has(key)) return;
+		queued.add(key);
+		pending.push({
+			url: parsed.href,
+			via,
+			from,
+			nesting
+		});
+	};
+	if (cancelled()) {
+		stopped = "cancelled";
+		return result();
+	}
+	let sitemapLines = [];
+	try {
+		const robots = await untilAbort(crawler.readRobots(start.href, marks), signal);
+		if (robots.robots.unreachable === "crawler closed") {
+			stopped = "closed";
+			return result();
+		}
+		if (robots.robots.unreachable !== void 0) {
+			stopped = "robots";
+			note = `robots.txt could not be read (${robots.robots.unreachable}), so nothing on the site is fetched`;
+			return result();
+		}
+		sitemapLines = robots.robots.sitemaps;
+	} catch {
+		stopped = cancelled() ? "cancelled" : "closed";
+		return result();
+	}
+	const fetchFile = (url, recipe) => crawler.fetch({
+		url,
+		mode: "page",
+		recipe,
+		accept: recipe === "feed" ? FEED_ACCEPT : SITEMAP_ACCEPT,
+		maxBytes,
+		maxAgeMs,
+		onBehalfOf: options.onBehalfOf,
+		signal,
+		scope: options.scope,
+		labels: options.labels,
+		deadline
+	});
+	if (options.linkedFeeds !== false) {
+		const page = await fetchAndIndex(runtime, {
+			url: start.href,
+			mode: "page",
+			recipe: "page",
+			accept: PAGE_ACCEPT$1,
+			maxAgeMs,
+			onBehalfOf: options.onBehalfOf,
+			signal,
+			scope: options.scope,
+			labels: options.labels,
+			deadline
+		});
+		if (stoppedBy(page.fetched)) return result();
+		const doc = page.docs[0];
+		const body = page.fetched.body;
+		const html = /html/i.test(page.fetched.contentType ?? "text/html");
+		if (body !== void 0 && html && doc !== void 0 && !isHeldBack(doc)) {
+			let linked = [];
+			try {
+				linked = extractPage(decodeBody(body, page.fetched.contentType ?? null), page.fetched.url).feeds ?? [];
+			} catch {}
+			let taken = 0;
+			for (const feed of linked) {
+				const parsed = parsedUrl(feed);
+				if (parsed === void 0 || !inScope(parsed)) {
+					dropped.scope += 1;
+					continue;
+				}
+				if (taken >= DISCOVER_LIMITS.maxLinkedFeeds) {
+					dropped.limit += 1;
+					continue;
+				}
+				taken += 1;
+				enqueue(parsed.href, "feed", "page");
+			}
+		}
+	}
+	for (const feed of options.feeds ?? []) enqueue(feed, "feed", "given");
+	for (const sitemap of options.sitemaps ?? []) enqueue(sitemap, "sitemap", "given");
+	for (const sitemap of sitemapLines) enqueue(sitemap, "sitemap", "robots.txt");
+	if ((options.sitemaps ?? []).length === 0 && sitemapLines.length === 0) enqueue(new URL("/sitemap.xml", start.origin).href, "sitemap", "default");
+	/** This function keeps the page URLs of one file that are new, in scope and permitted. */
+	const keep = async (read, source, via) => {
+		let kept = 0;
+		dropped.invalid += read.invalid;
+		for (const entry of read.entries) {
+			if (urls.size >= maxUrls) {
+				stopped = "max-urls";
+				return kept;
+			}
+			const known = urls.get(entry.url);
+			if (known !== void 0) {
+				dropped.duplicate += 1;
+				if (entry.lastmod !== null && (known.lastmod === null || Date.parse(entry.lastmod) > Date.parse(known.lastmod))) urls.set(entry.url, {
+					...known,
+					lastmod: entry.lastmod
+				});
+				continue;
+			}
+			const url = new URL(entry.url);
+			if (!inScope(url)) {
+				dropped.scope += 1;
+				continue;
+			}
+			if (!origins.has(url.origin)) {
+				if (origins.size >= DISCOVER_LIMITS.maxOrigins) {
+					dropped.limit += 1;
+					continue;
+				}
+				origins.add(url.origin);
+			}
+			let allowed;
+			try {
+				allowed = (await crawler.robotsAllow(entry.url, {
+					onBehalfOf: options.onBehalfOf,
+					signal,
+					...marks
+				})).allowed;
+			} catch {
+				stopped = "cancelled";
+				return kept;
+			}
+			if (!allowed) {
+				dropped.robots += 1;
+				continue;
+			}
+			urls.set(entry.url, {
+				url: entry.url,
+				lastmod: entry.lastmod,
+				changefreq: entry.changefreq,
+				via,
+				source
+			});
+			kept += 1;
+		}
+		return kept;
+	};
+	for (; next < pending.length && stopped === void 0; next += 1) {
+		if (cancelled()) {
+			stopped = "cancelled";
+			break;
+		}
+		if (now() >= deadline) {
+			stopped = "budget";
+			break;
+		}
+		if (files >= maxFiles) {
+			stopped = "max-files";
+			break;
+		}
+		const file = pending[next];
+		files += 1;
+		const fetched = await fetchFile(file.url, file.via);
+		const base = {
+			url: fetched.url,
+			via: file.via,
+			from: file.from,
+			outcome: fetched.outcome,
+			status: fetched.status,
+			capture: fetched.capture ?? null,
+			fetched_at: fetched.fetchedAt ?? null,
+			from_cache: fetched.fromCache
+		};
+		if (stoppedBy(fetched)) {
+			sources.push({
+				...base,
+				kind: null,
+				entries: 0,
+				urls: 0,
+				truncated: false,
+				note: fetched.note ?? null
+			});
+			break;
+		}
+		const body = fetched.body;
+		if (fetched.outcome !== "ok" && fetched.outcome !== "not-modified" || body === void 0) {
+			sources.push({
+				...base,
+				kind: null,
+				entries: 0,
+				urls: 0,
+				truncated: false,
+				note: fetched.note ?? `${fetched.outcome} ${String(fetched.status)}`
+			});
+			continue;
+		}
+		let bytes = body;
+		let truncated = fetched.truncated === true;
+		const notes = [];
+		if (isGzip(body)) {
+			const unzipped = await gunzipBounded(body, maxBytes);
+			bytes = unzipped.bytes;
+			truncated ||= unzipped.truncated;
+			if (unzipped.error !== void 0) notes.push(`the gzip data ended early (${unzipped.error})`);
+		}
+		const read = readSitemap(decodeBody(bytes, isGzip(body) ? null : fetched.contentType ?? null), fetched.url);
+		if (truncated) notes.push(`the file is longer than ${String(maxBytes)} bytes, so discovery read its first part`);
+		notes.push(...read.notes);
+		let kept = 0;
+		if (read.kind === "sitemapindex") {
+			const children = [...read.entries].sort((a, b) => (b.lastmod === null ? -Infinity : Date.parse(b.lastmod)) - (a.lastmod === null ? -Infinity : Date.parse(a.lastmod)));
+			for (const child of children) if (!inScope(new URL(child.url))) dropped.scope += 1;
+			else if (file.nesting + 1 > maxNesting) dropped.limit += 1;
+			else enqueue(child.url, "sitemap", "index", file.nesting + 1);
+			if (file.nesting + 1 > maxNesting && children.length > 0) notes.push(`the files of the index are deeper than maxNesting (${String(maxNesting)}), so discovery reads none of them`);
+		} else if (read.kind !== "unknown") kept = await keep(read, fetched.url, FEED_KINDS.has(read.kind) ? "feed" : "sitemap");
+		sources.push({
+			...base,
+			kind: read.kind,
+			entries: read.entries.length,
+			urls: kept,
+			truncated,
+			note: notes.length > 0 ? notes.join("; ") : null
+		});
+	}
+	if (stopped === void 0 && urls.size >= maxUrls) stopped = "max-urls";
+	return result();
+};
+//#endregion
+//#region src/frontier.ts
+/**
+* The frontier is a small crawl of a site. It starts at the seeds. It follows the links of each page
+* that it fetched, to a maximum of pages, a maximum depth and a maximum time.
+*
+* Each page is a fetch of the crawler, so each polite rule applies. These rules are robots.txt, one
+* request at a time for each host, the pace, the stops, the egress policy and the capture log. The
+* frontier does not start a second request to a host with a request in progress. The crawler
+* limits the hosts in progress (`maxActive`).
+*
+* The state of the frontier is in memory, for one call. A later version can make it resumable.
+*/
+/** These are the default limits of `crawl`, and the limits of the memory of the frontier. */
+const CRAWL_LIMITS = {
+	/** The most pages that one crawl gives. */
+	maxPages: 100,
+	/** The most links from a seed to a page. A seed has the depth 0. */
+	maxDepth: 3,
+	/** The longest time of one crawl: 10 minutes. */
+	budgetMs: 6e5,
+	/** The most hosts that get a request of the crawl at one time. */
+	hostsAtOnce: 16,
+	/** The most URLs that wait in the frontier. */
+	maxQueue: 1e4,
+	/** The most URLs that the frontier remembers, so it fetches none of them two times. */
+	maxSeen: 1e5
+};
+/** The lower depth first, then the lower tier, then the newer `lastmod`, then the first found. */
+const before = (a, b) => a.depth - b.depth || a.tier - b.tier || b.lastmod - a.lastmod || a.seq - b.seq;
+/** A binary heap of the URLs of one host, the first URL to fetch at the top. */
+var Heap = class {
+	items = [];
+	get size() {
+		return this.items.length;
+	}
+	peek() {
+		return this.items[0];
+	}
+	push(item) {
+		const items = this.items;
+		items.push(item);
+		let at = items.length - 1;
+		while (at > 0) {
+			const parent = at - 1 >> 1;
+			if (before(items[at], items[parent]) >= 0) break;
+			[items[at], items[parent]] = [items[parent], items[at]];
+			at = parent;
+		}
+	}
+	pop() {
+		const items = this.items;
+		const top = items[0];
+		const last = items.pop();
+		if (top === void 0 || last === void 0 || items.length === 0) return top;
+		items[0] = last;
+		let at = 0;
+		for (;;) {
+			const left = at * 2 + 1;
+			const right = left + 1;
+			let first = at;
+			if (left < items.length && before(items[left], items[first]) < 0) first = left;
+			if (right < items.length && before(items[right], items[first]) < 0) first = right;
+			if (first === at) return top;
+			[items[at], items[first]] = [items[first], items[at]];
+			at = first;
+		}
+	}
+};
+const hostOf = (url) => `${url.hostname.toLowerCase().replace(/\.$/, "")}${url.port === "" ? "" : `:${url.port}`}`;
+const matcher = (pattern) => {
+	if (pattern instanceof RegExp) return (url) => {
+		pattern.lastIndex = 0;
+		return pattern.test(url.href);
+	};
+	if (typeof pattern !== "string" || pattern.length === 0) throw new RangeError("a URL pattern is a regular expression or a text that is not empty");
+	return pattern.startsWith("/") ? (url) => matchesPattern(pattern, `${url.pathname}${url.search}`) : (url) => matchesPattern(pattern, url.href);
+};
+const whole = (name, value, fallback, min) => {
+	if (value === void 0) return fallback;
+	if (!Number.isInteger(value) || value < min) throw new RangeError(`${name} must be an integer of ${String(min)} or more, not ${String(value)}`);
+	return value;
+};
+const failedPage = (url, note) => ({
+	url,
+	doc: null,
+	outcome: "error",
+	status: 0,
+	title: null,
+	chars: 0,
+	rendered: false,
+	structured: [],
+	links: 0,
+	captures: [],
+	capture: null,
+	capture_count: 0,
+	fetched_at: null,
+	from_cache: false,
+	held_back: false,
+	note
+});
+const REFUSALS$1 = /* @__PURE__ */ new Set([
+	"skipped",
+	"blocked",
+	"challenge",
+	"payment-required"
+]);
+/**
+* This function crawls a site from its seeds. `PoliteCrawler.crawl` uses it. It gives each page
+* when its fetch ends, and a summary at the end. The seeds come first, then the pages of the
+* sitemaps (the newest first), then the links, the lower depth first.
+*/
+const crawlSite = (runtime, seeds, options = {}) => {
+	let resolveSummary = () => void 0;
+	const summary = new Promise((resolve) => {
+		resolveSummary = resolve;
+	});
+	let used = false;
+	async function* pages() {
+		const now = () => Date.parse(runtime.now());
+		const startedAt = now();
+		const abort = new AbortController();
+		const outer = options.signal;
+		const cancel = () => abort.abort(outer?.reason);
+		if (outer?.aborted === true) cancel();
+		outer?.addEventListener("abort", cancel, { once: true });
+		const inflight = /* @__PURE__ */ new Map();
+		const heaps = /* @__PURE__ */ new Map();
+		const seen = /* @__PURE__ */ new Set();
+		const counts = {
+			pages: 0,
+			fetched: 0,
+			from_cache: 0,
+			refused: 0,
+			failed: 0,
+			sitemap_urls: 0
+		};
+		const skips = {
+			scope: 0,
+			pattern: 0,
+			depth: 0,
+			held_back: 0,
+			limit: 0,
+			duplicate: 0
+		};
+		const got = /* @__PURE__ */ new Set();
+		let queued = 0;
+		let returned = 0;
+		let seq = 0;
+		let started = 0;
+		let stopping;
+		let reason;
+		let finished = false;
+		let maxPages = CRAWL_LIMITS.maxPages;
+		let budgetMs = CRAWL_LIMITS.budgetMs;
+		const crawler = runtime.crawler;
+		const marks = {
+			scope: options.scope,
+			labels: options.labels
+		};
+		try {
+			await options.ready?.();
+			maxPages = whole("maxPages", options.maxPages, CRAWL_LIMITS.maxPages, 1);
+			const maxDepth = whole("maxDepth", options.maxDepth, CRAWL_LIMITS.maxDepth, 0);
+			budgetMs = options.budgetMs ?? CRAWL_LIMITS.budgetMs;
+			if (!(budgetMs >= 0)) throw new RangeError(`budgetMs must be 0 or more, not ${String(budgetMs)}`);
+			const maxAge = options.maxAge ?? "24h";
+			const maxAgeMs = maxAge === "auto" ? "auto" : durationMs(maxAge);
+			const includes = (options.include ?? []).map(matcher);
+			const excludes = (options.exclude ?? []).map(matcher);
+			const deadline = startedAt + budgetMs;
+			const list = typeof seeds === "string" ? [seeds] : [...seeds];
+			if (list.length === 0) throw new RangeError("a crawl needs one seed URL or more");
+			const starts = list.flatMap((seed) => {
+				try {
+					const url = new URL(seed);
+					return url.protocol === "http:" || url.protocol === "https:" ? [url] : [];
+				} catch {
+					return [];
+				}
+			});
+			const inScope = scopeTest(starts, options);
+			const followed = (url) => (includes.length === 0 || includes.some((test) => test(url))) && !excludes.some((test) => test(url));
+			/** This function adds a URL to the frontier, when it is new, in scope and not too deep. */
+			const offer = (raw, depth, tier, lastmod, source, referrer) => {
+				let url;
+				try {
+					url = new URL(raw);
+				} catch {
+					url = void 0;
+				}
+				const web = url !== void 0 && (url.protocol === "http:" || url.protocol === "https:");
+				if (!web && source !== "seed") return;
+				const key = cacheKeyOf(url?.href ?? raw);
+				if (seen.has(key)) return;
+				if (seen.size >= CRAWL_LIMITS.maxSeen) {
+					skips.limit += 1;
+					return;
+				}
+				seen.add(key);
+				if (source !== "seed" && url !== void 0) {
+					if (!inScope(url)) {
+						skips.scope += 1;
+						return;
+					}
+					if (!followed(url)) {
+						skips.pattern += 1;
+						return;
+					}
+					if (depth > maxDepth) {
+						skips.depth += 1;
+						return;
+					}
+				}
+				if (queued >= CRAWL_LIMITS.maxQueue) {
+					skips.limit += 1;
+					return;
+				}
+				const host = url === void 0 || !web ? `not-web:${raw}` : hostOf(url);
+				let heap = heaps.get(host);
+				if (heap === void 0) {
+					heap = new Heap();
+					heaps.set(host, heap);
+				}
+				seq += 1;
+				heap.push({
+					url: raw,
+					key,
+					host,
+					depth,
+					tier,
+					lastmod,
+					seq,
+					source,
+					referrer
+				});
+				queued += 1;
+			};
+			/** This function gives the first URL of a host with no request of the crawl in progress. */
+			const pick = () => {
+				for (;;) {
+					let best;
+					for (const [host, heap] of heaps) {
+						if (inflight.has(host)) continue;
+						const top = heap.peek();
+						if (top === void 0) continue;
+						if (best === void 0 || before(top, best.heap.peek()) < 0) best = {
+							host,
+							heap
+						};
+					}
+					if (best === void 0) return void 0;
+					const item = best.heap.pop();
+					queued -= 1;
+					if (best.heap.size === 0) heaps.delete(best.host);
+					if (!got.has(item.key)) return item;
+					skips.duplicate += 1;
+				}
+			};
+			const fetchItem = async (item) => {
+				try {
+					if (item.source !== "seed") {
+						if (!(await crawler.robotsAllow(item.url, {
+							onBehalfOf: options.onBehalfOf,
+							signal: abort.signal,
+							...marks
+						})).allowed) return {
+							kind: "refused",
+							item
+						};
+					}
+					return {
+						kind: "page",
+						item,
+						page: await fetchPage(runtime, item.url, {
+							render: options.render ?? "auto",
+							maxAgeMs,
+							onBehalfOf: options.onBehalfOf,
+							signal: abort.signal,
+							scope: options.scope,
+							labels: options.labels,
+							deadline
+						})
+					};
+				} catch (error) {
+					if (abort.signal.aborted) return {
+						kind: "aborted",
+						item
+					};
+					const message = error instanceof Error ? error.message : String(error);
+					return {
+						kind: "page",
+						item,
+						page: failedPage(item.url, `crawl error: ${message}`)
+					};
+				}
+			};
+			/** This function adds the links of a page to the frontier. The canonical URL of the page is known after its fetch. */
+			const follow = (page, item) => {
+				if (page.doc === null) return;
+				const doc = runtime.index.get(page.doc);
+				if (doc === void 0) return;
+				for (const same of [doc.url, doc.meta["canonical"]]) {
+					if (typeof same !== "string") continue;
+					seen.add(cacheKeyOf(same));
+					got.add(cacheKeyOf(same));
+				}
+				const links = doc.meta["links"];
+				if (!Array.isArray(links) || links.length === 0) return;
+				if (isHeldBack(doc)) {
+					skips.held_back += 1;
+					return;
+				}
+				for (const link of links) if (typeof link?.href === "string") offer(link.href, item.depth + 1, 2, Number.NEGATIVE_INFINITY, "link", page.url);
+			};
+			for (const seed of list) offer(seed, 0, 0, Number.NEGATIVE_INFINITY, "seed", null);
+			if (options.useSitemaps === true) {
+				const sites = [...new Set(starts.map((start) => start.origin))];
+				for (const site of sites) {
+					if (abort.signal.aborted || now() >= deadline) break;
+					try {
+						const found = await discoverSite(runtime, site, {
+							sameSite: options.sameSite,
+							allowHosts: options.allowHosts,
+							maxUrls: Math.min(1e4, Math.max(50, maxPages * 4)),
+							linkedFeeds: false,
+							maxAge,
+							onBehalfOf: options.onBehalfOf,
+							signal: abort.signal,
+							scope: options.scope,
+							labels: options.labels,
+							deadline
+						});
+						counts.sitemap_urls += found.urls.length;
+						for (const url of found.urls) offer(url.url, 0, 1, url.lastmod === null ? Number.NEGATIVE_INFINITY : Date.parse(url.lastmod), "sitemap", null);
+					} catch {}
+				}
+			}
+			for (;;) {
+				if (stopping === void 0 && abort.signal.aborted) stopping = "cancelled";
+				if (stopping === void 0 && now() >= deadline) stopping = "budget";
+				while (stopping === void 0 && inflight.size < CRAWL_LIMITS.hostsAtOnce && started < maxPages) {
+					const item = pick();
+					if (item === void 0) break;
+					started += 1;
+					inflight.set(item.host, fetchItem(item));
+				}
+				if (inflight.size === 0) break;
+				const done = await Promise.race(inflight.values());
+				inflight.delete(done.item.host);
+				if (done.kind === "aborted") {
+					started -= 1;
+					stopping ??= "cancelled";
+					continue;
+				}
+				if (done.kind === "refused") {
+					started -= 1;
+					counts.refused += 1;
+					continue;
+				}
+				const page = done.page;
+				if (pastDeadline(page.note)) {
+					started -= 1;
+					returned += 1;
+					stopping ??= "budget";
+					continue;
+				}
+				if (abort.signal.aborted) {
+					started -= 1;
+					stopping ??= "cancelled";
+					continue;
+				}
+				if (page.outcome === "skipped" && page.note === "crawler closed") {
+					started -= 1;
+					stopping ??= "closed";
+					continue;
+				}
+				counts.pages += 1;
+				if (page.outcome === "ok" || page.outcome === "not-modified") {
+					counts.fetched += 1;
+					if (page.from_cache) counts.from_cache += 1;
+				} else if (REFUSALS$1.has(page.outcome)) counts.refused += 1;
+				else counts.failed += 1;
+				follow(page, done.item);
+				yield {
+					...page,
+					depth: done.item.depth,
+					source: done.item.source,
+					referrer: done.item.referrer
+				};
+			}
+			stopping ??= queued > 0 && started >= maxPages ? "max-pages" : "done";
+			finished = true;
+		} catch (error) {
+			stopping = "error";
+			reason = error instanceof Error ? error.message : String(error);
+			finished = true;
+			throw error;
+		} finally {
+			if (!finished) {
+				stopping ??= "cancelled";
+				reason ??= "the caller stopped the loop";
+			}
+			outer?.removeEventListener("abort", cancel);
+			abort.abort(/* @__PURE__ */ new Error("the crawl ended"));
+			await Promise.allSettled(inflight.values());
+			const stop = stopping ?? "done";
+			reason ??= stop === "done" ? "the crawl fetched each page in scope that it found" : stop === "max-pages" ? `the crawl gave ${String(maxPages)} pages, its limit (maxPages)` : stop === "budget" ? `the budget of ${String(budgetMs)} ms ended (budgetMs)` : stop === "cancelled" ? "the caller cancelled the crawl" : "the crawler closed";
+			resolveSummary({
+				...counts,
+				skipped: skips.scope + skips.pattern + skips.depth + skips.held_back + skips.limit + skips.duplicate,
+				skips,
+				queued: queued + returned,
+				stopped: stop,
+				reason,
+				elapsed_ms: Math.max(0, now() - startedAt)
+			});
+		}
+	}
+	return {
+		summary,
+		[Symbol.asyncIterator]() {
+			if (used) throw new Error("a crawl runs one time: call crawl again for a new crawl");
+			used = true;
+			return pages();
+		}
+	};
+};
+//#endregion
+//#region src/hoststate.ts
+/** A stop with no end goes into the table as this value, which is past each date that JavaScript can hold. */
+const NO_END = 9e15;
+/** A process that shares the store asks again after this wait, at most, during the lease of a different process. */
+const SHARED_WAIT_MS = 1e3;
+/** The store removes the rows of idle hosts after this many reservations. */
+const SWEEP_EVERY = 1e3;
+const verdictOf = (value) => value === "works" || value === "fails" ? value : "unknown";
+const memoOf = (row) => ({
+	skipEtag: row.skip_etag === 1,
+	etag: verdictOf(row.etag),
+	lastModified: verdictOf(row.last_modified),
+	etagMisses: row.etag_misses,
+	dateMisses: row.date_misses,
+	sinceProbe: row.since_probe,
+	changedAt: row.changed_at
+});
+/**
+* This store keeps the state of each host in a SQLite file. Two or more processes on one store
+* share it, so they share the pace and the stops of each host. Thus a host gets one request at a
+* time from all of them. Each method is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock
+* of a different process (`busy_timeout`).
+*
+* The lease of a request belongs to the store that gave it. A different process that asks during
+* the lease gets a wait of one second or less, and asks again. A process that stops during a
+* request holds its lease until the lease ends (5 minutes by default).
+*
+* The table `validators` keeps the validator memo of each host (`ValidatorMemo`), so the processes
+* share it and it stays after a restart. A file of version 0.1.0 gets the table when it opens.
+*
+* The state is not a projection of the capture log. A deletion of the file loses only the pace,
+* the counts, the stops and the memos, and the crawler starts them again.
+*/
+var SqliteHostStateStore = class {
+	path;
+	db;
+	statements = /* @__PURE__ */ new Map();
+	/** This id names the leases of this store. */
+	owner = randomUUID();
+	reservations = 0;
+	closed = false;
+	pollMs;
+	constructor(path, options = {}) {
+		this.path = path;
+		if (path !== ":memory:") mkdirSync(dirname(path), {
+			recursive: true,
+			mode: 448
+		});
+		this.pollMs = Math.max(1, Math.min(SHARED_WAIT_MS, options.pollMs ?? SHARED_WAIT_MS));
+		this.db = new DatabaseSync(path);
+		const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
+		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
+		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
+		this.db.exec("PRAGMA synchronous = NORMAL;");
+		this.migrate();
+	}
+	/**
+	* This method makes the tables, or adds the tables of a later version to an older file. It is one
+	* transaction, so two processes that open one old file at the same time do not collide.
+	*/
+	migrate() {
+		this.step(() => {
+			this.db.exec(`CREATE TABLE IF NOT EXISTS hosts (
+        host TEXT PRIMARY KEY,
+        next_at REAL NOT NULL DEFAULT 0,
+        leased_until REAL NOT NULL DEFAULT 0,
+        owner TEXT,
+        failures INTEGER NOT NULL DEFAULT 0,
+        waits INTEGER NOT NULL DEFAULT 0,
+        stop_until REAL,
+        stop_reason TEXT
+      )`);
+			this.db.exec(`CREATE TABLE IF NOT EXISTS validators (
+        host TEXT PRIMARY KEY,
+        skip_etag INTEGER NOT NULL DEFAULT 0,
+        etag TEXT NOT NULL DEFAULT 'unknown',
+        last_modified TEXT NOT NULL DEFAULT 'unknown',
+        etag_misses INTEGER NOT NULL DEFAULT 0,
+        date_misses INTEGER NOT NULL DEFAULT 0,
+        since_probe INTEGER NOT NULL DEFAULT 0,
+        changed_at REAL NOT NULL DEFAULT 0
+      )`);
+			const { user_version: version } = this.db.prepare("PRAGMA user_version").get();
+			if (version < 1) this.db.exec(`PRAGMA user_version = ${String(1)}`);
+		});
+	}
+	stmt(sql) {
+		let statement = this.statements.get(sql);
+		if (statement === void 0) {
+			statement = this.db.prepare(sql);
+			this.statements.set(sql, statement);
+		}
+		return statement;
+	}
+	/** This method does one step in one transaction. */
+	step(work) {
+		if (this.closed) throw new Error("the host state store is closed");
+		this.db.exec("BEGIN IMMEDIATE");
+		try {
+			const result = work();
+			this.db.exec("COMMIT");
+			return result;
+		} catch (error) {
+			if (this.db.isTransaction) this.db.exec("ROLLBACK");
+			throw error;
+		}
+	}
+	row(host) {
+		this.stmt("INSERT OR IGNORE INTO hosts (host) VALUES (?)").run(host);
+		return this.stmt("SELECT next_at, leased_until, owner, failures, waits, stop_until, stop_reason FROM hosts WHERE host = ?").get(host);
+	}
+	/** This method removes the hosts that have nothing to remember: no rest, no lease, no counts and no stop. */
+	sweep(now) {
+		this.stmt(`DELETE FROM hosts WHERE next_at <= ? AND leased_until <= ? AND failures = 0 AND waits = 0
+         AND (stop_until IS NULL OR stop_until <= ?)`).run(now, now, now);
+	}
+	reserve(host, now, leaseMs) {
+		return this.step(() => {
+			this.reservations += 1;
+			if (this.reservations % SWEEP_EVERY === 0) this.sweep(now);
+			const row = this.row(host);
+			if (row.leased_until > now) return row.owner === this.owner ? row.leased_until - now : Math.min(this.pollMs, row.leased_until - now);
+			if (row.next_at > now) return row.next_at - now;
+			this.stmt("UPDATE hosts SET leased_until = ?, owner = ? WHERE host = ?").run(now + leaseMs, this.owner, host);
+			return 0;
+		});
+	}
+	record(host, result) {
+		return this.step(() => {
+			const row = this.row(host);
+			let { failures, waits } = row;
+			if (result === "success") {
+				failures = 0;
+				waits = 0;
+			} else if (result === "failure") failures += 1;
+			else if (result === "wait") waits += 1;
+			if (failures !== row.failures || waits !== row.waits) this.stmt("UPDATE hosts SET failures = ?, waits = ? WHERE host = ?").run(failures, waits, host);
+			return {
+				failures,
+				waits
+			};
+		});
+	}
+	release(host, nextAt) {
+		this.step(() => {
+			this.row(host);
+			this.stmt(`UPDATE hosts SET next_at = max(next_at, ?),
+           leased_until = CASE WHEN owner = ? THEN 0 ELSE leased_until END,
+           owner = CASE WHEN owner = ? THEN NULL ELSE owner END
+         WHERE host = ?`).run(nextAt, this.owner, this.owner, host);
+		});
+	}
+	defer(host, until) {
+		this.step(() => {
+			this.row(host);
+			this.stmt("UPDATE hosts SET next_at = max(next_at, ?) WHERE host = ?").run(until, host);
+		});
+	}
+	stop(host, until, reason) {
+		const end = Number.isFinite(until) ? until : NO_END;
+		this.step(() => {
+			this.row(host);
+			this.stmt(`UPDATE hosts SET stop_until = ?, stop_reason = ?
+         WHERE host = ? AND (stop_until IS NULL OR stop_until < ?)`).run(end, reason, host, end);
+		});
+	}
+	stopped(host, now) {
+		if (this.closed) throw new Error("the host state store is closed");
+		const row = this.stmt("SELECT stop_until, stop_reason FROM hosts WHERE host = ?").get(host);
+		if (row === void 0 || row.stop_until === null || row.stop_until <= now) return void 0;
+		return {
+			until: row.stop_until >= NO_END ? Number.POSITIVE_INFINITY : row.stop_until,
+			reason: row.stop_reason ?? "stopped"
+		};
+	}
+	reset(host) {
+		this.step(() => {
+			if (host === void 0) this.stmt("UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL").run();
+			else this.stmt("UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL WHERE host = ?").run(host);
+		});
+	}
+	validators(host) {
+		if (this.closed) throw new Error("the host state store is closed");
+		const row = this.stmt(`SELECT skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at
+         FROM validators WHERE host = ?`).get(host);
+		return row === void 0 ? void 0 : memoOf(row);
+	}
+	noteValidators(host, report, now) {
+		return this.step(() => {
+			const memo = nextValidatorMemo(this.validators(host), report, now);
+			this.stmt(`INSERT INTO validators (host, skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(host) DO UPDATE SET skip_etag = excluded.skip_etag, etag = excluded.etag,
+           last_modified = excluded.last_modified, etag_misses = excluded.etag_misses,
+           date_misses = excluded.date_misses, since_probe = excluded.since_probe,
+           changed_at = excluded.changed_at`).run(host, memo.skipEtag ? 1 : 0, memo.etag, memo.lastModified, memo.etagMisses, memo.dateMisses, memo.sinceProbe, memo.changedAt);
+			return memo;
+		});
+	}
+	/** This value is the number of hosts that the store holds. */
+	get size() {
+		return this.stmt("SELECT count(*) AS n FROM hosts").get().n;
+	}
+	/** This method closes the file. A second close does nothing. */
+	close() {
+		if (this.closed) return;
+		this.closed = true;
+		this.statements.clear();
+		this.db.close();
+	}
+};
+//#endregion
+//#region src/embed.ts
+/** These are the limits of the chunks of a text. */
+const CHUNK_LIMITS = {
+	maxTokens: 512,
+	defaultTokens: 256,
+	fill: .9,
+	overlap: .1,
+	maxChunks: 64
+};
+/** These scripts write no space between words, so the estimate counts each character as a token. */
+const SEGMENTED = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;
+const WORD = /[\p{L}\p{N}\p{M}]/u;
+const SPACE = /\s/u;
+/** A word of this number of letters or fewer is one token in the estimate. A longer word is more tokens. */
+const LETTERS_PER_TOKEN = 6;
+const isAsciiWord = (code) => code >= 48 && code <= 57 || code >= 65 && code <= 90 || code >= 97 && code <= 122;
+const isAsciiSpace = (code) => code === 32 || code >= 9 && code <= 13;
+const kindOf = (point) => {
+	if (point < 128) return isAsciiWord(point) ? "word" : isAsciiSpace(point) ? "space" : "other";
+	const char = String.fromCodePoint(point);
+	if (SEGMENTED.test(char)) return "segmented";
+	if (WORD.test(char)) return "word";
+	return SPACE.test(char) ? "space" : "other";
+};
+/**
+* This function estimates the number of tokens of a part of a text. A word is one token for each
+* six letters. A Chinese, Japanese or Thai character is one token. Each other character that is
+* not a space is one token. The estimate is a little high for English text, so a chunk fits.
+*/
+const estimateTokens = (text, start = 0, end = text.length) => {
+	let tokens = 0;
+	let run = 0;
+	for (let at = start; at < end;) {
+		const point = text.codePointAt(at);
+		at += point > 65535 ? 2 : 1;
+		const kind = kindOf(point);
+		if (kind === "word") {
+			run += 1;
+			continue;
+		}
+		if (run > 0) tokens += Math.ceil(run / LETTERS_PER_TOKEN);
+		run = 0;
+		if (kind !== "space") tokens += 1;
+	}
+	if (run > 0) tokens += Math.ceil(run / LETTERS_PER_TOKEN);
+	return tokens;
+};
+/** These characters end a sentence when a space follows them. */
+const STOP = /[.!?…]+["'”’)\]]*(?=\s|$)|[。！？]+/gu;
+/** This function gives the ranges of the sentences of a part of a text, with no space at the ends. */
+const sentencesOf = (text, start, end) => {
+	const part = text.slice(start, end);
+	const out = [];
+	let from = 0;
+	for (const match of part.matchAll(STOP)) {
+		const to = match.index + match[0].length;
+		if (to > from) out.push([from, to]);
+		from = to;
+	}
+	if (from < part.length) out.push([from, part.length]);
+	return out.map(([a, b]) => {
+		let first = a;
+		let last = b;
+		while (first < last && SPACE.test(part[first])) first += 1;
+		while (last > first && SPACE.test(part[last - 1])) last -= 1;
+		return [start + first, start + last];
+	}).filter(([a, b]) => b > a);
+};
+/** This function cuts a part of a text that is longer than the limit. It cuts at the spaces, and in a word that is too long. */
+const cutLong = (text, start, end, size) => {
+	const out = [];
+	let from = start;
+	let tokens = 0;
+	let run = 0;
+	let lastSpace = -1;
+	let at = start;
+	const emit = (to) => {
+		let first = from;
+		while (first < to && SPACE.test(text[first])) first += 1;
+		let last = to;
+		while (last > first && SPACE.test(text[last - 1])) last -= 1;
+		if (last > first) out.push({
+			start: first,
+			end: last,
+			tokens: estimateTokens(text, first, last),
+			paragraph: false
+		});
+		from = to;
+		tokens = 0;
+		run = 0;
+		lastSpace = -1;
+	};
+	while (at < end) {
+		const point = text.codePointAt(at);
+		const width = point > 65535 ? 2 : 1;
+		const kind = kindOf(point);
+		let cost = 0;
+		if (kind === "word") {
+			run += 1;
+			if (run % LETTERS_PER_TOKEN === 1) cost = 1;
+		} else {
+			run = 0;
+			if (kind !== "space") cost = 1;
+		}
+		if (tokens + cost > size && at > from) {
+			if (lastSpace > from) {
+				const to = lastSpace;
+				emit(to);
+				at = to;
+				continue;
+			}
+			emit(at);
+		}
+		if (kind === "space") lastSpace = at;
+		tokens += cost;
+		at += width;
+	}
+	emit(end);
+	return out;
+};
+/** This function gives the pieces of a text: its sentences, in paragraphs, and the parts of a sentence that is longer than the limit. */
+const piecesOf = (text, size) => {
+	const pieces = [];
+	for (const line of text.matchAll(/[^\r\n]+/g)) {
+		let first = true;
+		for (const [start, end] of sentencesOf(text, line.index, line.index + line[0].length)) {
+			const tokens = estimateTokens(text, start, end);
+			const parts = tokens <= size ? [{
+				start,
+				end,
+				tokens,
+				paragraph: false
+			}] : cutLong(text, start, end, size);
+			for (const part of parts) {
+				pieces.push({
+					...part,
+					paragraph: first
+				});
+				first = false;
+			}
+		}
+	}
+	return pieces;
+};
+const positive = (name, value) => {
+	if (!Number.isFinite(value) || value < 1) throw new RangeError(`${name} must be a number of 1 or more, not ${String(value)}`);
+	return Math.floor(value);
+};
+/**
+* This function cuts a text into chunks for an embedder. A chunk is a window of whole sentences.
+* It ends at the end of a paragraph when the paragraph fills half of the window or more. A
+* sentence that is longer than the window is cut at its spaces. Two chunks next to each other share
+* a few sentences (`overlapTokens`), so a passage at a cut is in one chunk. Each chunk keeps its
+* position in the text, so a hit points at a passage.
+*/
+const chunkText = (text, options) => {
+	const size = positive("maxTokens", options.maxTokens);
+	const overlap = Math.min(Math.floor(size / 2), Math.max(0, Math.floor(options.overlapTokens ?? size * CHUNK_LIMITS.overlap)));
+	const maxChunks = positive("maxChunks", options.maxChunks ?? CHUNK_LIMITS.maxChunks);
+	const pieces = piecesOf(text, size);
+	const chunks = [];
+	let first = 0;
+	let fresh = 0;
+	while (first < pieces.length && chunks.length < maxChunks) {
+		let last = first;
+		let tokens = pieces[first].tokens;
+		while (last + 1 < pieces.length && tokens + pieces[last + 1].tokens <= size) {
+			last += 1;
+			tokens += pieces[last].tokens;
+		}
+		if (last + 1 < pieces.length && !pieces[last + 1].paragraph) {
+			let filled = 0;
+			let cut = -1;
+			for (let at = first; at < last; at += 1) {
+				filled += pieces[at].tokens;
+				if (at >= fresh && pieces[at + 1].paragraph && filled * 2 >= size) cut = at;
+			}
+			if (cut !== -1) last = cut;
+		}
+		fresh = last + 1;
+		const start = pieces[first].start;
+		const end = pieces[last].end;
+		chunks.push({
+			index: chunks.length,
+			start,
+			end,
+			text: text.slice(start, end)
+		});
+		if (last + 1 >= pieces.length) break;
+		let next = last + 1;
+		let shared = 0;
+		while (next - 1 > first && shared + pieces[next - 1].tokens <= overlap) {
+			next -= 1;
+			shared += pieces[next].tokens;
+		}
+		if (shared + pieces[last + 1].tokens > size) next = last + 1;
+		first = next;
+	}
+	return chunks;
+};
+/**
+* This function gives the number of tokens of a chunk for an embedder. It is the input limit with a
+* margin for the estimate, and `CHUNK_LIMITS.maxTokens` at most.
+*/
+const chunkTokensOf = (embedder) => {
+	const limit = embedder.maxInputTokens;
+	const tokens = limit === void 0 || !Number.isFinite(limit) || limit < 1 ? CHUNK_LIMITS.defaultTokens : Math.min(limit, CHUNK_LIMITS.maxTokens);
+	return Math.max(8, Math.floor(tokens * CHUNK_LIMITS.fill));
+};
+/** This function gives a copy of a vector with a length of 1. A vector of zeros stays zeros. A value that is not finite throws a `RangeError`. */
+const unitVector = (vector) => {
+	const out = Float32Array.from(vector);
+	let sum = 0;
+	for (const value of out) {
+		if (!Number.isFinite(value)) throw new RangeError("the embedder gave a value that is not finite");
+		sum += value * value;
+	}
+	if (sum === 0) return out;
+	const scale = 1 / Math.sqrt(sum);
+	for (let at = 0; at < out.length; at += 1) out[at] = out[at] * scale;
+	return out;
+};
+/** These are the defaults of a hybrid search. */
+const HYBRID_DEFAULTS = {
+	candidates: 50,
+	neighbors: 50,
+	fusionK: 60
+};
+/**
+* This function fuses ranked lists with reciprocal rank fusion. Each item gets the sum of
+* 1 / (k + rank) over the lists that hold it, with the rank from 1. The best item is first. A tie goes to
+* the item with the best rank in a list, then to the earlier list, then to the smaller id. An item
+* that is two times in one list counts at its first place.
+*/
+const fuseRanks = (lists, k = HYBRID_DEFAULTS.fusionK) => {
+	if (!Number.isFinite(k) || k < 0) throw new RangeError(`the fusion constant k must be a number of 0 or more, not ${String(k)}`);
+	const ranks = /* @__PURE__ */ new Map();
+	lists.forEach((list, which) => {
+		list.forEach((id, at) => {
+			let entry = ranks.get(id);
+			if (entry === void 0) {
+				entry = lists.map(() => null);
+				ranks.set(id, entry);
+			}
+			entry[which] ??= at + 1;
+		});
+	});
+	const keyed = [...ranks].map(([id, entry]) => {
+		let score = 0;
+		let best = Number.POSITIVE_INFINITY;
+		for (const rank of entry) {
+			if (rank === null) continue;
+			score += 1 / (k + rank);
+			best = Math.min(best, rank);
+		}
+		return {
+			id,
+			score,
+			ranks: entry,
+			best,
+			first: entry.findIndex((rank) => rank !== null)
+		};
+	});
+	keyed.sort((a, b) => b.score - a.score || a.best - b.best || a.first - b.first || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+	return keyed.map(({ id, score, ranks: entry }) => ({
+		id,
+		score,
+		ranks: entry
+	}));
+};
+//#endregion
+//#region src/hybrid.ts
+/** The embedder gets this number of texts in one call at most. */
+const BATCH = 32;
+/** The search keeps the vectors of this number of questions. */
+const QUERY_CACHE = 32;
+/** The search checks this number of texts of the vector search against the index at one time. */
+const PAGE = 64;
+const sha256 = (text) => createHash("sha256").update(text, "utf8").digest("hex");
+const count = (name, value, fallback, max) => {
+	if (value === void 0) return fallback;
+	if (!Number.isFinite(value) || value < 1) throw new RangeError(`${name} must be a number of 1 or more, not ${String(value)}`);
+	return Math.min(max, Math.floor(value));
+};
+/** This function checks the constant k of reciprocal rank fusion. */
+const fusionKOf = (value) => {
+	if (value === void 0) return HYBRID_DEFAULTS.fusionK;
+	if (!Number.isFinite(value) || value < 0) throw new RangeError(`fusionK must be a number of 0 or more, not ${String(value)}`);
+	return value;
+};
+/** This function gives a promise with its resolve and its reject. A rejection with no listener is not an error of the process. */
+const deferred = () => {
+	let resolve = () => void 0;
+	let reject = () => void 0;
+	const promise = new Promise((yes, no) => {
+		resolve = yes;
+		reject = no;
+	});
+	promise.catch(() => void 0);
+	return {
+		promise,
+		resolve,
+		reject
+	};
+};
+const round = (value, digits) => {
+	const scale = 10 ** digits;
+	return Math.round(value * scale) / scale;
+};
+/** This function makes a hybrid search over a runtime. The vectors go into the store, with the id of the embedder. */
+const createHybridSearch = (runtime, options) => {
+	const { embedder, store } = options;
+	const tokens = chunkTokensOf(embedder);
+	const plan = `chunks-1/${String(tokens)}/${String(Math.floor(tokens * CHUNK_LIMITS.overlap))}/${String(CHUNK_LIMITS.maxChunks)}`;
+	const working = /* @__PURE__ */ new Map();
+	const queries = /* @__PURE__ */ new Map();
+	let dimensions = embedder.dimensions > 0 ? embedder.dimensions : store.dimensions(embedder.id);
+	/** This function gives the heads of documents, from `digests` when the index has it. */
+	const headsOf = (docs) => {
+		const heads = /* @__PURE__ */ new Map();
+		if (docs.length === 0) return heads;
+		const index = runtime.index;
+		if (index.digests !== void 0) {
+			for (const head of index.digests({ docs })) heads.set(head.doc, head);
+			return heads;
+		}
+		for (const doc of docs) {
+			const found = index.get(doc);
+			if (found !== void 0) heads.set(doc, headOfDoc(found));
+		}
+		return heads;
+	};
+	const headOfDoc = (doc) => ({
+		doc: doc.doc,
+		kind: doc.kind,
+		ai_input: doc.ai_input,
+		text_sha: sha256(doc.text)
+	});
+	/** This function asks the embedder for the vectors of texts, in batches, and checks them. */
+	const vectorsOf = async (texts, purpose, signal) => {
+		const out = [];
+		for (let at = 0; at < texts.length; at += BATCH) {
+			signal?.throwIfAborted();
+			const part = texts.slice(at, at + BATCH);
+			const given = await embedder.embed(part, {
+				signal,
+				purpose
+			});
+			if (!Array.isArray(given) || given.length !== part.length) throw new Error(`the embedder ${embedder.id} gave ${String(Array.isArray(given) ? given.length : 0)} vectors for ${String(part.length)} texts`);
+			for (const vector of given) {
+				const unit = unitVector(vector);
+				dimensions ??= unit.length;
+				if (unit.length !== dimensions || unit.length === 0) throw new Error(`the embedder ${embedder.id} gave a vector of ${String(unit.length)} dimensions, not ${String(dimensions)}`);
+				out.push(unit);
+			}
+		}
+		return out;
+	};
+	const queryVector = async (q, signal) => {
+		const text = q.replace(/["*]/g, " ").replace(/\s+/g, " ").trim();
+		const known = queries.get(text);
+		if (known !== void 0) {
+			queries.delete(text);
+			queries.set(text, known);
+			return known;
+		}
+		const [vector] = await vectorsOf([text], "query", signal);
+		queries.set(text, vector);
+		if (queries.size > QUERY_CACHE) queries.delete(queries.keys().next().value);
+		return vector;
+	};
+	/**
+	* This function gives vectors to the texts of documents that have none, and points each document
+	* at its current text. It reads each text from the index, and it does not send the text
+	* of a held-back document. Two calls for one text share the work. It gives the number of new vectors.
+	*/
+	const ensure = async (docs, signal) => {
+		const tally = {
+			embedded: 0,
+			reused: 0,
+			held: 0,
+			chunks: 0
+		};
+		const now = runtime.now();
+		const heads = headsOf(docs);
+		const waiting = /* @__PURE__ */ new Map();
+		const shared = [];
+		for (const id of docs) {
+			const head = heads.get(id);
+			if (head === void 0) continue;
+			if (isHeldBack(head)) {
+				tally.held += 1;
+				if (store.pointed(embedder.id, id) !== void 0) store.forget(embedder.id, id);
+				continue;
+			}
+			if (store.plan(embedder.id, head.text_sha) === plan) {
+				tally.reused += 1;
+				if (store.pointed(embedder.id, id) !== head.text_sha) store.point(embedder.id, id, head.text_sha, now);
+				continue;
+			}
+			const doc = runtime.index.get(id);
+			if (doc === void 0 || isHeldBack(doc)) continue;
+			const sha = sha256(doc.text);
+			const entry = waiting.get(sha);
+			if (entry !== void 0) {
+				entry.docs.push(id);
+				continue;
+			}
+			const busy = working.get(sha);
+			if (busy !== void 0) {
+				shared.push(busy.then((made) => {
+					store.point(embedder.id, id, sha, now);
+					return made;
+				}));
+				continue;
+			}
+			waiting.set(sha, {
+				text: doc.text,
+				docs: [id]
+			});
+		}
+		const jobs = [...waiting].map(([sha, entry]) => {
+			const job = {
+				sha,
+				entry,
+				chunks: chunkText(entry.text, { maxTokens: tokens }),
+				...deferred()
+			};
+			working.set(sha, job.promise);
+			return job;
+		});
+		try {
+			for (let at = 0; at < jobs.length;) {
+				const batch = [];
+				let size = 0;
+				while (at < jobs.length && (batch.length === 0 || size + jobs[at].chunks.length <= BATCH)) {
+					batch.push(jobs[at]);
+					size += jobs[at].chunks.length;
+					at += 1;
+				}
+				const vectors = await vectorsOf(batch.flatMap((job) => job.chunks.map((chunk) => chunk.text)), "document", signal);
+				let next = 0;
+				for (const job of batch) {
+					const chunks = job.chunks.map((chunk) => ({
+						start: chunk.start,
+						end: chunk.end,
+						vector: vectors[next++]
+					}));
+					store.put({
+						embedder: embedder.id,
+						dimensions: dimensions ?? chunks[0]?.vector.length ?? 0,
+						sha: job.sha,
+						plan,
+						chars: job.entry.text.length,
+						chunks,
+						docs: job.entry.docs,
+						at: now
+					});
+					tally.embedded += 1;
+					tally.chunks += chunks.length;
+					working.delete(job.sha);
+					job.resolve(chunks.length);
+				}
+			}
+		} catch (error) {
+			for (const job of jobs) {
+				if (working.get(job.sha) === job.promise) working.delete(job.sha);
+				job.reject(error);
+			}
+			throw error;
+		}
+		await Promise.all(shared);
+		return tally;
+	};
+	/**
+	* This function gives the documents of the vector search, the best first, to `limit`. It checks
+	* each document against the index. Its current text must be the text of the hit, it must not be
+	* held back, and it must have the kind. With `within`, it ranks only those documents.
+	*/
+	const neighborsOf = (query, limit, kind, within) => {
+		const hits = store.search(embedder.id, query, within === void 0 ? {} : { within });
+		const out = [];
+		for (let at = 0; at < hits.length && out.length < limit; at += PAGE) {
+			const page = hits.slice(at, at + PAGE);
+			const docs = [...new Set(page.flatMap((hit) => within === void 0 ? hit.docs : hit.docs.filter((doc) => within.has(doc))))];
+			const heads = headsOf(docs);
+			for (const hit of page) for (const doc of hit.docs) {
+				if (out.length >= limit) break;
+				if (within !== void 0 && !within.has(doc)) continue;
+				const head = heads.get(doc);
+				if (head === void 0 || head.text_sha !== hit.sha || isHeldBack(head)) continue;
+				if (kind !== void 0 && kind !== null && head.kind !== kind) continue;
+				out.push({
+					doc,
+					hit
+				});
+			}
+		}
+		return out;
+	};
+	const passageOf = (hit) => ({
+		start: hit.start,
+		end: hit.end,
+		similarity: round(hit.similarity, 4)
+	});
+	return {
+		embedder,
+		async find(q, findOptions = {}) {
+			const query = parseQuery(q);
+			if (query === void 0) return [];
+			const limit = Math.max(1, Math.min(1e3, Math.floor(findOptions.limit ?? 10) || 10));
+			const candidates = count("candidates", findOptions.candidates, HYBRID_DEFAULTS.candidates, 1e3);
+			const neighbors = count("neighbors", findOptions.neighbors, HYBRID_DEFAULTS.neighbors, 1e3);
+			const k = fusionKOf(findOptions.fusionK);
+			const signal = findOptions.signal;
+			const vector = await queryVector(q, signal);
+			const kind = findOptions.kind ?? null;
+			const lexical = runtime.index.find(q, {
+				limit: candidates,
+				kind,
+				rank: "exact"
+			});
+			await ensure(lexical.map((hit) => hit.doc), signal);
+			const near = neighborsOf(vector, neighbors, kind, void 0);
+			const fused = fuseRanks([lexical.map((hit) => hit.doc), near.map((one) => one.doc)], k).slice(0, limit);
+			const byDoc = new Map(lexical.map((hit) => [hit.doc, hit]));
+			const nearBy = new Map(near.map((one) => [one.doc, one.hit]));
+			const results = [];
+			for (const one of fused) {
+				const doc = runtime.index.get(one.id);
+				if (doc === void 0 || isHeldBack(doc)) continue;
+				const best = nearBy.get(one.id) ?? store.best(embedder.id, sha256(doc.text), vector);
+				const lexicalHit = byDoc.get(one.id);
+				const archived = doc.kind === "archived" ? archivedOf(doc.meta) : void 0;
+				results.push({
+					doc: doc.doc,
+					kind: doc.kind,
+					title: doc.title,
+					url: doc.url,
+					snippet: best === void 0 ? lexicalHit?.snippet ?? snippetOf(doc.text, query.terms) : snippetOf(doc.text.slice(best.start, best.end), query.terms),
+					score: round(one.score, 6),
+					ai_input: doc.ai_input,
+					fetched_at: doc.fetched_at,
+					capture: doc.capture,
+					...archived === void 0 ? {} : { archived },
+					...best === void 0 ? {} : { passage: passageOf(best) }
+				});
+			}
+			return results;
+		},
+		async rankWithin(q, docs, rankOptions = {}) {
+			if (parseQuery(q) === void 0 || docs.length === 0) return [];
+			const candidates = count("candidates", rankOptions.candidates, HYBRID_DEFAULTS.candidates, 1e3);
+			const neighbors = count("neighbors", rankOptions.neighbors, HYBRID_DEFAULTS.neighbors, 1e5);
+			const k = fusionKOf(rankOptions.fusionK);
+			const signal = rankOptions.signal;
+			const vector = await queryVector(q, signal);
+			const lexical = runtime.index.rankWithin(q, docs, { rank: "exact" });
+			const first = [.../* @__PURE__ */ new Set([...lexical.map((hit) => hit.doc), ...docs])].slice(0, candidates);
+			await ensure(first, signal);
+			const within = new Set(docs);
+			const near = neighborsOf(vector, neighbors, void 0, within);
+			const nearBy = new Map(near.map((one) => [one.doc, one.hit]));
+			const heads = headsOf(lexical.map((hit) => hit.doc).filter((doc) => !nearBy.has(doc)));
+			return fuseRanks([lexical.map((hit) => hit.doc), near.map((one) => one.doc)], k).map((one) => {
+				const sha = heads.get(one.id)?.text_sha;
+				const best = nearBy.get(one.id) ?? (sha === void 0 ? void 0 : store.best(embedder.id, sha, vector));
+				return {
+					doc: one.id,
+					score: round(one.score, 6),
+					...best === void 0 ? {} : { passage: passageOf(best) }
+				};
+			});
+		},
+		async embed(embedOptions = {}) {
+			const started = Date.now();
+			const budgetMs = durationMs(embedOptions.budget ?? "10m");
+			const maxDocs = embedOptions.maxDocs === void 0 ? Number.POSITIVE_INFINITY : count("maxDocs", embedOptions.maxDocs, 1, Number.MAX_SAFE_INTEGER);
+			const kind = embedOptions.kind ?? null;
+			const timer = new AbortController();
+			const clock = setTimeout(() => timer.abort(/* @__PURE__ */ new Error("the budget of embed is over")), Math.min(budgetMs, 2147483647));
+			clock.unref();
+			const signal = embedOptions.signal === void 0 ? timer.signal : AbortSignal.any([embedOptions.signal, timer.signal]);
+			const tally = {
+				docs: 0,
+				embedded: 0,
+				reused: 0,
+				held_back: 0,
+				chunks: 0
+			};
+			let stopped = "done";
+			const stopOf = () => embedOptions.signal?.aborted === true ? "signal" : timer.signal.aborted || Date.now() - started >= budgetMs ? "budget" : tally.embedded >= maxDocs ? "max-docs" : void 0;
+			try {
+				for await (const page of pagesOf(runtime)) {
+					const docs = page.filter((head) => kind === null || head.kind === kind);
+					for (let at = 0; at < docs.length;) {
+						const stop = stopOf();
+						if (stop !== void 0) {
+							stopped = stop;
+							break;
+						}
+						const room = Math.max(1, Math.min(BATCH, maxDocs - tally.embedded));
+						const group = docs.slice(at, at + room).map((head) => head.doc);
+						at += group.length;
+						try {
+							const done = await ensure(group, signal);
+							tally.docs += group.length;
+							tally.embedded += done.embedded;
+							tally.reused += done.reused;
+							tally.held_back += done.held;
+							tally.chunks += done.chunks;
+						} catch (error) {
+							const stop = stopOf();
+							if (stop === void 0) throw error;
+							stopped = stop;
+							break;
+						}
+					}
+					if (stopped !== "done") break;
+				}
+			} finally {
+				clearTimeout(clock);
+			}
+			return {
+				embedder: embedder.id,
+				...tally,
+				stopped,
+				ms: Date.now() - started
+			};
+		}
+	};
+};
+/**
+* This function gives the heads of each stored document, in pages. It uses `digests` when the
+* index has it. Else it reads the capture log and asks the index for the documents of each capture.
+*/
+async function* pagesOf(runtime) {
+	const index = runtime.index;
+	if (index.digests !== void 0) {
+		let after = "";
+		for (;;) {
+			const page = index.digests({
+				after,
+				limit: 500
+			});
+			if (page.length === 0) return;
+			yield page;
+			after = page.at(-1).doc;
+		}
+	}
+	const seen = /* @__PURE__ */ new Set();
+	let page = [];
+	for await (const envelope of runtime.captures.scan({ source: "fetch" })) for (const doc of index.docsOfCapture(envelope.id)) {
+		if (seen.has(doc.doc)) continue;
+		seen.add(doc.doc);
+		page.push({
+			doc: doc.doc,
+			kind: doc.kind,
+			ai_input: doc.ai_input,
+			text_sha: sha256(doc.text)
+		});
+		if (page.length >= 500) {
+			yield page;
+			page = [];
+		}
+	}
+	if (page.length > 0) yield page;
+}
+//#endregion
+//#region src/recipe.ts
+/**
+* This function reads a body as JSON, in the encoding that the response declares. A body that is
+* not JSON (a page about maintenance, or a login page) throws an error that names the URL and the
+* content type. In an extractor, the runtime then reports that error.
+*/
+const readJson = (body, capture) => {
+	const text = decodeBody(body, capture.contentType);
+	try {
+		return JSON.parse(text);
+	} catch {
+		throw new Error(`${capture.url} answered ${capture.contentType ?? "a body of no stated type"}, not JSON`);
+	}
+};
+/**
+* These recipe names belong to the crawler: the page reader, the archived fallback, robots.txt,
+* and the sitemaps and feeds of `discover`. A recipe cannot use them.
+*/
+const RESERVED_RECIPES = [
+	"page",
+	"archive-copy",
+	"archive-index",
+	"robots.txt",
+	"sitemap",
+	"feed"
+];
+/** These are the limits of a recipe. */
+const RECIPE_LIMITS = {
+	nameChars: 64,
+	docIdChars: 2048
+};
+const NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
+const PAGE_ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
+const API_ACCEPT = "application/json";
+const AI_INPUTS = [
+	"yes",
+	"no",
+	"unset"
+];
+/** This function tells what is wrong with one document of an extractor, or gives undefined. */
+const faultOf = (value, seen) => {
+	if (value === null || typeof value !== "object" || Array.isArray(value)) return "a document must be an object";
+	const doc = value;
+	const id = doc["doc"];
+	if (typeof id !== "string" || id.length === 0 || id.length > RECIPE_LIMITS.docIdChars) return `doc must be a text of 1 to ${String(RECIPE_LIMITS.docIdChars)} characters`;
+	if (id.startsWith("archive:")) return `the id ${id} starts with archive:, which is for the archived copies`;
+	if (seen.has(id)) return `two documents have the id ${id}`;
+	seen.add(id);
+	const kind = doc["kind"];
+	if (typeof kind !== "string" || kind.length === 0) return "kind must be a text that is not empty";
+	if (kind === "archived") return "the kind archived is for the archived copies";
+	if (typeof doc["title"] !== "string") return "title must be a text";
+	if (typeof doc["text"] !== "string") return "text must be a text";
+	if (doc["url"] !== null && typeof doc["url"] !== "string") return "url must be a text or null";
+	const meta = doc["meta"];
+	if (meta === null || typeof meta !== "object" || Array.isArray(meta)) return "meta must be an object";
+	try {
+		JSON.stringify(meta);
+	} catch {
+		return "meta must be JSON: no cycle and no BigInt";
+	}
+	const merge = doc["merge"];
+	if (merge !== void 0 && merge !== "latest" && merge !== "richest") return "merge must be \"latest\" or \"richest\"";
+	if (doc["full_text"] !== void 0 && typeof doc["full_text"] !== "string") return "full_text must be a text";
+	const aiInput = doc["ai_input"];
+	if (aiInput !== void 0 && !AI_INPUTS.includes(aiInput)) return "ai_input must be \"yes\", \"no\" or \"unset\"";
+};
+/** This function gives the documents of an extractor, or throws an error that names the first fault. */
+const checkedDocs = (name, docs) => {
+	if (!Array.isArray(docs)) throw new TypeError(`the recipe ${name} gave ${typeof docs}, not an array of documents`);
+	const seen = /* @__PURE__ */ new Set();
+	for (const [index, doc] of docs.entries()) {
+		const fault = faultOf(doc, seen);
+		if (fault !== void 0) throw new TypeError(`the recipe ${name} gave a document that is not valid (${String(index)}): ${fault}`);
+	}
+	return docs;
+};
+/**
+* This function declares a recipe. It checks the definition at once: a name that is not valid, a
+* reserved name or a duration that it cannot read throws an error. The extractor of the recipe
+* checks each document of `extract`. A document that is not valid makes the extractor throw, and
+* the runtime reports it as an error of the extractor.
+*/
+const defineRecipe = (definition) => {
+	const name = definition.name;
+	if (typeof name !== "string" || name.length > RECIPE_LIMITS.nameChars || !NAME.test(name)) throw new RangeError(`a recipe name is 1 to ${String(RECIPE_LIMITS.nameChars)} letters, digits and . _ : -, and it starts with a letter or a digit, not ${JSON.stringify(name) ?? "undefined"}`);
+	if (RESERVED_RECIPES.includes(name)) throw new RangeError(`the recipe name ${name} belongs to the crawler: give the recipe a name of its own`);
+	if (typeof definition.extract !== "function") throw new TypeError(`the recipe ${name} needs extract, a function that reads a body into documents`);
+	const mode = definition.mode ?? "page";
+	if (mode !== "page" && mode !== "api") throw new RangeError(`the mode of the recipe ${name} must be "page" or "api", not ${String(mode)}`);
+	const accept = definition.accept ?? (mode === "api" ? API_ACCEPT : PAGE_ACCEPT);
+	const minIntervalMs = definition.minInterval === void 0 ? void 0 : durationMs(definition.minInterval);
+	const maxAgeOf = (value) => value === "auto" ? "auto" : durationMs(value);
+	const maxAge = maxAgeOf(definition.maxAge ?? "24h");
+	const maxBytes = definition.maxBytes;
+	if (maxBytes !== void 0 && (!Number.isSafeInteger(maxBytes) || maxBytes < 1)) throw new RangeError(`maxBytes of the recipe ${name} must be a whole number of 1 or more`);
+	const extract = definition.extract;
+	const extractor = async (body, capture) => checkedDocs(name, await extract(body, {
+		url: capture.url,
+		contentType: capture.contentType
+	}));
+	const fetch = definition.fetch;
+	return {
+		name,
+		description: definition.description ?? null,
+		mode,
+		accept,
+		extractor,
+		request(input, options = {}) {
+			const asked = fetch === void 0 ? input : fetch(input);
+			const target = typeof asked === "string" ? { url: asked } : asked;
+			const url = target?.url;
+			let parsed;
+			try {
+				parsed = typeof url === "string" ? new URL(url) : void 0;
+			} catch {
+				parsed = void 0;
+			}
+			if (parsed === void 0 || parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new TypeError(`the recipe ${name} needs an http or https URL for its input, not ${JSON.stringify(url) ?? String(url)}`);
+			return {
+				url: parsed.href,
+				mode,
+				recipe: name,
+				accept: target?.accept ?? accept,
+				headers: target?.headers,
+				secretParams: target?.secretParams,
+				minIntervalMs,
+				maxAgeMs: options.maxAge === void 0 ? maxAge : maxAgeOf(options.maxAge),
+				maxBytes,
+				onBehalfOf: options.onBehalfOf,
+				job: options.job,
+				signal: options.signal,
+				scope: options.scope,
+				labels: options.labels,
+				deadline: options.deadline
+			};
+		}
+	};
+};
+/**
+* This function gives the extractor of a recipe to the extractors of a runtime. A second
+* registration of the same recipe does nothing. A different extractor with the same name throws
+* an error, because `reindex` must read each capture with one extractor.
+*/
+const registerRecipe = (extractors, recipe) => {
+	const known = extractors.get(recipe.name);
+	if (known === recipe.extractor) return;
+	if (known !== void 0) throw new Error(`the recipe ${recipe.name} has a different extractor in this runtime: give each recipe a name of its own`);
+	extractors.register(recipe.name, recipe.extractor);
+};
+/** This function gives the runtime of a target. */
+const runtimeOf = (target) => "crawler" in target ? target : target.runtime;
+/** This function gives the deadline of a budget, on the clock of the runtime. */
+const deadlineOf = (runtime, budget) => budget === void 0 ? void 0 : Date.parse(runtime.now()) + durationMs(budget);
+/** This function gives one document of a run, with the rule for a held-back document. */
+const recipeDoc = async (runtime, doc, capture, userProvided, action) => {
+	const current = runtime.index.get(doc.doc) ?? doc;
+	const version = (capture === void 0 ? void 0 : runtime.index.versionText(doc.doc, capture)) ?? current;
+	const aiInput = current.ai_input === "no" || version.ai_input === "no" ? "no" : version.ai_input;
+	const held = isHeldBack({ ai_input: aiInput }, userProvided);
+	if (aiInput === "no" && userProvided) await recordOverride(runtime, current, action);
+	const archived = archivedOf(version.meta);
+	return {
+		doc: doc.doc,
+		kind: version.kind,
+		title: held ? "" : version.title,
+		url: version.url,
+		meta: held ? heldBackMeta(version.meta) : version.meta,
+		chars: version.text.length,
+		capture: version.capture,
+		fetched_at: version.fetched_at,
+		held_back: held,
+		...archived === void 0 ? {} : { archived }
+	};
+};
+const noteOf = (parts) => {
+	const kept = parts.filter((part) => typeof part === "string" && part.length > 0);
+	return kept.length > 0 ? kept.join("; ") : null;
+};
+/**
+* This function starts a recipe for one input. It fetches the request through the crawler, and it
+* reads the capture with the extractor of the recipe. It gives the documents of that capture.
+* These are its rules:
+*
+* - The runtime registers the recipe the first time that it gets it. Give the recipe to
+*   `openCrawler` too, so that `reindex` reads its captures before the first fetch.
+* - A stored copy younger than `maxAge` answers with no request. When the extractor cannot read a
+*   stored copy, the crawler asks the source one more time.
+* - An extractor that throws gives no document, and `error` tells why. The fetch still succeeds.
+* - A document that asked not to be AI input shows no title and only its structural metadata,
+*   unless `userProvided`. The crawler logs each `userProvided` read of a held-back document.
+* - With `fallback: "archive"`, a page that the site refuses gets an archived copy, with
+*   `archived`. The page reader reads the copy. The extractor of the recipe reads only live
+*   captures. Thus its documents do not mix a live answer and an old copy of a third party.
+*/
+const runRecipe = async (target, recipe, input, options = {}) => {
+	const runtime = runtimeOf(target);
+	if (options.fallback === "archive" && recipe.mode !== "page") throw new RangeError(`the recipe ${recipe.name} reads an API: an archived copy is for a page, so give fallback only to a recipe of the mode page`);
+	registerRecipe(runtime.extractors, recipe);
+	const deadline = deadlineOf(runtime, options.budget);
+	const request = recipe.request(input, {
+		...options,
+		deadline
+	});
+	let indexed = await fetchAndIndex(runtime, request);
+	if (indexed.error !== void 0 && indexed.blocked === void 0 && indexed.fetched.fromCache) indexed = await fetchAndIndex(runtime, {
+		...request,
+		maxAgeMs: 0
+	});
+	const userProvided = options.userProvided ?? false;
+	const action = options.job ?? `recipe ${recipe.name}`;
+	const fetched = indexed.fetched;
+	const notes = [fetched.note, indexed.error];
+	if (indexed.docs.length === 0 && options.fallback === "archive") {
+		const reason = fallbackReason(indexed);
+		if (reason !== void 0) {
+			const page = {
+				render: "never",
+				maxAgeMs: request.maxAgeMs,
+				onBehalfOf: request.onBehalfOf,
+				job: request.job,
+				signal: request.signal,
+				scope: request.scope,
+				labels: request.labels,
+				deadline,
+				fallback: "archive",
+				archiveDespiteRobots: options.archiveDespiteRobots
+			};
+			const answer = await archivedCopy(runtime, request.url, reason, page);
+			if (answer.doc !== void 0 && answer.archived !== void 0) {
+				const doc = await recipeDoc(runtime, answer.doc, answer.doc.capture, userProvided, action);
+				return {
+					recipe: recipe.name,
+					url: fetched.url,
+					outcome: fetched.outcome,
+					status: fetched.status,
+					docs: [doc],
+					capture: doc.capture,
+					fetched_at: doc.fetched_at,
+					from_cache: answer.fromCache,
+					held_back: doc.held_back,
+					error: indexed.error ?? null,
+					archived: answer.archived,
+					note: noteOf([
+						...notes,
+						...answer.notes,
+						"the page reader read the archived copy, not the extractor of the recipe",
+						doc.held_back ? HELD_BACK : void 0
+					])
+				};
+			}
+			notes.push(...answer.notes);
+		}
+	}
+	const capture = fetched.capture;
+	const docs = [];
+	for (const doc of indexed.docs) docs.push(await recipeDoc(runtime, doc, capture, userProvided, action));
+	const held = docs.some((doc) => doc.held_back);
+	return {
+		recipe: recipe.name,
+		url: fetched.url,
+		outcome: fetched.outcome,
+		status: fetched.status,
+		docs,
+		capture: docs[0]?.capture ?? capture ?? null,
+		fetched_at: docs[0]?.fetched_at ?? fetched.fetchedAt ?? null,
+		from_cache: fetched.fromCache,
+		held_back: held,
+		error: indexed.error ?? null,
+		note: noteOf([...notes, held ? HELD_BACK : void 0])
+	};
+};
+//#endregion
+//#region src/vectors.ts
+/**
+* The vector store keeps the vectors of hybrid search in `vectors.sqlite`, beside the index. It is
+* a projection, as the index is. Delete the file, and the next hybrid search or `embed` makes the
+* vectors again from the stored texts.
+*
+* - A row is one chunk of one text for one embedder. Its key is the SHA-256 of the text, the
+*   number of the chunk and the id of the embedder. Thus a reindex keeps each vector, and the same
+*   text gets its vectors one time for each model, also when two documents hold it.
+* - A text is complete when the table `texts` has it, with the plan of its chunks. A text with a
+*   different plan gets new chunks.
+* - The table `docs` points each document at the text that has its vectors. A search uses only
+*   the texts that a document points at, and the caller checks each document against the index.
+* - A search compares the vector of the question with each stored vector (a brute-force cosine).
+*   The store keeps the vectors of each embedder in memory after the first search. It reads them
+*   again when a different process writes the file.
+*
+* Each connection waits for the lock of a different process (`busy_timeout`), and each write is
+* one transaction (`BEGIN IMMEDIATE`). The file has a schema version (`VECTORS_SCHEMA`). A file of
+* a different version opens empty.
+*/
+/** This value is the version of the schema of the vector file. */
+const VECTORS_SCHEMA = 1;
+const DDL = `
+CREATE TABLE IF NOT EXISTS vectors_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+CREATE TABLE IF NOT EXISTS embedders (
+  id TEXT PRIMARY KEY,
+  dimensions INTEGER NOT NULL,
+  first_at TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS texts (
+  sha TEXT NOT NULL,
+  embedder TEXT NOT NULL,
+  plan TEXT NOT NULL,
+  chunks INTEGER NOT NULL,
+  chars INTEGER NOT NULL,
+  embedded_at TEXT NOT NULL,
+  PRIMARY KEY (sha, embedder)
+);
+CREATE TABLE IF NOT EXISTS chunks (
+  sha TEXT NOT NULL,
+  chunk INTEGER NOT NULL,
+  embedder TEXT NOT NULL,
+  char_start INTEGER NOT NULL,
+  char_end INTEGER NOT NULL,
+  vector BLOB NOT NULL,
+  PRIMARY KEY (sha, chunk, embedder)
+);
+CREATE INDEX IF NOT EXISTS chunks_embedder ON chunks (embedder, sha, chunk);
+CREATE TABLE IF NOT EXISTS docs (
+  doc TEXT NOT NULL,
+  embedder TEXT NOT NULL,
+  sha TEXT NOT NULL,
+  pointed_at TEXT NOT NULL,
+  PRIMARY KEY (doc, embedder)
+);
+CREATE INDEX IF NOT EXISTS docs_sha ON docs (embedder, sha);
+`;
+/** This function gives the bytes of a vector for a BLOB. The bytes are in the order of the platform (little-endian on each common platform). */
+const bytesOf = (vector) => new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength);
+/** This function gives the vector of the bytes of a BLOB. It copies the bytes, so an odd offset of the BLOB is not a problem. */
+const vectorOf = (bytes) => {
+	const out = new Float32Array(Math.floor(bytes.byteLength / 4));
+	new Uint8Array(out.buffer).set(bytes.subarray(0, out.length * 4));
+	return out;
+};
+const dot = (data, row, query, dims) => {
+	let sum = 0;
+	const base = row * dims;
+	for (let at = 0; at < dims; at += 1) sum += data[base + at] * query[at];
+	return sum;
+};
+var VectorStore = class {
+	path;
+	db;
+	statements = /* @__PURE__ */ new Map();
+	matrices = /* @__PURE__ */ new Map();
+	closed = false;
+	constructor(path, options = {}) {
+		this.path = path;
+		if (path !== ":memory:") mkdirSync(dirname(path), {
+			recursive: true,
+			mode: 448
+		});
+		this.db = new DatabaseSync(path);
+		const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
+		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
+		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
+		this.db.exec("PRAGMA synchronous = NORMAL;");
+		this.write(() => {
+			if (this.db.prepare("PRAGMA user_version").get().user_version !== 1) {
+				const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
+				for (const { name } of tables) this.db.exec(`DROP TABLE IF EXISTS "${name}"`);
+			}
+			this.db.exec(DDL);
+			this.db.exec(`PRAGMA user_version = ${String(1)}`);
+		});
+	}
+	stmt(sql) {
+		let statement = this.statements.get(sql);
+		if (statement === void 0) {
+			statement = this.db.prepare(sql);
+			this.statements.set(sql, statement);
+		}
+		return statement;
+	}
+	write(work) {
+		if (this.closed) throw new Error("the vector store is closed");
+		this.db.exec("BEGIN IMMEDIATE");
+		try {
+			const result = work();
+			this.db.exec("COMMIT");
+			return result;
+		} catch (error) {
+			if (this.db.isTransaction) this.db.exec("ROLLBACK");
+			throw error;
+		}
+	}
+	live() {
+		if (this.closed) throw new Error("the vector store is closed");
+	}
+	/** This method gives the plan of the chunks of a text when the store has each of its vectors, or undefined. */
+	plan(embedder, sha) {
+		this.live();
+		return this.stmt("SELECT plan FROM texts WHERE sha = ? AND embedder = ?").get(sha, embedder)?.plan;
+	}
+	/** This method gives the text that a document points at, or undefined. */
+	pointed(embedder, doc) {
+		this.live();
+		return this.stmt("SELECT sha FROM docs WHERE doc = ? AND embedder = ?").get(doc, embedder)?.sha;
+	}
+	/** This method gives the dimensions that the store keeps for an embedder, or undefined. */
+	dimensions(embedder) {
+		this.live();
+		return this.stmt("SELECT dimensions FROM embedders WHERE id = ?").get(embedder)?.dimensions;
+	}
+	/**
+	* This method writes the chunks of one text and points its documents at it, in one transaction.
+	* A vector with a length that is different from the dimensions of the embedder throws an error.
+	*/
+	put(input) {
+		const dims = input.dimensions;
+		for (const chunk of input.chunks) if (chunk.vector.length !== dims) throw new Error(`the embedder ${input.embedder} gave a vector of ${String(chunk.vector.length)} dimensions, not ${String(dims)}`);
+		let replaced = false;
+		this.write(() => {
+			const known = this.stmt("SELECT dimensions FROM embedders WHERE id = ?").get(input.embedder);
+			if (known === void 0 && input.chunks.length > 0) this.stmt("INSERT INTO embedders (id, dimensions, first_at) VALUES (?, ?, ?)").run(input.embedder, dims, input.at);
+			else if (known !== void 0 && input.chunks.length > 0 && known.dimensions !== dims) throw new Error(`vectors.sqlite holds vectors of ${String(known.dimensions)} dimensions for the embedder ${input.embedder}, and it gave ${String(dims)}: give the embedder a new id, or delete vectors.sqlite`);
+			const before = this.stmt("SELECT plan FROM texts WHERE sha = ? AND embedder = ?").get(input.sha, input.embedder);
+			replaced = before !== void 0 && before.plan !== input.plan;
+			this.stmt("DELETE FROM chunks WHERE sha = ? AND embedder = ?").run(input.sha, input.embedder);
+			const insert = this.stmt("INSERT INTO chunks (sha, chunk, embedder, char_start, char_end, vector) VALUES (?, ?, ?, ?, ?, ?)");
+			input.chunks.forEach((chunk, at) => {
+				insert.run(input.sha, at, input.embedder, chunk.start, chunk.end, bytesOf(chunk.vector));
+			});
+			this.stmt(`INSERT INTO texts (sha, embedder, plan, chunks, chars, embedded_at) VALUES (?, ?, ?, ?, ?, ?)
+         ON CONFLICT(sha, embedder) DO UPDATE SET plan = excluded.plan, chunks = excluded.chunks,
+           chars = excluded.chars, embedded_at = excluded.embedded_at`).run(input.sha, input.embedder, input.plan, input.chunks.length, input.chars, input.at);
+			for (const doc of input.docs) this.pointRow(input.embedder, doc, input.sha, input.at);
+		});
+		const matrix = this.matrices.get(input.embedder);
+		if (matrix === void 0) return;
+		if (replaced || matrix.texts.has(input.sha)) {
+			this.matrices.delete(input.embedder);
+			return;
+		}
+		this.append(matrix, input.sha, input.chunks.map((chunk, at) => ({
+			...chunk,
+			chunk: at
+		})));
+		for (const doc of input.docs) this.pointMatrix(matrix, doc, input.sha);
+	}
+	pointRow(embedder, doc, sha, at) {
+		this.stmt(`INSERT INTO docs (doc, embedder, sha, pointed_at) VALUES (?, ?, ?, ?)
+       ON CONFLICT(doc, embedder) DO UPDATE SET sha = excluded.sha, pointed_at = excluded.pointed_at`).run(doc, embedder, sha, at);
+	}
+	/** This method points a document at a text that the store has. */
+	point(embedder, doc, sha, at) {
+		this.write(() => this.pointRow(embedder, doc, sha, at));
+		const matrix = this.matrices.get(embedder);
+		if (matrix === void 0) return;
+		if (!matrix.texts.has(sha)) this.append(matrix, sha, this.chunks(embedder, sha));
+		this.pointMatrix(matrix, doc, sha);
+	}
+	/** This method removes the pointer of a document, for example when the document is held back. Its text keeps its vectors. */
+	forget(embedder, doc) {
+		this.write(() => this.stmt("DELETE FROM docs WHERE doc = ? AND embedder = ?").run(doc, embedder));
+		const matrix = this.matrices.get(embedder);
+		if (matrix === void 0) return;
+		const sha = matrix.shaOf.get(doc);
+		if (sha !== void 0) matrix.docsOf.get(sha)?.delete(doc);
+		matrix.shaOf.delete(doc);
+	}
+	/** This method gives the chunks of a text, in their sequence. */
+	chunks(embedder, sha) {
+		this.live();
+		return this.stmt("SELECT chunk, char_start, char_end, vector FROM chunks WHERE sha = ? AND embedder = ? ORDER BY chunk").all(sha, embedder).map((row) => ({
+			chunk: row.chunk,
+			start: row.char_start,
+			end: row.char_end,
+			vector: vectorOf(row.vector)
+		}));
+	}
+	pointMatrix(matrix, doc, sha) {
+		const before = matrix.shaOf.get(doc);
+		if (before !== void 0 && before !== sha) matrix.docsOf.get(before)?.delete(doc);
+		matrix.shaOf.set(doc, sha);
+		let docs = matrix.docsOf.get(sha);
+		if (docs === void 0) {
+			docs = /* @__PURE__ */ new Set();
+			matrix.docsOf.set(sha, docs);
+		}
+		docs.add(doc);
+	}
+	append(matrix, sha, chunks) {
+		const dims = matrix.dimensions;
+		const need = matrix.rows + chunks.length;
+		if (need > matrix.chunk.length) {
+			const size = Math.max(need, matrix.chunk.length * 2, 64);
+			const data = new Float32Array(size * dims);
+			data.set(matrix.data.subarray(0, matrix.rows * dims));
+			const grow = (old) => {
+				const next = new Int32Array(size);
+				next.set(old.subarray(0, matrix.rows));
+				return next;
+			};
+			matrix.data = data;
+			matrix.chunk = grow(matrix.chunk);
+			matrix.start = grow(matrix.start);
+			matrix.end = grow(matrix.end);
+		}
+		const from = matrix.rows;
+		for (const chunk of chunks) {
+			const row = matrix.rows;
+			if (chunk.vector.length === dims) matrix.data.set(chunk.vector, row * dims);
+			matrix.chunk[row] = chunk.chunk;
+			matrix.start[row] = chunk.start;
+			matrix.end[row] = chunk.end;
+			matrix.rows += 1;
+		}
+		matrix.texts.set(sha, {
+			from,
+			to: matrix.rows
+		});
+	}
+	/** This method gives the vectors of an embedder in memory. It reads them again when a different process wrote the file. */
+	matrix(embedder) {
+		this.live();
+		const version = this.db.prepare("PRAGMA data_version").get().data_version;
+		const cached = this.matrices.get(embedder);
+		if (cached !== void 0 && cached.version === version) return cached;
+		const dims = this.dimensions(embedder);
+		if (dims === void 0) return void 0;
+		this.db.exec("BEGIN");
+		try {
+			const matrix = this.load(embedder, dims, version);
+			this.matrices.set(embedder, matrix);
+			return matrix;
+		} finally {
+			if (this.db.isTransaction) this.db.exec("COMMIT");
+		}
+	}
+	load(embedder, dims, version) {
+		const count = this.db.prepare("SELECT count(*) AS n FROM chunks WHERE embedder = ? AND sha IN (SELECT sha FROM docs WHERE embedder = ?)").get(embedder, embedder).n;
+		const matrix = {
+			dimensions: dims,
+			rows: 0,
+			data: new Float32Array(Math.max(1, count) * dims),
+			chunk: new Int32Array(Math.max(1, count)),
+			start: new Int32Array(Math.max(1, count)),
+			end: new Int32Array(Math.max(1, count)),
+			texts: /* @__PURE__ */ new Map(),
+			docsOf: /* @__PURE__ */ new Map(),
+			shaOf: /* @__PURE__ */ new Map(),
+			version
+		};
+		const rows = this.db.prepare(`SELECT sha, chunk, char_start, char_end, vector FROM chunks
+          WHERE embedder = ? AND sha IN (SELECT sha FROM docs WHERE embedder = ?)
+          ORDER BY sha, chunk`).iterate(embedder, embedder);
+		let sha;
+		let from = 0;
+		for (const row of rows) {
+			if (row.sha !== sha) {
+				if (sha !== void 0) matrix.texts.set(sha, {
+					from,
+					to: matrix.rows
+				});
+				sha = row.sha;
+				from = matrix.rows;
+			}
+			if (matrix.rows >= matrix.chunk.length) break;
+			const at = matrix.rows;
+			if (row.vector.byteLength === dims * 4) new Uint8Array(matrix.data.buffer, at * dims * 4, dims * 4).set(row.vector);
+			matrix.chunk[at] = row.chunk;
+			matrix.start[at] = row.char_start;
+			matrix.end[at] = row.char_end;
+			matrix.rows += 1;
+		}
+		if (sha !== void 0) matrix.texts.set(sha, {
+			from,
+			to: matrix.rows
+		});
+		for (const row of this.db.prepare("SELECT doc, sha FROM docs WHERE embedder = ?").all(embedder)) this.pointMatrix(matrix, row.doc, row.sha);
+		return matrix;
+	}
+	/** This method gives the best chunk of one text for a question, or undefined when the store has no vector of the text. */
+	best(embedder, sha, query) {
+		const matrix = this.matrix(embedder);
+		const range = matrix?.texts.get(sha);
+		if (matrix === void 0 || range === void 0 || range.to === range.from) return void 0;
+		return this.hitOf(matrix, sha, range, query);
+	}
+	hitOf(matrix, sha, range, query) {
+		let bestRow = range.from;
+		let bestScore = Number.NEGATIVE_INFINITY;
+		for (let row = range.from; row < range.to; row += 1) {
+			const score = dot(matrix.data, row, query, matrix.dimensions);
+			if (score > bestScore) {
+				bestScore = score;
+				bestRow = row;
+			}
+		}
+		return {
+			sha,
+			chunk: matrix.chunk[bestRow],
+			start: matrix.start[bestRow],
+			end: matrix.end[bestRow],
+			similarity: bestScore,
+			docs: [...matrix.docsOf.get(sha) ?? []]
+		};
+	}
+	/**
+	* This method compares a question with each stored vector of an embedder (a brute-force cosine).
+	* It gives the best chunk of each text that a document points at, the best first. With `within`,
+	* it compares only the texts of those documents.
+	*/
+	search(embedder, query, options = {}) {
+		const matrix = this.matrix(embedder);
+		if (matrix === void 0) return [];
+		if (query.length !== matrix.dimensions) throw new Error(`the question has a vector of ${String(query.length)} dimensions, and the store holds ${String(matrix.dimensions)} for the embedder ${embedder}`);
+		const shas = /* @__PURE__ */ new Set();
+		if (options.within === void 0) {
+			for (const [sha, docs] of matrix.docsOf) if (docs.size > 0) shas.add(sha);
+		} else for (const doc of options.within) {
+			const sha = matrix.shaOf.get(doc);
+			if (sha !== void 0) shas.add(sha);
+		}
+		const hits = [];
+		for (const sha of shas) {
+			const range = matrix.texts.get(sha);
+			if (range === void 0 || range.to === range.from) continue;
+			hits.push(this.hitOf(matrix, sha, range, query));
+		}
+		return hits.sort((a, b) => b.similarity - a.similarity || (a.sha < b.sha ? -1 : 1));
+	}
+	/** This method gives the counts of the store, for one embedder or for each embedder. */
+	counts(embedder) {
+		this.live();
+		const where = embedder === void 0 ? "" : " WHERE embedder = ?";
+		const params = embedder === void 0 ? [] : [embedder];
+		const one = (sql) => this.stmt(sql).get(...params).n ?? 0;
+		return {
+			texts: one(`SELECT count(*) AS n FROM texts${where}`),
+			chunks: one(`SELECT count(*) AS n FROM chunks${where}`),
+			docs: one(`SELECT count(*) AS n FROM docs${where}`),
+			bytes: one(`SELECT sum(length(vector)) AS n FROM chunks${where}`)
+		};
+	}
+	/** This method closes the file. A second close does nothing. */
+	close() {
+		if (this.closed) return;
+		this.closed = true;
+		this.statements.clear();
+		this.matrices.clear();
+		this.db.close();
+	}
+};
+//#endregion
+//#region src/watch.ts
+/**
+* A watch list keeps the pages that the crawler checks again on a schedule. `watch.run` fetches the
+* pages that are due through the normal fetch, so each polite rule applies. It gives an event for
+* each page: new, changed, unchanged, gone, refused or error.
+*
+* These are the rules of a watch:
+*
+* - A watch compares the title, the text and the JSON-LD of the page with the last confirmed version.
+* - With `confirm: 2` (the default), a change is suspected when a check first sees it. The next
+*   check confirms it when it sees the same new version. A check that sees the earlier version
+*   again clears the suspicion, because one read can be a glitch. With `confirm: 1`, each check confirms.
+* - `every` has a floor of 1 hour by default. A check that a stop of the host refused is due again
+*   at the end of the stop, or later.
+* - The list is in `watch.sqlite` in the store, so it stays after a restart. A run claims each entry
+*   with a lease in that file, so two processes do not check one entry two times.
+*
+* A scheduler of the operator starts `watch.run`, or the daemon (`startDaemon`) starts it on a schedule.
+*/
+/** This value is the version of the schema of `watch.sqlite`. */
+const WATCH_SCHEMA = 1;
+/** These are the defaults and the limits of the watch list. */
+const WATCH_LIMITS = {
+	/** The shortest `every` of a watch by default: 1 hour, so a watch cannot hammer a site. */
+	floorMs: 36e5,
+	/** The default `every`: 24 hours. */
+	everyMs: 864e5,
+	/** The default time of one run: 10 minutes. No request starts after it. */
+	budgetMs: 6e5,
+	/** The default number of entries that one run checks. */
+	maxPages: 100,
+	/** The most hosts that get a check of one run at one time. */
+	hostsAtOnce: 8,
+	/** A run holds the lease of an entry until the end of its budget and this time more: 2 minutes. */
+	leaseMs: 12e4
+};
+const iso = (ms) => new Date(ms).toISOString();
+const parseJson = (text, fallback) => {
+	if (text === null) return fallback;
+	try {
+		return JSON.parse(text);
+	} catch {
+		return fallback;
+	}
+};
+const RENDERS = /* @__PURE__ */ new Set([
+	"auto",
+	"never",
+	"always"
+]);
+const entryOf = (row, now) => ({
+	url: row.target,
+	every_ms: row.every_ms,
+	render: row.render,
+	confirm: row.confirm === 1 ? 1 : 2,
+	scope: row.scope,
+	labels: parseJson(row.labels, null),
+	on_behalf_of: parseJson(row.on_behalf_of, []),
+	added_at: iso(row.added_at),
+	next_at: iso(row.next_at),
+	checked_at: row.last_at === null ? null : iso(row.last_at),
+	last_kind: row.last_kind,
+	last_note: row.last_note,
+	checks: row.checks,
+	changes: row.changes,
+	doc: row.doc,
+	capture: row.base_capture,
+	fetched_at: row.base_fetched_at,
+	gone: row.base_state === "gone",
+	pending: row.pend_state === null ? null : {
+		kind: row.pend_state === "gone" ? "gone" : row.base_state === "gone" ? "new" : "changed",
+		capture: row.pend_capture,
+		since: iso(row.pend_since ?? row.last_at ?? row.added_at)
+	},
+	leased_until: row.lease_until > now ? iso(row.lease_until) : null
+});
+/** This function gives the canonical key, the URL and the host of a page to watch. A URL that is not http(s) throws a `RangeError`. */
+const targetOf = (url) => {
+	let parsed;
+	try {
+		parsed = new URL(url);
+	} catch {
+		throw new RangeError(`not a valid URL: ${url}`);
+	}
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new RangeError(`a watch needs an http or https URL, not ${url}`);
+	parsed.hash = "";
+	return {
+		key: cacheKeyOf(parsed.href),
+		target: parsed.href,
+		host: parsed.host.toLowerCase()
+	};
+};
+/**
+* This store keeps the watch list in a SQLite file (`watch.sqlite`). Two or more processes on one
+* store share it. Each step is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock of a
+* different process (`busy_timeout`).
+*
+* The list is the decision of the operator, not a projection of the capture log. Thus it has its
+* own file: a deletion of `hosts.sqlite` or of the index does not lose it.
+*/
+var WatchStore = class {
+	path;
+	db;
+	statements = /* @__PURE__ */ new Map();
+	closed = false;
+	/** This value is the shortest `every` in milliseconds. */
+	floorMs;
+	constructor(path, options = {}) {
+		this.path = path;
+		if (path !== ":memory:") mkdirSync(dirname(path), {
+			recursive: true,
+			mode: 448
+		});
+		const floor = options.floorMs ?? WATCH_LIMITS.floorMs;
+		if (!Number.isFinite(floor) || floor < 0) throw new RangeError(`the floor of every must be 0 or more milliseconds, not ${String(floor)}`);
+		this.floorMs = floor;
+		this.db = new DatabaseSync(path);
+		const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
+		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
+		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
+		this.db.exec("PRAGMA synchronous = NORMAL;");
+		this.step(() => {
+			this.db.exec(`CREATE TABLE IF NOT EXISTS watches (
+        url TEXT PRIMARY KEY,
+        target TEXT NOT NULL,
+        host TEXT NOT NULL,
+        every_ms REAL NOT NULL,
+        render TEXT NOT NULL,
+        confirm INTEGER NOT NULL,
+        scope TEXT,
+        labels TEXT,
+        on_behalf_of TEXT,
+        added_at REAL NOT NULL,
+        next_at REAL NOT NULL,
+        lease_until REAL NOT NULL DEFAULT 0,
+        lease_owner TEXT,
+        checks INTEGER NOT NULL DEFAULT 0,
+        changes INTEGER NOT NULL DEFAULT 0,
+        last_at REAL,
+        last_kind TEXT,
+        last_note TEXT,
+        doc TEXT,
+        base_state TEXT,
+        base_sha TEXT,
+        base_capture TEXT,
+        base_fetched_at TEXT,
+        pend_state TEXT,
+        pend_sha TEXT,
+        pend_capture TEXT,
+        pend_fetched_at TEXT,
+        pend_since REAL
+      )`);
+			this.db.exec("CREATE INDEX IF NOT EXISTS watches_due ON watches (next_at)");
+			const { user_version: version } = this.db.prepare("PRAGMA user_version").get();
+			if (version < 1) this.db.exec(`PRAGMA user_version = ${String(1)}`);
+		});
+	}
+	stmt(sql) {
+		let statement = this.statements.get(sql);
+		if (statement === void 0) {
+			statement = this.db.prepare(sql);
+			this.statements.set(sql, statement);
+		}
+		return statement;
+	}
+	/** This method does one step in one transaction. */
+	step(work) {
+		if (this.closed) throw new Error("the watch store is closed");
+		this.db.exec("BEGIN IMMEDIATE");
+		try {
+			const result = work();
+			this.db.exec("COMMIT");
+			return result;
+		} catch (error) {
+			if (this.db.isTransaction) this.db.exec("ROLLBACK");
+			throw error;
+		}
+	}
+	row(key) {
+		return this.stmt("SELECT * FROM watches WHERE url = ?").get(key);
+	}
+	/**
+	* This method adds a page, or changes the options of its watch and keeps its state. A new entry
+	* is due at once. A changed entry is due one `every` after its last check. A bad option throws
+	* a `RangeError`.
+	*/
+	add(url, options, now) {
+		const { key, target, host } = targetOf(url);
+		const every = options.every === void 0 ? WATCH_LIMITS.everyMs : durationMs(options.every);
+		if (every < this.floorMs) throw new RangeError(`every must be ${String(this.floorMs / 6e4)} minutes or more, so a watch cannot hammer a site, not ${String(every / 6e4)} minutes`);
+		const render = options.render ?? "auto";
+		if (!RENDERS.has(render)) throw new RangeError(`render is auto, never or always, not ${JSON.stringify(render)}`);
+		const confirm = options.confirm ?? 2;
+		if (confirm !== 1 && confirm !== 2) throw new RangeError(`confirm is 1 or 2, not ${JSON.stringify(confirm)}`);
+		const labels = options.labels === void 0 || Object.keys(options.labels).length === 0 ? null : JSON.stringify(options.labels);
+		const agents = options.onBehalfOf === void 0 || options.onBehalfOf.length === 0 ? null : JSON.stringify([...options.onBehalfOf]);
+		return this.step(() => {
+			this.stmt(`INSERT INTO watches (url, target, host, every_ms, render, confirm, scope, labels, on_behalf_of, added_at, next_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(url) DO UPDATE SET target = excluded.target, host = excluded.host,
+           every_ms = excluded.every_ms, render = excluded.render, confirm = excluded.confirm,
+           scope = excluded.scope, labels = excluded.labels, on_behalf_of = excluded.on_behalf_of,
+           next_at = CASE WHEN watches.last_at IS NULL THEN watches.next_at
+                          ELSE watches.last_at + excluded.every_ms END`).run(key, target, host, every, render, confirm, options.scope ?? null, labels, agents, now, now);
+			return entryOf(this.row(key), now);
+		});
+	}
+	/** This method removes a page. It gives false when the list does not have the page. */
+	remove(url) {
+		const { key } = targetOf(url);
+		return this.step(() => Number(this.stmt("DELETE FROM watches WHERE url = ?").run(key).changes) > 0);
+	}
+	/** This method gives the entry of a page, or undefined. */
+	get(url, now) {
+		if (this.closed) throw new Error("the watch store is closed");
+		const row = this.row(targetOf(url).key);
+		return row === void 0 ? void 0 : entryOf(row, now);
+	}
+	/** This method gives the entries, the first due first. */
+	list(now) {
+		if (this.closed) throw new Error("the watch store is closed");
+		return this.stmt("SELECT * FROM watches ORDER BY next_at, url").all().map((row) => entryOf(row, now));
+	}
+	/** This method counts the entries that are due at a time. */
+	due(now) {
+		if (this.closed) throw new Error("the watch store is closed");
+		return this.stmt("SELECT count(*) AS n FROM watches WHERE next_at <= ?").get(now).n;
+	}
+	/**
+	* This method claims the first due entry with no lease, on a host that is not in `busy`. The
+	* lease of `owner` holds until `until`. Thus two processes do not get one entry at one time.
+	*/
+	claim(now, owner, until, busy) {
+		return this.step(() => {
+			const row = this.stmt(`SELECT * FROM watches WHERE next_at <= ? AND lease_until <= ?
+           AND host NOT IN (SELECT value FROM json_each(?)) ORDER BY next_at, url LIMIT 1`).get(now, now, JSON.stringify(busy));
+			if (row === void 0) return void 0;
+			this.stmt("UPDATE watches SET lease_until = ?, lease_owner = ? WHERE url = ?").run(until, owner, row.url);
+			return {
+				...row,
+				lease_until: until,
+				lease_owner: owner
+			};
+		});
+	}
+	/** This method ends the lease of `owner` and changes nothing else, so the entry stays due. A closed store does nothing. */
+	release(key, owner) {
+		if (this.closed) return;
+		this.step(() => {
+			this.stmt("UPDATE watches SET lease_until = 0, lease_owner = NULL WHERE url = ? AND lease_owner = ?").run(key, owner);
+		});
+	}
+	/**
+	* This method writes the result of a check and ends the lease. It gives false when `owner` does
+	* not hold the lease, or when the store is closed.
+	*/
+	settle(key, owner, update) {
+		if (this.closed) return false;
+		return this.step(() => {
+			const row = this.row(key);
+			if (row === void 0 || row.lease_owner !== owner) return false;
+			const base = update.base;
+			const pending = update.pending;
+			this.stmt(`UPDATE watches SET lease_until = 0, lease_owner = NULL, next_at = ?, last_at = ?, last_kind = ?,
+           last_note = ?, checks = checks + 1, changes = changes + ?, doc = ?,
+           base_state = ?, base_sha = ?, base_capture = ?, base_fetched_at = ?,
+           pend_state = ?, pend_sha = ?, pend_capture = ?, pend_fetched_at = ?, pend_since = ?
+         WHERE url = ?`).run(update.next_at, update.last_at, update.last_kind, update.last_note, update.changed ? 1 : 0, update.doc ?? row.doc, base === void 0 ? row.base_state : base.state, base === void 0 ? row.base_sha : base.sha, base === void 0 ? row.base_capture : base.capture, base === void 0 ? row.base_fetched_at : base.fetched_at, pending === void 0 ? row.pend_state : pending?.state ?? null, pending === void 0 ? row.pend_sha : pending?.sha ?? null, pending === void 0 ? row.pend_capture : pending?.capture ?? null, pending === void 0 ? row.pend_fetched_at : pending?.fetched_at ?? null, pending === void 0 ? row.pend_since : pending?.since ?? null, key);
+			return true;
+		});
+	}
+	/** This method closes the file. A second close does nothing. */
+	close() {
+		if (this.closed) return;
+		this.closed = true;
+		this.statements.clear();
+		this.db.close();
+	}
+};
+/** The outcomes of a check that the site or the crawler refused. */
+const REFUSALS = /* @__PURE__ */ new Set([
+	"skipped",
+	"blocked",
+	"challenge",
+	"payment-required"
+]);
+/** This function gives the fingerprint of a version: a hash of its title, its text and its JSON-LD. */
+const watchFingerprint = (version) => sha256$1(stableStringify([
+	version.title,
+	version.text,
+	version.meta["jsonld"] ?? null
+]));
+const sameState = (a, b) => a !== void 0 && a.state === b.state && (a.state === "gone" || a.sha === b.sha);
+/**
+* This function checks the entries of a watch store that are due. `PoliteCrawler.watch.run` uses
+* it. Each check is a fetch of the page with `fetchPage`, with no stored copy (`maxAgeMs: 0`), so
+* each polite rule applies. A check that the budget stops before its request stays due. The store
+* can be a function that opens it: the run uses it before the first check.
+*/
+const runWatch = (runtime, source, options = {}) => {
+	let resolveSummary = () => void 0;
+	const summary = new Promise((resolve) => {
+		resolveSummary = resolve;
+	});
+	let used = false;
+	async function* events() {
+		const now = () => Date.parse(runtime.now());
+		const startedAt = now();
+		const owner = randomUUID();
+		const abort = new AbortController();
+		const outer = options.signal;
+		const cancel = () => abort.abort(outer?.reason);
+		if (outer?.aborted === true) cancel();
+		outer?.addEventListener("abort", cancel, { once: true });
+		const inflight = /* @__PURE__ */ new Map();
+		const counts = {
+			checked: 0,
+			new: 0,
+			changed: 0,
+			unchanged: 0,
+			gone: 0,
+			refused: 0,
+			error: 0,
+			suspected: 0,
+			confirmed: 0
+		};
+		let started = 0;
+		let stopping;
+		let reason;
+		let finished = false;
+		let budgetMs = WATCH_LIMITS.budgetMs;
+		let maxPages = WATCH_LIMITS.maxPages;
+		let opened;
+		/** This function fetches one entry, compares it with its state, and writes the new state. */
+		const check = async (store, row, deadline) => {
+			const agents = [...options.onBehalfOf ?? [], ...parseJson(row.on_behalf_of, [])];
+			const labels = parseJson(row.labels, void 0);
+			let page;
+			let failure;
+			try {
+				page = await fetchPage(runtime, row.target, {
+					render: row.render,
+					maxAgeMs: 0,
+					onBehalfOf: agents,
+					signal: abort.signal,
+					scope: row.scope ?? void 0,
+					labels,
+					deadline
+				});
+			} catch (error) {
+				failure = `watch error: ${error instanceof Error ? error.message : String(error)}`;
+			}
+			const deferred = (stop) => {
+				store.release(row.url, owner);
+				return {
+					kind: "deferred",
+					host: row.host,
+					stop
+				};
+			};
+			if (abort.signal.aborted) return deferred("cancelled");
+			if (page !== void 0 && pastDeadline(page.note)) return deferred("budget");
+			if (page?.outcome === "skipped" && page.note === "crawler closed") return deferred("closed");
+			const at = now();
+			const base = row.base_state === null ? void 0 : {
+				state: row.base_state,
+				sha: row.base_sha,
+				capture: row.base_capture,
+				fetched_at: row.base_fetched_at
+			};
+			const pending = row.pend_state === null ? void 0 : {
+				state: row.pend_state,
+				sha: row.pend_sha,
+				capture: row.pend_capture,
+				fetched_at: row.pend_fetched_at
+			};
+			let seen;
+			let version;
+			let held = false;
+			let kind;
+			let note = page?.note ?? failure ?? null;
+			const ok = page !== void 0 && (page.outcome === "ok" || page.outcome === "not-modified");
+			if (page !== void 0 && ok && page.doc !== null && page.capture !== null) {
+				version = runtime.index.versionText(page.doc, page.capture);
+				const doc = runtime.index.get(page.doc);
+				held = doc !== void 0 && isHeldBack(doc) || version !== void 0 && isHeldBack(version);
+				if (version !== void 0) seen = {
+					state: "present",
+					sha: watchFingerprint(version),
+					capture: version.capture,
+					fetched_at: version.fetched_at
+				};
+			} else if (page?.outcome === "not-found") seen = {
+				state: "gone",
+				sha: null,
+				capture: null,
+				fetched_at: page.fetched_at
+			};
+			let update;
+			let suspected = false;
+			let confirmed = false;
+			if (seen === void 0) {
+				kind = page !== void 0 && REFUSALS.has(page.outcome) ? "refused" : "error";
+				if (ok) note = note ?? "the page gave no document, so the watch cannot compare it";
+				update = { changed: false };
+			} else if (base === void 0) {
+				kind = seen.state === "gone" ? "gone" : "new";
+				update = {
+					changed: false,
+					base: seen,
+					pending: null
+				};
+			} else if (sameState(base, seen)) {
+				kind = "unchanged";
+				if (pending !== void 0) note = "the change that the last check saw did not repeat, so the watch takes it for a glitch";
+				update = {
+					changed: false,
+					pending: null
+				};
+			} else {
+				kind = seen.state === "gone" ? "gone" : base.state === "gone" ? "new" : "changed";
+				const repeated = sameState(pending, seen) && (seen.state === "gone" || pending?.capture !== seen.capture);
+				if (row.confirm === 1 || repeated) {
+					confirmed = true;
+					update = {
+						changed: true,
+						base: seen,
+						pending: null
+					};
+				} else {
+					suspected = true;
+					note = pending === void 0 ? "the watch saw this change one time: the next check confirms it when it sees the same version" : "the watch saw a different change than the last check: the next check confirms it when it sees the same version";
+					update = {
+						changed: false,
+						pending: {
+							...seen,
+							since: at
+						}
+					};
+				}
+			}
+			let diff = null;
+			if (kind === "changed" && base?.state === "present" && base.capture !== null && version !== void 0 && !held) {
+				const docId = row.doc ?? version.doc;
+				const before = runtime.index.versionText(docId, base.capture);
+				if (before !== void 0 && !isHeldBack(before)) diff = diffTexts({
+					text: before.text,
+					fields: {
+						title: before.title,
+						meta: before.meta
+					}
+				}, {
+					text: version.text,
+					fields: {
+						title: version.title,
+						meta: version.meta
+					}
+				}).summary;
+			}
+			let next = at + row.every_ms;
+			const stop = runtime.crawler.hostStop(row.target);
+			if (stop !== void 0 && Number.isFinite(stop.until) && stop.until > next) next = stop.until;
+			store.settle(row.url, owner, {
+				...update,
+				next_at: next,
+				last_at: at,
+				last_kind: kind,
+				last_note: note,
+				doc: page?.doc ?? null
+			});
+			const event = {
+				url: row.target,
+				doc: page?.doc ?? row.doc,
+				kind,
+				suspected,
+				confirmed,
+				capture: seen?.capture ?? null,
+				fetched_at: seen?.fetched_at ?? null,
+				previous: base?.capture ?? null,
+				previous_fetched_at: base?.fetched_at ?? null,
+				outcome: page?.outcome ?? "error",
+				status: page?.status ?? 0,
+				diff,
+				held_back: held,
+				next_at: iso(next),
+				note
+			};
+			return {
+				kind: "event",
+				host: row.host,
+				event
+			};
+		};
+		try {
+			const store = typeof source === "function" ? await source() : source;
+			opened = store;
+			await options.ready?.();
+			budgetMs = options.budgetMs ?? WATCH_LIMITS.budgetMs;
+			if (!(budgetMs >= 0)) throw new RangeError(`budgetMs must be 0 or more, not ${String(budgetMs)}`);
+			maxPages = options.maxPages ?? WATCH_LIMITS.maxPages;
+			if (!Number.isInteger(maxPages) || maxPages < 1) throw new RangeError(`maxPages must be an integer of 1 or more, not ${String(maxPages)}`);
+			const deadline = startedAt + budgetMs;
+			for (;;) {
+				if (stopping === void 0 && abort.signal.aborted) stopping = "cancelled";
+				if (stopping === void 0 && options.drain?.aborted === true) {
+					stopping = "cancelled";
+					reason = "the run stopped for a shutdown: the checks in progress ended first";
+				}
+				if (stopping === void 0 && now() >= deadline) stopping = "budget";
+				while (stopping === void 0 && inflight.size < WATCH_LIMITS.hostsAtOnce && started < maxPages) {
+					const claimed = store.claim(now(), owner, Math.max(now(), deadline) + WATCH_LIMITS.leaseMs, [...inflight.keys()]);
+					if (claimed === void 0) break;
+					started += 1;
+					inflight.set(claimed.host, check(store, claimed, deadline));
+				}
+				if (inflight.size === 0) break;
+				const done = await Promise.race(inflight.values());
+				inflight.delete(done.host);
+				if (done.kind === "deferred") {
+					started -= 1;
+					stopping ??= done.stop;
+					continue;
+				}
+				const event = done.event;
+				counts.checked += 1;
+				counts[event.kind] += 1;
+				if (event.suspected) counts.suspected += 1;
+				if (event.confirmed) counts.confirmed += 1;
+				yield event;
+			}
+			stopping ??= started >= maxPages && store.due(now()) > 0 ? "max-pages" : "done";
+			finished = true;
+		} catch (error) {
+			stopping = "error";
+			reason = error instanceof Error ? error.message : String(error);
+			finished = true;
+			throw error;
+		} finally {
+			if (!finished) {
+				stopping ??= "cancelled";
+				reason ??= "the caller stopped the loop";
+			}
+			outer?.removeEventListener("abort", cancel);
+			abort.abort(/* @__PURE__ */ new Error("the watch run ended"));
+			await Promise.allSettled(inflight.values());
+			const stop = stopping ?? "done";
+			reason ??= stop === "done" ? "the run checked each entry that was due" : stop === "max-pages" ? `the run checked ${String(maxPages)} entries, its limit (maxPages)` : stop === "budget" ? `the budget of ${String(budgetMs)} ms ended (budgetMs)` : stop === "cancelled" ? "the caller cancelled the run" : "the crawler closed";
+			let due = 0;
+			try {
+				due = opened?.due(now()) ?? 0;
+			} catch {}
+			resolveSummary({
+				...counts,
+				due,
+				stopped: stop,
+				reason,
+				elapsed_ms: Math.max(0, now() - startedAt)
+			});
+		}
+	}
+	return {
+		summary,
+		[Symbol.asyncIterator]() {
+			if (used) throw new Error("a watch run operates one time: call run again for a new run");
+			used = true;
+			return events();
+		}
+	};
+};
+//#endregion
+//#region src/open.ts
+var open_exports = /* @__PURE__ */ __exportAll({
+	BOT_PAGE: () => BOT_PAGE,
+	DEFAULT_TOKEN: () => DEFAULT_TOKEN,
+	createRuntime: () => createRuntime,
+	defaultUserAgent: () => defaultUserAgent,
+	isMailbox: () => isMailbox,
+	openCrawler: () => openCrawler
+});
+/**
+* This function gives the default `User-Agent` header. The header names the crawler, its version
+* and the page about it, so a site owner can find out what made each request.
+*/
+const defaultUserAgent = (contact) => `${DEFAULT_TOKEN}/${VERSION} (+${BOT_PAGE}${contact === void 0 ? "" : `; ${contact}`})`;
+/** A mailbox is `name@host`, or a name and `<name@host>`. A URL is not a mailbox, also when it holds an `@`. */
+const MAILBOX = /^(?:[^\s<>@:/]+@[^\s<>@/]+|[^<>]*<[^\s<>@:/]+@[^\s<>@/]+>)$/;
+/**
+* This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,
+* section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in
+* the `User-Agent` only.
+*/
+const isMailbox = (contact) => MAILBOX.test(contact.trim());
+/** The parts of a store that can be a symbolic link to a different place only by mistake or by attack. */
+const STORE_PARTS = [
+	"captures",
+	"blobs",
+	"index.sqlite",
+	"hosts.sqlite",
+	"watch.sqlite",
+	"vectors.sqlite"
+];
+/**
+* This function makes the folder of a store for the owner only, where the system has modes. It
+* refuses a store folder of a different user, and a part of the store that is a symbolic link.
+* The store folder itself can be a link, for example to a larger disk.
+*/
+const guardStore = (dir) => {
+	mkdirSync(dir, {
+		recursive: true,
+		mode: 448
+	});
+	const real = lstatSync(dir).isSymbolicLink() ? realpathSync(dir) : dir;
+	const info = statSync(real);
+	if (!info.isDirectory()) throw new Error(`the store ${dir} is not a folder`);
+	if (process.platform !== "win32" && typeof process.getuid === "function" && info.uid !== process.getuid()) throw new Error(`the store ${dir} belongs to a different user: use a folder of your own`);
+	for (const part of STORE_PARTS) {
+		let link = false;
+		try {
+			link = lstatSync(join(dir, part)).isSymbolicLink();
+		} catch {}
+		if (link) throw new Error(`${part} in the store ${dir} is a symbolic link: the crawler does not follow it`);
+	}
+};
+/**
+* This function makes a runtime. It is the one place that puts the log, the blob store, the index,
+* the host state, the crawler and the extractors together. The function `openCrawler` uses it.
+* An application that needs the parts uses it too. The runtime and the crawler share one clock.
+*/
+const createRuntime = (options) => {
+	const dir = options.dir;
+	if (dir === void 0 && (options.log === void 0 || options.blobs === void 0 || options.index === void 0)) throw new Error("a runtime needs dir, or a log, a blob store and an index");
+	const contact = options.contact;
+	if (contact !== void 0 && !/^[\t -~]*$/.test(contact)) throw new Error(`contact "${contact}" has a character that an HTTP header cannot hold: use printable ASCII`);
+	const extractors = new Extractors();
+	for (const [recipe, extractor] of Object.entries(options.extractors ?? {})) extractors.register(recipe, extractor);
+	for (const recipe of options.recipes ?? []) registerRecipe(extractors, recipe);
+	if (dir !== void 0) guardStore(dir);
+	const owned = [];
+	const log = options.log ?? new CaptureLog(join(dir, "captures"));
+	const blobs = options.blobs ?? new FileBlobStore(join(dir, "blobs"));
+	let index = options.index;
+	if (index === void 0) {
+		const made = new DocIndex(join(dir, "index.sqlite"));
+		owned.push(() => made.close());
+		index = made;
+	}
+	let hostState = options.hostState;
+	if (hostState === void 0) {
+		if (dir === void 0) hostState = new MemoryHostStateStore();
+		else {
+			const made = new SqliteHostStateStore(join(dir, "hosts.sqlite"));
+			owned.push(() => made.close());
+			hostState = made;
+		}
+	}
+	const clock = options.clock ?? realClock;
+	const renderer = options.renderer === false ? void 0 : options.renderer ?? checkedRenderer(playwrightRenderer());
+	const crawler = new Crawler({
+		token: options.token ?? "PoliteCrawler",
+		userAgent: options.userAgent ?? defaultUserAgent(contact),
+		from: contact !== void 0 && isMailbox(contact) ? contact.trim() : void 0,
+		log,
+		blobs,
+		index,
+		renderer,
+		maxActive: options.maxActive,
+		maxRenders: options.maxRenders,
+		dispatcher: options.dispatcher,
+		egress: options.egress,
+		politeness: options.politeness,
+		hostState,
+		clock,
+		random: options.random,
+		scope: options.scope,
+		scopeRank: options.scopeRank,
+		obeyDefaultToken: options.obeyDefaultToken,
+		onEvent: options.onEvent,
+		signer: options.signer
+	});
+	let closing;
+	return {
+		crawler,
+		index,
+		blobs,
+		captures: log,
+		log,
+		extractors,
+		hostState,
+		now: () => new Date(clock.now()).toISOString(),
+		close() {
+			closing ??= (async () => {
+				await crawler.close();
+				for (const close of owned) await close();
+			})();
+			return closing;
+		}
+	};
+};
+/**
+* This function opens a crawler on a folder. The folder keeps the capture log (`captures/`), the
+* response bodies (`blobs/`), the search index (`index.sqlite`) and the state of each host
+* (`hosts.sqlite`). The index is a projection: `reindex` makes it again from the log and the bodies.
+* At open, `catchUpIndex` rebuilds an index of an earlier version, and gives an index that is
+* behind the log the captures that it missed.
+*
+* Each method gives a promise, and none throws before its promise. After `close`, each method
+* rejects.
+*/
+const openCrawler = (options) => {
+	const floorMs = options.watchFloor === void 0 ? WATCH_LIMITS.floorMs : durationMs(options.watchFloor);
+	const runtime = createRuntime(options);
+	let closed = false;
+	let rebuilt;
+	const ready = existsSync(join(options.dir, "captures")) ? catchUpIndex(runtime).then((done) => {
+		if (done?.mode === "rebuild") {
+			const { mode, ...counts } = done;
+			rebuilt = mode === "rebuild" ? counts : void 0;
+		}
+	}, () => void 0) : Promise.resolve();
+	const live = async () => {
+		if (closed) throw new Error("the crawler is closed");
+		await ready;
+		if (closed) throw new Error("the crawler is closed");
+	};
+	let watches;
+	const watchStore = async () => {
+		await live();
+		watches ??= new WatchStore(join(options.dir, "watch.sqlite"), { floorMs });
+		return watches;
+	};
+	const clockNow = () => Date.parse(runtime.now());
+	const embedder = options.embedder;
+	let vectors;
+	let search;
+	const hybridSearch = async () => {
+		await live();
+		if (embedder === void 0) throw new Error(NO_EMBEDDER);
+		vectors ??= new VectorStore(join(options.dir, "vectors.sqlite"));
+		search ??= createHybridSearch(runtime, {
+			embedder,
+			store: vectors
+		});
+		return search;
+	};
+	const hybrid = embedder === void 0 ? void 0 : {
+		embedder,
+		find: async (q, findOptions) => (await hybridSearch()).find(q, findOptions),
+		rankWithin: async (q, docs, rankOptions) => (await hybridSearch()).rankWithin(q, docs, rankOptions),
+		embed: async (embedOptions) => (await hybridSearch()).embed(embedOptions)
+	};
+	let closing;
+	return {
+		runtime,
+		async fetch(url, fetchOptions = {}) {
+			await live();
+			rebuilt = void 0;
+			const maxAge = fetchOptions.maxAge ?? "24h";
+			return fetchPage(runtime, url, {
+				render: fetchOptions.render ?? "auto",
+				maxAgeMs: maxAge === "auto" ? "auto" : durationMs(maxAge),
+				onBehalfOf: [...options.onBehalfOf ?? [], ...fetchOptions.onBehalfOf ?? []],
+				signal: fetchOptions.signal,
+				scope: fetchOptions.scope,
+				labels: fetchOptions.labels,
+				fallback: fetchOptions.fallback,
+				archiveDespiteRobots: fetchOptions.archiveDespiteRobots
+			});
+		},
+		async find(q, findOptions = {}) {
+			await live();
+			const { rank, ...rest } = findOptions;
+			if (rank === "hybrid") return (await hybridSearch()).find(q, rest);
+			return findDocuments(runtime, q, {
+				...rest,
+				...rank === void 0 ? {} : { rank }
+			});
+		},
+		hybrid,
+		async embed(embedOptions = {}) {
+			return (await hybridSearch()).embed(embedOptions);
+		},
+		async rankWithin(q, docs, rankOptions = {}) {
+			await live();
+			const { rank, ...rest } = rankOptions;
+			if (rank === "hybrid") return (await hybridSearch()).rankWithin(q, docs, rest);
+			return runtime.index.rankWithin(q, docs, rank === void 0 ? {} : { rank });
+		},
+		async read(id, readOptions = {}) {
+			await live();
+			return readDocument(runtime, id, readOptions);
+		},
+		async verify(id, quote, verifyOptions = {}) {
+			await live();
+			return verifyQuote(runtime, id, quote, verifyOptions);
+		},
+		async extract(id, fields, extractOptions = {}) {
+			await live();
+			return extractDocument(runtime, id, fields, extractOptions);
+		},
+		async versions(id, versionOptions = {}) {
+			await live();
+			const held = runtime.index.get(id)?.ai_input === "no";
+			return runtime.index.versions(id, versionOptions).map((version) => held || version.ai_input === "no" ? {
+				...version,
+				title: ""
+			} : version);
+		},
+		async discover(site, discoverOptions = {}) {
+			await live();
+			rebuilt = void 0;
+			return discoverSite(runtime, site, {
+				...discoverOptions,
+				onBehalfOf: [...options.onBehalfOf ?? [], ...discoverOptions.onBehalfOf ?? []]
+			});
+		},
+		crawl(seeds, crawlOptions = {}) {
+			return crawlSite(runtime, seeds, {
+				...crawlOptions,
+				onBehalfOf: [...options.onBehalfOf ?? [], ...crawlOptions.onBehalfOf ?? []],
+				ready: async () => {
+					await live();
+					rebuilt = void 0;
+				}
+			});
+		},
+		async reindex() {
+			await live();
+			const done = rebuilt;
+			rebuilt = void 0;
+			return done ?? reindex(runtime);
+		},
+		async diff(id, diffOptions = {}) {
+			await live();
+			return diffDocument(runtime, id, diffOptions);
+		},
+		watch: {
+			async add(url, watchOptions = {}) {
+				return (await watchStore()).add(url, watchOptions, clockNow());
+			},
+			async remove(url) {
+				return (await watchStore()).remove(url);
+			},
+			async list() {
+				return (await watchStore()).list(clockNow());
+			},
+			run(runOptions = {}) {
+				return runWatch(runtime, watchStore, {
+					...runOptions,
+					onBehalfOf: options.onBehalfOf,
+					ready: () => {
+						rebuilt = void 0;
+						return Promise.resolve();
+					}
+				});
+			}
+		},
+		close() {
+			closed = true;
+			closing ??= (async () => {
+				await ready;
+				await runtime.close();
+				watches?.close();
+				vectors?.close();
+				await embedder?.close?.();
+			})();
+			return closing;
+		}
+	};
+};
+//#endregion
+export { crawlSite as A, chunkText as C, unitVector as D, fuseRanks as E, readSitemap as F, DocIndex as I, FAST_RANK_SHARE as L, discoverSite as M, SITEMAP_LIMITS as N, SqliteHostStateStore as O, feedDate as P, NO_EMBEDDER as R, HYBRID_DEFAULTS as S, estimateTokens as T, registerRecipe as _, open_exports as a, createHybridSearch as b, WatchStore as c, VectorStore as d, RECIPE_LIMITS as f, readJson as g, defineRecipe as h, openCrawler as i, DISCOVER_LIMITS as j, CRAWL_LIMITS as k, runWatch as l, deadlineOf as m, defaultUserAgent as n, WATCH_LIMITS as o, RESERVED_RECIPES as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, VECTORS_SCHEMA as u, runRecipe as v, chunkTokensOf as w, CHUNK_LIMITS as x, runtimeOf as y };
+
+//# sourceMappingURL=open-C4fhXJgD.mjs.map