polite-crawler 0.1.1 → 0.1.3

package/build/cli.mjs

@@ -1,58 +1,29 @@
 #!/usr/bin/env node
-import { r as VERSION } from "./version-no12qgos.mjs";
+import { i as durationMs, r as VERSION } from "./version-ntbY-6wh.mjs";
+import { t as isLoopbackHost } from "./loopback-C_QYowSf.mjs";
 import { join, resolve } from "node:path";
-import { homedir } from "node:os";
+import { readFileSync } from "node:fs";
 import { parseArgs } from "node:util";
-//#region src/cli.ts
-const emitWarning = process.emitWarning.bind(process);
-process.emitWarning = ((warning, ...rest) => {
-	const first = rest[0];
-	const kind = typeof first === "string" ? first : typeof first === "object" && first !== null && "type" in first ? first.type : warning instanceof Error ? warning.name : void 0;
-	const text = typeof warning === "string" ? warning : warning.message;
-	if (kind === "ExperimentalWarning" && text.startsWith("SQLite is an experimental feature")) return;
-	emitWarning(warning, ...rest);
-});
-const HELP = `polite-crawler ${VERSION}: fetch a source once, keep it, read it many times.
-
-Usage:
-  polite-crawler fetch <url...>        fetch pages into the store (robots.txt obeyed)
-  polite-crawler find <query>          search the stored pages, with no request
-  polite-crawler read <doc>            print a range of a stored document
-  polite-crawler verify <doc> <quote>  check that a quote is in a stored document
-  polite-crawler reindex               rebuild the search index from the capture log
-  polite-crawler mcp                   serve the tools over MCP (standard input and output)
-
-Options:
-  --dir <path>       the store (default $POLITE_CRAWLER_DIR, or ~/.polite-crawler)
-  --contact <text>   an email (From and User-Agent) or a URL (User-Agent only)
-                     ($POLITE_CRAWLER_CONTACT)
-  --allow-private    fetch loopback and private addresses too ($POLITE_CRAWLER_ALLOW_PRIVATE=1)
-  --no-browser       never open the headless browser
-  --json             print JSON
-  -v, --version      print the version
-  -h, --help         print this help
-
-  fetch    --render <mode>   auto (default), never or always
-           --max-age <age>   reuse a stored copy this young: 30m, 24h (default), 7d; 0 fetches again
-  find     --limit <n>       how many results (default 10)
-           --kind <kind>     only this kind of document, such as page or pdf
-           --rank <mode>     exact (default): BM25 over each match; fast: leave common words
-                             out of the one-word part, faster for a long question
-  read     --from <n>        where the range starts (default 0)
-           --length <n>      how many characters (default 4000)
-           --user-provided   you gave this URL yourself: read a page that refuses AI input (logged)
-  reindex                    rebuild; the documents of recipes that this command cannot read stay
-  mcp      --allow-user-provided   let user_provided work when the client cannot ask you
-                                   ($POLITE_CRAWLER_ALLOW_USER_PROVIDED=1)
-
-Exit codes:
-  0  done
-  1  not done: a page not fetched, no match, a quote not found, a document unknown or
-     held back, or an error
-  2  the command line is wrong
-`;
+import { homedir } from "node:os";
+//#region src/cli-line.ts
 /** A usage error is a wrong command line. It exits with code 2. */
 var UsageError = class extends Error {};
+/** These options belong to the command `export`. */
+const EXPORT_OPTIONS = {
+	format: { type: "string" },
+	out: { type: "string" },
+	since: { type: "string" },
+	until: { type: "string" },
+	host: { type: "string" },
+	prefix: { type: "string" },
+	id: {
+		type: "string",
+		multiple: true
+	},
+	"include-refusals": { type: "boolean" },
+	overwrite: { type: "boolean" }
+};
+/** These are the options of the command line: the options of the commands and of `export`. */
 const OPTIONS = {
 	dir: { type: "string" },
 	contact: { type: "string" },
@@ -75,19 +46,119 @@ const OPTIONS = {
 	from: { type: "string" },
 	length: { type: "string" },
 	"user-provided": { type: "boolean" },
-	"allow-user-provided": { type: "boolean" }
+	"allow-user-provided": { type: "boolean" },
+	"max-urls": { type: "string" },
+	feed: {
+		type: "string",
+		multiple: true
+	},
+	"max-pages": { type: "string" },
+	"max-depth": { type: "string" },
+	budget: { type: "string" },
+	include: {
+		type: "string",
+		multiple: true
+	},
+	exclude: {
+		type: "string",
+		multiple: true
+	},
+	sitemaps: { type: "boolean" },
+	"allow-host": {
+		type: "string",
+		multiple: true
+	},
+	at: { type: "string" },
+	to: { type: "string" },
+	every: { type: "string" },
+	confirm: { type: "string" },
+	scope: { type: "string" },
+	label: {
+		type: "string",
+		multiple: true
+	},
+	"archive-fallback": { type: "boolean" },
+	"archive-despite-robots": { type: "boolean" },
+	"sign-key": { type: "string" },
+	"signature-agent": { type: "string" },
+	tick: { type: "string" },
+	jitter: { type: "string" },
+	"tick-budget": { type: "string" },
+	"tick-pages": { type: "string" },
+	grace: { type: "string" },
+	events: { type: "string" },
+	"on-change": { type: "string" },
+	"mcp-http": { type: "boolean" },
+	"mcp-port": { type: "string" },
+	"mcp-host": { type: "string" },
+	"mcp-token-file": { type: "string" },
+	"allow-remote": { type: "boolean" },
+	field: {
+		type: "string",
+		multiple: true
+	},
+	capture: { type: "string" },
+	embedder: { type: "string" },
+	"embed-model": { type: "string" },
+	"ollama-url": { type: "string" },
+	"allow-remote-ollama": { type: "boolean" },
+	"max-docs": { type: "string" },
+	...EXPORT_OPTIONS
 };
-/** These options belong to one command. */
+/** These options belong to `daemon`, and not to `daemon status`. */
+const DAEMON_RUN_OPTIONS = [
+	"tick",
+	"jitter",
+	"tick-budget",
+	"tick-pages",
+	"grace",
+	"events",
+	"on-change",
+	"mcp-http",
+	"mcp-port",
+	"mcp-host",
+	"mcp-token-file",
+	"allow-remote"
+];
+/** These options belong to one command, or to the commands that the text names. */
 const OWNER = {
-	render: "fetch",
-	"max-age": "fetch",
+	render: "fetch crawl watch",
+	"max-age": "fetch crawl discover",
 	limit: "find",
-	kind: "find",
+	kind: "find embed",
 	rank: "find",
-	from: "read",
+	from: "read diff",
 	length: "read",
-	"user-provided": "read",
-	"allow-user-provided": "mcp"
+	"user-provided": "read extract",
+	"allow-user-provided": "mcp daemon",
+	...Object.fromEntries(Object.keys(EXPORT_OPTIONS).map((option) => [option, "export"])),
+	"max-urls": "discover",
+	feed: "discover",
+	"max-pages": "crawl watch",
+	"max-depth": "crawl",
+	budget: "crawl watch embed",
+	include: "crawl",
+	exclude: "crawl",
+	sitemaps: "crawl",
+	"allow-host": "crawl discover",
+	at: "read verify extract",
+	to: "diff",
+	every: "watch",
+	confirm: "watch",
+	scope: "watch",
+	label: "watch",
+	"archive-fallback": "fetch",
+	"archive-despite-robots": "fetch",
+	"sign-key": "fetch discover crawl watch mcp key-directory daemon",
+	"signature-agent": "fetch discover crawl watch mcp key-directory daemon",
+	...Object.fromEntries(DAEMON_RUN_OPTIONS.map((option) => [option, "daemon"])),
+	field: "extract",
+	capture: "extract",
+	embedder: "find embed mcp",
+	"embed-model": "find embed mcp",
+	"ollama-url": "find embed mcp",
+	"allow-remote-ollama": "find embed mcp",
+	"max-docs": "embed"
 };
 /** These are the commands, with the smallest and the largest count of arguments. */
 const COMMANDS = {
@@ -120,6 +191,51 @@ const COMMANDS = {
 		min: 0,
 		max: 0,
 		wants: "mcp takes no argument"
+	},
+	export: {
+		min: 0,
+		max: 0,
+		wants: "export takes no argument: give the file with --out"
+	},
+	discover: {
+		min: 1,
+		max: 1,
+		wants: "discover wants one site URL"
+	},
+	crawl: {
+		min: 1,
+		max: Infinity,
+		wants: "crawl wants at least one URL"
+	},
+	diff: {
+		min: 1,
+		max: 1,
+		wants: "diff wants one document id"
+	},
+	watch: {
+		min: 1,
+		max: Infinity,
+		wants: "watch wants add, remove, list or run"
+	},
+	daemon: {
+		min: 0,
+		max: 1,
+		wants: "daemon takes no argument, or status"
+	},
+	"key-directory": {
+		min: 0,
+		max: 0,
+		wants: "key-directory takes no argument: give the key with --sign-key"
+	},
+	extract: {
+		min: 1,
+		max: 1,
+		wants: "extract wants one document id and --field name=value"
+	},
+	embed: {
+		min: 0,
+		max: 0,
+		wants: "embed takes no argument: give the model with --embedder"
 	}
 };
 const parse = (args) => {
@@ -144,8 +260,8 @@ const whole = (name, text, fallback, min) => {
 	return value;
 };
 /** An environment variable that switches a feature on is 1 or 0 (also true or false, yes or no). */
-const envFlag = (name) => {
-	const raw = process.env[name];
+const envFlag = (env, name) => {
+	const raw = env[name];
 	const value = raw?.trim().toLowerCase() ?? "";
 	if ([
 		"",
@@ -180,19 +296,48 @@ const FS_CODES = /* @__PURE__ */ new Set([
 	"EISDIR"
 ]);
 const ok = (page) => page.outcome === "ok" || page.outcome === "not-modified";
+/** A fetch is done when the page came, or when an archived copy came in its place. */
+const done = (page) => ok(page) || page.archived !== void 0;
 const describePage = (page) => {
+	const head = `${ok(page) ? "✓" : page.archived !== void 0 ? "~" : "✗"} ${page.url}`;
+	const facts = [
+		`${page.outcome} ${String(page.status)}`,
+		page.doc === null ? null : `doc ${page.doc}`,
+		page.chars > 0 ? `${String(page.chars)} chars` : null,
+		page.rendered ? "rendered" : null,
+		page.structured.length > 0 ? `JSON-LD: ${page.structured.join(", ")}` : null
+	].filter((fact) => fact !== null);
+	const archived = page.archived === void 0 ? [] : [`  archived copy (${page.archived.source}, captured ${page.archived.timestamp}): ${page.archived.url}`];
 	return [
-		`${ok(page) ? "✓" : "✗"} ${page.url}`,
-		`  ${[
-			`${page.outcome} ${String(page.status)}`,
-			page.doc === null ? null : `doc ${page.doc}`,
-			page.chars > 0 ? `${String(page.chars)} chars` : null,
-			page.rendered ? "rendered" : null,
-			page.structured.length > 0 ? `JSON-LD: ${page.structured.join(", ")}` : null
-		].filter((fact) => fact !== null).join(" · ")}`,
+		head,
+		`  ${facts.join(" · ")}`,
+		...archived,
 		...page.note === null ? [] : [`  ${page.note}`]
 	].join("\n");
 };
+/** This function reads the key file of `--sign-key`. An error names the file, and not the key. */
+const keyFile = (path) => {
+	const file = path === "~" ? homedir() : /^~[\\/]/.test(path) ? resolve(homedir(), path.slice(2)) : resolve(path);
+	try {
+		return readFileSync(file, "utf8");
+	} catch (error) {
+		const code = error.code;
+		throw new UsageError(`--sign-key: the file ${file} does not open (${typeof code === "string" ? code : "error"})`, { cause: error });
+	}
+};
+/**
+* This function gives the type of the URL of `--signature-agent`. An origin names a key directory at
+* the well-known path. A URL with a path names a JSON Web Key Set (`jwks_uri`), for a host that cannot
+* serve the well-known path, for example a project site of GitHub Pages.
+*/
+const agentTypeOf = (url) => {
+	try {
+		const parsed = new URL(url);
+		return parsed.pathname === "/" && parsed.search === "" ? "directory" : "jwks_uri";
+	} catch {
+		return "directory";
+	}
+};
 /** This is the result of a URL that the crawler did not fetch. */
 const failedPage = (url, note) => ({
 	url,
@@ -212,59 +357,841 @@ const failedPage = (url, note) => ({
 	held_back: false,
 	note
 });
-/** This function does the command of the command line, and gives the exit code. */
-const main = async (args) => {
-	const { values, positionals } = parse(args);
-	const out = (text) => void process.stdout.write(`${text}\n`);
-	if (values.version === true) {
-		out(VERSION);
-		return 0;
+const crawlOptionsOf = (values) => {
+	let budgetMs = 6e5;
+	if (values.budget !== void 0) try {
+		budgetMs = durationMs(values.budget);
+	} catch (error) {
+		throw new UsageError(`--budget: ${error.message}`, { cause: error });
 	}
-	const [command, ...rest] = positionals;
-	if (values.help === true || command === void 0) {
-		process.stdout.write(HELP);
-		return 0;
+	return {
+		maxUrls: whole("max-urls", values["max-urls"], 1e3, 1),
+		feeds: values.feed ?? [],
+		maxPages: whole("max-pages", values["max-pages"], 100, 1),
+		maxDepth: whole("max-depth", values["max-depth"], 3, 0),
+		budgetMs,
+		include: values.include ?? [],
+		exclude: values.exclude ?? [],
+		sitemaps: values.sitemaps === true,
+		allowHosts: values["allow-host"]
+	};
+};
+const describeSummary = (summary) => `${String(summary.pages)} pages: ${String(summary.fetched)} fetched (${String(summary.from_cache)} from the store), ${String(summary.refused)} refused, ${String(summary.failed)} failed. ${String(summary.skipped)} links not followed, ${String(summary.queued)} URLs waiting. Stopped (${summary.stopped}): ${summary.reason}.`;
+/** A text that starts with a date is a time. A different text is a capture id. */
+const DATE_START = /^\d{4}-\d{2}-\d{2}/;
+/** This function reads a time of the command line. A text that is not a time is a usage error. */
+const timeOf = (name, text) => {
+	if (text === void 0) return void 0;
+	if (!DATE_START.test(text) || !Number.isFinite(Date.parse(text))) throw new UsageError(`--${name} wants a time, such as 2026-10-01T12:00:00Z, not "${text}"`);
+	return text;
+};
+/** This function reads a version of `diff`: a time, or a capture id. */
+const versionOf = (name, text) => {
+	if (text === void 0) return void 0;
+	if (text.trim() === "") throw new UsageError(`--${name} wants a capture id or a time`);
+	return DATE_START.test(text) ? { at: timeOf(name, text) } : text;
+};
+const ACTIONS = {
+	add: [1, Infinity],
+	remove: [1, Infinity],
+	list: [0, 0],
+	run: [0, 0]
+};
+/** These options belong to one action of `watch`. */
+const WATCH_OWNER = {
+	every: "add",
+	confirm: "add",
+	scope: "add",
+	label: "add",
+	render: "add",
+	budget: "run",
+	"max-pages": "run",
+	"sign-key": "run",
+	"signature-agent": "run"
+};
+const watchLineOf = (values, rest) => {
+	const [action, ...urls] = rest;
+	const counts = action === void 0 ? void 0 : ACTIONS[action];
+	if (counts === void 0) throw new UsageError(`watch wants add, remove, list or run, not "${action ?? ""}"`);
+	if (urls.length < counts[0] || urls.length > counts[1]) throw new UsageError(counts[0] > 0 ? `watch ${action} wants at least one URL` : `watch ${action} takes no URL`);
+	for (const [option, owner] of Object.entries(WATCH_OWNER)) if (values[option] !== void 0 && owner !== action) throw new UsageError(`--${option} is an option of watch ${owner}`);
+	if (values.every !== void 0) {
+		let ms;
+		try {
+			ms = durationMs(values.every);
+		} catch (error) {
+			throw new UsageError(`--every: ${error.message}`, { cause: error });
+		}
+		if (ms < 36e5) throw new UsageError("--every wants 1h or more, so a watch cannot hammer a site");
+	}
+	if (values.confirm !== void 0 && values.confirm !== "1" && values.confirm !== "2") throw new UsageError(`--confirm is 1 or 2, not "${values.confirm}"`);
+	let labels;
+	for (const label of values.label ?? []) {
+		const at = label.indexOf("=");
+		if (at < 1) throw new UsageError(`--label wants name=value, not "${label}"`);
+		labels = {
+			...labels,
+			[label.slice(0, at)]: label.slice(at + 1)
+		};
+	}
+	for (const url of urls) if (!/^https?:\/\//i.test(url) || !URL.canParse(url)) throw new UsageError(`watch ${action} wants http or https URLs, not "${url}"`);
+	return {
+		action,
+		urls: [...new Set(urls)],
+		every: values.every,
+		confirm: values.confirm === void 0 ? void 0 : values.confirm === "1" ? 1 : 2,
+		labels
+	};
+};
+const everyText = (ms) => ms % 864e5 === 0 ? `${String(ms / 864e5)}d` : ms % 36e5 === 0 ? `${String(ms / 36e5)}h` : `${String(Math.round(ms / 6e4))}m`;
+const describeEntry = (entry) => {
+	const facts = [
+		`every ${everyText(entry.every_ms)}`,
+		`next ${entry.next_at}`,
+		entry.checked_at === null ? "not checked yet" : `last ${entry.last_kind ?? "check"} at ${entry.checked_at}`,
+		`${String(entry.checks)} checks, ${String(entry.changes)} changes`
+	];
+	return [
+		entry.url,
+		`  ${facts.join(" · ")}`,
+		...entry.pending === null ? [] : [`  suspected: ${entry.pending.kind} since ${entry.pending.since}`]
+	].join("\n");
+};
+const describeEvent = (event) => {
+	const state = event.suspected ? " (suspected)" : event.confirmed ? " (confirmed)" : "";
+	const facts = [
+		`${event.outcome} ${String(event.status)}`,
+		event.capture === null ? null : `capture ${event.capture}`,
+		event.previous === null || event.previous === event.capture ? null : `from ${event.previous}`,
+		event.diff === null ? null : `+${String(event.diff.lines_added)} −${String(event.diff.lines_removed)} lines, ${String(event.diff.meta_changes)} metadata changes`,
+		event.held_back ? "held back" : null
+	].filter((fact) => fact !== null);
+	return [
+		`${event.kind}${state} ${event.url}`,
+		`  ${facts.join(" · ")}`,
+		...event.note === null ? [] : [`  ${event.note}`]
+	].join("\n");
+};
+const describeWatchSummary = (summary) => `${String(summary.checked)} checked: ${String(summary.new)} new, ${String(summary.changed)} changed, ${String(summary.unchanged)} unchanged, ${String(summary.gone)} gone, ${String(summary.refused)} refused, ${String(summary.error)} errors. ${String(summary.suspected)} suspected, ${String(summary.confirmed)} confirmed, ${String(summary.due)} still due. Stopped (${summary.stopped}): ${summary.reason}.`;
+/** This function reads a file name of the command line: a leading `~` is the home folder. */
+const fileOf = (text) => text === "~" ? homedir() : /^~[\\/]/.test(text) ? resolve(homedir(), text.slice(2)) : resolve(text);
+/** This function reads a duration of `daemon`, with a floor in milliseconds. */
+const daemonDuration = (name, text, floor) => {
+	if (text === void 0) return void 0;
+	let ms;
+	try {
+		ms = durationMs(text);
+	} catch (error) {
+		throw new UsageError(`--${name}: ${error.message}`, { cause: error });
+	}
+	if (ms < floor) throw new UsageError(floor === 0 ? `--${name} wants 0 or more` : `--${name} wants ${String(floor / 1e3)}s or more`);
+	return ms;
+};
+/** These options of `daemon` belong to its MCP server, so they need `--mcp-http`. */
+const MCP_OPTIONS = [
+	"mcp-port",
+	"mcp-host",
+	"mcp-token-file",
+	"allow-remote"
+];
+/** This function checks the options of `daemon` and of `daemon status`. */
+const daemonLineOf = (values, rest) => {
+	const [action] = rest;
+	if (action !== void 0 && action !== "status") throw new UsageError(`daemon takes no argument, or status, not "${action}"`);
+	if (action === "status") {
+		const run = [
+			...DAEMON_RUN_OPTIONS,
+			"allow-user-provided",
+			"sign-key",
+			"signature-agent"
+		];
+		for (const option of run) if (values[option] !== void 0) throw new UsageError(`--${option} is an option of daemon, not of daemon status`);
+	}
+	if (values["mcp-http"] !== true) {
+		for (const option of MCP_OPTIONS) if (values[option] !== void 0) throw new UsageError(`--${option} goes with --mcp-http`);
+	}
+	const host = (values["mcp-host"] ?? "127.0.0.1").trim();
+	if (host === "") throw new UsageError("--mcp-host wants a host");
+	const allowRemote = values["allow-remote"] === true;
+	if (values["mcp-http"] === true && !isLoopbackHost(host) && !allowRemote) throw new UsageError(`--mcp-host ${host} is not a loopback address: add --allow-remote to serve other computers, after you read the warning in the README`);
+	const port = whole("mcp-port", values["mcp-port"], 9307, 0);
+	if (port > 65535) throw new UsageError("--mcp-port wants a port from 0 to 65535");
+	const onChange = values["on-change"];
+	if (onChange !== void 0 && onChange.trim() === "") throw new UsageError("--on-change wants a command");
+	const events = values.events;
+	if (events !== void 0 && events.trim() === "") throw new UsageError("--events wants a file");
+	const tokenFile = values["mcp-token-file"];
+	if (tokenFile !== void 0 && tokenFile.trim() === "") throw new UsageError("--mcp-token-file wants a file");
+	return {
+		action: action === "status" ? "status" : "run",
+		tickMs: daemonDuration("tick", values.tick, 1e3),
+		jitterMs: daemonDuration("jitter", values.jitter, 0),
+		budgetMs: daemonDuration("tick-budget", values["tick-budget"], 1e3),
+		maxPages: values["tick-pages"] === void 0 ? void 0 : whole("tick-pages", values["tick-pages"], 1, 1),
+		graceMs: daemonDuration("grace", values.grace, 0),
+		events: events === void 0 ? void 0 : fileOf(events),
+		onChange,
+		mcp: values["mcp-http"] === true ? {
+			port,
+			host,
+			allowRemote,
+			tokenFile: tokenFile === void 0 ? void 0 : fileOf(tokenFile)
+		} : void 0
+	};
+};
+/** This function gives the first line of `daemon status`: the daemon runs, stopped, or stopped with no clean stop. */
+const daemonHead = (state) => state.alive ? `the daemon runs (process ${String(state.pid)}) since ${state.started_at}, on the store ${state.store}` : state.running ? `the daemon of process ${String(state.pid)} does not run: it stopped with no clean stop. Its state is of ${state.updated_at}` : `the daemon stopped at ${state.stopped_at ?? state.updated_at}`;
+/** This function gives the line of the last tick of `daemon status`. */
+const lastTick = (state) => {
+	const tick = state.last_tick;
+	if (tick === null) return "last tick: none yet";
+	if (tick.summary === null) return `last tick: ${tick.ended_at}, failed: ${tick.error ?? "error"}`;
+	const summary = tick.summary;
+	return `last tick: ${tick.ended_at}, ${String(summary.checked)} checked, ${String(summary.confirmed)} changes, ${String(summary.refused)} refused, ${String(summary.error)} errors (${summary.stopped})`;
+};
+/** This function gives the text of `daemon status`. */
+const describeDaemonStatus = (state, dir) => {
+	if (state === void 0) return `no daemon has run on the store ${dir}`;
+	const lines = [daemonHead(state), lastTick(state)];
+	if (state.alive && state.next_tick_at !== null) lines.push(`next tick: ${state.next_tick_at}`);
+	lines.push(`${String(state.ticks)} ticks, ${String(state.checked)} checks, ${String(state.changes)} changes`);
+	if (state.next_due.length === 0) lines.push("next due: no watch");
+	else {
+		lines.push("next due:");
+		for (const due of state.next_due) lines.push(`  ${due.next_at}  ${due.url}  (every ${everyText(due.every_ms)}, last ${due.last_kind ?? "none"})`);
+	}
+	if (state.mcp !== null) {
+		const where = state.mcp.token_file === null ? "" : ` (the token is in ${state.mcp.token_file})`;
+		lines.push(`MCP: ${state.mcp.url}${where}`);
+		lines.push(state.tasks.length === 0 ? "tasks: none running" : `tasks: ${String(state.tasks.length)} running`);
+		for (const task of state.tasks) lines.push(`  ${task.id}  ${task.tool}  ${task.status}${task.message === null ? "" : `: ${task.message}`}`);
+	}
+	lines.push(state.last_error === null ? "last error: none" : `last error: ${state.last_error.at}: ${state.last_error.message}`);
+	return lines.join("\n");
+};
+/** These are the kinds of a field (`FIELD_KINDS` of `extract.ts`). The command line knows them before it loads the crawler. */
+const KINDS = [
+	"text",
+	"number",
+	"money",
+	"date",
+	"url",
+	"email",
+	"phone"
+];
+/** This function reads the fields of the command line: `name=value`, with `:kind` at the end. The name and the value cannot be empty. */
+const fieldsOf = (given) => {
+	if (given === void 0 || given.length === 0) throw new UsageError("extract wants at least one --field name=value[:kind]");
+	const fields = {};
+	for (const text of given) {
+		const equals = text.indexOf("=");
+		if (equals <= 0) throw new UsageError(`--field wants name=value[:kind], not "${text}"`);
+		const name = text.slice(0, equals);
+		let value = text.slice(equals + 1);
+		let kind;
+		const jsonLd = value.lastIndexOf(":json-ld:");
+		const colon = value.lastIndexOf(":");
+		if (jsonLd !== -1) {
+			kind = value.slice(jsonLd + 1);
+			value = value.slice(0, jsonLd);
+			if (kind === "json-ld:" || /\s/.test(kind)) throw new UsageError(`--field ${name} wants a path after json-ld:, such as json-ld:Product.offers.price`);
+		} else if (colon !== -1 && KINDS.includes(value.slice(colon + 1))) {
+			kind = value.slice(colon + 1);
+			value = value.slice(0, colon);
+		}
+		if (value.trim() === "") throw new UsageError(`--field ${name} wants a value`);
+		if (fields[name] !== void 0) throw new UsageError(`--field ${name} is given two times`);
+		fields[name] = kind === void 0 ? { value } : {
+			value,
+			kind
+		};
+	}
+	return fields;
+};
+/**
+* This function reads the embedder of the command line, or of the environment for the commands
+* that use one. `--embed-model` and `--ollama-url` need `--embedder`.
+*/
+const embedderOf = (values, env, command) => {
+	const uses = OWNER["embedder"].split(" ").includes(command);
+	const fromEnv = (name) => uses ? env[name] || void 0 : void 0;
+	const name = values.embedder ?? fromEnv("POLITE_CRAWLER_EMBEDDER");
+	const model = values["embed-model"] ?? fromEnv("POLITE_CRAWLER_EMBED_MODEL");
+	const url = values["ollama-url"] ?? fromEnv("POLITE_CRAWLER_OLLAMA_URL");
+	const allowRemote = values["allow-remote-ollama"] === true || uses && envFlag(env, "POLITE_CRAWLER_ALLOW_REMOTE_OLLAMA");
+	if (name === void 0) {
+		if (values["embed-model"] !== void 0 || values["ollama-url"] !== void 0) throw new UsageError("--embed-model and --ollama-url go with --embedder transformers or ollama");
+		return;
+	}
+	if (name !== "transformers" && name !== "ollama") throw new UsageError(`--embedder is transformers or ollama, not "${name}"`);
+	if (model !== void 0 && model.trim() === "") throw new UsageError("--embed-model wants a model");
+	if (values["ollama-url"] !== void 0 && name !== "ollama") throw new UsageError("--ollama-url goes with --embedder ollama");
+	return {
+		kind: name,
+		model,
+		url: name === "ollama" ? url : void 0,
+		allowRemote
+	};
+};
+/** This function gives the text of `embed`: the counts and the reason of the stop. */
+const describeEmbed = (result) => [`${String(result.embedded)} texts got vectors (${String(result.chunks)} chunks); ${String(result.reused)} documents had vectors; ${String(result.held_back)} held back, so not sent to the model`, `${String(result.docs)} documents examined in ${String(result.ms)} ms with ${result.embedder}. Stopped (${result.stopped})${result.stopped === "done" ? "" : ": start embed again for the rest"}.`].join("\n");
+/** This function gives the pages of `fetch`: a rejected fetch is an error of its URL only. A page that needs a browser names the reason when rendering is off. */
+const fetchedPages = (urls, settled, renderOff) => settled.map((result, at) => {
+	const url = urls[at];
+	if (result.status === "rejected") {
+		const reason = result.reason;
+		return failedPage(url, `${url}: ${reason instanceof Error ? reason.message : String(reason)}`);
+	}
+	const page = result.value;
+	return renderOff !== void 0 && page.note !== null && /no browser/i.test(page.note) ? {
+		...page,
+		note: `this page needs JavaScript, and rendering is off: ${renderOff}`
+	} : page;
+});
+/** This function gives the text of the results of `find`. An archived copy says so. */
+const describeFound = (found) => found.length === 0 ? "nothing stored matches" : found.map((hit) => `${hit.doc}  ${hit.title}${hit.archived === void 0 ? "" : `  [archived copy, captured ${hit.archived.timestamp}]`}${hit.passage === void 0 ? "" : `  [passage ${String(hit.passage.start)}–${String(hit.passage.end)}]`}\n  ${hit.snippet.replace(/\s+/g, " ")}`).join("\n");
+/** This function gives the text of `read`. With `--at`, the head names the capture and its fetch time. */
+const describeRead = (read, at) => {
+	const version = at === void 0 ? "" : ` ${read.capture} (${read.fetched_at})`;
+	return read.before_first === true ? read.note ?? "" : read.held_back ? `${read.note ?? ""}\nAdd --user-provided if you gave this URL yourself. The crawler logs that read.` : `${read.title}\n${read.url ?? ""}\n${read.archived === void 0 ? "" : `${read.note ?? ""}\n`}[${String(read.from)}–${String(read.to)} of ${String(read.total)}]${version}\n\n${read.text}`;
+};
+/** This function gives the text of `verify`. An archived copy and an earlier capture say so. */
+const describeVerify = (verified) => verified.found ? `found at ${String(verified.start)}–${String(verified.end)} in ${verified.capture ?? ""}${verified.stale ? " (an earlier capture)" : ""}${verified.archived === void 0 ? "" : ` (an archived copy from ${verified.archived.source}, captured ${verified.archived.timestamp})`}: ${verified.excerpt ?? ""}` : verified.held_back || verified.before_first === true ? verified.note ?? "held back" : "not found";
+/** This function gives the lines of `discover`: each URL with its date, then the count of the files. */
+const describeDiscover = (found) => {
+	const read = found.sources.filter((source) => source.kind !== null).length;
+	return [...found.urls.map((url) => url.lastmod === null ? url.url : `${url.url}  ${url.lastmod.slice(0, 10)}`), `${String(found.urls.length)} URLs from ${String(read)} of ${String(found.sources.length)} files (${found.stopped}${found.note === null ? "" : `: ${found.note}`})`];
+};
+/** This function gives the text of `reindex`: the counts, the documents that stay, and the captures that it skipped. */
+const describeReindex = (counts) => [
+	`${String(counts.captures)} captures replayed, ${String(counts.docs)} documents, ${String(counts.urls)} URLs`,
+	...counts.carried > 0 ? [`${String(counts.carried)} documents of other recipes kept: ${counts.unknown_recipes.join(", ")}`] : [],
+	...counts.failed + counts.missing + counts.damaged > 0 ? [`skipped: ${String(counts.failed)} captures whose reader failed, ${String(counts.missing)} with no stored body, ${String(counts.damaged)} damaged lines of the log`] : []
+].join("\n");
+/** This function gives the lines of `extract`: a line for each field, its candidates and its note, then the counts. */
+const describeExtract = (result) => {
+	const lines = [];
+	for (const [name, field] of Object.entries(result.fields)) {
+		const mark = field.status === "verified" || field.status === "normalized" ? "✓" : "✗";
+		const place = field.span !== null ? ` at ${String(field.span.start)}–${String(field.span.end)}${field.span.in === "full_text" ? " of the whole text" : ""}` : field.path !== null ? ` at ${field.path}` : "";
+		const how = field.normalization === null ? "" : ` (${field.normalization})`;
+		const stored = field.raw === null ? "" : `: ${JSON.stringify(field.raw)}`;
+		lines.push(`${mark} ${name}: ${field.status}${how}${place}${stored}`);
+		for (const candidate of field.candidates) {
+			const at = candidate.span !== null ? `${String(candidate.span.start)}–${String(candidate.span.end)}` : candidate.path ?? "";
+			lines.push(`    ${at}: ${JSON.stringify(candidate.raw)} can be ${candidate.readings.join(" or ")}`);
+		}
+		if (field.note !== null && field.status !== "held-back") lines.push(`    ${field.note}`);
 	}
+	const summary = result.summary;
+	lines.push(result.held_back ? `${result.note ?? "held back"}\nAdd --user-provided if you gave this URL yourself. The crawler logs that check.` : `${String(summary.fields)} fields: ${String(summary.verified)} verified, ${String(summary.normalized)} normalized, ${String(summary.ambiguous)} ambiguous, ${String(summary.not_found)} not found, ${String(summary.invalid)} not valid, in ${result.capture ?? "no capture"}${result.note === null ? "" : `. ${result.note}`}`);
+	return lines;
+};
+/** This function gives the exit code of `extract`: 0 when each field is verified or normalized. */
+const extractCode = (result) => {
+	const done = result.summary.verified + result.summary.normalized;
+	return done === result.summary.fields && done > 0 ? 0 : 1;
+};
+/** This function checks the options of `export`, and gives its format and its options. */
+const exportPlan = (values) => {
+	const out = values.out;
+	if (out === void 0 || out.trim() === "") throw new UsageError("export wants --out <file>");
+	const format = values.format ?? (/\.wacz$/i.test(out) ? "wacz" : /\.warc\.gz$/i.test(out) ? "warc" : void 0);
+	if (format !== "warc" && format !== "wacz") throw new UsageError(values.format === void 0 ? "export wants --format warc or wacz, or a file that ends in .warc.gz or .wacz" : `--format is warc or wacz, not "${values.format}"`);
+	for (const name of ["since", "until"]) {
+		const value = values[name];
+		if (value !== void 0 && !Number.isFinite(Date.parse(value))) throw new UsageError(`--${name} wants a time, such as 2026-10-01, not "${value}"`);
+	}
+	if (values.host !== void 0 && (values.host.trim() === "" || /[/:@\s]/.test(values.host))) throw new UsageError(`--host wants a host name, such as example.com, not "${values.host}"`);
+	return {
+		format,
+		options: {
+			out: out === "~" ? homedir() : /^~[\\/]/.test(out) ? resolve(homedir(), out.slice(2)) : resolve(out),
+			since: values.since,
+			until: values.until,
+			host: values.host,
+			urlPrefix: values.prefix,
+			ids: values.id,
+			includeRefusals: values["include-refusals"] === true,
+			overwrite: values.overwrite === true
+		}
+	};
+};
+/** This function gives the text of an export: the file, the counts, and what the archive left out. */
+const describeExport = (result, includeRefusals) => {
+	const left = [
+		result.bodiless > 0 && !includeRefusals ? `${String(result.bodiless)} captures with no body (add --include-refusals to keep them as metadata)` : null,
+		result.missing > 0 ? `${String(result.missing)} captures with no stored body` : null,
+		result.damaged > 0 ? `${String(result.damaged)} damaged lines of the log` : null
+	].filter((line) => line !== null);
+	return [
+		`wrote ${result.out} (${String(result.bytes)} bytes)`,
+		`  ${String(result.captures)} captures: ${String(result.responses)} responses, ${String(result.revisits)} revisits, ${String(result.conversions)} rendered pages, ${String(result.metadata)} metadata records${result.format === "wacz" ? `, ${String(result.pages)} pages` : ""}`,
+		...left.length > 0 ? [`  left out: ${left.join(", ")}`] : []
+	].join("\n");
+};
+/**
+* This function reads the whole command line before anything opens the store. A wrong command line
+* throws a `UsageError`, so the command exits with code 2 and makes no store.
+*/
+const readLine = (args, env) => {
+	const { values, positionals } = parse(args);
+	if (values.version === true) return { kind: "version" };
+	const [command, ...rest] = positionals;
+	if (values.help === true || command === void 0) return { kind: "help" };
 	const spec = COMMANDS[command];
 	if (spec === void 0) throw new UsageError(`no command "${command}"`);
 	if (rest.length < spec.min || rest.length > spec.max) throw new UsageError(spec.wants);
-	for (const [option, owner] of Object.entries(OWNER)) if (values[option] !== void 0 && owner !== command) throw new UsageError(`--${option} is an option of ${owner}`);
+	for (const [option, owner] of Object.entries(OWNER)) {
+		const owners = owner.split(" ");
+		if (values[option] !== void 0 && !owners.includes(command)) throw new UsageError(`--${option} is an option of ${owners.join(" or ")}`);
+	}
 	const render = values.render ?? "auto";
 	if (render !== "auto" && render !== "never" && render !== "always") throw new UsageError(`--render is auto, never or always, not "${render}"`);
 	const rank = values.rank ?? "exact";
-	if (rank !== "exact" && rank !== "fast") throw new UsageError(`--rank is exact or fast, not "${rank}"`);
+	if (rank !== "exact" && rank !== "fast" && rank !== "hybrid") throw new UsageError(`--rank is exact, fast or hybrid, not "${rank}"`);
+	const embedder = embedderOf(values, env, command);
+	if (rank === "hybrid" && embedder === void 0) throw new UsageError("--rank hybrid wants --embedder transformers or ollama");
+	if (command === "embed" && embedder === void 0) throw new UsageError("embed wants --embedder transformers or ollama");
+	const maxDocs = values["max-docs"] === void 0 ? void 0 : whole("max-docs", values["max-docs"], 1, 1);
 	const limit = whole("limit", values.limit, 10, 1);
-	const from = whole("from", values.from, 0, 0);
+	const from = command === "read" ? whole("from", values.from, 0, 0) : 0;
 	const length = whole("length", values.length, 4e3, 1);
-	const dir = storeDir(values.dir ?? (process.env["POLITE_CRAWLER_DIR"] || void 0));
-	const allowPrivate = values["allow-private"] === true || envFlag("POLITE_CRAWLER_ALLOW_PRIVATE");
-	const allowUserProvided = values["allow-user-provided"] === true || command === "mcp" && envFlag("POLITE_CRAWLER_ALLOW_USER_PROVIDED");
+	const crawlLimits = crawlOptionsOf(values);
+	const at = timeOf("at", values.at);
+	const diffFrom = command === "diff" ? versionOf("from", values.from) : void 0;
+	const diffTo = versionOf("to", values.to);
+	const watchLine = command === "watch" ? watchLineOf(values, rest) : void 0;
+	const extractFields = command === "extract" ? fieldsOf(values.field) : void 0;
+	const daemonLine = command === "daemon" ? daemonLineOf(values, rest) : void 0;
+	const dir = storeDir(values.dir ?? (env["POLITE_CRAWLER_DIR"] || void 0));
+	const allowPrivate = values["allow-private"] === true || envFlag(env, "POLITE_CRAWLER_ALLOW_PRIVATE");
+	const allowUserProvided = values["allow-user-provided"] === true || (command === "mcp" || daemonLine?.action === "run") && envFlag(env, "POLITE_CRAWLER_ALLOW_USER_PROVIDED");
+	if (values["archive-despite-robots"] === true && values["archive-fallback"] !== true) throw new UsageError("--archive-despite-robots works only with --archive-fallback");
+	const signs = OWNER["sign-key"].split(" ").includes(command) && (watchLine === void 0 || watchLine.action === "run") && (daemonLine === void 0 || daemonLine.action === "run");
+	const signKey = values["sign-key"] ?? (signs ? env["POLITE_CRAWLER_SIGN_KEY"] || void 0 : void 0);
+	const signatureAgent = values["signature-agent"] ?? (signs ? env["POLITE_CRAWLER_SIGNATURE_AGENT"] || void 0 : void 0);
+	if (command === "key-directory") {
+		if (signKey === void 0) throw new UsageError("key-directory wants --sign-key <file>");
+	} else if (signKey === void 0 !== (signatureAgent === void 0)) throw new UsageError("--sign-key and --signature-agent go together: the key signs, and the agent tells a site where its public key is");
+	let maxAge;
+	try {
+		maxAge = durationMs(values["max-age"] ?? "24h");
+	} catch (error) {
+		throw new UsageError(`--max-age: ${error.message}`, { cause: error });
+	}
+	return {
+		kind: "command",
+		command,
+		rest,
+		values,
+		render,
+		rank,
+		limit,
+		from,
+		length,
+		maxAge,
+		crawlLimits,
+		at,
+		diffFrom,
+		diffTo,
+		watchLine,
+		daemonLine,
+		extractFields,
+		exportPlan: command === "export" ? exportPlan(values) : void 0,
+		dir,
+		contact: values.contact ?? (env["POLITE_CRAWLER_CONTACT"] || void 0),
+		allowPrivate,
+		allowUserProvided,
+		signKey,
+		signatureAgent,
+		embedder,
+		maxDocs
+	};
+};
+//#endregion
+//#region src/cli.ts
+const emitWarning = process.emitWarning.bind(process);
+process.emitWarning = ((warning, ...rest) => {
+	const first = rest[0];
+	const kind = typeof first === "string" ? first : typeof first === "object" && first !== null && "type" in first ? first.type : warning instanceof Error ? warning.name : void 0;
+	const text = typeof warning === "string" ? warning : warning.message;
+	if (kind === "ExperimentalWarning" && text.startsWith("SQLite is an experimental feature")) return;
+	emitWarning(warning, ...rest);
+});
+const HELP = `polite-crawler ${VERSION}: fetch a source once, keep it, read it many times.
+
+Usage:
+  polite-crawler fetch <url...>        fetch pages into the store (robots.txt obeyed)
+  polite-crawler find <query>          search the stored pages, with no request
+  polite-crawler read <doc>            print a range of a stored document
+  polite-crawler verify <doc> <quote>  check that a quote is in a stored document
+  polite-crawler reindex               rebuild the search index from the capture log
+  polite-crawler mcp                   serve the tools over MCP (standard input and output)
+  polite-crawler export --out <file>   write the stored captures as a WARC or WACZ archive
+  polite-crawler discover <site>       list the pages of a site from its sitemaps and feeds
+  polite-crawler crawl <url...>        fetch a site and follow its links, within a budget
+  polite-crawler diff <doc>            show what changed between two versions of a document
+  polite-crawler watch add <url...>    check these pages again on a schedule
+  polite-crawler watch remove <url...> stop the watch of these pages
+  polite-crawler watch list            print the watch list
+  polite-crawler watch run             check the pages that are due, within a budget
+  polite-crawler daemon                check the watch list on a schedule, until SIGINT or SIGTERM;
+                                       with --mcp-http, serve the MCP tools over local HTTP too
+  polite-crawler daemon status         print the state of the daemon of the store
+  polite-crawler key-directory         print the key directory (JWKS) of --sign-key, to host
+  polite-crawler extract <doc> --field name=value[:kind]...
+                                       check proposed field values against a stored page
+  polite-crawler embed --embedder <name>  give vectors to the stored pages, for find --rank hybrid
+
+Options:
+  --dir <path>       the store (default $POLITE_CRAWLER_DIR, or ~/.polite-crawler)
+  --contact <text>   an email (From and User-Agent) or a URL (User-Agent only)
+                     ($POLITE_CRAWLER_CONTACT)
+  --allow-private    fetch loopback and private addresses too ($POLITE_CRAWLER_ALLOW_PRIVATE=1)
+  --no-browser       never open the headless browser
+  --sign-key <file>  sign each request (Web Bot Auth) with this Ed25519 key: PKCS#8 PEM or JWK
+                     ($POLITE_CRAWLER_SIGN_KEY)
+  --signature-agent <url>  the https origin of your key directory, or the URL of a
+                     JSON Web Key Set on a path, for --sign-key ($POLITE_CRAWLER_SIGNATURE_AGENT)
+  --embedder <name>  the local model of hybrid search, for find, embed and mcp: transformers
+                     (transformers.js in this process) or ollama ($POLITE_CRAWLER_EMBEDDER)
+  --embed-model <m>  the model: Xenova/bge-small-en-v1.5 (transformers) or nomic-embed-text
+                     (ollama) by default ($POLITE_CRAWLER_EMBED_MODEL)
+  --ollama-url <url> the Ollama server, http://127.0.0.1:11434 by default ($POLITE_CRAWLER_OLLAMA_URL)
+  --allow-remote-ollama  permit an Ollama server on a different computer: the page text goes
+                     to it ($POLITE_CRAWLER_ALLOW_REMOTE_OLLAMA=1)
+  --json             print JSON
+  -v, --version      print the version
+  -h, --help         print this help
+
+  fetch    --render <mode>   auto (default), never or always
+           --max-age <age>   reuse a stored copy this young: 30m, 24h (default), 7d; 0 fetches again
+           --archive-fallback  when the site refuses or is down, give a copy from the Wayback
+                             Machine or Common Crawl, marked archived (not a fresh fetch)
+           --archive-despite-robots  with --archive-fallback: also when the robots.txt of the
+                             site disallows the page now
+  find     --limit <n>       how many results (default 10)
+           --kind <kind>     only this kind of document, such as page or pdf
+           --rank <mode>     exact (default): BM25 over each match; fast: leave common words
+                             out of the one-word part, faster for a long question; hybrid: BM25
+                             and the meaning of the text, fused (needs --embedder)
+  read     --from <n>        where the range starts (default 0)
+           --length <n>      how many characters (default 4000)
+           --user-provided   you gave this URL yourself: read a page that refuses AI input (logged)
+  read, verify, extract  --at <time>  use the version that was current then, such as
+                             2026-10-01T12:00Z
+  reindex                    rebuild; the documents of recipes that this command cannot read stay
+  mcp      --allow-user-provided   let user_provided work when the client cannot ask you
+                                   ($POLITE_CRAWLER_ALLOW_USER_PROVIDED=1)
+  export   --out <file>      the archive to write: a name that ends in .warc.gz or .wacz
+           --format <f>      warc or wacz (default: from the name of the file)
+           --since <time>    only the captures from this time on, such as 2026-10-01
+           --until <time>    only the captures before this time
+           --host <host>     only the captures of this host and its subdomains
+           --prefix <url>    only the captures whose URL starts with this text
+           --id <capture>    only this capture; give --id again for more captures
+           --include-refusals  a metadata record for each capture with no body too
+           --overwrite       replace the file when it exists
+  discover --max-urls <n>    how many URLs (default 1000)
+           --feed <url>      read this RSS or Atom feed too (repeat it)
+           --max-age <age>   as for fetch
+  crawl    --max-pages <n>   how many pages (default 100)
+           --max-depth <n>   how many links from a seed (default 3; 0 fetches the seeds only)
+           --budget <time>   the longest time: 30s, 2m, 10m (default); no request starts after it
+           --include <pattern>  follow only the URLs that match, for example /blog/* (repeat it)
+           --exclude <pattern>  follow no URL that matches, for example *.pdf$ (repeat it)
+           --sitemaps        add the pages of the sitemaps of the site
+           --render, --max-age   as for fetch
+  discover, crawl  --allow-host <host>   keep the URLs on this host too (repeat it)
+  diff     --from <version>  the old version: a capture id or a time (default: the one before --to)
+           --to <version>    the new version: a capture id or a time (default: the newest)
+  watch add  --every <time>  the time between two checks: 1h or more (default 24h)
+             --confirm <n>   2 (default): a change counts when two checks see it; 1: at once
+             --scope <text>, --label <name=value>   marks for the captures (repeat --label)
+             --render <mode> as for fetch
+  watch run  --budget <time>  the longest time (default 10m); no request starts after it
+             --max-pages <n>  how many pages to check (default 100)
+  daemon   --tick <time>     the longest wait between two ticks: 1s or more (default 1m)
+           --jitter <time>   a random wait from 0 to this after each tick (default 10s)
+           --tick-budget <time>  the longest time of the checks of one tick (default 10m)
+           --tick-pages <n>  how many pages one tick checks at most (default 100)
+           --grace <time>    at a stop, the checks in progress get this time to end (default 10s)
+           --events <file>   write each change as one JSON line to this file (default: the
+                             standard output)
+           --on-change <command>  start this command of yours for each change: it gets the
+                             event as one JSON line on its standard input
+           --mcp-http        serve the MCP tools over HTTP; each request needs the bearer token
+           --mcp-port <n>    the port (default 9307)
+           --mcp-host <host> the host to bind (default 127.0.0.1)
+           --mcp-token-file <file>  read the token from this file (default: a new token at each
+                             start, in <store>/daemon/mcp-token)
+           --allow-remote    permit a --mcp-host that is not loopback: read the README first
+           --allow-user-provided  as for mcp
+  daemon status              exit code 0 when a daemon runs on the store, 1 when none runs
+  extract  --field <name=value[:kind]>  a value to check, never made up by the crawler (repeat it);
+                             kind: text (default), number, money, date, url, email, phone,
+                             or json-ld:<path> such as json-ld:Product.offers.price
+           --capture <id>    check the text of this capture, not the current text
+           --user-provided   as for read
+  embed    --budget <time>   the longest time (default 10m); no embedding starts after it
+           --max-docs <n>    how many texts get vectors at most
+           --kind <kind>     only this kind of document
+
+Exit codes:
+  0  done
+  1  not done: a page not fetched, no match, a quote not found, a document unknown or
+     held back, a watch check that failed or that is still due, a field value that extract
+     did not find or that is not clear, embed that stopped before each page had vectors,
+     or an error
+     (fetch: a page with an archived copy counts as done)
+  2  the command line is wrong
+`;
+/**
+* This function does the command `export`, and gives the exit code. `readLine` checked the whole
+* command line first. The export only reads the store: it makes no store, and it sends no request.
+*/
+const exportCommand = async (plan, dir, json) => {
+	const result = plan.format === "wacz" ? await (await import("./wacz-BFNG42Kt.mjs").then((n) => n.n)).exportWacz(dir, plan.options) : await (await import("./warc-CVS0t1fW.mjs").then((n) => n.a)).exportWarc(dir, plan.options);
+	process.stdout.write(`${json ? JSON.stringify(result, null, 2) : describeExport(result, plan.options.includeRefusals)}\n`);
+	return 0;
+};
+/** This function prints the key directory of a key file, and tells where to host it. */
+const keyDirectoryCommand = async (key, agent) => {
+	const { KEY_DIRECTORY_PATH, KEY_DIRECTORY_TYPE, keyDirectory } = await import("./webbotauth-CjJdTAfE.mjs").then((n) => n.s);
+	let directory;
+	try {
+		directory = keyDirectory([key]);
+	} catch (error) {
+		throw new UsageError(`--sign-key: ${error.message}`, { cause: error });
+	}
+	process.stdout.write(`${JSON.stringify(directory, null, 2)}\n`);
+	const where = agent !== void 0 && agentTypeOf(agent) === "jwks_uri" ? `${agent} (a JSON Web Key Set, the type jwks_uri)` : `${agent === void 0 ? "https://<your origin>" : agent.replace(/\/$/, "")}${KEY_DIRECTORY_PATH} with Content-Type: ${KEY_DIRECTORY_TYPE}`;
+	process.stderr.write(`polite-crawler: serve this JSON at ${where}\n`);
+	return 0;
+};
+/** This function lists the pages of a site. The exit code is 0 when it found a URL. */
+const runDiscover = async (crawler, site, limits, maxAge, json, out) => {
+	const found = await crawler.discover(site, {
+		maxUrls: limits.maxUrls,
+		feeds: limits.feeds,
+		maxAge,
+		...limits.allowHosts === void 0 ? {} : { allowHosts: limits.allowHosts }
+	});
+	if (json) out(JSON.stringify(found, null, 2));
+	else for (const line of describeDiscover(found)) out(line);
+	return found.urls.length > 0 ? 0 : 1;
+};
+/** This function crawls from the seeds. It prints each page when it ends. The exit code is 0 when the crawl got a page. */
+const runCrawl = async (crawler, seeds, limits, page, json, out) => {
+	const controller = new AbortController();
+	const stop = () => controller.abort(/* @__PURE__ */ new Error("interrupted"));
+	process.once("SIGINT", stop);
+	try {
+		const run = crawler.crawl([...new Set(seeds)], {
+			maxPages: limits.maxPages,
+			maxDepth: limits.maxDepth,
+			budgetMs: limits.budgetMs,
+			include: limits.include,
+			exclude: limits.exclude,
+			useSitemaps: limits.sitemaps,
+			render: page.render,
+			maxAge: page.maxAge,
+			signal: controller.signal,
+			...limits.allowHosts === void 0 ? {} : { allowHosts: limits.allowHosts }
+		});
+		const pages = [];
+		for await (const result of run) {
+			pages.push(result);
+			if (!json) out(`${describePage(result)}\n  depth ${String(result.depth)}, ${result.source}`);
+		}
+		const summary = await run.summary;
+		out(json ? JSON.stringify({
+			pages,
+			summary
+		}, null, 2) : describeSummary(summary));
+		return summary.fetched > 0 ? 0 : 1;
+	} finally {
+		process.removeListener("SIGINT", stop);
+	}
+};
+/** This function does the command `watch`, and gives the exit code. */
+const runWatchCommand = async (crawler, line, values, limits, json, out) => {
+	const print = (value, text) => out(json ? JSON.stringify(value, null, 2) : text());
+	switch (line.action) {
+		case "add": {
+			const added = [];
+			for (const url of line.urls) added.push(await crawler.watch.add(url, {
+				...line.every === void 0 ? {} : { every: line.every },
+				...line.confirm === void 0 ? {} : { confirm: line.confirm },
+				...values.scope === void 0 ? {} : { scope: values.scope },
+				...line.labels === void 0 ? {} : { labels: line.labels },
+				...values.render === void 0 ? {} : { render: values.render }
+			}));
+			print(added, () => added.map(describeEntry).join("\n"));
+			return 0;
+		}
+		case "remove": {
+			const removed = [];
+			for (const url of line.urls) removed.push({
+				url,
+				removed: await crawler.watch.remove(url)
+			});
+			print(removed, () => removed.map((one) => `${one.removed ? "removed" : "not on the list"} ${one.url}`).join("\n"));
+			return removed.every((one) => one.removed) ? 0 : 1;
+		}
+		case "list": {
+			const entries = await crawler.watch.list();
+			print(entries, () => entries.length === 0 ? "the watch list is empty" : entries.map(describeEntry).join("\n"));
+			return 0;
+		}
+		default: {
+			const controller = new AbortController();
+			const stop = () => controller.abort(/* @__PURE__ */ new Error("interrupted"));
+			process.once("SIGINT", stop);
+			try {
+				const run = crawler.watch.run({
+					budgetMs: values.budget === void 0 ? void 0 : limits.budgetMs,
+					maxPages: values["max-pages"] === void 0 ? void 0 : limits.maxPages,
+					signal: controller.signal
+				});
+				const events = [];
+				for await (const event of run) {
+					events.push(event);
+					if (!json) out(describeEvent(event));
+				}
+				const summary = await run.summary;
+				out(json ? JSON.stringify({
+					events,
+					summary
+				}, null, 2) : describeWatchSummary(summary));
+				return summary.refused + summary.error === 0 && summary.due === 0 ? 0 : 1;
+			} finally {
+				process.removeListener("SIGINT", stop);
+			}
+		}
+	}
+};
+/** This function gives the text of a diff for a person: the two captures, then the text of `formatDiff`. */
+const describeDiff = async (diff) => {
+	const { formatDiff } = await import("./diff-NQcQqKe9.mjs").then((n) => n.i);
+	return [
+		`${diff.doc}  ${diff.from.capture} (${diff.from.fetched_at}) -> ${diff.to.capture} (${diff.to.fetched_at})`,
+		formatDiff(diff),
+		...diff.same === true ? ["the two versions are the same"] : [],
+		...diff.note === null ? [] : [diff.note]
+	].join("\n");
+};
+/** This function checks the fields against a stored document, and prints a line for each field. The exit code is 0 when each field is verified or normalized. */
+const runExtract = async (crawler, doc, fields, options, json, out) => {
+	const { extractDocument } = await import("./extract-ByFfc8Zk.mjs").then((n) => n.i);
+	const claims = fields;
+	const result = crawler.extract === void 0 ? await extractDocument(crawler.runtime, doc, claims, options) : await crawler.extract(doc, claims, options);
+	if (json) out(JSON.stringify(result, null, 2));
+	else for (const line of describeExtract(result)) out(line);
+	return extractCode(result);
+};
+/**
+* This function makes the embedder of the command line. It loads no model: the model loads at the
+* first text. The transformers embedder keeps the files of its model in `<store>/models`.
+*/
+const embedderFor = async (choice, dir) => {
+	try {
+		if (choice.kind === "ollama") {
+			const { ollamaEmbedder } = await import("./ollama-BJxkjny4.mjs").then((n) => n.a);
+			return ollamaEmbedder({
+				...choice.model === void 0 ? {} : { model: choice.model },
+				...choice.url === void 0 ? {} : { baseUrl: choice.url },
+				allowRemote: choice.allowRemote
+			});
+		}
+		const { transformersEmbedder } = await import("./transformers-BQMlaGj4.mjs").then((n) => n.i);
+		return transformersEmbedder({
+			...choice.model === void 0 ? {} : { model: choice.model },
+			cacheDir: join(dir, "models")
+		});
+	} catch (error) {
+		throw new UsageError(`--embedder ${choice.kind}: ${error.message}`, { cause: error });
+	}
+};
+/** This function gives vectors to the stored pages within the budget. The exit code is 0 when each page has vectors. */
+const runEmbed = async (crawler, limits, options, json, out) => {
+	const controller = new AbortController();
+	const stop = () => controller.abort(/* @__PURE__ */ new Error("interrupted"));
+	process.once("SIGINT", stop);
+	try {
+		const result = await crawler.embed({
+			budget: limits.budgetMs,
+			signal: controller.signal,
+			...options.maxDocs === void 0 ? {} : { maxDocs: options.maxDocs },
+			...options.kind === void 0 ? {} : { kind: options.kind }
+		});
+		out(json ? JSON.stringify(result, null, 2) : describeEmbed(result));
+		return result.stopped === "done" ? 0 : 1;
+	} finally {
+		process.removeListener("SIGINT", stop);
+	}
+};
+/** This function does the command of the command line, and gives the exit code. */
+const main = async (args) => {
+	const out = (text) => void process.stdout.write(`${text}\n`);
+	const line = readLine(args, process.env);
+	if (line.kind === "version") {
+		out(VERSION);
+		return 0;
+	}
+	if (line.kind === "help") {
+		process.stdout.write(HELP);
+		return 0;
+	}
+	const { command, rest, values, render, rank, limit, from, length, maxAge, crawlLimits, at, diffFrom, diffTo, watchLine, extractFields, dir, signKey, signatureAgent } = line;
+	const embedder = line.embedder === void 0 ? void 0 : await embedderFor(line.embedder, dir);
+	if (command === "key-directory") return keyDirectoryCommand(keyFile(signKey), signatureAgent);
+	let signer;
+	if (signKey !== void 0 && signatureAgent !== void 0) {
+		const { createWebBotAuthSigner } = await import("./webbotauth-CjJdTAfE.mjs").then((n) => n.s);
+		const privateKey = keyFile(signKey);
+		try {
+			signer = createWebBotAuthSigner({
+				privateKey,
+				signatureAgent,
+				agentType: agentTypeOf(signatureAgent)
+			});
+		} catch (error) {
+			throw new UsageError(`--sign-key: ${error.message}`, { cause: error });
+		}
+	}
 	const options = {
 		dir,
-		contact: values.contact ?? (process.env["POLITE_CRAWLER_CONTACT"] || void 0),
+		contact: line.contact,
 		...values["no-browser"] === true ? { renderer: false } : {},
-		...allowPrivate ? { egress: { allowPrivate: true } } : {}
+		...line.allowPrivate ? { egress: { allowPrivate: true } } : {},
+		...signer === void 0 ? {} : { signer },
+		...embedder === void 0 ? {} : { embedder }
 	};
 	if (command === "mcp") {
 		const { startMcpServer } = await import("./mcp.mjs");
 		await startMcpServer({
 			...options,
-			allowUserProvided
+			allowUserProvided: line.allowUserProvided
 		});
 		return 0;
 	}
-	const { durationMs, openCrawler } = await import("./open-DYZ7KL2A.mjs").then((n) => n.o);
-	let maxAge;
-	try {
-		maxAge = durationMs(values["max-age"] ?? "24h");
-	} catch (error) {
-		throw new UsageError(`--max-age: ${error.message}`, { cause: error });
+	if (command === "daemon") {
+		const { daemonCommand } = await import("./cli-daemon--cTmq0Z5.mjs");
+		return daemonCommand(line.daemonLine, options, {
+			allowUserProvided: line.allowUserProvided,
+			json: values.json === true,
+			out
+		});
 	}
+	if (line.exportPlan !== void 0) return exportCommand(line.exportPlan, dir, values.json === true);
+	const { openCrawler } = await import("./open-C4fhXJgD.mjs").then((n) => n.a);
 	let renderOff;
 	let renderer = options.renderer;
-	if (command !== "fetch" || render === "never") renderer = false;
+	if (!(command === "fetch" || command === "crawl" || watchLine?.action === "run") || render === "never") renderer = false;
 	else if (renderer === void 0) {
-		const { browserMissing } = await import("./browser-gJ_h4AhB.mjs").then((n) => n.n);
+		const { browserMissing } = await import("./browser-CKRNbIDT.mjs").then((n) => n.n);
 		renderOff = await browserMissing();
 		if (renderOff !== void 0) renderer = false;
 	}
@@ -284,23 +1211,17 @@ const main = async (args) => {
 		switch (command) {
 			case "fetch": {
 				const urls = [...new Set(rest)];
-				const pages = (await Promise.allSettled(urls.map((url) => crawler.fetch(url, {
+				const fallback = values["archive-fallback"] === true ? "archive" : void 0;
+				const archiveDespiteRobots = values["archive-despite-robots"] === true;
+				const settled = await Promise.allSettled(urls.map((url) => crawler.fetch(url, {
 					render,
-					maxAge
-				})))).map((result, at) => {
-					const url = urls[at];
-					if (result.status === "rejected") {
-						const reason = result.reason;
-						return failedPage(url, `${url}: ${reason instanceof Error ? reason.message : String(reason)}`);
-					}
-					const page = result.value;
-					return renderOff !== void 0 && page.note !== null && /no browser/i.test(page.note) ? {
-						...page,
-						note: `this page needs JavaScript, and rendering is off: ${renderOff}`
-					} : page;
-				});
+					maxAge,
+					fallback,
+					archiveDespiteRobots
+				})));
+				const pages = fetchedPages(urls, settled, renderOff);
 				print(pages, () => pages.map(describePage).join("\n"));
-				return pages.every(ok) ? 0 : 1;
+				return pages.every(done) ? 0 : 1;
 			}
 			case "find": {
 				const found = await crawler.find(rest.join(" "), {
@@ -308,7 +1229,7 @@ const main = async (args) => {
 					kind: values.kind ?? null,
 					rank
 				});
-				print(found, () => found.length === 0 ? "nothing stored matches" : found.map((hit) => `${hit.doc}  ${hit.title}\n  ${hit.snippet.replace(/\s+/g, " ")}`).join("\n"));
+				print(found, () => describeFound(found));
 				return found.length === 0 ? 1 : 0;
 			}
 			case "read": {
@@ -316,24 +1237,45 @@ const main = async (args) => {
 				const read = await crawler.read(rest[0], {
 					from,
 					length,
-					userProvided
+					userProvided,
+					at
 				});
-				print(read, () => read.held_back ? `${read.note ?? ""}\nAdd --user-provided if you gave this URL yourself. The crawler logs that read.` : `${read.title}\n${read.url ?? ""}\n[${String(read.from)}–${String(read.to)} of ${String(read.total)}]\n\n${read.text}`);
-				return read.held_back ? 1 : 0;
+				print(read, () => describeRead(read, at));
+				return read.held_back || read.before_first === true ? 1 : 0;
 			}
 			case "verify": {
 				const [doc, ...quote] = rest;
-				const verified = await crawler.verify(doc, quote.join(" "));
-				print(verified, () => verified.found ? `found at ${String(verified.start)}–${String(verified.end)} in ${verified.capture ?? ""}${verified.stale ? " (an earlier capture)" : ""}: ${verified.excerpt ?? ""}` : verified.held_back ? verified.note ?? "held back" : "not found");
+				const verified = await crawler.verify(doc, quote.join(" "), { at });
+				print(verified, () => describeVerify(verified));
 				return verified.found ? 0 : 1;
 			}
+			case "discover": return await runDiscover(crawler, rest[0], crawlLimits, maxAge, values.json === true, out);
+			case "crawl": return await runCrawl(crawler, rest, crawlLimits, {
+				render,
+				maxAge
+			}, values.json === true, out);
+			case "diff": {
+				const diff = await crawler.diff(rest[0], {
+					...diffFrom === void 0 ? {} : { from: diffFrom },
+					...diffTo === void 0 ? {} : { to: diffTo }
+				});
+				const text = values.json === true ? "" : await describeDiff(diff);
+				print(diff, () => text);
+				return diff.held_back ? 1 : 0;
+			}
+			case "watch": return await runWatchCommand(crawler, watchLine, values, crawlLimits, values.json === true, out);
+			case "extract": return await runExtract(crawler, rest[0], extractFields, {
+				capture: values.capture,
+				userProvided: values["user-provided"] === true,
+				at
+			}, values.json === true, out);
+			case "embed": return await runEmbed(crawler, crawlLimits, {
+				maxDocs: line.maxDocs,
+				kind: values.kind
+			}, values.json === true, out);
 			default: {
 				const counts = await crawler.reindex();
-				print(counts, () => [
-					`${String(counts.captures)} captures replayed, ${String(counts.docs)} documents, ${String(counts.urls)} URLs`,
-					...counts.carried > 0 ? [`${String(counts.carried)} documents of other recipes kept: ${counts.unknown_recipes.join(", ")}`] : [],
-					...counts.failed + counts.missing + counts.damaged > 0 ? [`skipped: ${String(counts.failed)} captures whose reader failed, ${String(counts.missing)} with no stored body, ${String(counts.damaged)} damaged lines of the log`] : []
-				].join("\n"));
+				print(counts, () => describeReindex(counts));
 				return 0;
 			}
 		}
@@ -350,6 +1292,6 @@ main(process.argv.slice(2)).then((code) => {
 	process.exitCode = usage ? 2 : 1;
 });
 //#endregion
-export {};
+export { describeDaemonStatus as t };
 
 //# sourceMappingURL=cli.mjs.map