polite-crawler 0.1.3 → 0.2.0

package/build/cli.mjs

@@ -1,10 +1,26 @@
 #!/usr/bin/env node
-import { i as durationMs, r as VERSION } from "./version-ntbY-6wh.mjs";
-import { t as isLoopbackHost } from "./loopback-C_QYowSf.mjs";
+import { i as durationMs, r as VERSION } from "./version-Djva7JkN.mjs";
 import { join, resolve } from "node:path";
 import { readFileSync } from "node:fs";
+import { isIP } from "node:net";
 import { parseArgs } from "node:util";
 import { homedir } from "node:os";
+//#region src/loopback.ts
+/**
+* This function tells if a host name or an address is a loopback host: `localhost`, an address in
+* `127.0.0.0/8`, or `::1`. The MCP server over HTTP binds to a loopback host unless the operator
+* permits a different host. The function imports only Node.js, so the command line uses it before
+* the store opens.
+*/
+const isLoopbackHost = (host) => {
+	const name = host.trim().toLowerCase().replace(/^\[(.*)\]$/, "$1");
+	if (name === "localhost" || name.endsWith(".localhost")) return true;
+	const mapped = name.startsWith("::ffff:") ? name.slice(7) : name;
+	if (isIP(mapped) === 4) return mapped.startsWith("127.");
+	if (isIP(name) === 6) return /^(?:0{0,4}:){7}0{0,3}1$|^::1$/.test(name);
+	return false;
+};
+//#endregion
 //#region src/cli-line.ts
 /** A usage error is a wrong command line. It exits with code 2. */
 var UsageError = class extends Error {};
@@ -103,6 +119,18 @@ const OPTIONS = {
 	"ollama-url": { type: "string" },
 	"allow-remote-ollama": { type: "boolean" },
 	"max-docs": { type: "string" },
+	name: { type: "string" },
+	"require-name": { type: "boolean" },
+	address: { type: "string" },
+	lat: { type: "string" },
+	lon: { type: "string" },
+	osm: { type: "string" },
+	pages: { type: "string" },
+	compare: { type: "string" },
+	"on-behalf-of": {
+		type: "string",
+		multiple: true
+	},
 	...EXPORT_OPTIONS
 };
 /** These options belong to `daemon`, and not to `daemon status`. */
@@ -120,23 +148,34 @@ const DAEMON_RUN_OPTIONS = [
 	"mcp-token-file",
 	"allow-remote"
 ];
+/** These options belong to `venue` only. */
+const VENUE_OPTIONS = [
+	"name",
+	"require-name",
+	"address",
+	"lat",
+	"lon",
+	"osm",
+	"pages",
+	"compare"
+];
 /** These options belong to one command, or to the commands that the text names. */
 const OWNER = {
-	render: "fetch crawl watch",
-	"max-age": "fetch crawl discover",
+	render: "fetch crawl watch venue product",
+	"max-age": "fetch crawl discover venue product",
 	limit: "find",
 	kind: "find embed",
 	rank: "find",
 	from: "read diff",
 	length: "read",
-	"user-provided": "read extract",
+	"user-provided": "read extract venue product",
 	"allow-user-provided": "mcp daemon",
 	...Object.fromEntries(Object.keys(EXPORT_OPTIONS).map((option) => [option, "export"])),
 	"max-urls": "discover",
 	feed: "discover",
 	"max-pages": "crawl watch",
 	"max-depth": "crawl",
-	budget: "crawl watch embed",
+	budget: "crawl watch embed venue product",
 	include: "crawl",
 	exclude: "crawl",
 	sitemaps: "crawl",
@@ -147,10 +186,10 @@ const OWNER = {
 	confirm: "watch",
 	scope: "watch",
 	label: "watch",
-	"archive-fallback": "fetch",
-	"archive-despite-robots": "fetch",
-	"sign-key": "fetch discover crawl watch mcp key-directory daemon",
-	"signature-agent": "fetch discover crawl watch mcp key-directory daemon",
+	"archive-fallback": "fetch venue product",
+	"archive-despite-robots": "fetch venue product",
+	"sign-key": "fetch discover crawl watch mcp key-directory daemon venue product",
+	"signature-agent": "fetch discover crawl watch mcp key-directory daemon venue product",
 	...Object.fromEntries(DAEMON_RUN_OPTIONS.map((option) => [option, "daemon"])),
 	field: "extract",
 	capture: "extract",
@@ -158,7 +197,9 @@ const OWNER = {
 	"embed-model": "find embed mcp",
 	"ollama-url": "find embed mcp",
 	"allow-remote-ollama": "find embed mcp",
-	"max-docs": "embed"
+	"max-docs": "embed",
+	...Object.fromEntries(VENUE_OPTIONS.map((option) => [option, "venue"])),
+	"on-behalf-of": "venue product"
 };
 /** These are the commands, with the smallest and the largest count of arguments. */
 const COMMANDS = {
@@ -236,6 +277,16 @@ const COMMANDS = {
 		min: 0,
 		max: 0,
 		wants: "embed takes no argument: give the model with --embedder"
+	},
+	venue: {
+		min: 1,
+		max: 1,
+		wants: "venue wants the URL of the site of one venue"
+	},
+	product: {
+		min: 1,
+		max: 1,
+		wants: "product wants the URL of one product page"
 	}
 };
 const parse = (args) => {
@@ -692,6 +743,311 @@ const extractCode = (result) => {
 	const done = result.summary.verified + result.summary.normalized;
 	return done === result.summary.fields && done > 0 ? 0 : 1;
 };
+/** This function reads a latitude or a longitude of the command line. */
+const coordinateOf = (name, text, limit) => {
+	const value = /^\s*-?\d+(?:\.\d+)?\s*$/.test(text) ? Number(text) : NaN;
+	if (!Number.isFinite(value) || Math.abs(value) > limit) throw new UsageError(`--${name} wants a number from -${String(limit)} to ${String(limit)}, not "${text}"`);
+	return value;
+};
+/** This function checks the options of `venue` and `product`, and gives the options of the recipe. */
+const recipeLineOf = (recipe, values, rest, page) => {
+	const url = rest[0] ?? "";
+	if (!/^https?:\/\//i.test(url) || !URL.canParse(url)) throw new UsageError(`${recipe} wants an http or https URL, not "${url}"`);
+	for (const agent of values["on-behalf-of"] ?? []) if (!/^[A-Za-z0-9_-]+$/.test(agent)) throw new UsageError(`--on-behalf-of wants the robots.txt token of an agent, such as Claude-User, not "${agent}"`);
+	const name = values.name;
+	if (name !== void 0 && name.trim() === "") throw new UsageError("--name wants the name of the venue");
+	if (values["require-name"] === true && name === void 0) throw new UsageError("--require-name goes with --name");
+	const address = values.address;
+	if (address !== void 0 && address.trim() === "") throw new UsageError("--address wants a street address");
+	if (values.lat === void 0 !== (values.lon === void 0)) throw new UsageError("--lat and --lon go together");
+	const lat = values.lat === void 0 ? void 0 : coordinateOf("lat", values.lat, 90);
+	const lon = values.lon === void 0 ? void 0 : coordinateOf("lon", values.lon, 180);
+	const osm = values.osm;
+	if (osm !== void 0 && osm.trim() === "") throw new UsageError("--osm wants an opening_hours value, such as \"Mo-Fr 08:00-18:00\"");
+	const pages = values.pages === void 0 ? void 0 : whole("pages", values.pages, 0, 0);
+	if (pages !== void 0 && pages > 8) throw new UsageError(`--pages wants a whole number from 0 to ${String(8)}`);
+	const compare = values.compare;
+	if (compare !== void 0 && compare.trim() === "") throw new UsageError("--compare wants a file");
+	return {
+		recipe,
+		url,
+		render: page.render,
+		maxAge: page.maxAge,
+		budget: page.budget,
+		onBehalfOf: values["on-behalf-of"],
+		userProvided: values["user-provided"] === true,
+		fallback: values["archive-fallback"] === true ? "archive" : void 0,
+		archiveDespiteRobots: values["archive-despite-robots"] === true,
+		name,
+		requireName: values["require-name"] === true,
+		location: address === void 0 && lat === void 0 ? void 0 : {
+			...address === void 0 ? {} : { address },
+			...lat === void 0 ? {} : {
+				lat,
+				lon
+			}
+		},
+		osm,
+		pages,
+		compare
+	};
+};
+/** The days of a week in their order, as `DAYS` of `hours.ts`. */
+const WEEK_DAYS = [
+	"mon",
+	"tue",
+	"wed",
+	"thu",
+	"fri",
+	"sat",
+	"sun"
+];
+const DAY_LABELS = {
+	mon: "Mon",
+	tue: "Tue",
+	wed: "Wed",
+	thu: "Thu",
+	fri: "Fri",
+	sat: "Sat",
+	sun: "Sun"
+};
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+/** This function tells if a value is a time of a kept week: `HH:MM` or `H:MM`, to `24:00`. */
+const isClock = (value) => {
+	if (typeof value !== "string") return false;
+	const match = /^(\d{1,2}):(\d{2})$/.exec(value);
+	return match !== null && Number(match[1]) <= 24 && Number(match[2]) < 60;
+};
+const intervalOf = (value) => Array.isArray(value) && value.length === 2 && isClock(value[0]) && isClock(value[1]) ? [value[0], value[1]] : void 0;
+/**
+* This function reads the kept week of `--compare`, a JSON file. The file is a week, or an object
+* with the field `week` (a reading of `venue --json`) or `hours_week`. Each day gives a list of
+* intervals (`[]` for closed), one interval, or null for closed. An error names the file.
+*/
+const keptWeekOf = (path) => {
+	const file = fileOf(path);
+	let text;
+	try {
+		text = readFileSync(file, "utf8");
+	} catch (error) {
+		const code = error.code;
+		throw new UsageError(`--compare: the file ${file} does not open (${typeof code === "string" ? code : "error"})`, { cause: error });
+	}
+	let data;
+	try {
+		data = JSON.parse(text.charCodeAt(0) === 65279 ? text.slice(1) : text);
+	} catch (error) {
+		throw new UsageError(`--compare: the file ${file} is not JSON`, { cause: error });
+	}
+	if (isRecord(data) && (isRecord(data["week"]) || isRecord(data["hours_week"]))) data = isRecord(data["week"]) ? data["week"] : data["hours_week"];
+	const wants = `--compare: the file ${file} is not a week. Write a list of intervals, one interval or null for each day, such as {"mon": [["09:00", "17:00"]], "sun": []}`;
+	if (!isRecord(data)) throw new UsageError(wants);
+	const week = {};
+	for (const [key, value] of Object.entries(data)) {
+		const day = WEEK_DAYS.find((one) => one === key);
+		if (day === void 0) throw new UsageError(`--compare: the file ${file} names "${key}", not a day: use mon, tue, wed, thu, fri, sat or sun`);
+		const one = intervalOf(value);
+		const intervals = value === null ? [] : one !== void 0 ? [one] : Array.isArray(value) ? value.map(intervalOf) : [void 0];
+		if (intervals.some((interval) => interval === void 0)) throw new UsageError(`${wants} (the value of ${key} is not)`);
+		week[day] = intervals;
+	}
+	return {
+		file,
+		week
+	};
+};
+/** This function writes the intervals of one day, as `describeWeek` of `hours.ts` does. */
+const dayText = (intervals) => intervals === void 0 ? "not stated" : intervals.length === 0 ? "closed" : intervals.map(([open, close]) => open === "00:00" && close === "00:00" ? "open 24 hours" : `${open}–${close}`).join(", ");
+/** This function gives the days of a comparison in their order. A day that no week states is not there. */
+const compareDays = (kept, read, comparison) => {
+	const days = [];
+	for (const day of WEEK_DAYS) {
+		const one = {
+			day,
+			kept: kept[day],
+			read: read[day]
+		};
+		if (one.kept === void 0 && one.read === void 0) continue;
+		const status = comparison.status === "new" || comparison.added.includes(day) ? "new" : comparison.days.includes(day) ? "changed" : comparison.skipped.includes(day) ? "skipped" : one.read === void 0 ? "not read" : "same";
+		if (comparison.status === "new" && one.read === void 0) continue;
+		days.push({
+			...one,
+			status
+		});
+	}
+	return days;
+};
+const QUOTE_CHARS = 200;
+const shortText = (text) => text.length > QUOTE_CHARS ? `${text.slice(0, QUOTE_CHARS)}…` : text;
+/** This function gives the lines of one reading: its rank, its source, its week and its quote. */
+const readingLines = (one) => {
+	const marks = [
+		one.schedule === "other" ? "not the opening hours" : null,
+		one.inferred ? "am or pm guessed" : null,
+		one.partial ? "a rule left out" : null,
+		one.season === null ? null : `season ${one.season}`
+	].filter((mark) => mark !== null);
+	const where = [
+		one.url,
+		one.path,
+		one.capture === null ? null : `capture ${one.capture}`,
+		one.archived === void 0 ? null : `archived copy (${one.archived.source}, captured ${one.archived.timestamp})`
+	].filter((part) => part !== null);
+	return [
+		`  rank ${String(one.rank)}  ${one.from}${one.label === null ? "" : ` "${one.label}"`}  ${one.hours === "" ? "no day" : one.hours}${marks.length > 0 ? ` (${marks.join(", ")})` : ""}`,
+		...where.length > 0 ? [`    ${where.join(" · ")}`] : [],
+		one.from === "text" ? `    quote: ${JSON.stringify(shortText(one.raw))}` : `    value: ${shortText(one.raw.replace(/\s+/g, " "))}`
+	];
+};
+const pageLine = (page) => {
+	const facts = [
+		`${page.outcome} ${String(page.status)}`,
+		page.from_cache ? "from the store" : null,
+		page.rendered ? "rendered" : null,
+		page.held_back ? "held back" : null,
+		page.archived === void 0 ? null : `archived copy (${page.archived.source})`
+	].filter((fact) => fact !== null);
+	return `  ${page.outcome === "ok" || page.outcome === "not-modified" || page.archived !== void 0 ? "✓" : "✗"} ${page.url}  ${facts.join(" · ")}`;
+};
+const WALL_OUTCOMES = /* @__PURE__ */ new Set([
+	"blocked",
+	"challenge",
+	"payment-required"
+]);
+/**
+* This function tells if a site refused the crawler, with the rule of `wall` of `readProduct`. The
+* causes are a 401, a 402 or a 403, a challenge, a bot check, robots.txt or a stop of the host.
+*/
+const refusedPage = (page) => WALL_OUTCOMES.has(page.outcome) || page.outcome === "skipped" && /^(?:robots\.txt disallows|host stopped)/.test(page.note ?? "") || page.doc === null && /\bbot check\b/.test(page.note ?? "");
+/**
+* This function gives the exit code of `venue`. It is 0 when the site gave a reading of the opening
+* hours. It is `WALL` when the home page refused the crawler, and `NOTHING_READ` for a different
+* result. An OpenStreetMap value alone does not count.
+*/
+const venueCode = (result) => {
+	if (result.readings.some((one) => one.from !== "osm" && one.schedule === "opening")) return 0;
+	const home = result.pages[0];
+	return home !== void 0 && refusedPage(home) ? 4 : 3;
+};
+/** This function gives the exit code of `product`: 0 for Product data, `WALL` for a refusal, else `NOTHING_READ`. */
+const productCode = (result) => result.found ? 0 : result.wall ? 4 : 3;
+/** This line tells how to read a held-back page. */
+const HELD_BACK_LINE = "Add --user-provided if you gave this URL yourself. The crawler logs that read.";
+/** This function gives the line of a page that needs a browser, when rendering is off. */
+const renderLine = (notes, renderOff) => renderOff !== void 0 && notes.some((note) => note !== null && /no browser/i.test(note)) ? [`rendering is off: ${renderOff}`] : [];
+/**
+* This function gives the lines of `venue`: the week of the site, and each reading with its source.
+* Then come the days where the readings disagree, the menus, the phone, the pages and the notes.
+* With `--compare`, the lines give each day of the comparison.
+*/
+const describeVenue = (result, compare, renderOff) => {
+	const code = venueCode(result);
+	const lines = [`${code === 0 ? "✓" : "✗"} ${result.home ?? result.url}`];
+	const site = result.from_site;
+	const home = result.pages[0];
+	if (site !== null) {
+		const source = [
+			`${site.from}, rank ${String(site.rank)}`,
+			site.url,
+			site.capture === null ? null : `capture ${site.capture}`
+		].filter((part) => part !== null);
+		lines.push(`  the site: ${site.hours}`, `  from ${source.join(", ")}`);
+	} else if (result.best !== null) lines.push(`  no reading of the site with no guess. The best reading (${result.best.from}, rank ${String(result.best.rank)}): ${result.best.hours}`);
+	else if (code === 4 && home !== void 0) lines.push(`  wall: the site refused the crawler (${home.outcome} ${String(home.status)}). The crawler does not try a different identity.`);
+	else lines.push("  no opening hours found");
+	if (result.readings.length > 0) {
+		lines.push("readings, the best first:");
+		for (const one of result.readings) lines.push(...readingLines(one));
+	}
+	for (const day of result.disagree) {
+		const said = result.readings.filter((one) => one.rank <= 2 && one.week[day] !== void 0).map((one) => `${dayText(one.week[day])} (rank ${String(one.rank)} ${one.from})`);
+		lines.push(`the readings disagree on ${DAY_LABELS[day]}: ${said.join(" · ")}`);
+	}
+	if (result.menus.length > 0) lines.push("menus:", ...result.menus.map((menu) => `  ${menu.url}${menu.text === "" ? "" : `  ${menu.text}`}`));
+	if (result.phone !== null) lines.push(`phone: ${result.phone}`);
+	if (result.moved !== null) lines.push(`moved: the site now goes to ${result.moved}`);
+	if (result.names_venue !== null) lines.push(result.names_venue ? "names the venue: yes" : "names the venue: no. The domain can belong to a different business now.");
+	if (result.held_back) lines.push(`held back: a page asked not to be AI input, so the recipe read nothing from it. ${HELD_BACK_LINE}`);
+	if (result.pages.length > 0) lines.push(`pages (${String(result.pages.length)}):`, ...result.pages.map(pageLine));
+	if (compare !== void 0) {
+		const comparison = compare.comparison;
+		if (comparison === null || site === null) lines.push(`compare with ${compare.file}: no reading of the site with no guess, so nothing to compare`);
+		else {
+			const changed = comparison.days.length > 0 ? ` (${comparison.days.map((day) => DAY_LABELS[day]).join(", ")})` : "";
+			lines.push(`compare with ${compare.file}: ${comparison.status}${changed}`);
+			for (const one of compareDays(compare.kept, site.week, comparison)) {
+				const text = one.status === "changed" || one.status === "skipped" ? `${dayText(one.kept)} → ${dayText(one.read)}` : one.status === "not read" ? `kept ${dayText(one.kept)}` : dayText(one.read);
+				lines.push(`  ${DAY_LABELS[one.day]}  ${one.status.padEnd(8)}  ${text}`);
+			}
+		}
+	}
+	if (result.note !== null) lines.push(`note: ${result.note}`);
+	lines.push(...renderLine([result.note], renderOff));
+	return lines;
+};
+/** This function gives the number of digits after the decimal point of a currency: 2 for EUR, 0 for JPY. */
+const fractionOf = (currency) => {
+	try {
+		return new Intl.NumberFormat("en", {
+			style: "currency",
+			currency
+		}).resolvedOptions().maximumFractionDigits ?? 2;
+	} catch {
+		return 2;
+	}
+};
+/** This function writes an amount of money with its ISO 4217 code: `1,299.00 EUR`, `980 JPY`. */
+const moneyText = (money) => {
+	const digits = fractionOf(money.currency);
+	return `${new Intl.NumberFormat("en", {
+		minimumFractionDigits: digits,
+		maximumFractionDigits: Math.max(digits, 4)
+	}).format(money.amount)} ${money.currency}`;
+};
+/**
+* This function gives the lines of `product`. They are the name, the brand and the codes, the price
+* as money, the availability, the variant, the seller, the page and the notes. A wall and a
+* held-back page say so.
+*/
+const describeProduct = (result, renderOff) => {
+	const code = productCode(result);
+	const lines = [`${code === 0 ? "✓" : "✗"} ${result.name ?? result.url}`];
+	if (result.name !== null) lines.push(`  ${result.url}`);
+	if (code === 4) lines.push(`  wall: the shop refused the crawler (${result.outcome} ${String(result.status)}). The crawler does not try a different identity.`);
+	if (result.held_back) lines.push(`  held back: the page asked not to be AI input. ${HELD_BACK_LINE}`);
+	const codes = [
+		result.brand === null ? null : `brand ${result.brand}`,
+		result.sku === null ? null : `sku ${result.sku}`,
+		result.gtin === null ? null : `gtin ${result.gtin}`
+	].filter((part) => part !== null);
+	if (codes.length > 0) lines.push(`  ${codes.join(" · ")}`);
+	if (result.price !== null) lines.push(result.price_to === null ? `  price ${moneyText(result.price)}` : `  price ${moneyText(result.price)} to ${moneyText(result.price_to)}, the range of the offers`);
+	if (result.availability !== null) lines.push(`  availability ${result.availability}${result.availabilities.length > 1 ? ` (the offers: ${result.availabilities.join(", ")})` : ""}`);
+	const variant = result.variant;
+	if (variant !== null) {
+		const parts = [
+			variant.name,
+			variant.sku === null ? null : `sku ${variant.sku}`,
+			variant.price === null ? null : moneyText(variant.price),
+			variant.availability
+		].filter((part) => part !== null);
+		lines.push(`  variant ${parts.length > 0 ? parts.join(" · ") : "with no data"}`);
+	}
+	if (result.seller !== null) lines.push(`  seller ${result.seller}`);
+	if (result.image !== null) lines.push(`  image ${result.image}`);
+	if (result.archived !== void 0) lines.push(`  archived copy (${result.archived.source}, captured ${result.archived.timestamp}): ${result.archived.url}`);
+	const facts = [
+		`${result.outcome} ${String(result.status)}`,
+		result.doc === null ? null : `doc ${result.doc}`,
+		result.capture === null ? null : `capture ${result.capture}`,
+		result.rendered ? "rendered" : null,
+		result.from_cache ? "from the store" : null
+	].filter((fact) => fact !== null);
+	lines.push(`  ${facts.join(" · ")}`);
+	if (result.note !== null) lines.push(`  note: ${result.note}`);
+	lines.push(...renderLine([result.note], renderOff).map((line) => `  ${line}`));
+	return lines;
+};
 /** This function checks the options of `export`, and gives its format and its options. */
 const exportPlan = (values) => {
 	const out = values.out;
@@ -780,6 +1136,11 @@ const readLine = (args, env) => {
 	} catch (error) {
 		throw new UsageError(`--max-age: ${error.message}`, { cause: error });
 	}
+	const recipeLine = command === "venue" || command === "product" ? recipeLineOf(command, values, rest, {
+		render,
+		maxAge,
+		budget: values.budget === void 0 ? void 0 : crawlLimits.budgetMs
+	}) : void 0;
 	return {
 		kind: "command",
 		command,
@@ -806,7 +1167,8 @@ const readLine = (args, env) => {
 		signKey,
 		signatureAgent,
 		embedder,
-		maxDocs
+		maxDocs,
+		recipeLine
 	};
 };
 //#endregion
@@ -827,7 +1189,7 @@ Usage:
   polite-crawler read <doc>            print a range of a stored document
   polite-crawler verify <doc> <quote>  check that a quote is in a stored document
   polite-crawler reindex               rebuild the search index from the capture log
-  polite-crawler mcp                   serve the tools over MCP (standard input and output)
+  polite-crawler mcp                   start the MCP server of polite-crawler-mcp (deprecated)
   polite-crawler export --out <file>   write the stored captures as a WARC or WACZ archive
   polite-crawler discover <site>       list the pages of a site from its sitemaps and feeds
   polite-crawler crawl <url...>        fetch a site and follow its links, within a budget
@@ -843,6 +1205,8 @@ Usage:
   polite-crawler extract <doc> --field name=value[:kind]...
                                        check proposed field values against a stored page
   polite-crawler embed --embedder <name>  give vectors to the stored pages, for find --rank hybrid
+  polite-crawler venue <url>           read the opening hours of a venue from its own site
+  polite-crawler product <url>         read the schema.org Product data of a product page
 
 Options:
   --dir <path>       the store (default $POLITE_CRAWLER_DIR, or ~/.polite-crawler)
@@ -937,6 +1301,20 @@ Options:
   embed    --budget <time>   the longest time (default 10m); no embedding starts after it
            --max-docs <n>    how many texts get vectors at most
            --kind <kind>     only this kind of document
+  venue    --name <text>     the name of the venue: the result tells if the home page names it
+           --require-name    with --name: read no hours from a site that does not name the venue
+           --address <text>  the street address, for a page that lists more than one location
+           --lat <n> --lon <n>  the position of the venue, for the same use (within 150 m);
+                             write a value below 0 after =, such as --lon=-20.5
+           --osm <value>     an OpenStreetMap opening_hours value, read as one more reading;
+                             no request goes to OpenStreetMap
+           --pages <n>       the pages after the home page: 0 to 8 (default 3)
+           --compare <file>  compare the week of the site with a kept week (JSON), day by day
+  venue, product  --render, --max-age, --archive-fallback, --archive-despite-robots  as for fetch
+           --budget <time>   no request starts after this time, such as 2m (default: no limit)
+           --on-behalf-of <agent>  the robots.txt rules of this agent apply too, such as
+                             Claude-User (repeat it)
+           --user-provided   as for read
 
 Exit codes:
   0  done
@@ -946,19 +1324,24 @@ Exit codes:
      or an error
      (fetch: a page with an archived copy counts as done)
   2  the command line is wrong
+  3  venue, product: nothing read: no opening hours of the site or no Product data, a page
+     held back, or a page that did not come
+  4  venue, product: the site refused the crawler (a wall): a 401, a 402 or a 403, a
+     challenge, a bot check, robots.txt or a stopped host. The crawler does not try a
+     different identity
 `;
 /**
 * This function does the command `export`, and gives the exit code. `readLine` checked the whole
 * command line first. The export only reads the store: it makes no store, and it sends no request.
 */
 const exportCommand = async (plan, dir, json) => {
-	const result = plan.format === "wacz" ? await (await import("./wacz-BFNG42Kt.mjs").then((n) => n.n)).exportWacz(dir, plan.options) : await (await import("./warc-CVS0t1fW.mjs").then((n) => n.a)).exportWarc(dir, plan.options);
+	const result = plan.format === "wacz" ? await (await import("./wacz-DKk2HBmx.mjs").then((n) => n.n)).exportWacz(dir, plan.options) : await (await import("./warc-C_ew76bV.mjs").then((n) => n.a)).exportWarc(dir, plan.options);
 	process.stdout.write(`${json ? JSON.stringify(result, null, 2) : describeExport(result, plan.options.includeRefusals)}\n`);
 	return 0;
 };
 /** This function prints the key directory of a key file, and tells where to host it. */
 const keyDirectoryCommand = async (key, agent) => {
-	const { KEY_DIRECTORY_PATH, KEY_DIRECTORY_TYPE, keyDirectory } = await import("./webbotauth-CjJdTAfE.mjs").then((n) => n.s);
+	const { KEY_DIRECTORY_PATH, KEY_DIRECTORY_TYPE, keyDirectory } = await import("./webbotauth-Dq_dA_u3.mjs").then((n) => n.s);
 	let directory;
 	try {
 		directory = keyDirectory([key]);
@@ -1074,7 +1457,7 @@ const runWatchCommand = async (crawler, line, values, limits, json, out) => {
 };
 /** This function gives the text of a diff for a person: the two captures, then the text of `formatDiff`. */
 const describeDiff = async (diff) => {
-	const { formatDiff } = await import("./diff-NQcQqKe9.mjs").then((n) => n.i);
+	const { formatDiff } = await import("./diff-CpKN0Jdc.mjs").then((n) => n.i);
 	return [
 		`${diff.doc}  ${diff.from.capture} (${diff.from.fetched_at}) -> ${diff.to.capture} (${diff.to.fetched_at})`,
 		formatDiff(diff),
@@ -1084,7 +1467,7 @@ const describeDiff = async (diff) => {
 };
 /** This function checks the fields against a stored document, and prints a line for each field. The exit code is 0 when each field is verified or normalized. */
 const runExtract = async (crawler, doc, fields, options, json, out) => {
-	const { extractDocument } = await import("./extract-ByFfc8Zk.mjs").then((n) => n.i);
+	const { extractDocument } = await import("./extract-DqeqiNHb.mjs").then((n) => n.i);
 	const claims = fields;
 	const result = crawler.extract === void 0 ? await extractDocument(crawler.runtime, doc, claims, options) : await crawler.extract(doc, claims, options);
 	if (json) out(JSON.stringify(result, null, 2));
@@ -1098,14 +1481,14 @@ const runExtract = async (crawler, doc, fields, options, json, out) => {
 const embedderFor = async (choice, dir) => {
 	try {
 		if (choice.kind === "ollama") {
-			const { ollamaEmbedder } = await import("./ollama-BJxkjny4.mjs").then((n) => n.a);
+			const { ollamaEmbedder } = await import("./ollama-D1xRh3hY.mjs").then((n) => n.a);
 			return ollamaEmbedder({
 				...choice.model === void 0 ? {} : { model: choice.model },
 				...choice.url === void 0 ? {} : { baseUrl: choice.url },
 				allowRemote: choice.allowRemote
 			});
 		}
-		const { transformersEmbedder } = await import("./transformers-BQMlaGj4.mjs").then((n) => n.i);
+		const { transformersEmbedder } = await import("./transformers-BNiuiLxT.mjs").then((n) => n.i);
 		return transformersEmbedder({
 			...choice.model === void 0 ? {} : { model: choice.model },
 			cacheDir: join(dir, "models")
@@ -1149,7 +1532,7 @@ const main = async (args) => {
 	if (command === "key-directory") return keyDirectoryCommand(keyFile(signKey), signatureAgent);
 	let signer;
 	if (signKey !== void 0 && signatureAgent !== void 0) {
-		const { createWebBotAuthSigner } = await import("./webbotauth-CjJdTAfE.mjs").then((n) => n.s);
+		const { createWebBotAuthSigner } = await import("./webbotauth-Dq_dA_u3.mjs").then((n) => n.s);
 		const privateKey = keyFile(signKey);
 		try {
 			signer = createWebBotAuthSigner({
@@ -1170,15 +1553,24 @@ const main = async (args) => {
 		...embedder === void 0 ? {} : { embedder }
 	};
 	if (command === "mcp") {
-		const { startMcpServer } = await import("./mcp.mjs");
-		await startMcpServer({
+		const { McpPackageMissing, loadMcpPackage } = await import("./mcp-package-Y0bREnOS.mjs");
+		let mcp;
+		try {
+			mcp = await loadMcpPackage();
+		} catch (error) {
+			if (!(error instanceof McpPackageMissing)) throw error;
+			process.stderr.write(`polite-crawler: ${error.message}\n`);
+			return 2;
+		}
+		process.stderr.write("polite-crawler: the command mcp is deprecated, and 0.3.0 removes it. Start npx polite-crawler-mcp.\n");
+		await mcp.startMcpServer({
 			...options,
 			allowUserProvided: line.allowUserProvided
 		});
 		return 0;
 	}
 	if (command === "daemon") {
-		const { daemonCommand } = await import("./cli-daemon--cTmq0Z5.mjs");
+		const { daemonCommand } = await import("./cli-daemon-qS4PH2bM.mjs");
 		return daemonCommand(line.daemonLine, options, {
 			allowUserProvided: line.allowUserProvided,
 			json: values.json === true,
@@ -1186,12 +1578,15 @@ const main = async (args) => {
 		});
 	}
 	if (line.exportPlan !== void 0) return exportCommand(line.exportPlan, dir, values.json === true);
-	const { openCrawler } = await import("./open-C4fhXJgD.mjs").then((n) => n.a);
+	const recipeLine = line.recipeLine;
+	const recipes = recipeLine === void 0 ? void 0 : await import("./cli-recipes-qbLRVThF.mjs");
+	const recipeInput = recipes === void 0 || recipeLine === void 0 ? void 0 : recipes.prepareRecipe(recipeLine);
+	const { openCrawler } = await import("./open-CEo22Hpu.mjs").then((n) => n.a);
 	let renderOff;
 	let renderer = options.renderer;
-	if (!(command === "fetch" || command === "crawl" || watchLine?.action === "run") || render === "never") renderer = false;
+	if (!(command === "fetch" || command === "crawl" || recipeLine !== void 0 || watchLine?.action === "run") || render === "never") renderer = false;
 	else if (renderer === void 0) {
-		const { browserMissing } = await import("./browser-CKRNbIDT.mjs").then((n) => n.n);
+		const { browserMissing } = await import("./browser-56dBBLec.mjs").then((n) => n.n);
 		renderOff = await browserMissing();
 		if (renderOff !== void 0) renderer = false;
 	}
@@ -1273,6 +1668,15 @@ const main = async (args) => {
 				maxDocs: line.maxDocs,
 				kind: values.kind
 			}, values.json === true, out);
+			case "venue":
+			case "product": {
+				const io = {
+					json: values.json === true,
+					out,
+					renderOff
+				};
+				return command === "venue" ? await recipes.venueCommand(crawler, recipeLine, recipeInput, io) : await recipes.productCommand(crawler, recipeLine, io);
+			}
 			default: {
 				const counts = await crawler.reindex();
 				print(counts, () => describeReindex(counts));
@@ -1292,6 +1696,6 @@ main(process.argv.slice(2)).then((code) => {
 	process.exitCode = usage ? 2 : 1;
 });
 //#endregion
-export { describeDaemonStatus as t };
+export { keptWeekOf as a, describeVenue as i, describeDaemonStatus as n, productCode as o, describeProduct as r, venueCode as s, UsageError as t };
 
 //# sourceMappingURL=cli.mjs.map