polite-crawler 0.0.0-stage → 0.1.0

package/build/classify.mjs

@@ -0,0 +1,116 @@
+//#region src/classify.ts
+const MONTHS = [
+	"Jan",
+	"Feb",
+	"Mar",
+	"Apr",
+	"May",
+	"Jun",
+	"Jul",
+	"Aug",
+	"Sep",
+	"Oct",
+	"Nov",
+	"Dec"
+];
+const DAY = "(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)";
+const LONG_DAY = "(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)";
+const MONTH = `(${MONTHS.join("|")})`;
+const TIME = "(\\d{2}):(\\d{2}):(\\d{2})";
+const IMF_FIXDATE = new RegExp(`^${DAY}, (\\d{2}) ${MONTH} (\\d{4}) ${TIME} GMT$`);
+const RFC_850 = new RegExp(`^${LONG_DAY}, (\\d{2})-${MONTH}-(\\d{2}) ${TIME} GMT$`);
+const ASCTIME = new RegExp(`^${DAY} ${MONTH} ( \\d|\\d{2}) ${TIME} (\\d{4})$`);
+/**
+* This function reads an HTTP date in the three forms of RFC 9110 §5.6.7: the IMF-fixdate, the
+* RFC 850 form and the asctime form. It gives milliseconds since the epoch, or no value for a text
+* in a different form. It does not use `Date.parse`, which reads many texts that are not dates.
+*/
+const parseHttpDate = (value) => {
+	if (value === void 0) return void 0;
+	const text = value.trim();
+	let parts;
+	const fixdate = IMF_FIXDATE.exec(text);
+	const rfc850 = fixdate === null ? RFC_850.exec(text) : null;
+	const asctime = fixdate === null && rfc850 === null ? ASCTIME.exec(text) : null;
+	if (fixdate !== null) parts = fixdate.slice(1);
+	else if (rfc850 !== null) {
+		const short = Number(rfc850[3]);
+		const century = 2e3 + short > (/* @__PURE__ */ new Date()).getUTCFullYear() + 50 ? 1900 : 2e3;
+		parts = [
+			rfc850[1],
+			rfc850[2],
+			String(century + short),
+			...rfc850.slice(4)
+		];
+	} else if (asctime !== null) parts = [
+		asctime[2],
+		asctime[1],
+		asctime[6],
+		asctime[3],
+		asctime[4],
+		asctime[5]
+	];
+	if (parts === void 0) return void 0;
+	const [day, month, year, hour, minute, second] = parts;
+	const at = Date.UTC(Number(year), MONTHS.indexOf(month), Number(day.trim()), Number(hour), Number(minute), Number(second));
+	return Number.isFinite(at) ? at : void 0;
+};
+/**
+* Retry-After is either delta-seconds or an HTTP date (RFC 9110 §10.2.3). The function reads only
+* these two forms. A different text, for example `1.5` or `-5`, gives no value, so the backoff
+* applies.
+*/
+const parseRetryAfter = (value, now) => {
+	if (value === void 0) return void 0;
+	const trimmed = value.trim();
+	if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1e3;
+	const at = parseHttpDate(trimmed);
+	return at === void 0 ? void 0 : Math.max(0, at - now);
+};
+/** After these outcomes, the crawler does not ask the host again until the stop ends. */
+const STOPS_HOST = /* @__PURE__ */ new Set([
+	"challenge",
+	"blocked",
+	"payment-required"
+]);
+/**
+* This function tells if a status is a client error with no meaning for the host. Such a status
+* is a 4xx other than 401, 402, 403, 404, 408, 410 and 429. The request was wrong, and a
+* different request can succeed. Thus the failure count of the host does not change.
+*/
+const isClientError = (status) => status >= 400 && status < 500 && ![
+	401,
+	402,
+	403,
+	404,
+	408,
+	410,
+	429
+].includes(status);
+const classify = (status, header, now) => {
+	const retryAfterMs = parseRetryAfter(header("retry-after"), now);
+	const result = (outcome, note) => ({
+		outcome,
+		retryAfterMs,
+		note
+	});
+	if (header("cf-mitigated")?.trim().toLowerCase() === "challenge") return result("challenge", "Cloudflare challenge");
+	const waf = header("x-amzn-waf-action")?.trim().toLowerCase();
+	if (waf !== void 0 && (waf === "challenge" || waf === "captcha" || status === 202 || status === 405)) return result("challenge", `AWS WAF ${waf}`);
+	if (status === 304) return result("not-modified");
+	if (status >= 200 && status < 300) return result("ok");
+	if (status >= 300 && status < 400) return result("redirect");
+	if (status === 402) {
+		const price = header("crawler-price");
+		return result("payment-required", price === void 0 ? void 0 : `price ${price}`);
+	}
+	if (status === 401 || status === 403) return result("blocked");
+	if (status === 404 || status === 410) return result("not-found");
+	if (status === 429) return result("rate-limited");
+	if (status === 502 || status === 503 || status === 504) return result("unavailable");
+	return result("error", status === 0 ? "no response" : `status ${String(status)}`);
+};
+//#endregion
+export { STOPS_HOST, classify, isClientError, parseHttpDate, parseRetryAfter };
+
+//# sourceMappingURL=classify.mjs.map

package/build/classify.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"classify.mjs","names":[],"sources":["../src/classify.ts"],"sourcesContent":["/**\n * The outcome tells what a response means for the crawl. The crawler decides it before it keeps\n * the body as content. The crawler sees a refusal and stops. It does not try again with a\n * different identity.\n *\n * - `challenge`: a bot challenge page. Cloudflare marks it with `cf-mitigated: challenge`. AWS WAF\n *   answers 202 (challenge) or 405 (CAPTCHA) with an `x-amzn-waf-action` header. The host stops\n *   for 24 hours.\n * - `payment-required` (402): a site that wants payment or a license for a crawl. The crawler\n *   records the price and does not pay. The host stops for 24 hours.\n * - `blocked` (401, 403): RFC 9110 tells a client not to send the request again automatically.\n *   The host stops for 1 hour.\n * - `rate-limited` (429) and `unavailable` (502, 503, 504): the host waits until the time of\n *   Retry-After, or for a backoff with jitter. A busy gateway is a reason to wait, not a failure.\n * - `error`: a different status, or no answer. A 5xx or no answer counts toward the stop of the\n *   host. A different 4xx (a 400 or a 422) is a problem of the request, and it does not count.\n *\n * The politeness policy (`PolitenessPolicy`) sets each time of a stop.\n */\n\nexport type Outcome =\n  | \"ok\"\n  | \"not-modified\"\n  | \"redirect\"\n  | \"challenge\"\n  | \"payment-required\"\n  | \"blocked\"\n  | \"not-found\"\n  | \"rate-limited\"\n  | \"unavailable\"\n  | \"error\";\n\nexport interface Classified {\n  readonly outcome: Outcome;\n  /** This value is the wait that the server asked for, when it asked for one. */\n  readonly retryAfterMs: number | undefined;\n  readonly note: string | undefined;\n}\n\nconst MONTHS = [\"Jan\", \"Feb\", \"Mar\", \"Apr\", \"May\", \"Jun\", \"Jul\", \"Aug\", \"Sep\", \"Oct\", \"Nov\", \"Dec\"];\nconst DAY = \"(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\";\nconst LONG_DAY = \"(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\";\nconst MONTH = `(${MONTHS.join(\"|\")})`;\nconst TIME = \"(\\\\d{2}):(\\\\d{2}):(\\\\d{2})\";\nconst IMF_FIXDATE = new RegExp(`^${DAY}, (\\\\d{2}) ${MONTH} (\\\\d{4}) ${TIME} GMT$`);\nconst RFC_850 = new RegExp(`^${LONG_DAY}, (\\\\d{2})-${MONTH}-(\\\\d{2}) ${TIME} GMT$`);\nconst ASCTIME = new RegExp(`^${DAY} ${MONTH} ( \\\\d|\\\\d{2}) ${TIME} (\\\\d{4})$`);\n\n/**\n * This function reads an HTTP date in the three forms of RFC 9110 §5.6.7: the IMF-fixdate, the\n * RFC 850 form and the asctime form. It gives milliseconds since the epoch, or no value for a text\n * in a different form. It does not use `Date.parse`, which reads many texts that are not dates.\n */\nexport const parseHttpDate = (value: string | undefined): number | undefined => {\n  if (value === undefined) return undefined;\n  const text = value.trim();\n  // each form gives day, month, year, hour, minute and second\n  let parts: readonly string[] | undefined;\n  const fixdate = IMF_FIXDATE.exec(text);\n  const rfc850 = fixdate === null ? RFC_850.exec(text) : null;\n  const asctime = fixdate === null && rfc850 === null ? ASCTIME.exec(text) : null;\n  if (fixdate !== null) parts = fixdate.slice(1);\n  else if (rfc850 !== null) {\n    const short = Number(rfc850[3]);\n    // RFC 9110: a two-digit year more than 50 years in the future is in the past century\n    const century = 2000 + short > new Date().getUTCFullYear() + 50 ? 1900 : 2000;\n    parts = [rfc850[1]!, rfc850[2]!, String(century + short), ...rfc850.slice(4)];\n  } else if (asctime !== null) {\n    parts = [asctime[2]!, asctime[1]!, asctime[6]!, asctime[3]!, asctime[4]!, asctime[5]!];\n  }\n  if (parts === undefined) return undefined;\n  const [day, month, year, hour, minute, second] = parts as [\n    string,\n    string,\n    string,\n    string,\n    string,\n    string,\n  ];\n  const at = Date.UTC(\n    Number(year),\n    MONTHS.indexOf(month),\n    Number(day.trim()),\n    Number(hour),\n    Number(minute),\n    Number(second),\n  );\n  return Number.isFinite(at) ? at : undefined;\n};\n\n/**\n * Retry-After is either delta-seconds or an HTTP date (RFC 9110 §10.2.3). The function reads only\n * these two forms. A different text, for example `1.5` or `-5`, gives no value, so the backoff\n * applies.\n */\nexport const parseRetryAfter = (value: string | undefined, now: number): number | undefined => {\n  if (value === undefined) return undefined;\n  const trimmed = value.trim();\n  if (/^\\d+$/.test(trimmed)) return Number(trimmed) * 1000;\n  const at = parseHttpDate(trimmed);\n  return at === undefined ? undefined : Math.max(0, at - now);\n};\n\n/** After these outcomes, the crawler does not ask the host again until the stop ends. */\nexport const STOPS_HOST: ReadonlySet<Outcome> = new Set([\n  \"challenge\",\n  \"blocked\",\n  \"payment-required\",\n]);\n\n/**\n * This function tells if a status is a client error with no meaning for the host. Such a status\n * is a 4xx other than 401, 402, 403, 404, 408, 410 and 429. The request was wrong, and a\n * different request can succeed. Thus the failure count of the host does not change.\n */\nexport const isClientError = (status: number): boolean =>\n  status >= 400 && status < 500 && ![401, 402, 403, 404, 408, 410, 429].includes(status);\n\nexport const classify = (\n  status: number,\n  header: (name: string) => string | undefined,\n  now: number,\n): Classified => {\n  const retryAfterMs = parseRetryAfter(header(\"retry-after\"), now);\n  const result = (outcome: Outcome, note?: string): Classified => ({\n    outcome,\n    retryAfterMs,\n    note,\n  });\n  if (header(\"cf-mitigated\")?.trim().toLowerCase() === \"challenge\") {\n    return result(\"challenge\", \"Cloudflare challenge\");\n  }\n  const waf = header(\"x-amzn-waf-action\")?.trim().toLowerCase();\n  if (\n    waf !== undefined &&\n    (waf === \"challenge\" || waf === \"captcha\" || status === 202 || status === 405)\n  ) {\n    return result(\"challenge\", `AWS WAF ${waf}`);\n  }\n  if (status === 304) return result(\"not-modified\");\n  if (status >= 200 && status < 300) return result(\"ok\");\n  if (status >= 300 && status < 400) return result(\"redirect\");\n  if (status === 402) {\n    const price = header(\"crawler-price\");\n    return result(\"payment-required\", price === undefined ? undefined : `price ${price}`);\n  }\n  if (status === 401 || status === 403) return result(\"blocked\");\n  if (status === 404 || status === 410) return result(\"not-found\");\n  if (status === 429) return result(\"rate-limited\");\n  if (status === 502 || status === 503 || status === 504) return result(\"unavailable\");\n  return result(\"error\", status === 0 ? \"no response\" : `status ${String(status)}`);\n};\n"],"mappings":";AAuCA,MAAM,SAAS;CAAC;CAAO;CAAO;CAAO;CAAO;CAAO;CAAO;CAAO;CAAO;CAAO;CAAO;CAAO;AAAK;AAClG,MAAM,MAAM;AACZ,MAAM,WAAW;AACjB,MAAM,QAAQ,IAAI,OAAO,KAAK,GAAG,EAAE;AACnC,MAAM,OAAO;AACb,MAAM,cAAc,IAAI,OAAO,IAAI,IAAI,aAAa,MAAM,YAAY,KAAK,MAAM;AACjF,MAAM,UAAU,IAAI,OAAO,IAAI,SAAS,aAAa,MAAM,YAAY,KAAK,MAAM;AAClF,MAAM,UAAU,IAAI,OAAO,IAAI,IAAI,GAAG,MAAM,iBAAiB,KAAK,WAAW;;;;;;AAO7E,MAAa,iBAAiB,UAAkD;CAC9E,IAAI,UAAU,KAAA,GAAW,OAAO,KAAA;CAChC,MAAM,OAAO,MAAM,KAAK;CAExB,IAAI;CACJ,MAAM,UAAU,YAAY,KAAK,IAAI;CACrC,MAAM,SAAS,YAAY,OAAO,QAAQ,KAAK,IAAI,IAAI;CACvD,MAAM,UAAU,YAAY,QAAQ,WAAW,OAAO,QAAQ,KAAK,IAAI,IAAI;CAC3E,IAAI,YAAY,MAAM,QAAQ,QAAQ,MAAM,CAAC;MACxC,IAAI,WAAW,MAAM;EACxB,MAAM,QAAQ,OAAO,OAAO,EAAE;EAE9B,MAAM,UAAU,MAAO,yBAAQ,IAAI,KAAK,EAAA,CAAE,eAAe,IAAI,KAAK,OAAO;EACzE,QAAQ;GAAC,OAAO;GAAK,OAAO;GAAK,OAAO,UAAU,KAAK;GAAG,GAAG,OAAO,MAAM,CAAC;EAAC;CAC9E,OAAO,IAAI,YAAY,MACrB,QAAQ;EAAC,QAAQ;EAAK,QAAQ;EAAK,QAAQ;EAAK,QAAQ;EAAK,QAAQ;EAAK,QAAQ;CAAG;CAEvF,IAAI,UAAU,KAAA,GAAW,OAAO,KAAA;CAChC,MAAM,CAAC,KAAK,OAAO,MAAM,MAAM,QAAQ,UAAU;CAQjD,MAAM,KAAK,KAAK,IACd,OAAO,IAAI,GACX,OAAO,QAAQ,KAAK,GACpB,OAAO,IAAI,KAAK,CAAC,GACjB,OAAO,IAAI,GACX,OAAO,MAAM,GACb,OAAO,MAAM,CACf;CACA,OAAO,OAAO,SAAS,EAAE,IAAI,KAAK,KAAA;AACpC;;;;;;AAOA,MAAa,mBAAmB,OAA2B,QAAoC;CAC7F,IAAI,UAAU,KAAA,GAAW,OAAO,KAAA;CAChC,MAAM,UAAU,MAAM,KAAK;CAC3B,IAAI,QAAQ,KAAK,OAAO,GAAG,OAAO,OAAO,OAAO,IAAI;CACpD,MAAM,KAAK,cAAc,OAAO;CAChC,OAAO,OAAO,KAAA,IAAY,KAAA,IAAY,KAAK,IAAI,GAAG,KAAK,GAAG;AAC5D;;AAGA,MAAa,6BAAmC,IAAI,IAAI;CACtD;CACA;CACA;AACF,CAAC;;;;;;AAOD,MAAa,iBAAiB,WAC5B,UAAU,OAAO,SAAS,OAAO,CAAC;CAAC;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;AAAG,CAAC,CAAC,SAAS,MAAM;AAEvF,MAAa,YACX,QACA,QACA,QACe;CACf,MAAM,eAAe,gBAAgB,OAAO,aAAa,GAAG,GAAG;CAC/D,MAAM,UAAU,SAAkB,UAA+B;EAC/D;EACA;EACA;CACF;CACA,IAAI,OAAO,cAAc,CAAC,EAAE,KAAK,CAAC,CAAC,YAAY,MAAM,aACnD,OAAO,OAAO,aAAa,sBAAsB;CAEnD,MAAM,MAAM,OAAO,mBAAmB,CAAC,EAAE,KAAK,CAAC,CAAC,YAAY;CAC5D,IACE,QAAQ,KAAA,MACP,QAAQ,eAAe,QAAQ,aAAa,WAAW,OAAO,WAAW,MAE1E,OAAO,OAAO,aAAa,WAAW,KAAK;CAE7C,IAAI,WAAW,KAAK,OAAO,OAAO,cAAc;CAChD,IAAI,UAAU,OAAO,SAAS,KAAK,OAAO,OAAO,IAAI;CACrD,IAAI,UAAU,OAAO,SAAS,KAAK,OAAO,OAAO,UAAU;CAC3D,IAAI,WAAW,KAAK;EAClB,MAAM,QAAQ,OAAO,eAAe;EACpC,OAAO,OAAO,oBAAoB,UAAU,KAAA,IAAY,KAAA,IAAY,SAAS,OAAO;CACtF;CACA,IAAI,WAAW,OAAO,WAAW,KAAK,OAAO,OAAO,SAAS;CAC7D,IAAI,WAAW,OAAO,WAAW,KAAK,OAAO,OAAO,WAAW;CAC/D,IAAI,WAAW,KAAK,OAAO,OAAO,cAAc;CAChD,IAAI,WAAW,OAAO,WAAW,OAAO,WAAW,KAAK,OAAO,OAAO,aAAa;CACnF,OAAO,OAAO,SAAS,WAAW,IAAI,gBAAgB,UAAU,OAAO,MAAM,GAAG;AAClF"}

package/build/cli.mjs

@@ -0,0 +1,348 @@
+#!/usr/bin/env node
+import { r as VERSION } from "./version-rVx8_y1_.mjs";
+import { join, resolve } from "node:path";
+import { homedir } from "node:os";
+import { parseArgs } from "node:util";
+//#region src/cli.ts
+const emitWarning = process.emitWarning.bind(process);
+process.emitWarning = ((warning, ...rest) => {
+	const first = rest[0];
+	const kind = typeof first === "string" ? first : typeof first === "object" && first !== null && "type" in first ? first.type : warning instanceof Error ? warning.name : void 0;
+	const text = typeof warning === "string" ? warning : warning.message;
+	if (kind === "ExperimentalWarning" && text.startsWith("SQLite is an experimental feature")) return;
+	emitWarning(warning, ...rest);
+});
+const HELP = `polite-crawler ${VERSION}: fetch a source once, keep it, read it many times.
+
+Usage:
+  polite-crawler fetch <url...>        fetch pages into the store (robots.txt obeyed)
+  polite-crawler find <query>          search the stored pages, with no request
+  polite-crawler read <doc>            print a range of a stored document
+  polite-crawler verify <doc> <quote>  check that a quote is in a stored document
+  polite-crawler reindex               rebuild the search index from the capture log
+  polite-crawler mcp                   serve the tools over MCP (standard input and output)
+
+Options:
+  --dir <path>       the store (default $POLITE_CRAWLER_DIR, or ~/.polite-crawler)
+  --contact <text>   an email (From and User-Agent) or a URL (User-Agent only)
+                     ($POLITE_CRAWLER_CONTACT)
+  --allow-private    fetch loopback and private addresses too ($POLITE_CRAWLER_ALLOW_PRIVATE=1)
+  --no-browser       never open the headless browser
+  --json             print JSON
+  -v, --version      print the version
+  -h, --help         print this help
+
+  fetch    --render <mode>   auto (default), never or always
+           --max-age <age>   reuse a stored copy this young: 30m, 24h (default), 7d; 0 fetches again
+  find     --limit <n>       how many results (default 10)
+           --kind <kind>     only this kind of document, such as page or pdf
+  read     --from <n>        where the range starts (default 0)
+           --length <n>      how many characters (default 4000)
+           --user-provided   you gave this URL yourself: read a page that refuses AI input (logged)
+  reindex                    rebuild; the documents of recipes that this command cannot read stay
+  mcp      --allow-user-provided   let user_provided work when the client cannot ask you
+                                   ($POLITE_CRAWLER_ALLOW_USER_PROVIDED=1)
+
+Exit codes:
+  0  done
+  1  not done: a page not fetched, no match, a quote not found, a document unknown or
+     held back, or an error
+  2  the command line is wrong
+`;
+/** A usage error is a wrong command line. It exits with code 2. */
+var UsageError = class extends Error {};
+const OPTIONS = {
+	dir: { type: "string" },
+	contact: { type: "string" },
+	"allow-private": { type: "boolean" },
+	"no-browser": { type: "boolean" },
+	json: { type: "boolean" },
+	version: {
+		type: "boolean",
+		short: "v"
+	},
+	help: {
+		type: "boolean",
+		short: "h"
+	},
+	render: { type: "string" },
+	"max-age": { type: "string" },
+	limit: { type: "string" },
+	kind: { type: "string" },
+	from: { type: "string" },
+	length: { type: "string" },
+	"user-provided": { type: "boolean" },
+	"allow-user-provided": { type: "boolean" }
+};
+/** These options belong to one command. */
+const OWNER = {
+	render: "fetch",
+	"max-age": "fetch",
+	limit: "find",
+	kind: "find",
+	from: "read",
+	length: "read",
+	"user-provided": "read",
+	"allow-user-provided": "mcp"
+};
+/** These are the commands, with the smallest and the largest count of arguments. */
+const COMMANDS = {
+	fetch: {
+		min: 1,
+		max: Infinity,
+		wants: "fetch wants at least one URL"
+	},
+	find: {
+		min: 1,
+		max: Infinity,
+		wants: "find wants a query"
+	},
+	read: {
+		min: 1,
+		max: 1,
+		wants: "read wants one document id"
+	},
+	verify: {
+		min: 2,
+		max: Infinity,
+		wants: "verify wants a document id and a quote"
+	},
+	reindex: {
+		min: 0,
+		max: 0,
+		wants: "reindex takes no argument"
+	},
+	mcp: {
+		min: 0,
+		max: 0,
+		wants: "mcp takes no argument"
+	}
+};
+const parse = (args) => {
+	try {
+		return parseArgs({
+			args: [...args],
+			allowPositionals: true,
+			strict: true,
+			options: OPTIONS
+		});
+	} catch (error) {
+		const code = error.code;
+		if (typeof code === "string" && code.startsWith("ERR_PARSE_ARGS")) throw new UsageError(error.message, { cause: error });
+		throw error;
+	}
+};
+const whole = (name, text, fallback, min) => {
+	if (text === void 0) return fallback;
+	if (!/^\s*\d+\s*$/.test(text)) throw new UsageError(`--${name} wants a whole number, not "${text}"`);
+	const value = Number(text);
+	if (value < min) throw new UsageError(`--${name} wants ${String(min)} or more`);
+	return value;
+};
+/** An environment variable that switches a feature on is 1 or 0 (also true or false, yes or no). */
+const envFlag = (name) => {
+	const raw = process.env[name];
+	const value = raw?.trim().toLowerCase() ?? "";
+	if ([
+		"",
+		"0",
+		"false",
+		"no",
+		"off"
+	].includes(value)) return false;
+	if ([
+		"1",
+		"true",
+		"yes",
+		"on"
+	].includes(value)) return true;
+	throw new UsageError(`${name} is 1 or 0, not "${raw ?? ""}"`);
+};
+/** The store: a leading `~` is the home folder, and a relative path starts at the working folder. */
+const storeDir = (text) => {
+	if (text === void 0) return join(homedir(), ".polite-crawler");
+	if (text.trim() === "") throw new UsageError("--dir wants a folder");
+	if (text === "~") return homedir();
+	if (/^~[\\/]/.test(text)) return resolve(homedir(), text.slice(2));
+	return resolve(text);
+};
+const FS_CODES = /* @__PURE__ */ new Set([
+	"EEXIST",
+	"ENOENT",
+	"ENOTDIR",
+	"EACCES",
+	"EPERM",
+	"EROFS",
+	"EISDIR"
+]);
+const ok = (page) => page.outcome === "ok" || page.outcome === "not-modified";
+const describePage = (page) => {
+	return [
+		`${ok(page) ? "✓" : "✗"} ${page.url}`,
+		`  ${[
+			`${page.outcome} ${String(page.status)}`,
+			page.doc === null ? null : `doc ${page.doc}`,
+			page.chars > 0 ? `${String(page.chars)} chars` : null,
+			page.rendered ? "rendered" : null,
+			page.structured.length > 0 ? `JSON-LD: ${page.structured.join(", ")}` : null
+		].filter((fact) => fact !== null).join(" · ")}`,
+		...page.note === null ? [] : [`  ${page.note}`]
+	].join("\n");
+};
+/** This is the result of a URL that the crawler did not fetch. */
+const failedPage = (url, note) => ({
+	url,
+	doc: null,
+	outcome: "error",
+	status: 0,
+	title: null,
+	chars: 0,
+	rendered: false,
+	structured: [],
+	links: 0,
+	captures: [],
+	capture: null,
+	capture_count: 0,
+	fetched_at: null,
+	from_cache: false,
+	held_back: false,
+	note
+});
+/** This function does the command of the command line, and gives the exit code. */
+const main = async (args) => {
+	const { values, positionals } = parse(args);
+	const out = (text) => void process.stdout.write(`${text}\n`);
+	if (values.version === true) {
+		out(VERSION);
+		return 0;
+	}
+	const [command, ...rest] = positionals;
+	if (values.help === true || command === void 0) {
+		process.stdout.write(HELP);
+		return 0;
+	}
+	const spec = COMMANDS[command];
+	if (spec === void 0) throw new UsageError(`no command "${command}"`);
+	if (rest.length < spec.min || rest.length > spec.max) throw new UsageError(spec.wants);
+	for (const [option, owner] of Object.entries(OWNER)) if (values[option] !== void 0 && owner !== command) throw new UsageError(`--${option} is an option of ${owner}`);
+	const render = values.render ?? "auto";
+	if (render !== "auto" && render !== "never" && render !== "always") throw new UsageError(`--render is auto, never or always, not "${render}"`);
+	const limit = whole("limit", values.limit, 10, 1);
+	const from = whole("from", values.from, 0, 0);
+	const length = whole("length", values.length, 4e3, 1);
+	const dir = storeDir(values.dir ?? (process.env["POLITE_CRAWLER_DIR"] || void 0));
+	const allowPrivate = values["allow-private"] === true || envFlag("POLITE_CRAWLER_ALLOW_PRIVATE");
+	const allowUserProvided = values["allow-user-provided"] === true || command === "mcp" && envFlag("POLITE_CRAWLER_ALLOW_USER_PROVIDED");
+	const options = {
+		dir,
+		contact: values.contact ?? (process.env["POLITE_CRAWLER_CONTACT"] || void 0),
+		...values["no-browser"] === true ? { renderer: false } : {},
+		...allowPrivate ? { egress: { allowPrivate: true } } : {}
+	};
+	if (command === "mcp") {
+		const { startMcpServer } = await import("./mcp.mjs");
+		await startMcpServer({
+			...options,
+			allowUserProvided
+		});
+		return 0;
+	}
+	const { durationMs, openCrawler } = await import("./open-CJR4b44v.mjs").then((n) => n.o);
+	let maxAge;
+	try {
+		maxAge = durationMs(values["max-age"] ?? "24h");
+	} catch (error) {
+		throw new UsageError(`--max-age: ${error.message}`, { cause: error });
+	}
+	let renderOff;
+	let renderer = options.renderer;
+	if (command !== "fetch" || render === "never") renderer = false;
+	else if (renderer === void 0) {
+		const { browserMissing } = await import("./browser-gJ_h4AhB.mjs").then((n) => n.n);
+		renderOff = await browserMissing();
+		if (renderOff !== void 0) renderer = false;
+	}
+	let crawler;
+	try {
+		crawler = openCrawler({
+			...options,
+			renderer
+		});
+	} catch (error) {
+		const code = error.code;
+		if (typeof code === "string" && FS_CODES.has(code)) throw new Error(`the store ${dir} does not open: it is not a folder that polite-crawler can write (${code})`, { cause: error });
+		throw error;
+	}
+	const print = (value, text) => out(values.json === true ? JSON.stringify(value, null, 2) : text());
+	try {
+		switch (command) {
+			case "fetch": {
+				const urls = [...new Set(rest)];
+				const pages = (await Promise.allSettled(urls.map((url) => crawler.fetch(url, {
+					render,
+					maxAge
+				})))).map((result, at) => {
+					const url = urls[at];
+					if (result.status === "rejected") {
+						const reason = result.reason;
+						return failedPage(url, `${url}: ${reason instanceof Error ? reason.message : String(reason)}`);
+					}
+					const page = result.value;
+					return renderOff !== void 0 && page.note !== null && /no browser/i.test(page.note) ? {
+						...page,
+						note: `this page needs JavaScript, and rendering is off: ${renderOff}`
+					} : page;
+				});
+				print(pages, () => pages.map(describePage).join("\n"));
+				return pages.every(ok) ? 0 : 1;
+			}
+			case "find": {
+				const found = await crawler.find(rest.join(" "), {
+					limit,
+					kind: values.kind ?? null
+				});
+				print(found, () => found.length === 0 ? "nothing stored matches" : found.map((hit) => `${hit.doc}  ${hit.title}\n  ${hit.snippet.replace(/\s+/g, " ")}`).join("\n"));
+				return found.length === 0 ? 1 : 0;
+			}
+			case "read": {
+				const userProvided = values["user-provided"] === true;
+				const read = await crawler.read(rest[0], {
+					from,
+					length,
+					userProvided
+				});
+				print(read, () => read.held_back ? `${read.note ?? ""}\nAdd --user-provided if you gave this URL yourself. The crawler logs that read.` : `${read.title}\n${read.url ?? ""}\n[${String(read.from)}–${String(read.to)} of ${String(read.total)}]\n\n${read.text}`);
+				return read.held_back ? 1 : 0;
+			}
+			case "verify": {
+				const [doc, ...quote] = rest;
+				const verified = await crawler.verify(doc, quote.join(" "));
+				print(verified, () => verified.found ? `found at ${String(verified.start)}–${String(verified.end)} in ${verified.capture ?? ""}${verified.stale ? " (an earlier capture)" : ""}: ${verified.excerpt ?? ""}` : verified.held_back ? verified.note ?? "held back" : "not found");
+				return verified.found ? 0 : 1;
+			}
+			default: {
+				const counts = await crawler.reindex();
+				print(counts, () => [
+					`${String(counts.captures)} captures replayed, ${String(counts.docs)} documents, ${String(counts.urls)} URLs`,
+					...counts.carried > 0 ? [`${String(counts.carried)} documents of other recipes kept: ${counts.unknown_recipes.join(", ")}`] : [],
+					...counts.failed + counts.missing + counts.damaged > 0 ? [`skipped: ${String(counts.failed)} captures whose reader failed, ${String(counts.missing)} with no stored body, ${String(counts.damaged)} damaged lines of the log`] : []
+				].join("\n"));
+				return 0;
+			}
+		}
+	} finally {
+		await crawler.close();
+	}
+};
+main(process.argv.slice(2)).then((code) => {
+	process.exitCode = code;
+}, (error) => {
+	const message = error instanceof Error ? error.message : String(error);
+	const usage = error instanceof UsageError;
+	process.stderr.write(`polite-crawler: ${message}${usage ? " (polite-crawler --help lists the commands)" : ""}\n`);
+	process.exitCode = usage ? 2 : 1;
+});
+//#endregion
+export {};
+
+//# sourceMappingURL=cli.mjs.map

package/build/cli.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.mjs","names":[],"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { homedir } from \"node:os\";\nimport { join, resolve } from \"node:path\";\nimport { parseArgs } from \"node:util\";\nimport type { OpenOptions, PoliteCrawler } from \"./open.ts\";\nimport type { PageResult } from \"./runtime.ts\";\nimport { VERSION } from \"./version.ts\";\n\n// This file imports only Node.js and the version at start. A command loads the crawler, and only\n// `mcp` loads the MCP server, so `--version` and `--help` answer at once.\n\n// node:sqlite of Node.js 22 writes \"SQLite is an experimental feature\" to the standard error at\n// each start. The command removes only that warning, before the store loads node:sqlite.\nconst emitWarning = process.emitWarning.bind(process) as (\n  warning: string | Error,\n  ...rest: unknown[]\n) => void;\nprocess.emitWarning = ((warning: string | Error, ...rest: unknown[]): void => {\n  const first: unknown = rest[0];\n  const kind =\n    typeof first === \"string\"\n      ? first\n      : typeof first === \"object\" && first !== null && \"type\" in first\n        ? first.type\n        : warning instanceof Error\n          ? warning.name\n          : undefined;\n  const text = typeof warning === \"string\" ? warning : warning.message;\n  if (kind === \"ExperimentalWarning\" && text.startsWith(\"SQLite is an experimental feature\"))\n    return;\n  emitWarning(warning, ...rest);\n}) as typeof process.emitWarning;\n\nconst HELP = `polite-crawler ${VERSION}: fetch a source once, keep it, read it many times.\n\nUsage:\n  polite-crawler fetch <url...>        fetch pages into the store (robots.txt obeyed)\n  polite-crawler find <query>          search the stored pages, with no request\n  polite-crawler read <doc>            print a range of a stored document\n  polite-crawler verify <doc> <quote>  check that a quote is in a stored document\n  polite-crawler reindex               rebuild the search index from the capture log\n  polite-crawler mcp                   serve the tools over MCP (standard input and output)\n\nOptions:\n  --dir <path>       the store (default $POLITE_CRAWLER_DIR, or ~/.polite-crawler)\n  --contact <text>   an email (From and User-Agent) or a URL (User-Agent only)\n                     ($POLITE_CRAWLER_CONTACT)\n  --allow-private    fetch loopback and private addresses too ($POLITE_CRAWLER_ALLOW_PRIVATE=1)\n  --no-browser       never open the headless browser\n  --json             print JSON\n  -v, --version      print the version\n  -h, --help         print this help\n\n  fetch    --render <mode>   auto (default), never or always\n           --max-age <age>   reuse a stored copy this young: 30m, 24h (default), 7d; 0 fetches again\n  find     --limit <n>       how many results (default 10)\n           --kind <kind>     only this kind of document, such as page or pdf\n  read     --from <n>        where the range starts (default 0)\n           --length <n>      how many characters (default 4000)\n           --user-provided   you gave this URL yourself: read a page that refuses AI input (logged)\n  reindex                    rebuild; the documents of recipes that this command cannot read stay\n  mcp      --allow-user-provided   let user_provided work when the client cannot ask you\n                                   ($POLITE_CRAWLER_ALLOW_USER_PROVIDED=1)\n\nExit codes:\n  0  done\n  1  not done: a page not fetched, no match, a quote not found, a document unknown or\n     held back, or an error\n  2  the command line is wrong\n`;\n\n/** A usage error is a wrong command line. It exits with code 2. */\nclass UsageError extends Error {}\n\nconst OPTIONS = {\n  dir: { type: \"string\" },\n  contact: { type: \"string\" },\n  \"allow-private\": { type: \"boolean\" },\n  \"no-browser\": { type: \"boolean\" },\n  json: { type: \"boolean\" },\n  version: { type: \"boolean\", short: \"v\" },\n  help: { type: \"boolean\", short: \"h\" },\n  render: { type: \"string\" },\n  \"max-age\": { type: \"string\" },\n  limit: { type: \"string\" },\n  kind: { type: \"string\" },\n  from: { type: \"string\" },\n  length: { type: \"string\" },\n  \"user-provided\": { type: \"boolean\" },\n  \"allow-user-provided\": { type: \"boolean\" },\n} as const;\n\n/** These options belong to one command. */\nconst OWNER: Readonly<Record<string, string>> = {\n  render: \"fetch\",\n  \"max-age\": \"fetch\",\n  limit: \"find\",\n  kind: \"find\",\n  from: \"read\",\n  length: \"read\",\n  \"user-provided\": \"read\",\n  \"allow-user-provided\": \"mcp\",\n};\n\n/** These are the commands, with the smallest and the largest count of arguments. */\nconst COMMANDS: Readonly<\n  Record<string, { readonly min: number; readonly max: number; readonly wants: string }>\n> = {\n  fetch: { min: 1, max: Infinity, wants: \"fetch wants at least one URL\" },\n  find: { min: 1, max: Infinity, wants: \"find wants a query\" },\n  read: { min: 1, max: 1, wants: \"read wants one document id\" },\n  verify: { min: 2, max: Infinity, wants: \"verify wants a document id and a quote\" },\n  reindex: { min: 0, max: 0, wants: \"reindex takes no argument\" },\n  mcp: { min: 0, max: 0, wants: \"mcp takes no argument\" },\n};\n\nconst parse = (args: readonly string[]) => {\n  try {\n    return parseArgs({ args: [...args], allowPositionals: true, strict: true, options: OPTIONS });\n  } catch (error) {\n    // parseArgs gives a TypeError with an ERR_PARSE_ARGS code for a wrong command line\n    const code = (error as { code?: unknown }).code;\n    if (typeof code === \"string\" && code.startsWith(\"ERR_PARSE_ARGS\")) {\n      throw new UsageError((error as Error).message, { cause: error });\n    }\n    throw error;\n  }\n};\n\nconst whole = (name: string, text: string | undefined, fallback: number, min: number): number => {\n  if (text === undefined) return fallback;\n  if (!/^\\s*\\d+\\s*$/.test(text))\n    throw new UsageError(`--${name} wants a whole number, not \"${text}\"`);\n  const value = Number(text);\n  if (value < min) throw new UsageError(`--${name} wants ${String(min)} or more`);\n  return value;\n};\n\n/** An environment variable that switches a feature on is 1 or 0 (also true or false, yes or no). */\nconst envFlag = (name: string): boolean => {\n  const raw = process.env[name];\n  const value = raw?.trim().toLowerCase() ?? \"\";\n  if ([\"\", \"0\", \"false\", \"no\", \"off\"].includes(value)) return false;\n  if ([\"1\", \"true\", \"yes\", \"on\"].includes(value)) return true;\n  throw new UsageError(`${name} is 1 or 0, not \"${raw ?? \"\"}\"`);\n};\n\n/** The store: a leading `~` is the home folder, and a relative path starts at the working folder. */\nconst storeDir = (text: string | undefined): string => {\n  if (text === undefined) return join(homedir(), \".polite-crawler\");\n  if (text.trim() === \"\") throw new UsageError(\"--dir wants a folder\");\n  if (text === \"~\") return homedir();\n  if (/^~[\\\\/]/.test(text)) return resolve(homedir(), text.slice(2));\n  return resolve(text);\n};\n\nconst FS_CODES = new Set([\"EEXIST\", \"ENOENT\", \"ENOTDIR\", \"EACCES\", \"EPERM\", \"EROFS\", \"EISDIR\"]);\n\nconst ok = (page: PageResult): boolean => page.outcome === \"ok\" || page.outcome === \"not-modified\";\n\nconst describePage = (page: PageResult): string => {\n  const head = `${ok(page) ? \"✓\" : \"✗\"} ${page.url}`;\n  const facts = [\n    `${page.outcome} ${String(page.status)}`,\n    page.doc === null ? null : `doc ${page.doc}`,\n    page.chars > 0 ? `${String(page.chars)} chars` : null,\n    page.rendered ? \"rendered\" : null,\n    page.structured.length > 0 ? `JSON-LD: ${page.structured.join(\", \")}` : null,\n  ].filter((fact) => fact !== null);\n  return [head, `  ${facts.join(\" · \")}`, ...(page.note === null ? [] : [`  ${page.note}`])].join(\n    \"\\n\",\n  );\n};\n\n/** This is the result of a URL that the crawler did not fetch. */\nconst failedPage = (url: string, note: string): PageResult => ({\n  url,\n  doc: null,\n  outcome: \"error\",\n  status: 0,\n  title: null,\n  chars: 0,\n  rendered: false,\n  structured: [],\n  links: 0,\n  captures: [],\n  capture: null,\n  capture_count: 0,\n  fetched_at: null,\n  from_cache: false,\n  held_back: false,\n  note,\n});\n\n/** This function does the command of the command line, and gives the exit code. */\nconst main = async (args: readonly string[]): Promise<number> => {\n  const { values, positionals } = parse(args);\n  const out = (text: string): void => void process.stdout.write(`${text}\\n`);\n  if (values.version === true) {\n    out(VERSION);\n    return 0;\n  }\n  const [command, ...rest] = positionals;\n  if (values.help === true || command === undefined) {\n    process.stdout.write(HELP);\n    return 0;\n  }\n\n  // the whole command line is valid before anything opens the store\n  const spec = COMMANDS[command];\n  if (spec === undefined) throw new UsageError(`no command \"${command}\"`);\n  if (rest.length < spec.min || rest.length > spec.max) throw new UsageError(spec.wants);\n  for (const [option, owner] of Object.entries(OWNER)) {\n    if (values[option as keyof typeof values] !== undefined && owner !== command)\n      throw new UsageError(`--${option} is an option of ${owner}`);\n  }\n  const render = values.render ?? \"auto\";\n  if (render !== \"auto\" && render !== \"never\" && render !== \"always\")\n    throw new UsageError(`--render is auto, never or always, not \"${render}\"`);\n  const limit = whole(\"limit\", values.limit, 10, 1);\n  const from = whole(\"from\", values.from, 0, 0);\n  const length = whole(\"length\", values.length, 4000, 1);\n  const dir = storeDir(values.dir ?? (process.env[\"POLITE_CRAWLER_DIR\"] || undefined));\n  const allowPrivate = values[\"allow-private\"] === true || envFlag(\"POLITE_CRAWLER_ALLOW_PRIVATE\");\n  const allowUserProvided =\n    values[\"allow-user-provided\"] === true ||\n    (command === \"mcp\" && envFlag(\"POLITE_CRAWLER_ALLOW_USER_PROVIDED\"));\n  const options: OpenOptions = {\n    dir,\n    contact: values.contact ?? (process.env[\"POLITE_CRAWLER_CONTACT\"] || undefined),\n    ...(values[\"no-browser\"] === true ? { renderer: false as const } : {}),\n    ...(allowPrivate ? { egress: { allowPrivate: true } } : {}),\n  };\n\n  if (command === \"mcp\") {\n    const { startMcpServer } = await import(\"./mcp.ts\");\n    await startMcpServer({ ...options, allowUserProvided });\n    return 0;\n  }\n\n  const { durationMs, openCrawler } = await import(\"./open.ts\");\n  let maxAge: number;\n  try {\n    maxAge = durationMs(values[\"max-age\"] ?? \"24h\");\n  } catch (error) {\n    throw new UsageError(`--max-age: ${(error as Error).message}`, { cause: error });\n  }\n  // only fetch renders; a missing browser turns rendering off one time, before the first page\n  let renderOff: string | undefined;\n  let renderer = options.renderer;\n  if (command !== \"fetch\" || render === \"never\") renderer = false;\n  else if (renderer === undefined) {\n    const { browserMissing } = await import(\"./browser.ts\");\n    renderOff = await browserMissing();\n    if (renderOff !== undefined) renderer = false;\n  }\n\n  let crawler: PoliteCrawler;\n  try {\n    crawler = openCrawler({ ...options, renderer });\n  } catch (error) {\n    const code = (error as { code?: unknown }).code;\n    if (typeof code === \"string\" && FS_CODES.has(code)) {\n      throw new Error(\n        `the store ${dir} does not open: it is not a folder that polite-crawler can write (${code})`,\n        {\n          cause: error,\n        },\n      );\n    }\n    throw error;\n  }\n  const print = (value: unknown, text: () => string): void =>\n    out(values.json === true ? JSON.stringify(value, null, 2) : text());\n  try {\n    switch (command) {\n      case \"fetch\": {\n        // the same URL twice is one fetch; one bad URL is an error of that URL only\n        const urls = [...new Set(rest)];\n        const settled = await Promise.allSettled(\n          urls.map((url) => crawler.fetch(url, { render, maxAge })),\n        );\n        const pages = settled.map((result, at) => {\n          const url = urls[at]!;\n          if (result.status === \"rejected\") {\n            const reason: unknown = result.reason;\n            return failedPage(\n              url,\n              `${url}: ${reason instanceof Error ? reason.message : String(reason)}`,\n            );\n          }\n          const page = result.value;\n          return renderOff !== undefined && page.note !== null && /no browser/i.test(page.note)\n            ? { ...page, note: `this page needs JavaScript, and rendering is off: ${renderOff}` }\n            : page;\n        });\n        print(pages, () => pages.map(describePage).join(\"\\n\"));\n        return pages.every(ok) ? 0 : 1;\n      }\n      case \"find\": {\n        const found = await crawler.find(rest.join(\" \"), { limit, kind: values.kind ?? null });\n        print(found, () =>\n          found.length === 0\n            ? \"nothing stored matches\"\n            : found\n                .map((hit) => `${hit.doc}  ${hit.title}\\n  ${hit.snippet.replace(/\\s+/g, \" \")}`)\n                .join(\"\\n\"),\n        );\n        return found.length === 0 ? 1 : 0;\n      }\n      case \"read\": {\n        const userProvided = values[\"user-provided\"] === true;\n        const read = await crawler.read(rest[0]!, { from, length, userProvided });\n        print(read, () =>\n          read.held_back\n            ? `${read.note ?? \"\"}\\nAdd --user-provided if you gave this URL yourself. The crawler logs that read.`\n            : `${read.title}\\n${read.url ?? \"\"}\\n[${String(read.from)}–${String(read.to)} of ${String(read.total)}]\\n\\n${read.text}`,\n        );\n        return read.held_back ? 1 : 0;\n      }\n      case \"verify\": {\n        const [doc, ...quote] = rest;\n        const verified = await crawler.verify(doc!, quote.join(\" \"));\n        print(verified, () =>\n          verified.found\n            ? `found at ${String(verified.start)}–${String(verified.end)} in ${verified.capture ?? \"\"}${verified.stale ? \" (an earlier capture)\" : \"\"}: ${verified.excerpt ?? \"\"}`\n            : verified.held_back\n              ? (verified.note ?? \"held back\")\n              : \"not found\",\n        );\n        return verified.found ? 0 : 1;\n      }\n      default: {\n        // the documents of a recipe that this command cannot read stay, from the old index\n        const counts = await crawler.reindex();\n        print(counts, () =>\n          [\n            `${String(counts.captures)} captures replayed, ${String(counts.docs)} documents, ${String(counts.urls)} URLs`,\n            ...(counts.carried > 0\n              ? [\n                  `${String(counts.carried)} documents of other recipes kept: ${counts.unknown_recipes.join(\", \")}`,\n                ]\n              : []),\n            ...(counts.failed + counts.missing + counts.damaged > 0\n              ? [\n                  `skipped: ${String(counts.failed)} captures whose reader failed, ${String(counts.missing)} with no stored body, ${String(counts.damaged)} damaged lines of the log`,\n                ]\n              : []),\n          ].join(\"\\n\"),\n        );\n        return 0;\n      }\n    }\n  } finally {\n    await crawler.close();\n  }\n};\n\nmain(process.argv.slice(2)).then(\n  (code) => {\n    process.exitCode = code;\n  },\n  (error: unknown) => {\n    const message = error instanceof Error ? error.message : String(error);\n    const usage = error instanceof UsageError;\n    process.stderr.write(\n      `polite-crawler: ${message}${usage ? \" (polite-crawler --help lists the commands)\" : \"\"}\\n`,\n    );\n    process.exitCode = usage ? 2 : 1;\n  },\n);\n"],"mappings":";;;;;;AAaA,MAAM,cAAc,QAAQ,YAAY,KAAK,OAAO;AAIpD,QAAQ,gBAAgB,SAAyB,GAAG,SAA0B;CAC5E,MAAM,QAAiB,KAAK;CAC5B,MAAM,OACJ,OAAO,UAAU,WACb,QACA,OAAO,UAAU,YAAY,UAAU,QAAQ,UAAU,QACvD,MAAM,OACN,mBAAmB,QACjB,QAAQ,OACR,KAAA;CACV,MAAM,OAAO,OAAO,YAAY,WAAW,UAAU,QAAQ;CAC7D,IAAI,SAAS,yBAAyB,KAAK,WAAW,mCAAmC,GACvF;CACF,YAAY,SAAS,GAAG,IAAI;AAC9B;AAEA,MAAM,OAAO,kBAAkB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCvC,IAAM,aAAN,cAAyB,MAAM,CAAC;AAEhC,MAAM,UAAU;CACd,KAAK,EAAE,MAAM,SAAS;CACtB,SAAS,EAAE,MAAM,SAAS;CAC1B,iBAAiB,EAAE,MAAM,UAAU;CACnC,cAAc,EAAE,MAAM,UAAU;CAChC,MAAM,EAAE,MAAM,UAAU;CACxB,SAAS;EAAE,MAAM;EAAW,OAAO;CAAI;CACvC,MAAM;EAAE,MAAM;EAAW,OAAO;CAAI;CACpC,QAAQ,EAAE,MAAM,SAAS;CACzB,WAAW,EAAE,MAAM,SAAS;CAC5B,OAAO,EAAE,MAAM,SAAS;CACxB,MAAM,EAAE,MAAM,SAAS;CACvB,MAAM,EAAE,MAAM,SAAS;CACvB,QAAQ,EAAE,MAAM,SAAS;CACzB,iBAAiB,EAAE,MAAM,UAAU;CACnC,uBAAuB,EAAE,MAAM,UAAU;AAC3C;;AAGA,MAAM,QAA0C;CAC9C,QAAQ;CACR,WAAW;CACX,OAAO;CACP,MAAM;CACN,MAAM;CACN,QAAQ;CACR,iBAAiB;CACjB,uBAAuB;AACzB;;AAGA,MAAM,WAEF;CACF,OAAO;EAAE,KAAK;EAAG,KAAK;EAAU,OAAO;CAA+B;CACtE,MAAM;EAAE,KAAK;EAAG,KAAK;EAAU,OAAO;CAAqB;CAC3D,MAAM;EAAE,KAAK;EAAG,KAAK;EAAG,OAAO;CAA6B;CAC5D,QAAQ;EAAE,KAAK;EAAG,KAAK;EAAU,OAAO;CAAyC;CACjF,SAAS;EAAE,KAAK;EAAG,KAAK;EAAG,OAAO;CAA4B;CAC9D,KAAK;EAAE,KAAK;EAAG,KAAK;EAAG,OAAO;CAAwB;AACxD;AAEA,MAAM,SAAS,SAA4B;CACzC,IAAI;EACF,OAAO,UAAU;GAAE,MAAM,CAAC,GAAG,IAAI;GAAG,kBAAkB;GAAM,QAAQ;GAAM,SAAS;EAAQ,CAAC;CAC9F,SAAS,OAAO;EAEd,MAAM,OAAQ,MAA6B;EAC3C,IAAI,OAAO,SAAS,YAAY,KAAK,WAAW,gBAAgB,GAC9D,MAAM,IAAI,WAAY,MAAgB,SAAS,EAAE,OAAO,MAAM,CAAC;EAEjE,MAAM;CACR;AACF;AAEA,MAAM,SAAS,MAAc,MAA0B,UAAkB,QAAwB;CAC/F,IAAI,SAAS,KAAA,GAAW,OAAO;CAC/B,IAAI,CAAC,cAAc,KAAK,IAAI,GAC1B,MAAM,IAAI,WAAW,KAAK,KAAK,8BAA8B,KAAK,EAAE;CACtE,MAAM,QAAQ,OAAO,IAAI;CACzB,IAAI,QAAQ,KAAK,MAAM,IAAI,WAAW,KAAK,KAAK,SAAS,OAAO,GAAG,EAAE,SAAS;CAC9E,OAAO;AACT;;AAGA,MAAM,WAAW,SAA0B;CACzC,MAAM,MAAM,QAAQ,IAAI;CACxB,MAAM,QAAQ,KAAK,KAAK,CAAC,CAAC,YAAY,KAAK;CAC3C,IAAI;EAAC;EAAI;EAAK;EAAS;EAAM;CAAK,CAAC,CAAC,SAAS,KAAK,GAAG,OAAO;CAC5D,IAAI;EAAC;EAAK;EAAQ;EAAO;CAAI,CAAC,CAAC,SAAS,KAAK,GAAG,OAAO;CACvD,MAAM,IAAI,WAAW,GAAG,KAAK,mBAAmB,OAAO,GAAG,EAAE;AAC9D;;AAGA,MAAM,YAAY,SAAqC;CACrD,IAAI,SAAS,KAAA,GAAW,OAAO,KAAK,QAAQ,GAAG,iBAAiB;CAChE,IAAI,KAAK,KAAK,MAAM,IAAI,MAAM,IAAI,WAAW,sBAAsB;CACnE,IAAI,SAAS,KAAK,OAAO,QAAQ;CACjC,IAAI,UAAU,KAAK,IAAI,GAAG,OAAO,QAAQ,QAAQ,GAAG,KAAK,MAAM,CAAC,CAAC;CACjE,OAAO,QAAQ,IAAI;AACrB;AAEA,MAAM,2BAAW,IAAI,IAAI;CAAC;CAAU;CAAU;CAAW;CAAU;CAAS;CAAS;AAAQ,CAAC;AAE9F,MAAM,MAAM,SAA8B,KAAK,YAAY,QAAQ,KAAK,YAAY;AAEpF,MAAM,gBAAgB,SAA6B;CASjD,OAAO;EAAC,GARQ,GAAG,IAAI,IAAI,MAAM,IAAI,GAAG,KAAK;EAQ/B,KAPA;GACZ,GAAG,KAAK,QAAQ,GAAG,OAAO,KAAK,MAAM;GACrC,KAAK,QAAQ,OAAO,OAAO,OAAO,KAAK;GACvC,KAAK,QAAQ,IAAI,GAAG,OAAO,KAAK,KAAK,EAAE,UAAU;GACjD,KAAK,WAAW,aAAa;GAC7B,KAAK,WAAW,SAAS,IAAI,YAAY,KAAK,WAAW,KAAK,IAAI,MAAM;EAC1E,CAAC,CAAC,QAAQ,SAAS,SAAS,IACL,CAAC,CAAC,KAAK,KAAK;EAAK,GAAI,KAAK,SAAS,OAAO,CAAC,IAAI,CAAC,KAAK,KAAK,MAAM;CAAE,CAAC,CAAC,KACzF,IACF;AACF;;AAGA,MAAM,cAAc,KAAa,UAA8B;CAC7D;CACA,KAAK;CACL,SAAS;CACT,QAAQ;CACR,OAAO;CACP,OAAO;CACP,UAAU;CACV,YAAY,CAAC;CACb,OAAO;CACP,UAAU,CAAC;CACX,SAAS;CACT,eAAe;CACf,YAAY;CACZ,YAAY;CACZ,WAAW;CACX;AACF;;AAGA,MAAM,OAAO,OAAO,SAA6C;CAC/D,MAAM,EAAE,QAAQ,gBAAgB,MAAM,IAAI;CAC1C,MAAM,OAAO,SAAuB,KAAK,QAAQ,OAAO,MAAM,GAAG,KAAK,GAAG;CACzE,IAAI,OAAO,YAAY,MAAM;EAC3B,IAAI,OAAO;EACX,OAAO;CACT;CACA,MAAM,CAAC,SAAS,GAAG,QAAQ;CAC3B,IAAI,OAAO,SAAS,QAAQ,YAAY,KAAA,GAAW;EACjD,QAAQ,OAAO,MAAM,IAAI;EACzB,OAAO;CACT;CAGA,MAAM,OAAO,SAAS;CACtB,IAAI,SAAS,KAAA,GAAW,MAAM,IAAI,WAAW,eAAe,QAAQ,EAAE;CACtE,IAAI,KAAK,SAAS,KAAK,OAAO,KAAK,SAAS,KAAK,KAAK,MAAM,IAAI,WAAW,KAAK,KAAK;CACrF,KAAK,MAAM,CAAC,QAAQ,UAAU,OAAO,QAAQ,KAAK,GAChD,IAAI,OAAO,YAAmC,KAAA,KAAa,UAAU,SACnE,MAAM,IAAI,WAAW,KAAK,OAAO,mBAAmB,OAAO;CAE/D,MAAM,SAAS,OAAO,UAAU;CAChC,IAAI,WAAW,UAAU,WAAW,WAAW,WAAW,UACxD,MAAM,IAAI,WAAW,2CAA2C,OAAO,EAAE;CAC3E,MAAM,QAAQ,MAAM,SAAS,OAAO,OAAO,IAAI,CAAC;CAChD,MAAM,OAAO,MAAM,QAAQ,OAAO,MAAM,GAAG,CAAC;CAC5C,MAAM,SAAS,MAAM,UAAU,OAAO,QAAQ,KAAM,CAAC;CACrD,MAAM,MAAM,SAAS,OAAO,QAAQ,QAAQ,IAAI,yBAAyB,KAAA,EAAU;CACnF,MAAM,eAAe,OAAO,qBAAqB,QAAQ,QAAQ,8BAA8B;CAC/F,MAAM,oBACJ,OAAO,2BAA2B,QACjC,YAAY,SAAS,QAAQ,oCAAoC;CACpE,MAAM,UAAuB;EAC3B;EACA,SAAS,OAAO,YAAY,QAAQ,IAAI,6BAA6B,KAAA;EACrE,GAAI,OAAO,kBAAkB,OAAO,EAAE,UAAU,MAAe,IAAI,CAAC;EACpE,GAAI,eAAe,EAAE,QAAQ,EAAE,cAAc,KAAK,EAAE,IAAI,CAAC;CAC3D;CAEA,IAAI,YAAY,OAAO;EACrB,MAAM,EAAE,mBAAmB,MAAM,OAAO;EACxC,MAAM,eAAe;GAAE,GAAG;GAAS;EAAkB,CAAC;EACtD,OAAO;CACT;CAEA,MAAM,EAAE,YAAY,gBAAgB,MAAM,OAAO,sBAAY,CAAA,MAAA,MAAA,EAAA,CAAA;CAC7D,IAAI;CACJ,IAAI;EACF,SAAS,WAAW,OAAO,cAAc,KAAK;CAChD,SAAS,OAAO;EACd,MAAM,IAAI,WAAW,cAAe,MAAgB,WAAW,EAAE,OAAO,MAAM,CAAC;CACjF;CAEA,IAAI;CACJ,IAAI,WAAW,QAAQ;CACvB,IAAI,YAAY,WAAW,WAAW,SAAS,WAAW;MACrD,IAAI,aAAa,KAAA,GAAW;EAC/B,MAAM,EAAE,mBAAmB,MAAM,OAAO,yBAAe,CAAA,MAAA,MAAA,EAAA,CAAA;EACvD,YAAY,MAAM,eAAe;EACjC,IAAI,cAAc,KAAA,GAAW,WAAW;CAC1C;CAEA,IAAI;CACJ,IAAI;EACF,UAAU,YAAY;GAAE,GAAG;GAAS;EAAS,CAAC;CAChD,SAAS,OAAO;EACd,MAAM,OAAQ,MAA6B;EAC3C,IAAI,OAAO,SAAS,YAAY,SAAS,IAAI,IAAI,GAC/C,MAAM,IAAI,MACR,aAAa,IAAI,oEAAoE,KAAK,IAC1F,EACE,OAAO,MACT,CACF;EAEF,MAAM;CACR;CACA,MAAM,SAAS,OAAgB,SAC7B,IAAI,OAAO,SAAS,OAAO,KAAK,UAAU,OAAO,MAAM,CAAC,IAAI,KAAK,CAAC;CACpE,IAAI;EACF,QAAQ,SAAR;GACE,KAAK,SAAS;IAEZ,MAAM,OAAO,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC;IAI9B,MAAM,SAAQ,MAHQ,QAAQ,WAC5B,KAAK,KAAK,QAAQ,QAAQ,MAAM,KAAK;KAAE;KAAQ;IAAO,CAAC,CAAC,CAC1D,EAAA,CACsB,KAAK,QAAQ,OAAO;KACxC,MAAM,MAAM,KAAK;KACjB,IAAI,OAAO,WAAW,YAAY;MAChC,MAAM,SAAkB,OAAO;MAC/B,OAAO,WACL,KACA,GAAG,IAAI,IAAI,kBAAkB,QAAQ,OAAO,UAAU,OAAO,MAAM,GACrE;KACF;KACA,MAAM,OAAO,OAAO;KACpB,OAAO,cAAc,KAAA,KAAa,KAAK,SAAS,QAAQ,cAAc,KAAK,KAAK,IAAI,IAChF;MAAE,GAAG;MAAM,MAAM,qDAAqD;KAAY,IAClF;IACN,CAAC;IACD,MAAM,aAAa,MAAM,IAAI,YAAY,CAAC,CAAC,KAAK,IAAI,CAAC;IACrD,OAAO,MAAM,MAAM,EAAE,IAAI,IAAI;GAC/B;GACA,KAAK,QAAQ;IACX,MAAM,QAAQ,MAAM,QAAQ,KAAK,KAAK,KAAK,GAAG,GAAG;KAAE;KAAO,MAAM,OAAO,QAAQ;IAAK,CAAC;IACrF,MAAM,aACJ,MAAM,WAAW,IACb,2BACA,MACG,KAAK,QAAQ,GAAG,IAAI,IAAI,IAAI,IAAI,MAAM,MAAM,IAAI,QAAQ,QAAQ,QAAQ,GAAG,GAAG,CAAC,CAC/E,KAAK,IAAI,CAClB;IACA,OAAO,MAAM,WAAW,IAAI,IAAI;GAClC;GACA,KAAK,QAAQ;IACX,MAAM,eAAe,OAAO,qBAAqB;IACjD,MAAM,OAAO,MAAM,QAAQ,KAAK,KAAK,IAAK;KAAE;KAAM;KAAQ;IAAa,CAAC;IACxE,MAAM,YACJ,KAAK,YACD,GAAG,KAAK,QAAQ,GAAG,oFACnB,GAAG,KAAK,MAAM,IAAI,KAAK,OAAO,GAAG,KAAK,OAAO,KAAK,IAAI,EAAE,GAAG,OAAO,KAAK,EAAE,EAAE,MAAM,OAAO,KAAK,KAAK,EAAE,OAAO,KAAK,MACtH;IACA,OAAO,KAAK,YAAY,IAAI;GAC9B;GACA,KAAK,UAAU;IACb,MAAM,CAAC,KAAK,GAAG,SAAS;IACxB,MAAM,WAAW,MAAM,QAAQ,OAAO,KAAM,MAAM,KAAK,GAAG,CAAC;IAC3D,MAAM,gBACJ,SAAS,QACL,YAAY,OAAO,SAAS,KAAK,EAAE,GAAG,OAAO,SAAS,GAAG,EAAE,MAAM,SAAS,WAAW,KAAK,SAAS,QAAQ,0BAA0B,GAAG,IAAI,SAAS,WAAW,OAChK,SAAS,YACN,SAAS,QAAQ,cAClB,WACR;IACA,OAAO,SAAS,QAAQ,IAAI;GAC9B;GACA,SAAS;IAEP,MAAM,SAAS,MAAM,QAAQ,QAAQ;IACrC,MAAM,cACJ;KACE,GAAG,OAAO,OAAO,QAAQ,EAAE,sBAAsB,OAAO,OAAO,IAAI,EAAE,cAAc,OAAO,OAAO,IAAI,EAAE;KACvG,GAAI,OAAO,UAAU,IACjB,CACE,GAAG,OAAO,OAAO,OAAO,EAAE,oCAAoC,OAAO,gBAAgB,KAAK,IAAI,GAChG,IACA,CAAC;KACL,GAAI,OAAO,SAAS,OAAO,UAAU,OAAO,UAAU,IAClD,CACE,YAAY,OAAO,OAAO,MAAM,EAAE,iCAAiC,OAAO,OAAO,OAAO,EAAE,wBAAwB,OAAO,OAAO,OAAO,EAAE,0BAC3I,IACA,CAAC;IACP,CAAC,CAAC,KAAK,IAAI,CACb;IACA,OAAO;GACT;EACF;CACF,UAAU;EACR,MAAM,QAAQ,MAAM;CACtB;AACF;AAEA,KAAK,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,MACzB,SAAS;CACR,QAAQ,WAAW;AACrB,IACC,UAAmB;CAClB,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;CACrE,MAAM,QAAQ,iBAAiB;CAC/B,QAAQ,OAAO,MACb,mBAAmB,UAAU,QAAQ,gDAAgD,GAAG,GAC1F;CACA,QAAQ,WAAW,QAAQ,IAAI;AACjC,CACF"}

package/build/hours.d.mts

@@ -0,0 +1,113 @@
+//#region src/hours.d.ts
+/**
+ * The hours reader reads opening hours from the three places where they occur:
+ *
+ * - schema.org JSON-LD: the `openingHours` strings or `openingHoursSpecification`.
+ * - The `opening_hours` tag of OpenStreetMap, in its common subset. It does not guess a rule
+ *   that it cannot fully read (holidays, months, sunrise). It leaves that rule out and says so.
+ * - The text of the page: "Mon–Thu 11am–10pm, Fri–Sat 11am–1am", "Lunch Mon-Fri 11:30-2:30 /
+ *   Dinner 5-10", "Sunday: Closed". The text reader reads English.
+ *
+ * A day can have more than one interval (lunch and dinner). A day that is not in a week is
+ * unknown, not closed. When a time has no am or pm, the parser guesses it and the result says so
+ * (`inferred`). Thus a guess does not look like a fact. A guess is not longer than 16 hours.
+ *
+ * The time of each reader grows in proportion to the length of its input.
+ */
+export type Day = "mon" | "tue" | "wed" | "thu" | "fri" | "sat" | "sun";
+export declare const DAYS: readonly Day[];
+/**
+ * An interval is [open, close] as HH:MM. A close at or before the open is after midnight, and
+ * ["00:00", "00:00"] is open 24 hours.
+ */
+export type Interval = readonly [string, string];
+/** A week gives the intervals of each day, and [] for a closed day. A day that is not there is unknown. */
+export type Week = Partial<Record<Day, readonly Interval[]>>;
+export interface Hours {
+  readonly week: Week;
+  readonly from: "jsonld" | "osm" | "text";
+  /** This text is the words that the hours came from. */
+  readonly raw: string;
+  /** This value is true when the parser guessed an am or a pm ("11-9"). */
+  readonly inferred: boolean;
+  /**
+   * This value is true when the reader left out a rule or a line. The rule is not about the week (a
+   * holiday, a date) or the reader cannot read it ("sunrise").
+   */
+  readonly partial?: boolean;
+  /** This text names the season or the dates of the hours, for example "Summer" or "2026-06-01/2026-08-31". */
+  readonly season?: string;
+}
+/** This function gives the days from one day to another, with the two days. It continues past Sunday (Fri-Mon). */
+export declare const dayRange: (from: Day, to: Day) => Day[];
+/**
+ * This function reads the `opening_hours` notation of OpenStreetMap, for example
+ * "Mo-Th 11:00-23:00; Fr,Sa 11:00-01:00; Su off". The `openingHours` of schema.org uses the same
+ * notation. A later rule replaces an earlier rule for the same days, as OpenStreetMap specifies.
+ * The reader leaves out a rule for dates or holidays, and the days of a rule that it cannot read
+ * stay unknown. In both cases the result is `partial`.
+ */
+export declare const fromCompact: (value: string, from?: "osm" | "jsonld") => Hours | undefined;
+/**
+ * This function reads the hours of one JSON-LD object, for example a `Restaurant`. It reads the
+ * `openingHoursSpecification` first, and the `openingHours` when there is no specification. In
+ * schema.org, a closed day has the same opening and closing time, 00:00, and 00:00 to 24:00 is
+ * open 24 hours.
+ *
+ * A specification for less than a week (`validFrom` to `validThrough`) is a holiday: the reader
+ * leaves it out. When specifications hold for different seasons, the option `at` picks the
+ * season of that date. With no date, the reader keeps the specifications with no season, or else
+ * the first season, and it names the season.
+ */
+export declare const fromSchemaOrg: (record: Readonly<Record<string, unknown>>, options?: {
+  readonly at?: string | Date | undefined;
+}) => Hours | undefined;
+/**
+ * This function reads a range of two times, as a restaurant writes it:
+ *
+ * - "5-10pm" is 17:00-22:00, and "11-3pm" is 11:00-15:00. The opening gets the am or pm of the
+ *   close, but not when that puts the opening after the close.
+ * - "11-1am" is 11:00-01:00, because 23:00-01:00 is too short to be the meaning. The result is
+ *   then `inferred`.
+ * - "11am-10" closes at 22:00.
+ * - With no am, pm or 24-hour time ("11-9"), an opening before 7 is pm, but "6-2" is 06:00-14:00.
+ *   A close at or before the opening is pm, and "7-12" closes at noon. The result is then
+ *   `inferred`, and it is not longer than 16 hours.
+ * - "21-14" is no range: a 24-hour opening past midnight does not end after noon.
+ */
+export declare const readRange: (start: string, end: string) => {
+  readonly interval: Interval;
+  readonly inferred: boolean;
+} | undefined;
+/**
+ * This function reads the hours in the text of a page. It reads the lines with days and times as
+ * one stream. Thus "Monday – Thursday" on one line and "11:30am – 9pm" on the next line go
+ * together. A list of days ("Mon, Wed & Fri") waits for a time or for "closed".
+ *
+ * Each group of these lines is a candidate. Its label is the short line above it ("Hours",
+ * "Dinner", "Happy hour"). A page often shows more than one schedule, and the choice between them
+ * is a judgment, not a parse. A season ("Summer:", "June-August") starts a new candidate.
+ *
+ * A line about a date or an occasion ("Dec 24: 9am-2pm", "New Year's Eve 8pm-1am", "Private events
+ * 11am-3pm") is not the week. A line with no day that names a service before its hours ("Happy hour
+ * 6-8pm") under a line with days and hours is not the week either. The reader leaves it out, and
+ * the candidate is `partial`.
+ */
+export declare const fromText: (text: string) => Array<Hours & {
+  readonly label: string | null;
+}>;
+/** This function writes a week as text, for example `Mon–Thu 11:00–22:00, Fri–Sat 11:00–23:00`. */
+export declare const describeWeek: (week: Week) => string;
+/**
+ * This function gives one interval for each day (`hours_week`), and null for a closed day. One
+ * interval cannot show a day with two intervals and a gap between them. Thus such a day is not in
+ * `hours_week`, and `split` names it.
+ */
+export declare const toHoursWeek: (week: Week) => {
+  readonly hours_week: Partial<Record<Day, readonly [string, string] | null>>;
+  readonly split: readonly Day[];
+};
+/** This function gives the days where two readings disagree, from the days that the two readings state. */
+export declare const disagreements: (a: Week, b: Week) => Day[];
+//#endregion
+//# sourceMappingURL=hours.d.mts.map