polite-crawler 0.1.2 → 0.1.4

package/build/recipe-GiFQojmo.mjs

@@ -0,0 +1,279 @@
+import { i as durationMs } from "./version-BPem-XZd.mjs";
+import { C as fallbackReason, S as archivedOf, c as isHeldBack, i as fetchAndIndex, n as HELD_BACK, p as recordOverride, s as heldBackMeta, y as archivedCopy } from "./runtime-D_r2SYge.mjs";
+import { o as decodeBody } from "./html-DkQUAE_d.mjs";
+//#region src/recipe.ts
+/**
+* This function reads a body as JSON, in the encoding that the response declares. A body that is
+* not JSON (a page about maintenance, or a login page) throws an error that names the URL and the
+* content type. In an extractor, the runtime then reports that error.
+*/
+const readJson = (body, capture) => {
+	const text = decodeBody(body, capture.contentType);
+	try {
+		return JSON.parse(text);
+	} catch {
+		throw new Error(`${capture.url} answered ${capture.contentType ?? "a body of no stated type"}, not JSON`);
+	}
+};
+/**
+* These recipe names belong to the crawler: the page reader, the archived fallback, robots.txt,
+* and the sitemaps and feeds of `discover`. A recipe cannot use them.
+*/
+const RESERVED_RECIPES = [
+	"page",
+	"archive-copy",
+	"archive-index",
+	"robots.txt",
+	"sitemap",
+	"feed"
+];
+/** These are the limits of a recipe. */
+const RECIPE_LIMITS = {
+	nameChars: 64,
+	docIdChars: 2048
+};
+const NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
+const PAGE_ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
+const API_ACCEPT = "application/json";
+const AI_INPUTS = [
+	"yes",
+	"no",
+	"unset"
+];
+/** This function tells what is wrong with one document of an extractor, or gives undefined. */
+const faultOf = (value, seen) => {
+	if (value === null || typeof value !== "object" || Array.isArray(value)) return "a document must be an object";
+	const doc = value;
+	const id = doc["doc"];
+	if (typeof id !== "string" || id.length === 0 || id.length > RECIPE_LIMITS.docIdChars) return `doc must be a text of 1 to ${String(RECIPE_LIMITS.docIdChars)} characters`;
+	if (id.startsWith("archive:")) return `the id ${id} starts with archive:, which is for the archived copies`;
+	if (seen.has(id)) return `two documents have the id ${id}`;
+	seen.add(id);
+	const kind = doc["kind"];
+	if (typeof kind !== "string" || kind.length === 0) return "kind must be a text that is not empty";
+	if (kind === "archived") return "the kind archived is for the archived copies";
+	if (typeof doc["title"] !== "string") return "title must be a text";
+	if (typeof doc["text"] !== "string") return "text must be a text";
+	if (doc["url"] !== null && typeof doc["url"] !== "string") return "url must be a text or null";
+	const meta = doc["meta"];
+	if (meta === null || typeof meta !== "object" || Array.isArray(meta)) return "meta must be an object";
+	try {
+		JSON.stringify(meta);
+	} catch {
+		return "meta must be JSON: no cycle and no BigInt";
+	}
+	const merge = doc["merge"];
+	if (merge !== void 0 && merge !== "latest" && merge !== "richest") return "merge must be \"latest\" or \"richest\"";
+	if (doc["full_text"] !== void 0 && typeof doc["full_text"] !== "string") return "full_text must be a text";
+	const aiInput = doc["ai_input"];
+	if (aiInput !== void 0 && !AI_INPUTS.includes(aiInput)) return "ai_input must be \"yes\", \"no\" or \"unset\"";
+};
+/** This function gives the documents of an extractor, or throws an error that names the first fault. */
+const checkedDocs = (name, docs) => {
+	if (!Array.isArray(docs)) throw new TypeError(`the recipe ${name} gave ${typeof docs}, not an array of documents`);
+	const seen = /* @__PURE__ */ new Set();
+	for (const [index, doc] of docs.entries()) {
+		const fault = faultOf(doc, seen);
+		if (fault !== void 0) throw new TypeError(`the recipe ${name} gave a document that is not valid (${String(index)}): ${fault}`);
+	}
+	return docs;
+};
+/**
+* This function declares a recipe. It checks the definition at once: a name that is not valid, a
+* reserved name or a duration that it cannot read throws an error. The extractor of the recipe
+* checks each document of `extract`. A document that is not valid makes the extractor throw, and
+* the runtime reports it as an error of the extractor.
+*/
+const defineRecipe = (definition) => {
+	const name = definition.name;
+	if (typeof name !== "string" || name.length > RECIPE_LIMITS.nameChars || !NAME.test(name)) throw new RangeError(`a recipe name is 1 to ${String(RECIPE_LIMITS.nameChars)} letters, digits and . _ : -, and it starts with a letter or a digit, not ${JSON.stringify(name) ?? "undefined"}`);
+	if (RESERVED_RECIPES.includes(name)) throw new RangeError(`the recipe name ${name} belongs to the crawler: give the recipe a name of its own`);
+	if (typeof definition.extract !== "function") throw new TypeError(`the recipe ${name} needs extract, a function that reads a body into documents`);
+	const mode = definition.mode ?? "page";
+	if (mode !== "page" && mode !== "api") throw new RangeError(`the mode of the recipe ${name} must be "page" or "api", not ${String(mode)}`);
+	const accept = definition.accept ?? (mode === "api" ? API_ACCEPT : PAGE_ACCEPT);
+	const minIntervalMs = definition.minInterval === void 0 ? void 0 : durationMs(definition.minInterval);
+	const maxAgeOf = (value) => value === "auto" ? "auto" : durationMs(value);
+	const maxAge = maxAgeOf(definition.maxAge ?? "24h");
+	const maxBytes = definition.maxBytes;
+	if (maxBytes !== void 0 && (!Number.isSafeInteger(maxBytes) || maxBytes < 1)) throw new RangeError(`maxBytes of the recipe ${name} must be a whole number of 1 or more`);
+	const extract = definition.extract;
+	const extractor = async (body, capture) => checkedDocs(name, await extract(body, {
+		url: capture.url,
+		contentType: capture.contentType
+	}));
+	const fetch = definition.fetch;
+	return {
+		name,
+		description: definition.description ?? null,
+		mode,
+		accept,
+		extractor,
+		request(input, options = {}) {
+			const asked = fetch === void 0 ? input : fetch(input);
+			const target = typeof asked === "string" ? { url: asked } : asked;
+			const url = target?.url;
+			let parsed;
+			try {
+				parsed = typeof url === "string" ? new URL(url) : void 0;
+			} catch {
+				parsed = void 0;
+			}
+			if (parsed === void 0 || parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new TypeError(`the recipe ${name} needs an http or https URL for its input, not ${JSON.stringify(url) ?? String(url)}`);
+			return {
+				url: parsed.href,
+				mode,
+				recipe: name,
+				accept: target?.accept ?? accept,
+				headers: target?.headers,
+				secretParams: target?.secretParams,
+				minIntervalMs,
+				maxAgeMs: options.maxAge === void 0 ? maxAge : maxAgeOf(options.maxAge),
+				maxBytes,
+				onBehalfOf: options.onBehalfOf,
+				job: options.job,
+				signal: options.signal,
+				scope: options.scope,
+				labels: options.labels,
+				deadline: options.deadline
+			};
+		}
+	};
+};
+/**
+* This function gives the extractor of a recipe to the extractors of a runtime. A second
+* registration of the same recipe does nothing. A different extractor with the same name throws
+* an error, because `reindex` must read each capture with one extractor.
+*/
+const registerRecipe = (extractors, recipe) => {
+	const known = extractors.get(recipe.name);
+	if (known === recipe.extractor) return;
+	if (known !== void 0) throw new Error(`the recipe ${recipe.name} has a different extractor in this runtime: give each recipe a name of its own`);
+	extractors.register(recipe.name, recipe.extractor);
+};
+/** This function gives the runtime of a target. */
+const runtimeOf = (target) => "crawler" in target ? target : target.runtime;
+/** This function gives the deadline of a budget, on the clock of the runtime. */
+const deadlineOf = (runtime, budget) => budget === void 0 ? void 0 : Date.parse(runtime.now()) + durationMs(budget);
+/** This function gives one document of a run, with the rule for a held-back document. */
+const recipeDoc = async (runtime, doc, capture, userProvided, action) => {
+	const current = runtime.index.get(doc.doc) ?? doc;
+	const version = (capture === void 0 ? void 0 : runtime.index.versionText(doc.doc, capture)) ?? current;
+	const aiInput = current.ai_input === "no" || version.ai_input === "no" ? "no" : version.ai_input;
+	const held = isHeldBack({ ai_input: aiInput }, userProvided);
+	if (aiInput === "no" && userProvided) await recordOverride(runtime, current, action);
+	const archived = archivedOf(version.meta);
+	return {
+		doc: doc.doc,
+		kind: version.kind,
+		title: held ? "" : version.title,
+		url: version.url,
+		meta: held ? heldBackMeta(version.meta) : version.meta,
+		chars: version.text.length,
+		capture: version.capture,
+		fetched_at: version.fetched_at,
+		held_back: held,
+		...archived === void 0 ? {} : { archived }
+	};
+};
+const noteOf = (parts) => {
+	const kept = parts.filter((part) => typeof part === "string" && part.length > 0);
+	return kept.length > 0 ? kept.join("; ") : null;
+};
+/**
+* This function starts a recipe for one input. It fetches the request through the crawler, and it
+* reads the capture with the extractor of the recipe. It gives the documents of that capture.
+* These are its rules:
+*
+* - The runtime registers the recipe the first time that it gets it. Give the recipe to
+*   `openCrawler` too, so that `reindex` reads its captures before the first fetch.
+* - A stored copy younger than `maxAge` answers with no request. When the extractor cannot read a
+*   stored copy, the crawler asks the source one more time.
+* - An extractor that throws gives no document, and `error` tells why. The fetch still succeeds.
+* - A document that asked not to be AI input shows no title and only its structural metadata,
+*   unless `userProvided`. The crawler logs each `userProvided` read of a held-back document.
+* - With `fallback: "archive"`, a page that the site refuses gets an archived copy, with
+*   `archived`. The page reader reads the copy. The extractor of the recipe reads only live
+*   captures. Thus its documents do not mix a live answer and an old copy of a third party.
+*/
+const runRecipe = async (target, recipe, input, options = {}) => {
+	const runtime = runtimeOf(target);
+	if (options.fallback === "archive" && recipe.mode !== "page") throw new RangeError(`the recipe ${recipe.name} reads an API: an archived copy is for a page, so give fallback only to a recipe of the mode page`);
+	registerRecipe(runtime.extractors, recipe);
+	const deadline = deadlineOf(runtime, options.budget);
+	const request = recipe.request(input, {
+		...options,
+		deadline
+	});
+	let indexed = await fetchAndIndex(runtime, request);
+	if (indexed.error !== void 0 && indexed.blocked === void 0 && indexed.fetched.fromCache) indexed = await fetchAndIndex(runtime, {
+		...request,
+		maxAgeMs: 0
+	});
+	const userProvided = options.userProvided ?? false;
+	const action = options.job ?? `recipe ${recipe.name}`;
+	const fetched = indexed.fetched;
+	const notes = [fetched.note, indexed.error];
+	if (indexed.docs.length === 0 && options.fallback === "archive") {
+		const reason = fallbackReason(indexed);
+		if (reason !== void 0) {
+			const page = {
+				render: "never",
+				maxAgeMs: request.maxAgeMs,
+				onBehalfOf: request.onBehalfOf,
+				job: request.job,
+				signal: request.signal,
+				scope: request.scope,
+				labels: request.labels,
+				deadline,
+				fallback: "archive",
+				archiveDespiteRobots: options.archiveDespiteRobots
+			};
+			const answer = await archivedCopy(runtime, request.url, reason, page);
+			if (answer.doc !== void 0 && answer.archived !== void 0) {
+				const doc = await recipeDoc(runtime, answer.doc, answer.doc.capture, userProvided, action);
+				return {
+					recipe: recipe.name,
+					url: fetched.url,
+					outcome: fetched.outcome,
+					status: fetched.status,
+					docs: [doc],
+					capture: doc.capture,
+					fetched_at: doc.fetched_at,
+					from_cache: answer.fromCache,
+					held_back: doc.held_back,
+					error: indexed.error ?? null,
+					archived: answer.archived,
+					note: noteOf([
+						...notes,
+						...answer.notes,
+						"the page reader read the archived copy, not the extractor of the recipe",
+						doc.held_back ? HELD_BACK : void 0
+					])
+				};
+			}
+			notes.push(...answer.notes);
+		}
+	}
+	const capture = fetched.capture;
+	const docs = [];
+	for (const doc of indexed.docs) docs.push(await recipeDoc(runtime, doc, capture, userProvided, action));
+	const held = docs.some((doc) => doc.held_back);
+	return {
+		recipe: recipe.name,
+		url: fetched.url,
+		outcome: fetched.outcome,
+		status: fetched.status,
+		docs,
+		capture: docs[0]?.capture ?? capture ?? null,
+		fetched_at: docs[0]?.fetched_at ?? fetched.fetchedAt ?? null,
+		from_cache: fetched.fromCache,
+		held_back: held,
+		error: indexed.error ?? null,
+		note: noteOf([...notes, held ? HELD_BACK : void 0])
+	};
+};
+//#endregion
+export { readJson as a, runtimeOf as c, defineRecipe as i, RESERVED_RECIPES as n, registerRecipe as o, deadlineOf as r, runRecipe as s, RECIPE_LIMITS as t };
+
+//# sourceMappingURL=recipe-GiFQojmo.mjs.map

package/build/recipe-GiFQojmo.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"recipe-GiFQojmo.mjs","names":[],"sources":["../src/recipe.ts"],"sourcesContent":["/*\n * The recipe SDK. A recipe tells the crawler what to fetch and how to read it into documents.\n *\n * - `defineRecipe` declares a recipe: its name, the request for an input, and its extractor.\n * - `runRecipe` fetches one input through the crawler and gives the documents of the capture.\n * - `registerRecipe` gives the extractor to the extractors of a runtime. `openCrawler` and\n *   `createRuntime` take `recipes` and register them at open, so `reindex` reads their captures.\n *\n * Each fetch of a recipe is a fetch of the crawler. Thus robots.txt (for a page), the pace of each\n * host, the stops, the egress policy and the capture log apply. The documents are versions in the\n * index, so `find`, `read`, `verify`, `extract`, `diff` and the time travel use them.\n */\nimport type { FetchRequest } from \"./crawler.ts\";\nimport type { Doc, ExtractedDoc, VersionText } from \"./docs.ts\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport type { Archived } from \"./fallback.ts\";\nimport { archivedCopy, archivedOf, fallbackReason } from \"./fallback.ts\";\nimport type { CrawlRuntime, Extractor, Extractors, Indexed, PageOptions } from \"./runtime.ts\";\nimport { HELD_BACK, fetchAndIndex, heldBackMeta, isHeldBack, recordOverride } from \"./runtime.ts\";\nimport type { AiInput } from \"./signals.ts\";\nimport { decodeBody } from \"./text.ts\";\n\n/** The capture that an extractor reads: the final URL of the response and its content type. */\nexport interface RecipeCapture {\n  readonly url: string;\n  readonly contentType: string | null;\n}\n\n/**\n * This function reads one captured body into the documents that it holds. It is pure: the same\n * body and capture always give the same documents. It sends no request. It can answer later.\n */\nexport type RecipeExtract = (\n  body: Uint8Array,\n  capture: RecipeCapture,\n) => ReadonlyArray<ExtractedDoc> | Promise<ReadonlyArray<ExtractedDoc>>;\n\n/** The request of a recipe for one input: the URL, and the parts that change with the input. */\nexport interface RecipeRequest {\n  readonly url: string;\n  /** This value replaces the `accept` of the recipe for this request. */\n  readonly accept?: string | undefined;\n  /** These headers go with the request. The crawler does not write them into the capture. */\n  readonly headers?: Readonly<Record<string, string>> | undefined;\n  /** These query parameters hold secrets. The capture shows `REDACTED` in their place. */\n  readonly secretParams?: readonly string[] | undefined;\n}\n\n/** These values declare a recipe. Only `name` and `extract` are necessary. */\nexport interface RecipeDefinition<Input = string> {\n  /**\n   * This name goes into each capture record of the recipe, so it is part of each capture id.\n   * `reindex` finds the extractor by it. Do not change it after the first capture.\n   */\n  readonly name: string;\n  /** This text tells what the recipe reads. */\n  readonly description?: string | undefined;\n  /**\n   * The value `page` (the default) is a web page: robots.txt applies. The value `api` is a\n   * documented API with terms of its own: robots.txt does not apply, and the pace still does.\n   */\n  readonly mode?: \"api\" | \"page\" | undefined;\n  /** This value is the `Accept` header. The defaults are HTML for a page and JSON for an API. */\n  readonly accept?: string | undefined;\n  /** This value is the documented pace of the source: the shortest time between two of its requests. */\n  readonly minInterval?: Duration | undefined;\n  /** A stored copy younger than this age answers with no request. The default is `24h`. */\n  readonly maxAge?: Duration | undefined;\n  /** This value is the most bytes of a body that the crawler keeps. */\n  readonly maxBytes?: number | undefined;\n  /** This function gives the request for an input. With no function, the input is the URL. */\n  readonly fetch?: ((input: Input) => string | RecipeRequest) | undefined;\n  /** This function reads a captured body into documents. It is pure, and it sends no request. */\n  readonly extract: RecipeExtract;\n}\n\n/** These options build the request of a recipe for one input (`Recipe.request`). */\nexport interface RecipeRequestOptions {\n  /** This value replaces the `maxAge` of the recipe. The value `0` asks the source again. */\n  readonly maxAge?: Duration | undefined;\n  /** These agents get their robots.txt rules applied too, for a recipe of the mode `page`. */\n  readonly onBehalfOf?: readonly string[] | undefined;\n  /** This name connects the captures to the job that asked for them. */\n  readonly job?: string | undefined;\n  /** When this signal aborts, the crawler stops the fetch. */\n  readonly signal?: AbortSignal | undefined;\n  /** This label goes into each capture record of the request. */\n  readonly scope?: string | undefined;\n  /** These labels go into each capture record of the request. They are not part of the capture id. */\n  readonly labels?: Readonly<Record<string, string>> | undefined;\n  /** No request starts at or after this time (milliseconds since the epoch, on the clock of the crawler). */\n  readonly deadline?: number | undefined;\n}\n\n/** A recipe that `defineRecipe` made. Give it to `runRecipe`, to `registerRecipe` or to `openCrawler`. */\nexport interface Recipe<Input = string> {\n  readonly name: string;\n  readonly description: string | null;\n  readonly mode: \"api\" | \"page\";\n  readonly accept: string;\n  /** This extractor reads the captures of the recipe. It checks the documents of the definition. */\n  readonly extractor: Extractor;\n  /** This method gives the request of the crawler for one input. A value that is not a URL throws a `TypeError`. */\n  request(input: Input, options?: RecipeRequestOptions): FetchRequest;\n}\n\n/**\n * This function reads a body as JSON, in the encoding that the response declares. A body that is\n * not JSON (a page about maintenance, or a login page) throws an error that names the URL and the\n * content type. In an extractor, the runtime then reports that error.\n */\nexport const readJson = (body: Uint8Array, capture: RecipeCapture): unknown => {\n  const text = decodeBody(body, capture.contentType);\n  try {\n    return JSON.parse(text) as unknown;\n  } catch {\n    throw new Error(\n      `${capture.url} answered ${capture.contentType ?? \"a body of no stated type\"}, not JSON`,\n    );\n  }\n};\n\n/**\n * These recipe names belong to the crawler: the page reader, the archived fallback, robots.txt,\n * and the sitemaps and feeds of `discover`. A recipe cannot use them.\n */\nexport const RESERVED_RECIPES: readonly string[] = [\n  \"page\",\n  \"archive-copy\",\n  \"archive-index\",\n  \"robots.txt\",\n  \"sitemap\",\n  \"feed\",\n];\n\n/** These are the limits of a recipe. */\nexport const RECIPE_LIMITS: {\n  /** The longest recipe name. */\n  readonly nameChars: number;\n  /** The longest document id. */\n  readonly docIdChars: number;\n} = { nameChars: 64, docIdChars: 2048 };\n\nconst NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;\nconst PAGE_ACCEPT = \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.5\";\nconst API_ACCEPT = \"application/json\";\nconst AI_INPUTS: readonly AiInput[] = [\"yes\", \"no\", \"unset\"];\n\n/** This function tells what is wrong with one document of an extractor, or gives undefined. */\nconst faultOf = (value: unknown, seen: Set<string>): string | undefined => {\n  if (value === null || typeof value !== \"object\" || Array.isArray(value))\n    return \"a document must be an object\";\n  const doc = value as Record<string, unknown>;\n  const id = doc[\"doc\"];\n  if (typeof id !== \"string\" || id.length === 0 || id.length > RECIPE_LIMITS.docIdChars)\n    return `doc must be a text of 1 to ${String(RECIPE_LIMITS.docIdChars)} characters`;\n  // the archived fallback labels the copies of third parties, and only it makes these\n  if (id.startsWith(\"archive:\"))\n    return `the id ${id} starts with archive:, which is for the archived copies`;\n  if (seen.has(id)) return `two documents have the id ${id}`;\n  seen.add(id);\n  const kind = doc[\"kind\"];\n  if (typeof kind !== \"string\" || kind.length === 0) return \"kind must be a text that is not empty\";\n  if (kind === \"archived\") return \"the kind archived is for the archived copies\";\n  if (typeof doc[\"title\"] !== \"string\") return \"title must be a text\";\n  if (typeof doc[\"text\"] !== \"string\") return \"text must be a text\";\n  if (doc[\"url\"] !== null && typeof doc[\"url\"] !== \"string\") return \"url must be a text or null\";\n  const meta = doc[\"meta\"];\n  if (meta === null || typeof meta !== \"object\" || Array.isArray(meta))\n    return \"meta must be an object\";\n  try {\n    JSON.stringify(meta);\n  } catch {\n    return \"meta must be JSON: no cycle and no BigInt\";\n  }\n  const merge = doc[\"merge\"];\n  if (merge !== undefined && merge !== \"latest\" && merge !== \"richest\")\n    return 'merge must be \"latest\" or \"richest\"';\n  if (doc[\"full_text\"] !== undefined && typeof doc[\"full_text\"] !== \"string\")\n    return \"full_text must be a text\";\n  const aiInput = doc[\"ai_input\"];\n  if (aiInput !== undefined && !AI_INPUTS.includes(aiInput as AiInput))\n    return 'ai_input must be \"yes\", \"no\" or \"unset\"';\n  return undefined;\n};\n\n/** This function gives the documents of an extractor, or throws an error that names the first fault. */\nconst checkedDocs = (name: string, docs: unknown): ReadonlyArray<ExtractedDoc> => {\n  if (!Array.isArray(docs))\n    throw new TypeError(`the recipe ${name} gave ${typeof docs}, not an array of documents`);\n  const seen = new Set<string>();\n  for (const [index, doc] of docs.entries()) {\n    const fault = faultOf(doc, seen);\n    if (fault !== undefined)\n      throw new TypeError(\n        `the recipe ${name} gave a document that is not valid (${String(index)}): ${fault}`,\n      );\n  }\n  return docs as ReadonlyArray<ExtractedDoc>;\n};\n\n/**\n * This function declares a recipe. It checks the definition at once: a name that is not valid, a\n * reserved name or a duration that it cannot read throws an error. The extractor of the recipe\n * checks each document of `extract`. A document that is not valid makes the extractor throw, and\n * the runtime reports it as an error of the extractor.\n */\nexport const defineRecipe = <Input = string>(\n  definition: RecipeDefinition<Input>,\n): Recipe<Input> => {\n  const name = definition.name;\n  if (typeof name !== \"string\" || name.length > RECIPE_LIMITS.nameChars || !NAME.test(name)) {\n    throw new RangeError(\n      `a recipe name is 1 to ${String(RECIPE_LIMITS.nameChars)} letters, digits and . _ : -, and it starts with a letter or a digit, not ${JSON.stringify(name) ?? \"undefined\"}`,\n    );\n  }\n  if (RESERVED_RECIPES.includes(name)) {\n    throw new RangeError(\n      `the recipe name ${name} belongs to the crawler: give the recipe a name of its own`,\n    );\n  }\n  if (typeof definition.extract !== \"function\") {\n    throw new TypeError(\n      `the recipe ${name} needs extract, a function that reads a body into documents`,\n    );\n  }\n  const mode = definition.mode ?? \"page\";\n  if (mode !== \"page\" && mode !== \"api\") {\n    throw new RangeError(\n      `the mode of the recipe ${name} must be \"page\" or \"api\", not ${String(mode)}`,\n    );\n  }\n  const accept = definition.accept ?? (mode === \"api\" ? API_ACCEPT : PAGE_ACCEPT);\n  const minIntervalMs =\n    definition.minInterval === undefined ? undefined : durationMs(definition.minInterval);\n  const maxAgeOf = (value: Duration): number | \"auto\" =>\n    value === \"auto\" ? \"auto\" : durationMs(value);\n  const maxAge = maxAgeOf(definition.maxAge ?? \"24h\");\n  const maxBytes = definition.maxBytes;\n  if (maxBytes !== undefined && (!Number.isSafeInteger(maxBytes) || maxBytes < 1)) {\n    throw new RangeError(`maxBytes of the recipe ${name} must be a whole number of 1 or more`);\n  }\n  const extract = definition.extract;\n  const extractor: Extractor = async (body, capture) =>\n    checkedDocs(name, await extract(body, { url: capture.url, contentType: capture.contentType }));\n  const fetch = definition.fetch;\n  return {\n    name,\n    description: definition.description ?? null,\n    mode,\n    accept,\n    extractor,\n    request(input, options = {}) {\n      const asked: unknown = fetch === undefined ? input : fetch(input);\n      const target = typeof asked === \"string\" ? { url: asked } : (asked as RecipeRequest | null);\n      const url = target?.url;\n      let parsed: URL | undefined;\n      try {\n        parsed = typeof url === \"string\" ? new URL(url) : undefined;\n      } catch {\n        parsed = undefined;\n      }\n      if (parsed === undefined || (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\")) {\n        throw new TypeError(\n          `the recipe ${name} needs an http or https URL for its input, not ${JSON.stringify(url) ?? String(url)}`,\n        );\n      }\n      return {\n        url: parsed.href,\n        mode,\n        recipe: name,\n        accept: target?.accept ?? accept,\n        headers: target?.headers,\n        secretParams: target?.secretParams,\n        minIntervalMs,\n        maxAgeMs: options.maxAge === undefined ? maxAge : maxAgeOf(options.maxAge),\n        maxBytes,\n        onBehalfOf: options.onBehalfOf,\n        job: options.job,\n        signal: options.signal,\n        scope: options.scope,\n        labels: options.labels,\n        deadline: options.deadline,\n      };\n    },\n  };\n};\n\n/**\n * This function gives the extractor of a recipe to the extractors of a runtime. A second\n * registration of the same recipe does nothing. A different extractor with the same name throws\n * an error, because `reindex` must read each capture with one extractor.\n */\nexport const registerRecipe = <Input>(extractors: Extractors, recipe: Recipe<Input>): void => {\n  const known = extractors.get(recipe.name);\n  if (known === recipe.extractor) return;\n  if (known !== undefined) {\n    throw new Error(\n      `the recipe ${recipe.name} has a different extractor in this runtime: give each recipe a name of its own`,\n    );\n  }\n  extractors.register(recipe.name, recipe.extractor);\n};\n\n/** These options control one run of a recipe. */\nexport interface RecipeRunOptions extends Omit<RecipeRequestOptions, \"deadline\"> {\n  /** No request starts after this time from the start of the run, for example `\"30s\"`. */\n  readonly budget?: Duration | undefined;\n  /** This value is true when a person gave this input directly. It is the only way to see a held-back document. The crawler logs it. */\n  readonly userProvided?: boolean | undefined;\n  /**\n   * The value `archive` gives a copy from a public web archive when the site refuses the page or is\n   * down. Only a recipe of the mode `page` takes it. The page reader reads the copy.\n   */\n  readonly fallback?: \"archive\" | \"none\" | undefined;\n  /** The value true also gives an archived copy of a page that the current robots.txt disallows. */\n  readonly archiveDespiteRobots?: boolean | undefined;\n}\n\n/** One document of a run of a recipe: the version of the capture of the run. */\nexport interface RecipeDoc {\n  readonly doc: string;\n  readonly kind: string;\n  /** This value is the title. A held-back document shows no title. */\n  readonly title: string;\n  readonly url: string | null;\n  /** This value is the metadata. A held-back document shows only the structural fields. */\n  readonly meta: Readonly<Record<string, unknown>>;\n  /** This value is the length of the text. `read` gives the text. */\n  readonly chars: number;\n  /** This id is the capture of this version. A citation names it. */\n  readonly capture: string;\n  readonly fetched_at: string;\n  readonly held_back: boolean;\n  /** This value tells that the document is an archived copy, with the archive and the date of the copy. */\n  readonly archived?: Archived | undefined;\n}\n\n/** The result of `runRecipe`. */\nexport interface RecipeResult {\n  readonly recipe: string;\n  /** This URL is the URL of the capture. Its secrets show as `REDACTED`. */\n  readonly url: string;\n  readonly outcome: string;\n  readonly status: number;\n  /** These documents came from the capture of the run, in the order of the extractor. */\n  readonly docs: readonly RecipeDoc[];\n  /** This id is the capture of the documents. */\n  readonly capture: string | null;\n  readonly fetched_at: string | null;\n  /** This value is true when the stored copy answered, and the crawler sent no request. */\n  readonly from_cache: boolean;\n  /** This value is true when a document is held back. */\n  readonly held_back: boolean;\n  /** This text is the error of the extractor, when it threw for the capture. */\n  readonly error: string | null;\n  /** This value tells that the documents are an archived copy, not a fresh fetch. */\n  readonly archived?: Archived | undefined;\n  readonly note: string | null;\n}\n\n/** A recipe operates on a runtime, or on a crawler that has one (`crawler.runtime`). */\nexport type RecipeTarget = CrawlRuntime | { readonly runtime: CrawlRuntime };\n\n/** This function gives the runtime of a target. */\nexport const runtimeOf = (target: RecipeTarget): CrawlRuntime =>\n  \"crawler\" in target ? target : target.runtime;\n\n/** This function gives the deadline of a budget, on the clock of the runtime. */\nexport const deadlineOf = (\n  runtime: CrawlRuntime,\n  budget: Duration | undefined,\n): number | undefined =>\n  budget === undefined ? undefined : Date.parse(runtime.now()) + durationMs(budget);\n\n/** This function gives one document of a run, with the rule for a held-back document. */\nconst recipeDoc = async (\n  runtime: CrawlRuntime,\n  doc: Doc,\n  capture: string | undefined,\n  userProvided: boolean,\n  action: string,\n): Promise<RecipeDoc> => {\n  const current = runtime.index.get(doc.doc) ?? doc;\n  const version: VersionText | Doc =\n    (capture === undefined ? undefined : runtime.index.versionText(doc.doc, capture)) ?? current;\n  const aiInput: AiInput =\n    current.ai_input === \"no\" || version.ai_input === \"no\" ? \"no\" : version.ai_input;\n  const held = isHeldBack({ ai_input: aiInput }, userProvided);\n  if (aiInput === \"no\" && userProvided) await recordOverride(runtime, current, action);\n  const archived = archivedOf(version.meta);\n  return {\n    doc: doc.doc,\n    kind: version.kind,\n    title: held ? \"\" : version.title,\n    url: version.url,\n    meta: held ? heldBackMeta(version.meta) : version.meta,\n    chars: version.text.length,\n    capture: version.capture,\n    fetched_at: version.fetched_at,\n    held_back: held,\n    ...(archived === undefined ? {} : { archived }),\n  };\n};\n\nconst noteOf = (parts: ReadonlyArray<string | null | undefined>): string | null => {\n  const kept = parts.filter((part): part is string => typeof part === \"string\" && part.length > 0);\n  return kept.length > 0 ? kept.join(\"; \") : null;\n};\n\n/**\n * This function starts a recipe for one input. It fetches the request through the crawler, and it\n * reads the capture with the extractor of the recipe. It gives the documents of that capture.\n * These are its rules:\n *\n * - The runtime registers the recipe the first time that it gets it. Give the recipe to\n *   `openCrawler` too, so that `reindex` reads its captures before the first fetch.\n * - A stored copy younger than `maxAge` answers with no request. When the extractor cannot read a\n *   stored copy, the crawler asks the source one more time.\n * - An extractor that throws gives no document, and `error` tells why. The fetch still succeeds.\n * - A document that asked not to be AI input shows no title and only its structural metadata,\n *   unless `userProvided`. The crawler logs each `userProvided` read of a held-back document.\n * - With `fallback: \"archive\"`, a page that the site refuses gets an archived copy, with\n *   `archived`. The page reader reads the copy. The extractor of the recipe reads only live\n *   captures. Thus its documents do not mix a live answer and an old copy of a third party.\n */\nexport const runRecipe = async <Input>(\n  target: RecipeTarget,\n  recipe: Recipe<Input>,\n  input: Input,\n  options: RecipeRunOptions = {},\n): Promise<RecipeResult> => {\n  const runtime = runtimeOf(target);\n  if (options.fallback === \"archive\" && recipe.mode !== \"page\") {\n    throw new RangeError(\n      `the recipe ${recipe.name} reads an API: an archived copy is for a page, so give fallback only to a recipe of the mode page`,\n    );\n  }\n  registerRecipe(runtime.extractors, recipe);\n  const deadline = deadlineOf(runtime, options.budget);\n  const request = recipe.request(input, { ...options, deadline });\n  let indexed: Indexed = await fetchAndIndex(runtime, request);\n  // a stored answer that the extractor cannot read is no answer: it would give the same error until maxAge\n  if (indexed.error !== undefined && indexed.blocked === undefined && indexed.fetched.fromCache) {\n    indexed = await fetchAndIndex(runtime, { ...request, maxAgeMs: 0 });\n  }\n  const userProvided = options.userProvided ?? false;\n  const action = options.job ?? `recipe ${recipe.name}`;\n  const fetched = indexed.fetched;\n  const notes: Array<string | undefined> = [fetched.note, indexed.error];\n  if (indexed.docs.length === 0 && options.fallback === \"archive\") {\n    const reason = fallbackReason(indexed);\n    if (reason !== undefined) {\n      const page: PageOptions = {\n        render: \"never\",\n        maxAgeMs: request.maxAgeMs,\n        onBehalfOf: request.onBehalfOf,\n        job: request.job,\n        signal: request.signal,\n        scope: request.scope,\n        labels: request.labels,\n        deadline,\n        fallback: \"archive\",\n        archiveDespiteRobots: options.archiveDespiteRobots,\n      };\n      const answer = await archivedCopy(runtime, request.url, reason, page);\n      if (answer.doc !== undefined && answer.archived !== undefined) {\n        const doc = await recipeDoc(runtime, answer.doc, answer.doc.capture, userProvided, action);\n        return {\n          recipe: recipe.name,\n          url: fetched.url,\n          outcome: fetched.outcome,\n          status: fetched.status,\n          docs: [doc],\n          capture: doc.capture,\n          fetched_at: doc.fetched_at,\n          from_cache: answer.fromCache,\n          held_back: doc.held_back,\n          error: indexed.error ?? null,\n          archived: answer.archived,\n          note: noteOf([\n            ...notes,\n            ...answer.notes,\n            \"the page reader read the archived copy, not the extractor of the recipe\",\n            doc.held_back ? HELD_BACK : undefined,\n          ]),\n        };\n      }\n      notes.push(...answer.notes);\n    }\n  }\n  const capture = fetched.capture;\n  const docs: RecipeDoc[] = [];\n  for (const doc of indexed.docs)\n    docs.push(await recipeDoc(runtime, doc, capture, userProvided, action));\n  const held = docs.some((doc) => doc.held_back);\n  return {\n    recipe: recipe.name,\n    url: fetched.url,\n    outcome: fetched.outcome,\n    status: fetched.status,\n    docs,\n    capture: docs[0]?.capture ?? capture ?? null,\n    fetched_at: docs[0]?.fetched_at ?? fetched.fetchedAt ?? null,\n    from_cache: fetched.fromCache,\n    held_back: held,\n    error: indexed.error ?? null,\n    note: noteOf([...notes, held ? HELD_BACK : undefined]),\n  };\n};\n"],"mappings":";;;;;;;;;AAgHA,MAAa,YAAY,MAAkB,YAAoC;CAC7E,MAAM,OAAO,WAAW,MAAM,QAAQ,WAAW;CACjD,IAAI;EACF,OAAO,KAAK,MAAM,IAAI;CACxB,QAAQ;EACN,MAAM,IAAI,MACR,GAAG,QAAQ,IAAI,YAAY,QAAQ,eAAe,2BAA2B,WAC/E;CACF;AACF;;;;;AAMA,MAAa,mBAAsC;CACjD;CACA;CACA;CACA;CACA;CACA;AACF;;AAGA,MAAa,gBAKT;CAAE,WAAW;CAAI,YAAY;AAAK;AAEtC,MAAM,OAAO;AACb,MAAM,cAAc;AACpB,MAAM,aAAa;AACnB,MAAM,YAAgC;CAAC;CAAO;CAAM;AAAO;;AAG3D,MAAM,WAAW,OAAgB,SAA0C;CACzE,IAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GACpE,OAAO;CACT,MAAM,MAAM;CACZ,MAAM,KAAK,IAAI;CACf,IAAI,OAAO,OAAO,YAAY,GAAG,WAAW,KAAK,GAAG,SAAS,cAAc,YACzE,OAAO,8BAA8B,OAAO,cAAc,UAAU,EAAE;CAExE,IAAI,GAAG,WAAW,UAAU,GAC1B,OAAO,UAAU,GAAG;CACtB,IAAI,KAAK,IAAI,EAAE,GAAG,OAAO,6BAA6B;CACtD,KAAK,IAAI,EAAE;CACX,MAAM,OAAO,IAAI;CACjB,IAAI,OAAO,SAAS,YAAY,KAAK,WAAW,GAAG,OAAO;CAC1D,IAAI,SAAS,YAAY,OAAO;CAChC,IAAI,OAAO,IAAI,aAAa,UAAU,OAAO;CAC7C,IAAI,OAAO,IAAI,YAAY,UAAU,OAAO;CAC5C,IAAI,IAAI,WAAW,QAAQ,OAAO,IAAI,WAAW,UAAU,OAAO;CAClE,MAAM,OAAO,IAAI;CACjB,IAAI,SAAS,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GACjE,OAAO;CACT,IAAI;EACF,KAAK,UAAU,IAAI;CACrB,QAAQ;EACN,OAAO;CACT;CACA,MAAM,QAAQ,IAAI;CAClB,IAAI,UAAU,KAAA,KAAa,UAAU,YAAY,UAAU,WACzD,OAAO;CACT,IAAI,IAAI,iBAAiB,KAAA,KAAa,OAAO,IAAI,iBAAiB,UAChE,OAAO;CACT,MAAM,UAAU,IAAI;CACpB,IAAI,YAAY,KAAA,KAAa,CAAC,UAAU,SAAS,OAAkB,GACjE,OAAO;AAEX;;AAGA,MAAM,eAAe,MAAc,SAA+C;CAChF,IAAI,CAAC,MAAM,QAAQ,IAAI,GACrB,MAAM,IAAI,UAAU,cAAc,KAAK,QAAQ,OAAO,KAAK,4BAA4B;CACzF,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,CAAC,OAAO,QAAQ,KAAK,QAAQ,GAAG;EACzC,MAAM,QAAQ,QAAQ,KAAK,IAAI;EAC/B,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,UACR,cAAc,KAAK,sCAAsC,OAAO,KAAK,EAAE,KAAK,OAC9E;CACJ;CACA,OAAO;AACT;;;;;;;AAQA,MAAa,gBACX,eACkB;CAClB,MAAM,OAAO,WAAW;CACxB,IAAI,OAAO,SAAS,YAAY,KAAK,SAAS,cAAc,aAAa,CAAC,KAAK,KAAK,IAAI,GACtF,MAAM,IAAI,WACR,yBAAyB,OAAO,cAAc,SAAS,EAAE,4EAA4E,KAAK,UAAU,IAAI,KAAK,aAC/J;CAEF,IAAI,iBAAiB,SAAS,IAAI,GAChC,MAAM,IAAI,WACR,mBAAmB,KAAK,2DAC1B;CAEF,IAAI,OAAO,WAAW,YAAY,YAChC,MAAM,IAAI,UACR,cAAc,KAAK,4DACrB;CAEF,MAAM,OAAO,WAAW,QAAQ;CAChC,IAAI,SAAS,UAAU,SAAS,OAC9B,MAAM,IAAI,WACR,0BAA0B,KAAK,gCAAgC,OAAO,IAAI,GAC5E;CAEF,MAAM,SAAS,WAAW,WAAW,SAAS,QAAQ,aAAa;CACnE,MAAM,gBACJ,WAAW,gBAAgB,KAAA,IAAY,KAAA,IAAY,WAAW,WAAW,WAAW;CACtF,MAAM,YAAY,UAChB,UAAU,SAAS,SAAS,WAAW,KAAK;CAC9C,MAAM,SAAS,SAAS,WAAW,UAAU,KAAK;CAClD,MAAM,WAAW,WAAW;CAC5B,IAAI,aAAa,KAAA,MAAc,CAAC,OAAO,cAAc,QAAQ,KAAK,WAAW,IAC3E,MAAM,IAAI,WAAW,0BAA0B,KAAK,qCAAqC;CAE3F,MAAM,UAAU,WAAW;CAC3B,MAAM,YAAuB,OAAO,MAAM,YACxC,YAAY,MAAM,MAAM,QAAQ,MAAM;EAAE,KAAK,QAAQ;EAAK,aAAa,QAAQ;CAAY,CAAC,CAAC;CAC/F,MAAM,QAAQ,WAAW;CACzB,OAAO;EACL;EACA,aAAa,WAAW,eAAe;EACvC;EACA;EACA;EACA,QAAQ,OAAO,UAAU,CAAC,GAAG;GAC3B,MAAM,QAAiB,UAAU,KAAA,IAAY,QAAQ,MAAM,KAAK;GAChE,MAAM,SAAS,OAAO,UAAU,WAAW,EAAE,KAAK,MAAM,IAAK;GAC7D,MAAM,MAAM,QAAQ;GACpB,IAAI;GACJ,IAAI;IACF,SAAS,OAAO,QAAQ,WAAW,IAAI,IAAI,GAAG,IAAI,KAAA;GACpD,QAAQ;IACN,SAAS,KAAA;GACX;GACA,IAAI,WAAW,KAAA,KAAc,OAAO,aAAa,WAAW,OAAO,aAAa,UAC9E,MAAM,IAAI,UACR,cAAc,KAAK,iDAAiD,KAAK,UAAU,GAAG,KAAK,OAAO,GAAG,GACvG;GAEF,OAAO;IACL,KAAK,OAAO;IACZ;IACA,QAAQ;IACR,QAAQ,QAAQ,UAAU;IAC1B,SAAS,QAAQ;IACjB,cAAc,QAAQ;IACtB;IACA,UAAU,QAAQ,WAAW,KAAA,IAAY,SAAS,SAAS,QAAQ,MAAM;IACzE;IACA,YAAY,QAAQ;IACpB,KAAK,QAAQ;IACb,QAAQ,QAAQ;IAChB,OAAO,QAAQ;IACf,QAAQ,QAAQ;IAChB,UAAU,QAAQ;GACpB;EACF;CACF;AACF;;;;;;AAOA,MAAa,kBAAyB,YAAwB,WAAgC;CAC5F,MAAM,QAAQ,WAAW,IAAI,OAAO,IAAI;CACxC,IAAI,UAAU,OAAO,WAAW;CAChC,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MACR,cAAc,OAAO,KAAK,+EAC5B;CAEF,WAAW,SAAS,OAAO,MAAM,OAAO,SAAS;AACnD;;AA+DA,MAAa,aAAa,WACxB,aAAa,SAAS,SAAS,OAAO;;AAGxC,MAAa,cACX,SACA,WAEA,WAAW,KAAA,IAAY,KAAA,IAAY,KAAK,MAAM,QAAQ,IAAI,CAAC,IAAI,WAAW,MAAM;;AAGlF,MAAM,YAAY,OAChB,SACA,KACA,SACA,cACA,WACuB;CACvB,MAAM,UAAU,QAAQ,MAAM,IAAI,IAAI,GAAG,KAAK;CAC9C,MAAM,WACH,YAAY,KAAA,IAAY,KAAA,IAAY,QAAQ,MAAM,YAAY,IAAI,KAAK,OAAO,MAAM;CACvF,MAAM,UACJ,QAAQ,aAAa,QAAQ,QAAQ,aAAa,OAAO,OAAO,QAAQ;CAC1E,MAAM,OAAO,WAAW,EAAE,UAAU,QAAQ,GAAG,YAAY;CAC3D,IAAI,YAAY,QAAQ,cAAc,MAAM,eAAe,SAAS,SAAS,MAAM;CACnF,MAAM,WAAW,WAAW,QAAQ,IAAI;CACxC,OAAO;EACL,KAAK,IAAI;EACT,MAAM,QAAQ;EACd,OAAO,OAAO,KAAK,QAAQ;EAC3B,KAAK,QAAQ;EACb,MAAM,OAAO,aAAa,QAAQ,IAAI,IAAI,QAAQ;EAClD,OAAO,QAAQ,KAAK;EACpB,SAAS,QAAQ;EACjB,YAAY,QAAQ;EACpB,WAAW;EACX,GAAI,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS;CAC/C;AACF;AAEA,MAAM,UAAU,UAAmE;CACjF,MAAM,OAAO,MAAM,QAAQ,SAAyB,OAAO,SAAS,YAAY,KAAK,SAAS,CAAC;CAC/F,OAAO,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI;AAC7C;;;;;;;;;;;;;;;;;AAkBA,MAAa,YAAY,OACvB,QACA,QACA,OACA,UAA4B,CAAC,MACH;CAC1B,MAAM,UAAU,UAAU,MAAM;CAChC,IAAI,QAAQ,aAAa,aAAa,OAAO,SAAS,QACpD,MAAM,IAAI,WACR,cAAc,OAAO,KAAK,kGAC5B;CAEF,eAAe,QAAQ,YAAY,MAAM;CACzC,MAAM,WAAW,WAAW,SAAS,QAAQ,MAAM;CACnD,MAAM,UAAU,OAAO,QAAQ,OAAO;EAAE,GAAG;EAAS;CAAS,CAAC;CAC9D,IAAI,UAAmB,MAAM,cAAc,SAAS,OAAO;CAE3D,IAAI,QAAQ,UAAU,KAAA,KAAa,QAAQ,YAAY,KAAA,KAAa,QAAQ,QAAQ,WAClF,UAAU,MAAM,cAAc,SAAS;EAAE,GAAG;EAAS,UAAU;CAAE,CAAC;CAEpE,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,SAAS,QAAQ,OAAO,UAAU,OAAO;CAC/C,MAAM,UAAU,QAAQ;CACxB,MAAM,QAAmC,CAAC,QAAQ,MAAM,QAAQ,KAAK;CACrE,IAAI,QAAQ,KAAK,WAAW,KAAK,QAAQ,aAAa,WAAW;EAC/D,MAAM,SAAS,eAAe,OAAO;EACrC,IAAI,WAAW,KAAA,GAAW;GACxB,MAAM,OAAoB;IACxB,QAAQ;IACR,UAAU,QAAQ;IAClB,YAAY,QAAQ;IACpB,KAAK,QAAQ;IACb,QAAQ,QAAQ;IAChB,OAAO,QAAQ;IACf,QAAQ,QAAQ;IAChB;IACA,UAAU;IACV,sBAAsB,QAAQ;GAChC;GACA,MAAM,SAAS,MAAM,aAAa,SAAS,QAAQ,KAAK,QAAQ,IAAI;GACpE,IAAI,OAAO,QAAQ,KAAA,KAAa,OAAO,aAAa,KAAA,GAAW;IAC7D,MAAM,MAAM,MAAM,UAAU,SAAS,OAAO,KAAK,OAAO,IAAI,SAAS,cAAc,MAAM;IACzF,OAAO;KACL,QAAQ,OAAO;KACf,KAAK,QAAQ;KACb,SAAS,QAAQ;KACjB,QAAQ,QAAQ;KAChB,MAAM,CAAC,GAAG;KACV,SAAS,IAAI;KACb,YAAY,IAAI;KAChB,YAAY,OAAO;KACnB,WAAW,IAAI;KACf,OAAO,QAAQ,SAAS;KACxB,UAAU,OAAO;KACjB,MAAM,OAAO;MACX,GAAG;MACH,GAAG,OAAO;MACV;MACA,IAAI,YAAY,YAAY,KAAA;KAC9B,CAAC;IACH;GACF;GACA,MAAM,KAAK,GAAG,OAAO,KAAK;EAC5B;CACF;CACA,MAAM,UAAU,QAAQ;CACxB,MAAM,OAAoB,CAAC;CAC3B,KAAK,MAAM,OAAO,QAAQ,MACxB,KAAK,KAAK,MAAM,UAAU,SAAS,KAAK,SAAS,cAAc,MAAM,CAAC;CACxE,MAAM,OAAO,KAAK,MAAM,QAAQ,IAAI,SAAS;CAC7C,OAAO;EACL,QAAQ,OAAO;EACf,KAAK,QAAQ;EACb,SAAS,QAAQ;EACjB,QAAQ,QAAQ;EAChB;EACA,SAAS,KAAK,EAAE,EAAE,WAAW,WAAW;EACxC,YAAY,KAAK,EAAE,EAAE,cAAc,QAAQ,aAAa;EACxD,YAAY,QAAQ;EACpB,WAAW;EACX,OAAO,QAAQ,SAAS;EACxB,MAAM,OAAO,CAAC,GAAG,OAAO,OAAO,YAAY,KAAA,CAAS,CAAC;CACvD;AACF"}