polite-crawler 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/{cli-daemon-C2_Pd4v_.mjs → cli-daemon-CehzGYMQ.mjs} +2 -2
- package/build/{cli-daemon-C2_Pd4v_.mjs.map → cli-daemon-CehzGYMQ.mjs.map} +1 -1
- package/build/{cli-recipes-XlxoNo75.mjs → cli-recipes-InEU5y1L.mjs} +2 -2
- package/build/{cli-recipes-XlxoNo75.mjs.map → cli-recipes-InEU5y1L.mjs.map} +1 -1
- package/build/cli.mjs +7 -7
- package/build/{daemon-B0Cp8xeB.mjs → daemon-1RNbvE5p.mjs} +3 -3
- package/build/{daemon-B0Cp8xeB.mjs.map → daemon-1RNbvE5p.mjs.map} +1 -1
- package/build/{diff-DHpF9lYY.mjs → diff-DLyQ2mkg.mjs} +2 -2
- package/build/{diff-DHpF9lYY.mjs.map → diff-DLyQ2mkg.mjs.map} +1 -1
- package/build/{extract-BgcEkiQr.mjs → extract-mnKsYeaM.mjs} +3 -3
- package/build/{extract-BgcEkiQr.mjs.map → extract-mnKsYeaM.mjs.map} +1 -1
- package/build/{html-DzobO0fy.mjs → html-Dir5QEi0.mjs} +2 -2
- package/build/html-Dir5QEi0.mjs.map +1 -0
- package/build/html.mjs +1 -1
- package/build/index.d.mts +1 -1
- package/build/index.mjs +11 -11
- package/build/{open-YGJPjbDc.mjs → open-CDFgV-FC.mjs} +7 -7
- package/build/{open-YGJPjbDc.mjs.map → open-CDFgV-FC.mjs.map} +1 -1
- package/build/{recipe-CPmYb91t.mjs → recipe-BNjpjJ_8.mjs} +4 -4
- package/build/{recipe-CPmYb91t.mjs.map → recipe-BNjpjJ_8.mjs.map} +1 -1
- package/build/{recipe-product-B6XmkpB3.mjs → recipe-product-BqDdxZxL.mjs} +5 -5
- package/build/{recipe-product-B6XmkpB3.mjs.map → recipe-product-BqDdxZxL.mjs.map} +1 -1
- package/build/{runtime-j2NpXEiN.mjs → runtime-p5WN2UAA.mjs} +3 -3
- package/build/{runtime-j2NpXEiN.mjs.map → runtime-p5WN2UAA.mjs.map} +1 -1
- package/build/{version-Dw3pZOVT.mjs → version-BVrf4OJP.mjs} +2 -2
- package/build/{version-Dw3pZOVT.mjs.map → version-BVrf4OJP.mjs.map} +1 -1
- package/build/{wacz-ixsUXYCs.mjs → wacz-BXMxWyfg.mjs} +3 -3
- package/build/{wacz-ixsUXYCs.mjs.map → wacz-BXMxWyfg.mjs.map} +1 -1
- package/build/{warc-B47Bdjzk.mjs → warc-DmzB-FkC.mjs} +2 -2
- package/build/{warc-B47Bdjzk.mjs.map → warc-DmzB-FkC.mjs.map} +1 -1
- package/package.json +1 -1
- package/build/html-DzobO0fy.mjs.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runtime-j2NpXEiN.mjs","names":["systemLookup","messageOf","MAX_INFLATED_BYTES"],"sources":["../src/egress.ts","../src/record.ts","../src/crawler.ts","../src/fold.ts","../src/pdf.ts","../src/fallback.ts","../src/runtime.ts"],"sourcesContent":["import type { LookupAddress, LookupAllOptions, LookupOptions } from \"node:dns\";\nimport { lookup as systemLookup } from \"node:dns\";\nimport type { LookupFunction } from \"node:net\";\nimport { isIP } from \"node:net\";\n\n/**\n * The egress policy keeps the crawler on the public internet. A page or an agent can ask for an\n * internal address. Examples are a loopback service, a router on the LAN and the metadata\n * service of a cloud host. The crawler refuses these addresses by default.\n *\n * - The crawler fetches only `http:` and `https:` URLs.\n * - The check uses the address after DNS. The `lookup` hook of the connection checks the address\n * that the socket then uses, so a DNS answer that changes cannot pass the check.\n * - The crawler examines a literal IP host directly, in each spelling that the URL parser reads.\n * - The crawler examines each redirect hop again.\n *\n * A refusal is a skipped fetch with a clear note. It does not throw an error.\n */\n\n/** These options change the egress policy. With no options, the crawler fetches only public addresses. */\nexport interface EgressOptions {\n /** The value true lets the crawler fetch loopback, private and link-local addresses. The default is false. */\n readonly allowPrivate?: boolean | undefined;\n /**\n * These hosts are permitted although their address is not public. An entry is a host name, a\n * name pattern (`*.corp.example`), an IP address or a CIDR range (`10.0.0.0/8`).\n */\n readonly allowHosts?: readonly string[] | undefined;\n}\n\n/**\n * The connection hook gives this error when the policy refuses each address of a host.\n *\n * @beta\n */\nexport class EgressRefused extends Error {\n readonly code: string = \"EGRESS_REFUSED\";\n constructor(\n readonly address: string,\n readonly kind: string,\n ) {\n super(egressNote(address, kind));\n this.name = \"EgressRefused\";\n }\n}\n\n/** This function gives the note of a refusal, with the address and the kind of address. */\nexport const egressNote = (address: string, kind: string): string =>\n `egress policy: ${address} is ${/^[aeiou]/i.test(kind) ? \"an\" : \"a\"} ${kind} address; set allowPrivate to fetch it`;\n\ntype Bytes = readonly number[];\n\n/**\n * This function reads an IPv4 address the way the URL parser reads a host. The address has one to\n * four parts, and each part is decimal, octal (`0177`) or hexadecimal (`0x7f`). Thus\n * `2130706433` and `0x7f.1` both give 127.0.0.1. A text that is not an IPv4 address gives no value.\n */\nexport const parseIPv4 = (text: string): Bytes | undefined => {\n const parts = text.split(\".\");\n if (parts.length > 1 && parts[parts.length - 1] === \"\") parts.pop();\n if (parts.length === 0 || parts.length > 4) return undefined;\n const numbers: number[] = [];\n for (const part of parts) {\n let value: number;\n if (/^0[xX][0-9a-fA-F]*$/.test(part))\n value = part.length === 2 ? 0 : Number.parseInt(part.slice(2), 16);\n else if (/^0[0-7]+$/.test(part)) value = Number.parseInt(part.slice(1), 8);\n else if (/^(0|[1-9][0-9]*)$/.test(part)) value = Number(part);\n else return undefined;\n numbers.push(value);\n }\n const last = numbers.pop()!;\n if (numbers.some((value) => value > 255)) return undefined;\n if (last >= 256 ** (4 - numbers.length)) return undefined;\n let rest = last;\n const tail: number[] = [];\n for (let index = numbers.length; index < 4; index += 1) {\n tail.unshift(rest % 256);\n rest = Math.floor(rest / 256);\n }\n return [...numbers, ...tail];\n};\n\n/** This function reads an IPv6 address, with or without brackets and a zone. It gives 16 bytes. */\nexport const parseIPv6 = (text: string): Bytes | undefined => {\n let value = text.startsWith(\"[\") && text.endsWith(\"]\") ? text.slice(1, -1) : text;\n const zone = value.indexOf(\"%\");\n if (zone !== -1) value = value.slice(0, zone);\n if (isIP(value) !== 6) return undefined;\n const lastColon = value.lastIndexOf(\":\");\n const tail = value.slice(lastColon + 1);\n if (tail.includes(\".\")) {\n const v4 = parseIPv4(tail);\n if (v4 === undefined) return undefined;\n const high = (v4[0]! << 8) | v4[1]!;\n const low = (v4[2]! << 8) | v4[3]!;\n value = `${value.slice(0, lastColon + 1)}${high.toString(16)}:${low.toString(16)}`;\n }\n const [head = \"\", rest] = value.split(\"::\");\n const groupsOf = (part: string): number[] =>\n part.length === 0 ? [] : part.split(\":\").map((group) => Number.parseInt(group, 16));\n const front = groupsOf(head);\n const back = rest === undefined ? [] : groupsOf(rest);\n const groups = [...front, ...Array<number>(8 - front.length - back.length).fill(0), ...back];\n if (groups.length !== 8) return undefined;\n return groups.flatMap((group) => [group >> 8, group & 0xff]);\n};\n\nconst v4 = (text: string): Bytes => parseIPv4(text)!;\nconst v6 = (text: string): Bytes => parseIPv6(text)!;\n\n/** This function tells if the first `bits` bits of two addresses agree. */\nconst inPrefix = (address: Bytes, prefix: Bytes, bits: number): boolean => {\n if (address.length !== prefix.length) return false;\n for (let index = 0; index < address.length; index += 1) {\n const take = Math.max(0, Math.min(8, bits - index * 8));\n if (take === 0) return true;\n const mask = (0xff << (8 - take)) & 0xff;\n if ((address[index]! & mask) !== (prefix[index]! & mask)) return false;\n }\n return true;\n};\n\n// the more specific ranges come first, so a metadata address does not read as link-local\nconst V4_RANGES: ReadonlyArray<readonly [Bytes, number, string]> = [\n [v4(\"169.254.169.254\"), 32, \"metadata\"],\n [v4(\"168.63.129.16\"), 32, \"metadata\"],\n [v4(\"100.100.100.200\"), 32, \"metadata\"],\n [v4(\"0.0.0.0\"), 8, \"unspecified\"],\n [v4(\"10.0.0.0\"), 8, \"private\"],\n [v4(\"100.64.0.0\"), 10, \"carrier-grade NAT\"],\n [v4(\"127.0.0.0\"), 8, \"loopback\"],\n [v4(\"169.254.0.0\"), 16, \"link-local\"],\n [v4(\"172.16.0.0\"), 12, \"private\"],\n [v4(\"192.0.0.0\"), 24, \"reserved\"],\n [v4(\"192.0.2.0\"), 24, \"documentation\"],\n [v4(\"192.168.0.0\"), 16, \"private\"],\n [v4(\"198.18.0.0\"), 15, \"benchmark\"],\n [v4(\"198.51.100.0\"), 24, \"documentation\"],\n [v4(\"203.0.113.0\"), 24, \"documentation\"],\n [v4(\"224.0.0.0\"), 4, \"multicast\"],\n [v4(\"240.0.0.0\"), 4, \"reserved\"],\n];\n\nconst V6_RANGES: ReadonlyArray<readonly [Bytes, number, string]> = [\n [v6(\"fd00:ec2::254\"), 128, \"metadata\"],\n [v6(\"::\"), 128, \"unspecified\"],\n [v6(\"::1\"), 128, \"loopback\"],\n [v6(\"64:ff9b:1::\"), 48, \"reserved\"],\n [v6(\"100::\"), 64, \"reserved\"],\n [v6(\"2001::\"), 32, \"reserved\"],\n [v6(\"2001:db8::\"), 32, \"documentation\"],\n [v6(\"fc00::\"), 7, \"unique local\"],\n [v6(\"fe80::\"), 10, \"link-local\"],\n [v6(\"fec0::\"), 10, \"private\"],\n [v6(\"ff00::\"), 8, \"multicast\"],\n];\n\nconst kindOfV4 = (address: Bytes): string | undefined =>\n V4_RANGES.find(([prefix, bits]) => inPrefix(address, prefix, bits))?.[2];\n\nconst kindOfV6 = (address: Bytes): string | undefined => {\n const zeroes = (from: number, to: number): boolean =>\n address.slice(from, to).every((byte) => byte === 0);\n const embedded = (from: number): string | undefined => kindOfV4(address.slice(from, from + 4));\n // an IPv4 address in an IPv6 form has the kind of the IPv4 address\n if (zeroes(0, 10) && address[10] === 0xff && address[11] === 0xff) return embedded(12);\n if (inPrefix(address, v6(\"64:ff9b::\"), 96)) return embedded(12);\n if (address[0] === 0x20 && address[1] === 0x02) return embedded(2);\n const listed = V6_RANGES.find(([prefix, bits]) => inPrefix(address, prefix, bits))?.[2];\n if (listed !== undefined) return listed;\n // the old IPv4-compatible form (::a.b.c.d) and the rest of ::/8 are not public\n if (zeroes(0, 12)) return embedded(12) ?? \"reserved\";\n if (address[0] === 0) return \"reserved\";\n return undefined;\n};\n\n/**\n * This function gives the kind of an address that is not public, for example `loopback` or\n * `private`. It reads each IPv4 spelling and each IPv6 form. A public address, or a text that is not\n * an address, gives no value.\n */\nexport const addressKind = (address: string): string | undefined => {\n const four = parseIPv4(address);\n if (four !== undefined) return kindOfV4(four);\n const six = parseIPv6(address);\n return six === undefined ? undefined : kindOfV6(six);\n};\n\nconst nameOf = (host: string): string => host.trim().toLowerCase().replace(/\\.$/, \"\");\n\n/** These names have a fixed meaning. They do not go to DNS. */\nconst NAME_KINDS: ReadonlyArray<readonly [RegExp, string]> = [\n [/(^|\\.)localhost$/, \"loopback\"],\n [/^metadata\\.google\\.internal$/, \"metadata\"],\n [/^metadata\\.goog$/, \"metadata\"],\n];\n\nconst literalOf = (host: string): Bytes | undefined => parseIPv6(host) ?? parseIPv4(host);\n\nconst inRange = (entry: string, address: Bytes): boolean => {\n const slash = entry.indexOf(\"/\");\n const base = literalOf(slash === -1 ? entry : entry.slice(0, slash));\n if (base === undefined || base.length !== address.length) return false;\n const bits = slash === -1 ? base.length * 8 : Number(entry.slice(slash + 1));\n return (\n Number.isInteger(bits) && bits >= 0 && bits <= base.length * 8 && inPrefix(address, base, bits)\n );\n};\n\n/** This function tells if an entry of `allowHosts` permits this host name or this address. */\nconst permitted = (policy: EgressOptions, host: string, address: string | undefined): boolean => {\n if (policy.allowPrivate === true) return true;\n const name = nameOf(host);\n const bytes = address === undefined ? literalOf(name) : literalOf(address);\n for (const raw of policy.allowHosts ?? []) {\n const entry = nameOf(raw);\n if (entry === name) return true;\n if (entry.startsWith(\"*.\") && name.endsWith(entry.slice(1))) return true;\n if (bytes !== undefined && inRange(entry, bytes)) return true;\n }\n return false;\n};\n\n/** This function gives a short form of an address for a note: IPv6 in brackets, IPv4 with dots. */\nconst shown = (host: string): string => {\n const bytes = literalOf(host);\n if (bytes === undefined) return host;\n if (bytes.length === 4) return bytes.join(\".\");\n const groups: string[] = [];\n for (let index = 0; index < 16; index += 2)\n groups.push(((bytes[index]! << 8) | bytes[index + 1]!).toString(16));\n return groups.join(\":\").replace(/(^|:)0(:0)+(:|$)/, \"::\");\n};\n\n/**\n * This function examines a host before DNS: a literal IP address in any spelling, and the names\n * with a fixed meaning (`localhost`). It gives the note of a refusal, or no value. A host name that\n * needs DNS gets its check in the connection hook.\n */\nexport const egressRefusal = (policy: EgressOptions, host: string): string | undefined => {\n if (policy.allowPrivate === true) return undefined;\n const name = nameOf(host);\n const literal = literalOf(name);\n if (literal !== undefined) {\n const kind = addressKind(name);\n return kind === undefined || permitted(policy, name, undefined)\n ? undefined\n : egressNote(shown(name), kind);\n }\n for (const [pattern, kind] of NAME_KINDS) {\n if (pattern.test(name) && !permitted(policy, name, undefined)) return egressNote(name, kind);\n }\n return undefined;\n};\n\n/**\n * This is the form of `dns.lookup` that the egress hook uses. Tests give a resolver of their own.\n *\n * @beta\n */\nexport type EgressResolve = (\n host: string,\n options: LookupAllOptions,\n callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void,\n) => void;\n\nconst resolveAll: EgressResolve = (host, options, callback) =>\n systemLookup(host, options, callback);\n\n/** This function keeps the addresses that the policy permits. When it keeps none, it gives the refusal. */\nconst screen = (\n policy: EgressOptions,\n host: string,\n addresses: readonly LookupAddress[],\n): { readonly usable: LookupAddress[]; readonly refusal: EgressRefused | undefined } => {\n const usable: LookupAddress[] = [];\n let refusal: EgressRefused | undefined;\n for (const entry of addresses) {\n const kind = addressKind(entry.address);\n if (kind === undefined || permitted(policy, host, entry.address)) usable.push(entry);\n else refusal ??= new EgressRefused(shown(entry.address), kind);\n }\n return {\n usable,\n refusal: usable.length === 0 ? (refusal ?? new EgressRefused(host, \"unresolved\")) : undefined,\n };\n};\n\n/**\n * This function gives a `lookup` hook for the connection options of an undici `Agent`. The hook\n * resolves the name, removes each address that the policy refuses, and gives the rest to the\n * socket. When no address stays, the connection fails with `EgressRefused`.\n *\n * @beta\n */\nexport const egressLookup =\n (policy: EgressOptions, resolve: EgressResolve = resolveAll): LookupFunction =>\n (host: string, options: LookupOptions, callback) => {\n resolve(host, { ...options, all: true }, (error, addresses) => {\n if (error !== null) {\n callback(error, []);\n return;\n }\n const { usable, refusal } =\n policy.allowPrivate === true\n ? { usable: addresses, refusal: undefined }\n : screen(policy, host, addresses);\n if (refusal !== undefined) {\n callback(refusal, []);\n return;\n }\n if (options.all === true) callback(null, usable);\n else callback(null, usable[0]!.address, usable[0]!.family);\n });\n };\n\n/**\n * This function examines a URL with DNS, for a client that does its own DNS, for example the\n * browser. It refuses a host when one of its addresses is not public. A name that DNS cannot\n * resolve gets no refusal, because the client cannot connect to it either. The function gives\n * the note of a refusal, or no value.\n *\n * The client resolves the name again, so a DNS answer that changes between the two lookups can\n * pass. The connection hook (`egressLookup`) does not have this gap.\n */\nexport const checkEgress = async (\n policy: EgressOptions,\n url: URL,\n resolve: EgressResolve = resolveAll,\n): Promise<string | undefined> => {\n if (policy.allowPrivate === true) return undefined;\n const before = egressRefusal(policy, url.hostname);\n if (before !== undefined) return before;\n const name = nameOf(url.hostname);\n if (literalOf(name) !== undefined || permitted(policy, name, undefined)) return undefined;\n const addresses = await new Promise<LookupAddress[] | Error>((settle) =>\n resolve(name, { all: true }, (error, found) => settle(error ?? found)),\n );\n if (addresses instanceof Error) return undefined;\n for (const entry of addresses) {\n const kind = addressKind(entry.address);\n if (kind !== undefined && !permitted(policy, name, entry.address))\n return egressNote(shown(entry.address), kind);\n }\n return undefined;\n};\n\n/** This function finds an `EgressRefused` in an error, its cause or its group of errors. */\nexport const egressError = (error: unknown): EgressRefused | undefined => {\n for (\n let current = error, depth = 0;\n current !== undefined && current !== null && depth < 5;\n depth += 1\n ) {\n if (current instanceof EgressRefused) return current;\n if (current instanceof AggregateError) {\n const inner = current.errors.find((item: unknown) => item instanceof EgressRefused) as\n EgressRefused | undefined;\n if (inner !== undefined) return inner;\n }\n current = (current as { cause?: unknown }).cause;\n }\n return undefined;\n};\n","import { createHash } from \"node:crypto\";\n\n/**\n * A capture record is one line of the capture log. Each fetch decision makes one record, and a\n * fetch that the crawler declines makes one too.\n *\n * The record id comes from a hash of its content, so the same decision always gets the same id.\n * A citation names this id.\n *\n * The id has 48 bits: the source and the first 12 hexadecimal digits of the hash. Approximately\n * 1 million records give a chance of a collision of 0.2 percent. The package keeps this length on purpose,\n * because the ids of the earlier capture logs and their citations must stay the same. A later\n * version can make the ids of new records longer behind a record version. It must still read a\n * 12-digit id.\n */\nexport interface CaptureRecord {\n readonly id: string;\n /** The source of the record. The crawler writes `fetch`, and an application can add its own sources. */\n readonly source: string;\n readonly kind: string;\n readonly external_id?: string | undefined;\n readonly occurred_at?: string | undefined;\n readonly received_at: string;\n readonly payload: unknown;\n readonly content_hash: string;\n /** This label comes from the application. The crawler writes it as it is and does not read it. */\n readonly scope?: string | undefined;\n /**\n * These labels come from the application, for example the lineage of a request. The crawler writes\n * them as they are and does not read them. Like `scope`, they are not part of the hash.\n */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/**\n * These are the fields of a new record. The function `makeRecord` calculates the id and the hash.\n *\n * @beta\n */\nexport interface RecordInput {\n readonly source: string;\n readonly kind: string;\n readonly payload: unknown;\n readonly received_at: string;\n readonly external_id?: string | undefined;\n readonly occurred_at?: string | undefined;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/**\n * This function writes a value as JSON with the keys of each object in sorted order. Equal values\n * thus always give equal text, and equal text gives an equal hash.\n */\nexport const stableStringify = (value: unknown): string => {\n if (value === null || typeof value !== \"object\") {\n return JSON.stringify(value) ?? \"null\";\n }\n if (Array.isArray(value)) {\n return `[${value.map((item) => stableStringify(item)).join(\",\")}]`;\n }\n const entries = Object.entries(value as Record<string, unknown>)\n .filter(([, item]) => item !== undefined)\n .sort(([a], [b]) => a.localeCompare(b))\n .map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`);\n return `{${entries.join(\",\")}}`;\n};\n\n/** This function gives the SHA-256 hash of a text as hexadecimal digits. */\nexport const sha256 = (text: string): string =>\n createHash(\"sha256\").update(text, \"utf8\").digest(\"hex\");\n\n/**\n * This function makes a capture record. The hash covers the source, the kind, the external id\n * and the payload. The id is the source and the first 12 digits of the hash. The times, `scope`\n * and `labels` are not part of the hash.\n *\n * @beta\n */\nexport const makeRecord = (input: RecordInput): CaptureRecord => {\n const content_hash = sha256(\n stableStringify({\n source: input.source,\n kind: input.kind,\n external_id: input.external_id ?? null,\n payload: input.payload,\n }),\n );\n return {\n id: `${input.source}.${content_hash.slice(0, 12)}`,\n source: input.source,\n kind: input.kind,\n received_at: input.received_at,\n payload: input.payload,\n content_hash,\n ...(input.external_id !== undefined ? { external_id: input.external_id } : {}),\n ...(input.occurred_at !== undefined ? { occurred_at: input.occurred_at } : {}),\n ...(input.scope !== undefined ? { scope: input.scope } : {}),\n ...(input.labels !== undefined && Object.keys(input.labels).length > 0\n ? { labels: { ...input.labels } }\n : {}),\n };\n};\n","import type { Readable } from \"node:stream\";\nimport { Transform } from \"node:stream\";\nimport { createBrotliDecompress, createGunzip, createInflate, createInflateRaw } from \"node:zlib\";\nimport type { Dispatcher } from \"undici\";\nimport { Agent, request } from \"undici\";\nimport type { BlobStore } from \"./blobs.ts\";\nimport type { Classified, Outcome } from \"./classify.ts\";\nimport { classify, isClientError, parseHttpDate } from \"./classify.ts\";\nimport type { Variant } from \"./docs.ts\";\nimport type { EgressOptions } from \"./egress.ts\";\nimport { checkEgress, egressError, egressLookup, egressRefusal } from \"./egress.ts\";\nimport type {\n Clock,\n HostResult,\n HostStateStore,\n Paced,\n PolicyOptions,\n PolitenessPolicy,\n RestInput,\n ValidatorAnswer,\n ValidatorMemo,\n} from \"./politeness.ts\";\nimport {\n DEFAULT_POLICY,\n HostScheduler,\n MAX_STOP_MS,\n HostStopped,\n checkPolicy,\n equalJitterMs,\n policyFor,\n realClock,\n restAfter,\n validatorsToSend,\n} from \"./politeness.ts\";\nimport type { CaptureRecord } from \"./record.ts\";\nimport { makeRecord } from \"./record.ts\";\nimport type { Rendered, Renderer } from \"./render.ts\";\nimport type { RefusedReason } from \"./render.ts\";\nimport { NO_PLAYWRIGHT, RenderRefused } from \"./render.ts\";\nimport type { Robots, RobotsDecision } from \"./robots.ts\";\nimport {\n decide,\n isProductToken,\n robotsFromResponse,\n robotsFromStatus,\n signalsFor,\n unreachableRobots,\n} from \"./robots.ts\";\nimport type { AiInput, Signals } from \"./signals.ts\";\nimport { NO_SIGNALS, metaSignals, readSignals } from \"./signals.ts\";\nimport { BOT_PAGE, DEFAULT_TOKEN } from \"./version.ts\";\nimport { cacheKeyOf } from \"./urls.ts\";\n// ── signed requests (Web Bot Auth) ──\nimport type { RequestSigner, WebBotAuthOptions } from \"./webbotauth.ts\";\nimport { signerOf } from \"./webbotauth.ts\";\n// ── end of signed requests ──\n\n/**\n * The crawler is a polite fetcher. It sends one request at a time to each host, and it obeys\n * robots.txt for web pages. Its `User-Agent` header tells what it is. It asks again with the\n * validators of the stored copy, and it stops when a site refuses. Each decision goes into the\n * capture log, and a fetch that the crawler declines goes there too.\n *\n * There are two kinds of request. A documented public API (`api`) has its own terms and rate\n * limits. A web page (`page`) obeys robots.txt. The settings come from measurements:\n *\n * - The `request` function of undici on one `Agent`.\n * - A keep-alive time longer than the delay for each host. With the default of 4 seconds, each\n * polite request opens a new connection.\n * - A backoff with jitter in the queue, not the retry handler of undici. That handler limits\n * Retry-After to 30 seconds and adds no jitter.\n * - A maximum size for each body, after decompression. The crawler keeps the first part of a\n * longer body and marks it as truncated.\n *\n * A fetch and a render go through one admission. It examines the URL, the egress policy, the\n * product tokens, robots.txt, the cache and the stops of the host. Only the transport is\n * different: an HTTP request or the headless browser.\n *\n * The egress policy (`EgressOptions`) keeps the crawler on public addresses by default. The\n * politeness policy (`PolitenessPolicy`) holds each limit, and a host can have its own values.\n * The fetch and the render do not throw an error: each problem is an outcome with a note.\n */\n\n/**\n * The outcome of a fetch or a render. A `skipped` request did not go out. A render of a URL that\n * gives a file and not a page is `not-html`.\n */\nexport type FetchOutcome = Outcome | \"skipped\" | \"not-html\";\n\n/** @beta */\nexport interface FetchRequest {\n readonly url: string;\n /** The value `api` is a documented API with its own terms. The value `page` is a web page, and robots.txt applies. */\n readonly mode: \"api\" | \"page\";\n /** This recipe reads the capture again. The function `reindex` uses it to replay the capture. */\n readonly recipe: string;\n readonly accept?: string | undefined;\n /**\n * These are more request headers. The crawler does not write them into the capture. They\n * cannot replace `User-Agent`, `From`, `Accept-Encoding` or the validators. A redirect to a\n * different origin removes `Authorization`, `Cookie` and `Proxy-Authorization`.\n */\n readonly headers?: Readonly<Record<string, string>> | undefined;\n /**\n * These query parameters contain secrets. Each capture shows `REDACTED` in their place. The\n * names in `DEFAULT_SECRET_PARAMS` get the same treatment with no list.\n */\n readonly secretParams?: readonly string[] | undefined;\n /** This value is the documented pace of the source: the shortest time between two of its requests. */\n readonly minIntervalMs?: number | undefined;\n /**\n * A stored capture younger than this age answers, and the crawler sends no request. The value\n * `auto` uses the freshness that the server gave (RFC 9111): `Cache-Control`, `Expires`, or 10\n * percent of the age of `Last-Modified`.\n */\n readonly maxAgeMs?: number | \"auto\" | undefined;\n readonly maxBytes?: number | undefined;\n /**\n * These are the product tokens of an agent that the fetch acts for, for example `Claude-User`.\n * The robots.txt rules for these tokens apply too.\n */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** This name connects the captures to the job that asked for them. */\n readonly job?: string | undefined;\n /** When this signal aborts, the crawler stops the fetch. The result is then `skipped` or `error`, with a note. */\n readonly signal?: AbortSignal | undefined;\n /** This label goes into each capture record of the request. It replaces the `scope` of the crawler. */\n readonly scope?: string | undefined;\n /** These labels go into each capture record of the request, for example its lineage. They are not part of the capture id. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n /**\n * This time (milliseconds since the epoch, on the clock of the crawler) is the end of a budget.\n * The request does not start at or after it. When the host is not free before it, the result is\n * `skipped`, and its note starts with `DEADLINE_NOTE`.\n */\n readonly deadline?: number | undefined;\n // ── archived fallback: the parts of a request to a web archive ──\n /**\n * This byte range is the part of the resource to fetch (a `Range` request). The crawler then sends\n * no validators, keeps no stored copy for the URL, and reads `length` bytes at most. The capture\n * records the range.\n */\n readonly range?: { readonly offset: number; readonly length: number } | undefined;\n /** These lines of usage signals apply to the response too. A copy from a web archive gets the content signals of the robots.txt of its site. */\n readonly signalLines?: readonly string[] | undefined;\n /**\n * This prefix names the response headers that hold the headers of an archived response, for\n * example `x-archive-orig-` of the Wayback Machine. The usage signals of those headers apply too.\n */\n readonly originalHeaderPrefix?: string | undefined;\n // ── end of archived fallback ──\n}\n\n/** @beta */\nexport interface Fetched {\n /** This URL is the URL of the capture. Its secrets show as `REDACTED`. */\n readonly url: string;\n readonly status: number;\n readonly outcome: FetchOutcome;\n /** This value is true when a stored capture answered, with no request. */\n readonly fromCache: boolean;\n /** This id is the capture record behind the body. A citation names it. */\n readonly capture: string | undefined;\n readonly blob: string | undefined;\n readonly body: Uint8Array | undefined;\n readonly contentType: string | undefined;\n readonly signals: Signals;\n readonly note: string | undefined;\n /** This value is true when the body was longer than the maximum, so the crawler kept only its first part. */\n readonly truncated?: boolean | undefined;\n /** This time (ISO 8601) is the time of the fetch of the body. For a stored copy, it is the time of that copy. */\n readonly fetchedAt?: string | undefined;\n /** This label is the scope of the capture. */\n readonly scope?: string | undefined;\n /** These labels are the labels of the capture. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/**\n * The crawler writes each record to a capture writer. The class `CaptureLog` is one.\n *\n * @beta\n */\nexport interface CaptureWriter {\n append(record: CaptureRecord): Promise<void>;\n close?(): Promise<void>;\n}\n\n/**\n * A cache entry is the last answer for a URL: its capture, its body and its validators.\n *\n * @beta\n */\nexport interface CacheEntry {\n /** This text is the cache key: the canonical URL, with its secrets as `REDACTED`. */\n readonly url: string;\n readonly capture: string;\n readonly blob: string | null;\n readonly etag: string | null;\n readonly last_modified: string | null;\n readonly content_type: string | null;\n readonly fetched_at: string;\n readonly status: number;\n /** This value is what the page said about AI input at the time of the fetch. */\n readonly ai_input: AiInput;\n /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */\n readonly location?: string | null | undefined;\n /** This value is true when the stored body is only the first part of the response. */\n readonly truncated?: boolean | undefined;\n /** This time (ISO 8601) is the end of the freshness that the server gave. */\n readonly fresh_until?: string | null | undefined;\n /** This text holds the request headers that the response varies on (`Vary`), with their values. */\n readonly vary_key?: string | null | undefined;\n /** This value tells a plain response from a page that the browser rendered. The default is `plain`. */\n readonly variant?: Variant | undefined;\n}\n\n/**\n * The cache index holds the last answer for each URL. The crawler reads it before a request, and\n * it writes it after an answer. The class `DocIndex` is one. An index that keeps only the\n * necessary fields of `CacheEntry` works, and the crawler keeps the other fields in memory.\n *\n * @beta\n */\nexport interface CacheIndex {\n latest(url: string, variant?: Variant): CacheEntry | undefined;\n setLatest(entry: CacheEntry): void;\n}\n\n/** An event tells a listener what the crawler does. The listener cannot change it. */\nexport type CrawlEvent =\n | {\n readonly type: \"request\";\n readonly url: string;\n readonly host: string;\n readonly transport: \"http\" | \"browser\";\n }\n | {\n readonly type: \"response\";\n readonly url: string;\n readonly host: string;\n readonly status: number;\n readonly outcome: FetchOutcome;\n readonly elapsedMs: number;\n }\n | { readonly type: \"wait\"; readonly host: string; readonly ms: number; readonly reason: string }\n | {\n readonly type: \"stop\";\n readonly host: string;\n readonly until: number;\n readonly reason: string;\n }\n | { readonly type: \"skip\"; readonly url: string; readonly note: string };\n\n/** @beta */\nexport interface CrawlerOptions {\n /** This product token is the name that robots.txt rules match. It contains only letters, `_` and `-` (RFC 9309). */\n readonly token: string;\n /** This text is the full `User-Agent` header. */\n readonly userAgent: string;\n /** This contact goes into the `From` header. Without it, the crawler sends no `From` header. */\n readonly from?: string | undefined;\n readonly log: CaptureWriter;\n readonly blobs: BlobStore;\n readonly index: CacheIndex;\n readonly clock?: Clock | undefined;\n readonly random?: (() => number) | undefined;\n /** This value is the number of hosts that get requests at one time. The default is 16. */\n readonly maxActive?: number | undefined;\n /**\n * This value tells if the crawler also obeys the robots.txt group of `PoliteCrawler`. That applies\n * when the `User-Agent` links the bot page of polite-crawler (`BOT_PAGE`) and the token is different.\n * Site owners read that page, and they write their rules for `PoliteCrawler`. The most restrictive\n * of the groups applies, as for `onBehalfOf`. The default is true. Set false for a bot page of your own.\n */\n readonly obeyDefaultToken?: boolean | undefined;\n /**\n * This value is the number of pages that the browser renders at one time. The default is 2. Each\n * render is a browser context of its own, which holds much memory. A render waits for a slot before\n * the lease of its host, so the wait does not count toward the time of the host.\n */\n readonly maxRenders?: number | undefined;\n /**\n * This undici dispatcher sends the requests. With a dispatcher of your own, the crawler checks\n * only literal addresses. Put `egressLookup` in its connection options to check the address after DNS.\n */\n readonly dispatcher?: Dispatcher | undefined;\n /** This label goes into each capture record as it is. */\n readonly scope?: string | undefined;\n /**\n * This function ranks the scopes of the requests, the most private scope highest. One fetch of a\n * robots.txt can serve requests of different scopes. Its record then takes the scope with the\n * highest rank, and the labels of each request. With no function, the first request gives the scope.\n */\n readonly scopeRank?: ((scope: string) => number) | undefined;\n /** This browser renders the pages that need JavaScript. Without it, the crawler renders no page. */\n readonly renderer?: Renderer | undefined;\n /** This policy controls the addresses that the crawler fetches. By default, the crawler fetches only public addresses. */\n readonly egress?: EgressOptions | undefined;\n /** These values change the limits of the politeness policy, for each host or for some hosts. */\n readonly politeness?: PolicyOptions | undefined;\n /** This store keeps the pace, the counts and the stops of each host. The default keeps them in memory. */\n readonly hostState?: HostStateStore | undefined;\n /** This function gets an event for each request, response, wait, stop and skip. An error in it does not stop the crawler. */\n readonly onEvent?: ((event: CrawlEvent) => void) | undefined;\n // ── signed requests (Web Bot Auth) ──\n /**\n * This signer signs each request, robots.txt and the requests of a rendered page too. Give the\n * options of `createWebBotAuthSigner`, or a `RequestSigner`. Without it, the crawler signs nothing.\n * A signature adds an identity: the `User-Agent` and `From` headers stay as they are.\n */\n readonly signer?: RequestSigner | WebBotAuthOptions | undefined;\n // ── end of signed requests ──\n}\n\n/** @beta */\nexport interface RenderRequest {\n readonly url: string;\n readonly recipe: string;\n /** A stored render younger than this age answers. The value `auto` uses the freshness that the server gave. */\n readonly maxAgeMs?: number | \"auto\" | undefined;\n readonly onBehalfOf?: readonly string[] | undefined;\n readonly job?: string | undefined;\n /** When this signal aborts, the crawler stops the render, and the browser stops the page. */\n readonly signal?: AbortSignal | undefined;\n /** This label goes into the capture record of the render. It replaces the `scope` of the crawler. */\n readonly scope?: string | undefined;\n /** These labels go into the capture record of the render. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n /** This time (milliseconds since the epoch) is the end of a budget. The render does not start at or after it (`FetchRequest.deadline`). */\n readonly deadline?: number | undefined;\n}\n\n/**\n * The note of a request starts with this text when its host was not free before its deadline, so the request did not start.\n *\n * @beta\n */\nexport const DEADLINE_NOTE = \"deadline reached\";\n\n/**\n * This function gives the key of the cache entry of a rendered page in the form of version 0.1.\n * The index keeps the variant in a column of its own, and it reads this form too.\n *\n * This export is internal: the root entry point does not have it since 0.2.0.\n */\nexport const renderedKey = (url: string): string => `${url}#rendered`;\n\n/**\n * This value is the version of the payload of the records that the crawler writes. A record with no `schema` is version 1.\n *\n * @beta\n */\nexport const CAPTURE_SCHEMA = 2;\n\n/** The content type of a rendered page: the browser gives its text as UTF-8. */\nexport const RENDERED_TYPE = \"text/html; charset=utf-8\";\n\n/**\n * These query parameters hold secrets in many APIs. The crawler writes `REDACTED` in place of\n * their values in each capture, in addition to the names of `secretParams`.\n *\n * @beta\n */\nexport const DEFAULT_SECRET_PARAMS: readonly string[] = [\n \"access_token\",\n \"api-key\",\n \"api_key\",\n \"apikey\",\n \"auth\",\n \"auth_token\",\n \"client_secret\",\n \"id_token\",\n \"passwd\",\n \"password\",\n \"pwd\",\n \"refresh_token\",\n \"secret\",\n \"session\",\n \"session_id\",\n \"sessionid\",\n \"sig\",\n \"signature\",\n \"token\",\n \"x-amz-credential\",\n \"x-amz-security-token\",\n \"x-amz-signature\",\n \"x-goog-credential\",\n \"x-goog-signature\",\n];\n\nconst ERROR_BODY_BYTES = 64 * 1024;\nconst ROBOTS_BYTES = 512 * 1024;\nconst LAST_DATE_MS = 8.64e15;\nconst WEB_PROTOCOLS: ReadonlySet<string> = new Set([\"http:\", \"https:\"]);\nconst HEADER_TEXT = /^[\\t -~]*$/;\nconst PERMANENT_REDIRECTS: ReadonlySet<number> = new Set([301, 308]);\nconst HEURISTIC_STATUSES: ReadonlySet<number> = new Set([\n 200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501,\n]);\n/** A caller cannot set these headers. The crawler sets them, or HTTP does. */\nconst RESERVED_HEADERS: ReadonlySet<string> = new Set([\n \"user-agent\",\n \"from\",\n \"accept-encoding\",\n \"if-none-match\",\n \"if-modified-since\",\n \"if-match\",\n \"if-unmodified-since\",\n \"if-range\",\n \"range\",\n \"host\",\n \"connection\",\n \"content-length\",\n \"transfer-encoding\",\n \"te\",\n \"upgrade\",\n \"keep-alive\",\n]);\n/** A redirect to a different origin removes these headers, as the Fetch standard does. */\nconst CREDENTIAL_HEADERS: ReadonlySet<string> = new Set([\n \"authorization\",\n \"cookie\",\n \"proxy-authorization\",\n]);\n/** The crawler sends the same value of these headers in each request, so a `Vary` on them changes nothing. */\nconst CONSTANT_HEADERS: ReadonlySet<string> = new Set([\n \"accept-encoding\",\n \"user-agent\",\n \"from\",\n // ── signed requests: the identity is the same in each request ──\n \"signature\",\n \"signature-input\",\n \"signature-agent\",\n]);\n// ── signed requests (Web Bot Auth) ──\n/** With a signer, the crawler sets these headers, and a caller cannot. */\nconst SIGNATURE_HEADERS: ReadonlySet<string> = new Set([\n \"signature\",\n \"signature-input\",\n \"signature-agent\",\n]);\n// ── end of signed requests ──\n// ── archived fallback ──\n/** These response headers hold usage signals. With `originalHeaderPrefix`, the archived forms of them count too. */\nconst SIGNAL_HEADERS = [\n \"content-usage\",\n \"content-signal\",\n \"x-robots-tag\",\n \"tdm-reservation\",\n \"tdm-policy\",\n] as const;\n// ── end of archived fallback ──\n/**\n * These content codings change nothing in the body. A server that stored an object with\n * `aws-chunked` can send the name with the decoded body.\n */\nconst IDENTITY_CODINGS: ReadonlySet<string> = new Set([\"identity\", \"aws-chunked\"]);\n\n/**\n * This function tells if an answer is a web page that the request did not ask for. The answer is\n * HTML, and the `Accept` of the request admits no HTML (`application/json`). A login or an error\n * page in place of the data of an API is such an answer. A media range with `q=0` admits nothing.\n */\nconst unaskedPage = (accept: string | undefined, contentType: string | undefined): boolean => {\n const type = (contentType ?? \"\").split(\";\")[0]!.trim().toLowerCase();\n if (\n accept === undefined ||\n accept.trim() === \"\" ||\n (type !== \"text/html\" && type !== \"application/xhtml+xml\")\n )\n return false;\n return !accept.split(\",\").some((range) => {\n const [media = \"\", ...params] = range.split(\";\").map((part) => part.trim().toLowerCase());\n const q = params.find((param) => param.startsWith(\"q=\"));\n if (q !== undefined && Number(q.slice(2)) === 0) return false;\n return (\n media === \"*/*\" ||\n media === type ||\n (media.endsWith(\"/*\") && type.startsWith(media.slice(0, -1)))\n );\n });\n};\n\n/** This function reads the TDMRep `<meta>` tags at the start of an HTML body. */\nconst pageSignals = (body: Uint8Array | undefined, contentType: string | undefined): string[] =>\n body === undefined || !/html/i.test(contentType ?? \"\")\n ? []\n : metaSignals(new TextDecoder(\"latin1\").decode(body.subarray(0, 65_536)));\n\n/** A count of slots: `max` holders at one time at most, and the others wait in sequence. */\nclass Slots {\n private used = 0;\n private readonly waiting: Array<() => void> = [];\n\n constructor(private readonly max: number) {}\n\n /** This method gives a function that frees the slot. It rejects when the signal aborts before a slot is free. */\n async acquire(signal: AbortSignal): Promise<() => void> {\n if (this.used < this.max) {\n this.used += 1;\n return this.releaser();\n }\n await new Promise<void>((resolve, reject) => {\n const stop = (): void => {\n const at = this.waiting.indexOf(go);\n if (at !== -1) this.waiting.splice(at, 1);\n reject(signal.reason instanceof Error ? signal.reason : new Error(\"cancelled\"));\n };\n const go = (): void => {\n signal.removeEventListener(\"abort\", stop);\n resolve();\n };\n if (signal.aborted) {\n stop();\n return;\n }\n this.waiting.push(go);\n signal.addEventListener(\"abort\", stop, { once: true });\n });\n // the slot of the holder that freed it passes to this waiter, so the count stays\n return this.releaser();\n }\n\n private releaser(): () => void {\n let freed = false;\n return () => {\n if (freed) return;\n freed = true;\n const next = this.waiting.shift();\n if (next !== undefined) next();\n else this.used -= 1;\n };\n }\n}\n\n/** This function tells if an error of a renderer says that it has no browser: no `playwright-core`, or no browser on the disk. */\nconst missingBrowser = (message: string): boolean =>\n message.startsWith(NO_PLAYWRIGHT) ||\n /Executable doesn't exist|Cannot find (?:module|package) 'playwright-core'/.test(message);\n\n/** This text tells why the crawler did not fetch a page: a rule of the site, or no robots.txt to read. */\nconst robotsRefusal = (robots: Robots, by: string): string =>\n robots.unreachable === undefined\n ? `robots.txt disallows it for ${by}`\n : `robots.txt could not be read (${robots.unreachable}), so nothing on the site is fetched`;\n\nconst messageOf = (error: unknown): string =>\n error instanceof Error ? (error.message.split(/\\r?\\n/)[0] ?? \"\") : String(error);\n\nconst isoOf = (ms: number): string | undefined =>\n Number.isFinite(ms) && Math.abs(ms) < LAST_DATE_MS ? new Date(ms).toISOString() : undefined;\n\nconst stopNote = (reason: string, until: number): string => {\n const end = isoOf(until);\n return end === undefined ? `host stopped: ${reason}` : `host stopped until ${end}: ${reason}`;\n};\n\n/** A request did not start before its deadline. The crawler gives a skip with `DEADLINE_NOTE` for it. */\nclass DeadlineReached extends Error {\n constructor(\n /** This value is the earliest start of the request on its host, in milliseconds since the epoch. */\n readonly readyAt: number,\n ) {\n super(DEADLINE_NOTE);\n }\n}\n\nconst deadlineNote = (error: DeadlineReached, deadline: number | undefined): string =>\n `${DEADLINE_NOTE}: the host is free at ${isoOf(error.readyAt) ?? \"a later time\"}, and the request must start before ${isoOf(deadline ?? Number.NaN) ?? \"its deadline\"}`;\n\nconst secretSet = (secretParams: readonly string[]): Set<string> =>\n new Set([...DEFAULT_SECRET_PARAMS, ...secretParams].map((name) => name.toLowerCase()));\n\nconst nameOfParam = (raw: string): string => {\n try {\n return decodeURIComponent(raw.replace(/\\+/g, \" \")).toLowerCase();\n } catch {\n return raw.toLowerCase();\n }\n};\n\n/**\n * This function gives the URL of a capture. It removes a user name and a password, and it writes\n * `REDACTED` in place of the value of each secret parameter. The other parameters stay as they\n * were written.\n */\nexport const redact = (url: URL, secretParams: readonly string[] = []): string => {\n const copy = new URL(url.href);\n copy.username = \"\";\n copy.password = \"\";\n if (copy.search.length > 1) {\n const secrets = secretSet(secretParams);\n const parts = copy.search\n .slice(1)\n .split(\"&\")\n .map((part) => {\n const equals = part.indexOf(\"=\");\n if (equals === -1) return part;\n const name = part.slice(0, equals);\n return secrets.has(nameOfParam(name)) ? `${name}=REDACTED` : part;\n });\n copy.search = `?${parts.join(\"&\")}`;\n }\n return copy.href;\n};\n\n/** This function tells if a URL holds a secret: a user name, a password or a secret parameter. */\nconst holdsSecrets = (url: URL, secretParams: readonly string[]): boolean => {\n if (url.username !== \"\" || url.password !== \"\") return true;\n const secrets = secretSet(secretParams);\n return url.search\n .slice(1)\n .split(\"&\")\n .some(\n (part) => part.includes(\"=\") && secrets.has(nameOfParam(part.slice(0, part.indexOf(\"=\")))),\n );\n};\n\n/** The pace, the stops and the robots.txt of a host do not change with a trailing dot or the case of the name. */\nconst hostKey = (url: URL): string =>\n `${url.hostname.toLowerCase().replace(/\\.$/, \"\")}${url.port === \"\" ? \"\" : `:${url.port}`}`;\n\nconst originKey = (url: URL): string => `${url.protocol}//${hostKey(url)}`;\n\nconst headerOf =\n (headers: Readonly<Record<string, string | string[] | undefined>>) =>\n (name: string): string | undefined => {\n const value = headers[name.toLowerCase()];\n return Array.isArray(value) ? value.join(\", \") : value;\n };\n\nclass Lru<K, V> {\n private readonly map = new Map<K, V>();\n constructor(private readonly max: number) {}\n\n get(key: K): V | undefined {\n const value = this.map.get(key);\n if (value !== undefined) {\n this.map.delete(key);\n this.map.set(key, value);\n }\n return value;\n }\n\n peek(key: K): V | undefined {\n return this.map.get(key);\n }\n\n set(key: K, value: V): void {\n this.map.delete(key);\n this.map.set(key, value);\n if (this.map.size > this.max) this.map.delete(this.map.keys().next().value as K);\n }\n\n delete(key: K): void {\n this.map.delete(key);\n }\n\n keys(): K[] {\n return [...this.map.keys()];\n }\n\n get size(): number {\n return this.map.size;\n }\n}\n\n/** A body failed in transport (a reset, a timeout) or in decompression. */\nclass BodyError extends Error {\n constructor(\n readonly kind: \"transport\" | \"decode\",\n cause: unknown,\n ) {\n super(messageOf(cause), { cause });\n }\n}\n\n/** RFC 9110 says that `deflate` is the zlib format, but some servers send raw deflate. This stream reads both. */\nconst inflateAny = (): Transform => {\n let inner: Transform | undefined;\n const outer: Transform = new Transform({\n transform(chunk: Buffer, _encoding, done) {\n if (inner === undefined) {\n const zlib =\n chunk.length >= 2 &&\n (chunk[0]! & 0x0f) === 8 &&\n ((chunk[0]! << 8) | chunk[1]!) % 31 === 0;\n inner = zlib ? createInflate() : createInflateRaw();\n inner.on(\"data\", (data: Buffer) => outer.push(data));\n inner.on(\"error\", (error) => outer.destroy(error));\n }\n inner.write(chunk, () => done());\n },\n flush(done) {\n if (inner === undefined) {\n done();\n return;\n }\n inner.once(\"end\", () => done());\n inner.end();\n },\n destroy(error, callback) {\n inner?.destroy();\n callback(error);\n },\n });\n return outer;\n};\n\nconst decoderFor = (coding: string): Transform => {\n if (coding === \"gzip\" || coding === \"x-gzip\") return createGunzip();\n if (coding === \"br\") return createBrotliDecompress();\n if (coding === \"deflate\") return inflateAny();\n throw new BodyError(\"decode\", new Error(`unsupported content-encoding \"${coding}\"`));\n};\n\n/** This function tells if a decoder stopped because the compressed body ended early: the server cut it. */\nconst endedEarly = (error: unknown): boolean =>\n typeof error === \"object\" &&\n error !== null &&\n \"code\" in error &&\n (error.code === \"Z_BUF_ERROR\" || /unexpected end of file/i.test(messageOf(error)));\n\n/**\n * This function reads a body as a stream and decodes each content coding in sequence. It counts\n * the decoded bytes and keeps at most `maxBytes`. Thus a decompression bomb stops at the\n * maximum, and a long page keeps its first part. A body that fails gives a `BodyError`. A\n * compressed body that ends early gives the part that it decoded, marked `cut`.\n */\nconst readBody = async (\n body: Readable,\n encoding: string | undefined,\n maxBytes: number,\n): Promise<{ readonly bytes: Uint8Array; readonly truncated: boolean; readonly cut: boolean }> => {\n const codings = (encoding ?? \"\")\n .split(\",\")\n .map((coding) => coding.trim().toLowerCase())\n .filter((coding) => coding.length > 0 && !IDENTITY_CODINGS.has(coding))\n .reverse();\n // undici gives an error when a body that did not end is destroyed; the loop below reports real errors\n body.on(\"error\", () => undefined);\n let decoders: Transform[];\n try {\n decoders = codings.map(decoderFor);\n } catch (error) {\n body.destroy();\n throw error;\n }\n // a compressed stream that gives almost no output stops at this length of input\n const rawCap = maxBytes * 4 + 65_536;\n let rawBytes = 0;\n let stream: Readable = body;\n if (decoders.length > 0) {\n body.on(\"data\", (chunk: Buffer) => {\n rawBytes += chunk.length;\n });\n for (const decoder of decoders) {\n const source = stream;\n source.on(\"error\", (error) => decoder.destroy(error));\n stream = source.pipe(decoder);\n }\n }\n const chunks: Buffer[] = [];\n let size = 0;\n let truncated = false;\n let cut = false;\n try {\n for await (const chunk of stream as AsyncIterable<Buffer>) {\n const room = maxBytes - size;\n if (chunk.length > room) {\n // the chunk that crosses the maximum gives its first part\n if (room > 0) chunks.push(chunk.subarray(0, room));\n size += Math.max(0, room);\n truncated = true;\n break;\n }\n chunks.push(chunk);\n size += chunk.length;\n if (rawBytes > rawCap) {\n truncated = true;\n break;\n }\n }\n } catch (error) {\n const transport = body.errored !== null || decoders.length === 0;\n // a server that closes a compressed body early: the part before the end is still the page\n if (!transport && size > 0 && endedEarly(error)) {\n cut = true;\n truncated = true;\n } else {\n throw new BodyError(transport ? \"transport\" : \"decode\", error);\n }\n } finally {\n body.destroy();\n for (const decoder of decoders) decoder.destroy();\n }\n // one copy into a buffer of its own, so no reader sees the shared pool of Node.js\n const bytes = new Uint8Array(size);\n let at = 0;\n for (const chunk of chunks) {\n bytes.set(chunk, at);\n at += chunk.length;\n }\n return { bytes, truncated, cut };\n};\n\nconst directives = (value: string | undefined): Map<string, string> => {\n const out = new Map<string, string>();\n for (const part of (value ?? \"\").split(\",\")) {\n const equals = part.indexOf(\"=\");\n const name = (equals === -1 ? part : part.slice(0, equals)).trim().toLowerCase();\n if (name.length === 0) continue;\n const raw = equals === -1 ? \"\" : part.slice(equals + 1).trim();\n out.set(\n name,\n raw.length >= 2 && raw.startsWith('\"') && raw.endsWith('\"') ? raw.slice(1, -1) : raw,\n );\n }\n return out;\n};\n\nconst TEN_YEARS_MS = 10 * 365 * 24 * 60 * 60 * 1000;\n\n/**\n * This function gives the end of the freshness of a response, as RFC 9111 §4.2 calculates it.\n * The sources in sequence are `max-age`, `Expires`, and 10 percent of the age of\n * `Last-Modified`, with a maximum. A response with `no-store` or `no-cache` is stale at once. A\n * response with no freshness gives null.\n */\nconst freshUntil = (\n header: (name: string) => string | undefined,\n status: number,\n fetchedAt: number,\n lastModified: string | null | undefined,\n capMs: number,\n): string | null => {\n const control = directives(header(\"cache-control\"));\n if (control.has(\"no-store\") || control.has(\"no-cache\")) return isoOf(fetchedAt) ?? null;\n const ageText = header(\"age\")?.trim() ?? \"\";\n const age = /^\\d+$/.test(ageText) ? Number(ageText) * 1000 : 0;\n const date = parseHttpDate(header(\"date\")) ?? fetchedAt;\n let lifetime: number | undefined;\n const maxAge = control.get(\"max-age\");\n const expires = header(\"expires\");\n if (maxAge !== undefined && /^\\d+$/.test(maxAge)) lifetime = Number(maxAge) * 1000;\n else if (expires !== undefined) {\n // an Expires that is not a date means that the response is already stale\n const at = parseHttpDate(expires);\n lifetime = at === undefined ? 0 : at - date;\n } else if (HEURISTIC_STATUSES.has(status)) {\n const modified = parseHttpDate(lastModified ?? undefined);\n if (modified !== undefined && modified < date)\n lifetime = Math.min(capMs, (date - modified) / 10);\n else if (PERMANENT_REDIRECTS.has(status)) lifetime = capMs;\n }\n if (lifetime === undefined) return null;\n return isoOf(fetchedAt + Math.max(0, Math.min(lifetime, TEN_YEARS_MS) - age)) ?? null;\n};\n\n/** This function gives the request headers that a response varies on, with their values. A `Vary: *` gives `*`. */\nconst varyKeyOf = (\n vary: string | undefined,\n sent: Readonly<Record<string, string>>,\n): string | null => {\n if (vary === undefined) return null;\n const names = vary\n .split(\",\")\n .map((name) => name.trim().toLowerCase())\n .filter((name) => name.length > 0 && !CONSTANT_HEADERS.has(name));\n if (names.includes(\"*\")) return \"*\";\n if (names.length === 0) return null;\n return [...new Set(names)]\n .sort()\n .map((name) => `${name}=${sent[name] ?? \"\"}`)\n .join(\"\\n\");\n};\n\nconst varyMatches = (\n stored: string | null | undefined,\n sent: Readonly<Record<string, string>>,\n): boolean => {\n if (stored === undefined || stored === null) return true;\n if (stored === \"*\") return false;\n return stored.split(\"\\n\").every((line) => {\n const equals = line.indexOf(\"=\");\n return (sent[line.slice(0, equals)] ?? \"\") === line.slice(equals + 1);\n });\n};\n\nconst withoutCredentials = (\n headers: Readonly<Record<string, string>> | undefined,\n): Readonly<Record<string, string>> | undefined =>\n headers === undefined\n ? undefined\n : Object.fromEntries(\n Object.entries(headers).filter(([name]) => !CREDENTIAL_HEADERS.has(name.toLowerCase())),\n );\n\nconst invalidNumber = (\n name: string,\n value: number | \"auto\" | undefined,\n auto: boolean,\n): string | undefined => {\n if (value === undefined || (auto && value === \"auto\")) return undefined;\n return typeof value === \"number\" && value >= 0\n ? undefined\n : `invalid option: ${name} is ${String(value)}`;\n};\n\nconst closedResult = (url: string): Fetched => ({\n url,\n status: 0,\n outcome: \"skipped\",\n fromCache: false,\n capture: undefined,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note: \"crawler closed\",\n});\n\n/** One step of a fetch: the answer for one URL, and the next URL when the answer is a redirect to follow. */\ninterface Step {\n readonly fetched: Fetched;\n /** This URL is the target of the redirect, with its secrets. It stays in memory. */\n readonly next?: string | undefined;\n readonly retryAfterMs?: number | undefined;\n}\n\n/** A robots verdict is a parsed robots.txt and the capture that holds it. */\ninterface RobotsVerdict {\n readonly robots: Robots;\n readonly capture: string | undefined;\n /** When the crawler could not read robots.txt and used a stored copy, this text gives the reason. */\n readonly stale?: string | undefined;\n}\n\n/** The scope and the labels of the requests that wait for one fetch of a robots.txt. The record of the fetch reads them when it is written. */\ninterface Waiting {\n scope: string | undefined;\n labels: Record<string, string> | undefined;\n}\n\ninterface RobotsEntry {\n readonly at: number;\n readonly waiting: Waiting;\n ttlMs: number;\n verdict: Promise<RobotsVerdict>;\n value: RobotsVerdict | undefined;\n /** This value counts the unreachable verdicts in sequence, for the backoff. */\n misses: number;\n /** This verdict is the last good robots.txt, with its time. */\n good: { readonly verdict: RobotsVerdict; readonly at: number } | undefined;\n}\n\ninterface Settle {\n readonly outcome: FetchOutcome;\n readonly status: number;\n readonly retryAfterMs: number | undefined;\n /** The pace of the host: the kind of the request, its duration, the crawl delay and the documented interval. */\n readonly rest: RestInput;\n /** A 401 or a 403 on robots.txt means \"no robots.txt\" (RFC 9309), so it does not stop the host. */\n readonly robotsFile: boolean;\n /** This result replaces the result that the outcome gives. */\n readonly result?: HostResult | undefined;\n}\n\ntype RenderDone =\n | {\n readonly kind: \"rendered\";\n readonly rendered: Rendered;\n readonly classified: Classified;\n readonly elapsedMs: number;\n }\n | { readonly kind: \"failed\"; readonly note: string; readonly elapsedMs: number }\n | {\n readonly kind: \"refused\";\n readonly reason: RefusedReason;\n readonly note: string;\n /** This URL is the request that the renderer refused. */\n readonly url?: string | undefined;\n readonly elapsedMs: number;\n }\n | { readonly kind: \"cancelled\"; readonly elapsedMs: number }\n | { readonly kind: \"closed\" };\n\n/** These are the parts of a request that the admission reads. */\ninterface AdmitSpec {\n readonly transport: \"http\" | \"browser\";\n readonly url: string;\n readonly mode: \"api\" | \"page\";\n readonly base: Record<string, unknown>;\n readonly secretParams: readonly string[];\n readonly invalid: string | undefined;\n readonly onBehalfOf: readonly string[] | undefined;\n readonly scope: string | undefined;\n readonly labels: Readonly<Record<string, string>> | undefined;\n /** This function answers from the cache, when the stored copy is young enough. */\n readonly cached: (url: string) => Promise<Step | undefined>;\n}\n\n/** A request that the admission let through: the facts that the transport needs. */\ninterface Admitted {\n readonly target: URL;\n readonly url: string;\n readonly host: string;\n readonly policy: PolitenessPolicy;\n readonly tokens: readonly string[];\n readonly verdict: RobotsVerdict | undefined;\n readonly check: Record<string, unknown> | undefined;\n readonly robotsLines: readonly string[];\n readonly crawlDelaySeconds: number;\n readonly skip: (note: string, extra?: Record<string, unknown>) => Promise<Fetched>;\n}\n\nconst robotsCheck = (\n verdict: RobotsVerdict,\n tokens: readonly string[],\n decision: RobotsDecision,\n): Record<string, unknown> => ({\n capture: verdict.capture ?? null,\n tokens,\n rule:\n decision.rule === undefined\n ? null\n : {\n line: decision.rule.line ?? null,\n pattern: decision.rule.pattern,\n allow: decision.rule.allow,\n },\n group_line: decision.groups?.[0]?.line ?? null,\n ...(verdict.robots.unreachable !== undefined ? { unreachable: verdict.robots.unreachable } : {}),\n ...(verdict.stale !== undefined ? { stale: verdict.stale } : {}),\n});\n\nconst raceAbort = <T>(work: Promise<T>, signal: AbortSignal): Promise<T> => {\n if (signal.aborted) {\n work.catch(() => undefined);\n return Promise.reject(signal.reason);\n }\n return new Promise<T>((resolve, reject) => {\n const cancel = (): void => reject(signal.reason);\n signal.addEventListener(\"abort\", cancel, { once: true });\n work.then(\n (value) => {\n signal.removeEventListener(\"abort\", cancel);\n resolve(value);\n },\n (error: unknown) => {\n signal.removeEventListener(\"abort\", cancel);\n reject(error);\n },\n );\n });\n};\n\n/** The result of reading robots.txt: the rules, the capture that holds them, and the wait that the server asked for. */\nexport interface RobotsRead {\n readonly robots: Robots;\n readonly capture: string | undefined;\n readonly retryAfterMs: number | undefined;\n}\n\n/** @beta */\nexport class Crawler {\n private readonly clock: Clock;\n private readonly random: () => number;\n private readonly scheduler: HostScheduler;\n private readonly dispatcher: Dispatcher;\n private readonly ownsDispatcher: boolean;\n private readonly egress: EgressOptions;\n private readonly robots = new Lru<string, RobotsEntry>(4096);\n /** The fields of a cache entry that the index does not keep, and the raw target of a redirect. */\n private readonly meta = new Lru<\n string,\n { readonly entry: CacheEntry; readonly location?: string | undefined }\n >(10_000);\n private readonly inFlight = new Map<string, Promise<Step>>();\n private readonly running = new Set<Promise<unknown>>();\n private readonly closing = new AbortController();\n private closed = false;\n private closingDone: Promise<void> | undefined;\n /** This note tells why the renderer has no browser. After the first page that finds it, the crawler renders nothing. */\n private renderOff: string | undefined;\n /** The renders that run at one time (`maxRenders`). */\n private readonly renders: Slots;\n /** These tokens apply because the `User-Agent` links the bot page (`obeyDefaultToken`). */\n private readonly pageTokens: readonly string[];\n /** The deadlines of the requests that have one, by the signal that the scheduler gets for each of them. */\n private readonly deadlines = new WeakMap<AbortSignal, number>();\n // ── signed requests (Web Bot Auth) ──\n /** This signer signs each request. It holds the key, and the crawler writes only its key id and its agent. */\n private readonly signer: RequestSigner | undefined;\n // ── end of signed requests ──\n\n constructor(private readonly options: CrawlerOptions) {\n // ── signed requests: a bad key throws here, as each bad option does ──\n this.signer = signerOf(options.signer);\n if (!isProductToken(options.token)) {\n throw new Error(\n `the token \"${options.token}\" is not a product token: use only letters, \"_\" and \"-\" (RFC 9309)`,\n );\n }\n for (const [name, value] of [\n [\"userAgent\", options.userAgent],\n [\"from\", options.from],\n ] as const) {\n if (value !== undefined && !HEADER_TEXT.test(value)) {\n throw new Error(\n `${name} \"${value}\" has a character that an HTTP header cannot hold: use printable ASCII`,\n );\n }\n }\n checkPolicy(options.politeness);\n this.clock = options.clock ?? realClock;\n this.random = options.random ?? Math.random;\n this.egress = options.egress ?? {};\n // the scheduler waits on this clock: a wait that ends at the deadline of its request or later\n // does not start, so a request with a deadline does not wait for nothing\n const clock = this.clock;\n const deadlines = this.deadlines;\n this.scheduler = new HostScheduler(\n {\n now: () => clock.now(),\n sleep: (ms, signal) => {\n const deadline = signal === undefined ? undefined : deadlines.get(signal);\n const readyAt = clock.now() + ms;\n return deadline !== undefined && readyAt >= deadline\n ? Promise.reject(new DeadlineReached(readyAt))\n : clock.sleep(ms, signal);\n },\n },\n options.maxActive ?? 16,\n { store: options.hostState },\n );\n // the bot page tells site owners to write rules for PoliteCrawler, so an application that links it obeys them\n this.pageTokens =\n options.obeyDefaultToken !== false &&\n options.token.toLowerCase() !== DEFAULT_TOKEN.toLowerCase() &&\n options.userAgent.includes(BOT_PAGE)\n ? [DEFAULT_TOKEN]\n : [];\n const maxRenders = options.maxRenders ?? 2;\n if (!Number.isInteger(maxRenders) || maxRenders < 1) {\n throw new RangeError(`maxRenders must be an integer of 1 or more, not ${String(maxRenders)}`);\n }\n this.renders = new Slots(maxRenders);\n this.ownsDispatcher = options.dispatcher === undefined;\n this.dispatcher =\n options.dispatcher ??\n new Agent({\n // past the per-host delay, or every polite request reconnects\n keepAliveTimeout: 60_000,\n keepAliveMaxTimeout: 120_000,\n allowH2: true,\n // the egress policy checks each address after DNS, at the time of the connection\n connect: { timeout: 10_000, lookup: egressLookup(this.egress) },\n headersTimeout: 30_000,\n bodyTimeout: 60_000,\n });\n }\n\n private iso(): string {\n return new Date(this.clock.now()).toISOString();\n }\n\n private policyOf(url: URL): PolitenessPolicy {\n return policyFor(this.options.politeness, url.hostname);\n }\n\n /**\n * This method gives the signal of a request for the scheduler. A request with a deadline gets a\n * signal of its own, and the clock of the scheduler reads the deadline of that signal.\n */\n private paced(signal: AbortSignal, deadline: number | undefined): AbortSignal {\n if (deadline === undefined) return signal;\n const own = AbortSignal.any([signal]);\n this.deadlines.set(own, deadline);\n return own;\n }\n\n /** This method throws `DeadlineReached` when the deadline of a request came before its slot. */\n private beforeDeadline(deadline: number | undefined): void {\n const now = this.clock.now();\n if (deadline !== undefined && now >= deadline) throw new DeadlineReached(now);\n }\n\n private emit(event: CrawlEvent): void {\n try {\n this.options.onEvent?.(event);\n } catch {\n // a listener that fails does not change the crawl\n }\n }\n\n private track<T>(work: Promise<T>): Promise<T> {\n this.running.add(work);\n void work.then(\n () => this.running.delete(work),\n () => this.running.delete(work),\n );\n return work;\n }\n\n /** This method gives the validator memo of a host. The memo is a hint, so an error of its store gives no memo. */\n private memoOf(host: string): ValidatorMemo | undefined {\n try {\n return this.scheduler.store.validators?.(host);\n } catch {\n return undefined;\n }\n }\n\n private stopHost(host: string, until: number, reason: string): void {\n this.scheduler.store.stop(host, until, reason);\n this.emit({ type: \"stop\", host, until, reason });\n }\n\n private async record(\n kind: \"capture\" | \"revisit\" | \"skipped\",\n payload: Record<string, unknown>,\n url: string,\n labels: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<string | undefined> {\n // a closed crawler writes nothing, so its log stays closed\n if (this.closed) return undefined;\n const at = this.iso();\n const scope = labels.scope ?? this.options.scope;\n const record = makeRecord({\n source: \"fetch\",\n kind,\n payload: { schema: CAPTURE_SCHEMA, ...payload, fetched_at: at },\n received_at: at,\n occurred_at: at,\n external_id: url,\n ...(scope !== undefined ? { scope } : {}),\n ...(labels.labels !== undefined ? { labels: labels.labels } : {}),\n });\n await this.options.log.append(record);\n return record.id;\n }\n\n private async skip(\n base: Record<string, unknown>,\n url: string,\n note: string,\n extra: Record<string, unknown> = {},\n labels: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<Fetched> {\n this.emit({ type: \"skip\", url, note });\n return {\n url,\n status: 0,\n outcome: \"skipped\",\n fromCache: false,\n capture: await this.record(\"skipped\", { ...base, url, note, ...extra }, url, labels),\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note,\n };\n }\n\n /** This method reads the cache entry of a URL, with the fields that the crawler keeps in memory. */\n private lookup(url: string, variant: Variant = \"plain\"): CacheEntry | undefined {\n const key = cacheKeyOf(url);\n const row = this.options.index.latest(key, variant);\n if (row === undefined) return undefined;\n const meta = this.meta.get(`${variant} ${key}`);\n if (meta === undefined || meta.entry.capture !== row.capture) return row;\n const kept = Object.fromEntries(Object.entries(row).filter(([, value]) => value !== undefined));\n return { ...meta.entry, ...kept } as CacheEntry;\n }\n\n private store(entry: CacheEntry, location?: string): void {\n const variant = entry.variant ?? \"plain\";\n const keyed = { ...entry, url: cacheKeyOf(entry.url), variant };\n this.options.index.setLatest(keyed);\n this.meta.set(`${variant} ${keyed.url}`, {\n entry: keyed,\n ...(location !== undefined ? { location } : {}),\n });\n }\n\n /** This method gives the target of a stored redirect, when the crawler can follow it. */\n private followable(entry: CacheEntry, secretParams: readonly string[]): string | undefined {\n const meta = this.meta.peek(`${entry.variant ?? \"plain\"} ${cacheKeyOf(entry.url)}`);\n if (meta?.location !== undefined && meta.entry.capture === entry.capture) return meta.location;\n if (entry.location === undefined || entry.location === null) return undefined;\n try {\n const target = new URL(entry.location);\n // a stored target with secrets holds REDACTED in their place, and is not the real target\n return holdsSecrets(target, secretParams) ? undefined : target.href;\n } catch {\n return undefined;\n }\n }\n\n /** This method answers from the cache, when the stored entry is young or fresh enough. */\n private async fromCache(\n url: string,\n variant: Variant,\n maxAge: number | \"auto\" | undefined,\n sent: Readonly<Record<string, string>>,\n secretParams: readonly string[],\n ): Promise<Step | undefined> {\n if (maxAge === undefined) return undefined;\n const entry = this.lookup(url, variant);\n if (entry === undefined || !varyMatches(entry.vary_key, sent)) return undefined;\n const now = this.clock.now();\n const fetchedAt = Date.parse(entry.fetched_at);\n if (!Number.isFinite(fetchedAt)) return undefined;\n const age = now - fetchedAt;\n const until =\n entry.fresh_until === undefined || entry.fresh_until === null\n ? Number.NaN\n : Date.parse(entry.fresh_until);\n const fresh = Number.isFinite(until) && until > now;\n const redirect = entry.status >= 300 && entry.status < 400;\n // an explicit age wins; `auto` uses the freshness of the server; a temporary redirect needs both\n const young =\n maxAge === \"auto\"\n ? fresh\n : redirect && !PERMANENT_REDIRECTS.has(entry.status)\n ? fresh && age < maxAge\n : age < maxAge;\n if (!young) return undefined;\n if (redirect) {\n const next = this.followable(entry, secretParams);\n if (next === undefined) return undefined;\n return {\n fetched: {\n url,\n status: entry.status,\n outcome: \"redirect\",\n fromCache: true,\n capture: entry.capture,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note: `location ${entry.location ?? redact(new URL(next), secretParams)}`,\n fetchedAt: entry.fetched_at,\n },\n next,\n };\n }\n if (entry.blob === null || entry.status < 200 || entry.status >= 300) return undefined;\n const body = await this.options.blobs.get(entry.blob);\n if (body === undefined) return undefined;\n return {\n fetched: {\n url,\n status: entry.status,\n outcome: \"ok\",\n fromCache: true,\n capture: entry.capture,\n blob: entry.blob,\n body,\n contentType: entry.content_type ?? undefined,\n signals: { ai_input: entry.ai_input ?? \"unset\", raw: [] },\n note: entry.truncated === true ? \"a truncated copy\" : undefined,\n ...(entry.truncated === true ? { truncated: true } : {}),\n fetchedAt: entry.fetched_at,\n },\n };\n }\n\n // ── signed requests (Web Bot Auth) ──\n /**\n * This method gives the signature headers of one request, and the facts of the capture: the key id\n * and the agent. With no signer, it gives nothing. A signer that throws gives the note of a skip,\n * because the crawler sends no unsigned request when it has a signer.\n */\n private signFor(target: URL):\n | {\n readonly headers: Readonly<Record<string, string>>;\n readonly facts: Record<string, unknown>;\n }\n | { readonly error: string }\n | undefined {\n if (this.signer === undefined) return undefined;\n let given: Readonly<Record<string, string>>;\n try {\n given = this.signer.sign(target, this.clock.now());\n } catch (error) {\n return { error: `the signer did not sign the request: ${messageOf(error)}` };\n }\n // a signer adds headers: it cannot replace the User-Agent, From or another header of the crawler\n const headers: Record<string, string> = {};\n for (const [name, value] of Object.entries(given)) {\n const lower = name.toLowerCase();\n if (RESERVED_HEADERS.has(lower)) continue;\n if (typeof value !== \"string\" || !HEADER_TEXT.test(value))\n return { error: `the signer gave a ${lower} header that HTTP cannot send` };\n headers[lower] = value;\n }\n return {\n headers,\n facts: { signed: { key_id: this.signer.keyId, agent: this.signer.signatureAgent } },\n };\n }\n // ── end of signed requests ──\n\n private requestHeaders(request_: FetchRequest): Record<string, string> {\n const headers: Record<string, string> = {};\n // the caller's headers go first, so the crawler's own headers replace them\n for (const [name, value] of Object.entries(request_.headers ?? {})) {\n const lower = name.toLowerCase();\n if (RESERVED_HEADERS.has(lower)) continue;\n // with a signer, the signature headers are the crawler's own\n if (this.signer !== undefined && SIGNATURE_HEADERS.has(lower)) continue;\n headers[lower] = value;\n }\n headers[\"accept\"] = request_.accept ?? headers[\"accept\"] ?? \"*/*\";\n headers[\"user-agent\"] = this.options.userAgent;\n headers[\"accept-encoding\"] = \"gzip, deflate, br\";\n if (this.options.from !== undefined) headers[\"from\"] = this.options.from;\n return headers;\n }\n\n /**\n * This method records the result of a request for the host, and it gives the rest of the host.\n * The rest comes from `restAfter`. A 429 or 503 waits, an error counts toward a stop, and a\n * refusal stops the host for a time.\n */\n private settle(host: string, policy: PolitenessPolicy, step: Settle): number {\n const store = this.scheduler.store;\n const now = this.clock.now();\n const rest = restAfter(policy, step.rest);\n const delayMs = rest.ms;\n const waited = (ms: number, reason: string): number => {\n if (ms > delayMs) this.emit({ type: \"wait\", host, ms, reason });\n return ms;\n };\n if (step.result !== undefined) {\n store.record(host, step.result);\n return delayMs;\n }\n const outcome = step.outcome;\n if (outcome === \"rate-limited\" || outcome === \"unavailable\") {\n const { waits } = store.record(host, \"wait\");\n const asked = step.retryAfterMs;\n if (asked !== undefined && asked > policy.maxRetryAfterMs) {\n // a wait longer than the budget stops the host, so no request sleeps for days\n this.stopHost(\n host,\n now + Math.min(asked, MAX_STOP_MS),\n `asked to wait ${String(Math.round(asked / 1000))} s`,\n );\n return delayMs;\n }\n return waited(\n Math.max(\n delayMs,\n asked ?? equalJitterMs(waits, policy.backoffBaseMs, policy.backoffCapMs, this.random),\n ),\n asked === undefined ? \"backoff\" : \"retry-after\",\n );\n }\n if (outcome === \"error\") {\n if (isClientError(step.status)) {\n store.record(host, \"neutral\");\n return delayMs;\n }\n const { failures } = store.record(host, \"failure\");\n if (failures >= policy.maxFailures)\n this.stopHost(host, now + policy.failureStopMs, `${String(failures)} failures in a row`);\n return waited(\n Math.max(\n delayMs,\n equalJitterMs(failures, policy.backoffBaseMs, policy.backoffCapMs, this.random),\n ),\n \"backoff\",\n );\n }\n if (outcome === \"challenge\") this.stopHost(host, now + policy.challengeStopMs, outcome);\n else if (outcome === \"payment-required\")\n this.stopHost(host, now + policy.paymentStopMs, outcome);\n else if (outcome === \"blocked\" && !step.robotsFile)\n this.stopHost(host, now + policy.blockedStopMs, outcome);\n store.record(host, \"success\");\n return delayMs;\n }\n\n /**\n * This method stops the host of a URL as a challenge does: for the `challengeStopMs` of the policy.\n * The runtime uses it when a page that answered 200 is a bot check (`meta.blocked`).\n */\n reportBlocked(url: string, what: string): void {\n let target: URL;\n try {\n target = new URL(url);\n } catch {\n return;\n }\n const policy = this.policyOf(target);\n this.stopHost(\n hostKey(target),\n this.clock.now() + policy.challengeStopMs,\n `challenge (${what})`,\n );\n }\n\n /**\n * This method applies the rules that a fetch and a render share. The sequence is the URL, the\n * options, the egress policy, the product tokens and the rules for an agent. Then come the\n * cache, the stops of the host and robots.txt. It gives the answer when a rule decides, or the\n * facts that the transport needs.\n */\n private async admit(\n spec: AdmitSpec,\n extra?: (\n admitted: Omit<Admitted, \"verdict\" | \"check\" | \"robotsLines\" | \"crawlDelaySeconds\">,\n ) => Promise<Fetched | undefined>,\n ): Promise<{ readonly answer: Step } | { readonly admitted: Admitted }> {\n const marks = { scope: spec.scope, labels: spec.labels };\n let target: URL;\n try {\n target = new URL(spec.url);\n } catch {\n return {\n answer: { fetched: await this.skip(spec.base, spec.url, \"not a valid URL\", {}, marks) },\n };\n }\n if (!WEB_PROTOCOLS.has(target.protocol)) {\n return {\n answer: {\n fetched: await this.skip(\n spec.base,\n redact(target, spec.secretParams),\n \"not an http(s) URL\",\n {},\n marks,\n ),\n },\n };\n }\n // a fragment never goes to the server, so it is not part of the cache key\n target.hash = \"\";\n const url = redact(target, spec.secretParams);\n const skip = (note: string, extra_?: Record<string, unknown>): Promise<Fetched> =>\n this.skip(spec.base, url, note, extra_, marks);\n const answer = async (note: string, extra_?: Record<string, unknown>) => ({\n answer: { fetched: await skip(note, extra_) },\n });\n if (spec.invalid !== undefined) return answer(spec.invalid);\n const refused = egressRefusal(this.egress, target.hostname);\n if (refused !== undefined) return answer(refused);\n const tokens = [this.options.token, ...(spec.onBehalfOf ?? []), ...this.pageTokens];\n const badToken = tokens.find((token) => !isProductToken(token));\n if (spec.mode === \"page\" && badToken !== undefined)\n return answer(`not a valid product token: \"${badToken}\"`);\n const host = hostKey(target);\n const policy = this.policyOf(target);\n const path = `${target.pathname}${target.search}`;\n\n // the rules for an agent apply before a stored copy answers it\n let verdict: RobotsVerdict | undefined;\n if (spec.mode === \"page\") {\n verdict =\n tokens.length > 1 ? await this.robotsFor(target, policy, spec) : this.knownRobots(target);\n if (verdict !== undefined) {\n const decision = decide(verdict.robots, tokens, path);\n if (!decision.allowed) {\n return answer(robotsRefusal(verdict.robots, decision.by ?? this.options.token), {\n robots_check: robotsCheck(verdict, tokens, decision),\n });\n }\n }\n }\n const cached = await spec.cached(url);\n if (cached !== undefined) return { answer: cached };\n\n const stop = this.scheduler.stopOf(host);\n if (stop !== undefined) return answer(stopNote(stop.reason, stop.until));\n const early = await extra?.({ target, url, host, policy, tokens, skip });\n if (early !== undefined) return { answer: { fetched: early } };\n\n let robotsLines: readonly string[] = [];\n let crawlDelaySeconds = 0;\n let check: Record<string, unknown> | undefined;\n if (spec.mode === \"page\") {\n verdict ??= await this.robotsFor(target, policy, spec);\n const decision = decide(verdict.robots, tokens, path);\n check = robotsCheck(verdict, tokens, decision);\n if (!decision.allowed) {\n return answer(robotsRefusal(verdict.robots, decision.by ?? this.options.token), {\n robots_check: check,\n });\n }\n robotsLines = signalsFor(verdict.robots, tokens, path);\n crawlDelaySeconds = decision.crawlDelaySeconds ?? 0;\n }\n if (this.closed) return { answer: { fetched: closedResult(url) } };\n return {\n admitted: {\n target,\n url,\n host,\n policy,\n tokens,\n verdict,\n check,\n robotsLines,\n crawlDelaySeconds,\n skip,\n },\n };\n }\n\n /** This method fetches a URL. It follows the redirects up to the maximum, and each step is a separate decision. */\n async fetch(request_: FetchRequest): Promise<Fetched> {\n if (this.closed) return closedResult(request_.url);\n return this.track(this.follow(request_));\n }\n\n private async follow(request_: FetchRequest): Promise<Fetched> {\n const signal =\n request_.signal === undefined\n ? this.closing.signal\n : AbortSignal.any([request_.signal, this.closing.signal]);\n let url = request_.url;\n let headers = request_.headers;\n let origin: string | undefined;\n let maxRedirects = DEFAULT_POLICY.maxRedirects;\n try {\n const first = new URL(url);\n origin = first.origin;\n maxRedirects = this.policyOf(first).maxRedirects;\n } catch {\n // fetchOnce records the URL that does not parse\n }\n try {\n for (let hop = 0; ; hop += 1) {\n const step = await this.hop({ ...request_, url, headers }, signal);\n if (step.next === undefined) return step.fetched;\n if (hop >= maxRedirects) {\n return {\n ...step.fetched,\n outcome: \"error\",\n note: `more than ${String(maxRedirects)} redirects`,\n };\n }\n const next = new URL(step.next);\n // a credential for one origin does not go to a different origin\n if (next.origin !== origin) headers = withoutCredentials(headers);\n origin = next.origin;\n url = step.next;\n }\n } catch (error) {\n if (this.closed) return closedResult(request_.url);\n return {\n ...closedResult(request_.url),\n outcome: \"error\",\n note: `crawler error: ${messageOf(error)}`,\n };\n }\n }\n\n /** This method joins a request to an equal request in progress, so one URL gets one request at a time. */\n private hop(request_: FetchRequest, signal: AbortSignal): Promise<Step> {\n // a request with a signal of its own runs alone, so its cancellation stops only that request\n if (request_.signal !== undefined) return this.fetchOnce(request_, signal);\n const key = JSON.stringify([\n cacheKeyOf(request_.url.split(\"#\")[0] ?? request_.url),\n request_.mode,\n request_.recipe,\n request_.accept ?? null,\n request_.headers ?? null,\n request_.secretParams ?? null,\n request_.minIntervalMs ?? null,\n request_.maxAgeMs ?? null,\n request_.maxBytes ?? null,\n request_.onBehalfOf ?? null,\n request_.job ?? null,\n request_.scope ?? null,\n request_.labels ?? null,\n request_.deadline ?? null,\n // ── archived fallback ──\n request_.range ?? null,\n request_.signalLines ?? null,\n request_.originalHeaderPrefix ?? null,\n ]);\n const running = this.inFlight.get(key);\n if (running !== undefined) return running;\n const work = this.fetchOnce(request_, signal).finally(() => this.inFlight.delete(key));\n this.inFlight.set(key, work);\n return work;\n }\n\n private knownRobots(target: URL): RobotsVerdict | undefined {\n const entry = this.robots.peek(originKey(target));\n return entry !== undefined && this.clock.now() - entry.at < entry.ttlMs\n ? entry.value\n : undefined;\n }\n\n /**\n * This method gives the robots.txt of the origin of a URL. A good copy stays for the TTL of the\n * policy. An unreachable verdict stays for some minutes, with a backoff and the `Retry-After`\n * of the server. During that time the last good copy applies, when there is one (RFC 9309 §2.4).\n */\n private robotsFor(\n target: URL,\n policy: PolitenessPolicy,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<RobotsVerdict> {\n const key = originKey(target);\n const now = this.clock.now();\n const cached = this.robots.get(key);\n if (cached !== undefined && now - cached.at < cached.ttlMs) {\n // a request that joins a fetch in progress adds its scope and its labels to the record\n if (cached.value === undefined) this.join(cached.waiting, marks);\n return cached.verdict;\n }\n const waiting: Waiting = { scope: undefined, labels: undefined };\n this.join(waiting, marks);\n const entry: RobotsEntry = {\n at: now,\n waiting,\n ttlMs: policy.robotsTtlMs,\n verdict: Promise.resolve({ robots: unreachableRobots(\"not read yet\"), capture: undefined }),\n value: undefined,\n misses: cached?.misses ?? 0,\n good: cached?.good,\n };\n entry.verdict = this.fetchRobots(target, policy, waiting)\n .catch((error: unknown) => ({\n robots: unreachableRobots(`robots.txt failed: ${messageOf(error)}`),\n capture: undefined,\n retryAfterMs: undefined,\n }))\n .then(async (read): Promise<RobotsVerdict> => {\n let verdict: RobotsVerdict = { robots: read.robots, capture: read.capture };\n if (read.robots.unreachable === undefined) {\n entry.misses = 0;\n entry.good = { verdict, at: this.clock.now() };\n } else {\n entry.misses += 1;\n const backoff = equalJitterMs(\n entry.misses,\n policy.robotsRetryMs,\n policy.robotsRetryCapMs,\n this.random,\n );\n entry.ttlMs = Math.min(\n policy.robotsRetryCapMs,\n Math.max(read.retryAfterMs ?? 0, backoff),\n );\n const good =\n entry.good !== undefined && this.clock.now() - entry.good.at <= policy.robotsStaleMs\n ? entry.good.verdict\n : await this.storedRobots(target, policy);\n if (good !== undefined) verdict = { ...good, stale: read.robots.unreachable };\n }\n entry.value = verdict;\n return verdict;\n });\n this.robots.set(key, entry);\n return entry.verdict;\n }\n\n /**\n * This method gives the robots.txt rules that apply to a URL, as the crawler applies them. They\n * come from its memory, or from a fetch of `/robots.txt` with its redirects. A robots.txt that\n * the crawler cannot read gives rules that permit nothing, and `unreachable` gives the reason.\n */\n async readRobots(\n url: string,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<{\n readonly robots: Robots;\n readonly capture: string | undefined;\n readonly stale?: string | undefined;\n }> {\n const target = new URL(url);\n if (!WEB_PROTOCOLS.has(target.protocol))\n return { robots: unreachableRobots(\"not an http(s) URL\"), capture: undefined };\n return this.robotsFor(target, this.policyOf(target), marks);\n }\n\n /**\n * This method tells if robots.txt lets the crawler fetch a URL as a page. The rules of the token\n * of the crawler and of the agents of `onBehalfOf` apply, as for a fetch. It reads robots.txt as\n * a fetch does, so a robots.txt that the crawler cannot read permits nothing. When the signal\n * aborts, the promise rejects, and a fetch of robots.txt in progress continues.\n */\n async robotsAllow(\n url: string,\n options: {\n readonly onBehalfOf?: readonly string[] | undefined;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n readonly signal?: AbortSignal | undefined;\n } = {},\n ): Promise<{\n readonly allowed: boolean;\n readonly note: string | undefined;\n /** These are the lines of usage signals (`Content-Signal`) of robots.txt that apply to the URL, when there are any. The archived fallback applies them to a copy. */\n readonly signals?: readonly string[] | undefined;\n }> {\n let target: URL;\n try {\n target = new URL(url);\n } catch {\n return { allowed: false, note: \"not a valid URL\" };\n }\n if (!WEB_PROTOCOLS.has(target.protocol)) return { allowed: false, note: \"not an http(s) URL\" };\n if (this.closed) return { allowed: false, note: \"crawler closed\" };\n const tokens = [this.options.token, ...(options.onBehalfOf ?? []), ...this.pageTokens];\n const badToken = tokens.find((token) => !isProductToken(token));\n if (badToken !== undefined)\n return { allowed: false, note: `not a valid product token: \"${badToken}\"` };\n const work = this.robotsFor(target, this.policyOf(target), options);\n const verdict = await (options.signal === undefined ? work : raceAbort(work, options.signal));\n const path = `${target.pathname}${target.search}`;\n const decision = decide(verdict.robots, tokens, path);\n // ── archived fallback: the content signals of the site apply to an archived copy ──\n // (only when robots.txt has such lines, so the result of a plain robots.txt stays the same)\n const lines = signalsFor(verdict.robots, tokens, path);\n const signals = lines.length > 0 ? { signals: lines } : {};\n return decision.allowed\n ? { allowed: true, note: undefined, ...signals }\n : {\n allowed: false,\n note: robotsRefusal(verdict.robots, decision.by ?? this.options.token),\n ...signals,\n };\n }\n\n /** This method reads the stored copy of a robots.txt, when it is younger than the limit of the policy. */\n private async storedRobots(\n target: URL,\n policy: PolitenessPolicy,\n ): Promise<RobotsVerdict | undefined> {\n try {\n const row = this.lookup(redact(new URL(\"/robots.txt\", target.origin)));\n if (row === undefined || row.blob === null || row.status < 200 || row.status >= 300)\n return undefined;\n if (this.clock.now() - Date.parse(row.fetched_at) > policy.robotsStaleMs) return undefined;\n const body = await this.options.blobs.get(row.blob);\n if (body === undefined) return undefined;\n const robots = robotsFromResponse(200, row.content_type, body);\n return robots.unreachable === undefined ? { robots, capture: row.capture } : undefined;\n } catch {\n return undefined;\n }\n }\n\n /** This method adds the scope and the labels of one request to the requests that wait for a robots.txt. The most private scope wins. */\n private join(\n waiting: Waiting,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n },\n ): void {\n const scope = marks.scope ?? this.options.scope;\n const rank = this.options.scopeRank;\n if (waiting.scope === undefined) waiting.scope = scope;\n else if (scope !== undefined && rank !== undefined && rank(scope) > rank(waiting.scope))\n waiting.scope = scope;\n if (marks.labels !== undefined) waiting.labels = { ...marks.labels, ...waiting.labels };\n }\n\n private async fetchRobots(\n target: URL,\n policy: PolitenessPolicy,\n waiting: Waiting,\n ): Promise<RobotsRead> {\n let url = new URL(\"/robots.txt\", target.origin);\n const unreachable = (\n reason: string,\n capture: string | undefined,\n retryAfterMs?: number,\n ): RobotsRead => ({\n robots: unreachableRobots(reason),\n capture,\n retryAfterMs,\n });\n for (let hop = 0; hop <= policy.maxRedirects; hop += 1) {\n const step = await this.fetchOnce(\n {\n url: url.href,\n mode: \"api\",\n recipe: \"robots.txt\",\n accept: \"text/plain\",\n maxBytes: ROBOTS_BYTES,\n minIntervalMs: policy.apiIntervalMs,\n // the record reads the requests that wait at the time that it is written, not at the start\n get scope() {\n return waiting.scope;\n },\n get labels() {\n return waiting.labels;\n },\n },\n this.closing.signal,\n );\n const result = step.fetched;\n if (result.outcome === \"redirect\") {\n if (step.next === undefined)\n return unreachable(result.note ?? \"a redirect with no Location header\", result.capture);\n url = new URL(step.next);\n continue;\n }\n if (result.outcome === \"skipped\" || result.status === 0)\n return unreachable(result.note ?? \"no response\", result.capture);\n // a body that the crawler cannot read is not an empty robots.txt: the site stays closed\n if (result.outcome === \"error\" && result.status >= 200 && result.status < 300)\n return unreachable(result.note ?? \"the body could not be read\", result.capture);\n // a body that the server cut can have lost a rule\n if (result.note?.startsWith(\"the compressed body ended early\") === true)\n return unreachable(result.note, result.capture);\n if (result.outcome === \"not-modified\") {\n return result.body === undefined\n ? unreachable(\"status 304, and no stored copy\", result.capture)\n : {\n robots: robotsFromResponse(200, result.contentType ?? null, result.body),\n capture: result.capture,\n retryAfterMs: undefined,\n };\n }\n const robots =\n result.status >= 200 && result.status < 300\n ? robotsFromResponse(\n result.status,\n result.contentType ?? null,\n result.body ?? new Uint8Array(),\n )\n : robotsFromStatus(result.status, undefined);\n return { robots, capture: result.capture, retryAfterMs: step.retryAfterMs };\n }\n return unreachable(`more than ${String(policy.maxRedirects)} redirects`, undefined);\n }\n\n private async fetchOnce(request_: FetchRequest, signal: AbortSignal): Promise<Step> {\n const base = {\n recipe: request_.recipe,\n mode: request_.mode,\n ...(request_.job ? { job: request_.job } : {}),\n };\n const secretParams = request_.secretParams ?? [];\n const sent = this.requestHeaders(request_);\n // read when a record is written: more requests can wait for the fetch of a robots.txt by then\n const marks = {\n get scope() {\n return request_.scope;\n },\n get labels() {\n return request_.labels;\n },\n };\n // ── archived fallback: a range is a part of a resource, never its stored copy ──\n const range = request_.range;\n const badRange =\n range === undefined ||\n (Number.isSafeInteger(range.offset) &&\n range.offset >= 0 &&\n Number.isSafeInteger(range.length) &&\n range.length >= 1)\n ? undefined\n : `invalid option: range is ${JSON.stringify(range)}`;\n // ── end of archived fallback ──\n const admission = await this.admit({\n transport: \"http\",\n url: request_.url,\n mode: request_.mode,\n base,\n secretParams,\n invalid:\n invalidNumber(\"minIntervalMs\", request_.minIntervalMs, false) ??\n invalidNumber(\"maxBytes\", request_.maxBytes, false) ??\n invalidNumber(\"maxAgeMs\", request_.maxAgeMs, true) ??\n badRange,\n onBehalfOf: request_.onBehalfOf,\n scope: request_.scope,\n labels: request_.labels,\n cached: (url) =>\n range !== undefined\n ? Promise.resolve(undefined)\n : this.fromCache(url, \"plain\", request_.maxAgeMs, sent, secretParams),\n });\n if (\"answer\" in admission) return admission.answer;\n const { target, url, host, policy, check, robotsLines, crawlDelaySeconds, skip } =\n admission.admitted;\n const maxBytes = Math.min(request_.maxBytes ?? policy.maxBytes, range?.length ?? Infinity);\n const rest = (elapsedMs: number): RestInput => ({\n kind: request_.mode,\n elapsedMs,\n crawlDelaySeconds,\n minIntervalMs: request_.minIntervalMs,\n });\n const robotsFile = request_.recipe === \"robots.txt\";\n\n try {\n return await this.scheduler.run(\n host,\n async (): Promise<Paced<Step>> => {\n // an equal request that waited in the queue can find a fresh copy now\n const again =\n range === undefined\n ? await this.fromCache(url, \"plain\", request_.maxAgeMs, sent, secretParams)\n : undefined;\n if (again !== undefined) return { value: again, delayMs: 0 };\n // no request starts at or after its deadline\n this.beforeDeadline(request_.deadline);\n // a range sends no validators: a 304 for a part has no stored body\n const latest = range === undefined ? this.lookup(url) : undefined;\n // validators go only with a stored body of the same variant, so a 304 always has a body\n const usable =\n latest !== undefined && latest.truncated !== true && varyMatches(latest.vary_key, sent)\n ? latest\n : undefined;\n let stored: Uint8Array | undefined;\n if (usable?.blob && (usable.etag || usable.last_modified))\n stored = await this.options.blobs.get(usable.blob);\n const headers: Record<string, string> = { ...sent };\n // the memo of the host can leave out an ETag that never matches; robots.txt is often a\n // static file with validators of its own, so it keeps the two and teaches the memo nothing\n const send =\n stored === undefined || usable === undefined\n ? { etag: false, lastModified: false }\n : validatorsToSend(\n robotsFile ? undefined : this.memoOf(host),\n { etag: Boolean(usable.etag), lastModified: Boolean(usable.last_modified) },\n this.clock.now(),\n );\n if (send.etag && usable?.etag) headers[\"if-none-match\"] = usable.etag;\n if (send.lastModified && usable?.last_modified)\n headers[\"if-modified-since\"] = usable.last_modified;\n const validators = [\"if-none-match\", \"if-modified-since\"].filter(\n (name) => headers[name] !== undefined,\n );\n // ── archived fallback: the part of a WARC file ──\n const rangeText =\n range === undefined\n ? undefined\n : `bytes=${String(range.offset)}-${String(range.offset + range.length - 1)}`;\n if (rangeText !== undefined) headers[\"range\"] = rangeText;\n // ── signed requests: the signature is made last, for this URL at this time ──\n const signed = this.signFor(target);\n if (signed !== undefined && \"error\" in signed)\n return { value: { fetched: await skip(signed.error) }, delayMs: 0 };\n if (signed !== undefined) Object.assign(headers, signed.headers);\n const signedFacts = signed?.facts ?? {};\n // ── end of signed requests ──\n const started = this.clock.now();\n const timeout = AbortSignal.timeout(policy.requestTimeoutMs);\n const abort = AbortSignal.any([timeout, signal]);\n const cancelled = (): boolean => request_.signal?.aborted === true;\n const failedNote = (prefix: string, error: unknown): string =>\n timeout.aborted\n ? `${prefix}: timed out after ${String(Math.round(policy.requestTimeoutMs / 1000))} s`\n : `${prefix}: ${messageOf(error)}`;\n\n this.emit({ type: \"request\", url, host, transport: \"http\" });\n let response: Dispatcher.ResponseData;\n try {\n response = await request(target, {\n method: \"GET\",\n headers,\n dispatcher: this.dispatcher,\n signal: abort,\n });\n } catch (error) {\n const egress = egressError(error);\n if (egress !== undefined)\n return { value: { fetched: await skip(egress.message) }, delayMs: 0 };\n if (this.closed) return { value: { fetched: closedResult(url) }, delayMs: 0 };\n const elapsedMs = this.clock.now() - started;\n const note = cancelled() ? \"cancelled by the caller\" : failedNote(\"no response\", error);\n const delayMs = this.settle(host, policy, {\n outcome: \"error\",\n status: 0,\n retryAfterMs: undefined,\n rest: rest(elapsedMs),\n robotsFile,\n ...(cancelled() ? { result: \"neutral\" as const } : {}),\n });\n this.emit({ type: \"response\", url, host, status: 0, outcome: \"error\", elapsedMs });\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n status: 0,\n outcome: \"error\",\n note,\n elapsed_ms: elapsedMs,\n ...(check ? { robots_check: check } : {}),\n ...(rangeText !== undefined ? { range: rangeText } : {}),\n ...signedFacts,\n },\n url,\n marks,\n );\n return {\n value: {\n fetched: {\n url,\n status: 0,\n outcome: \"error\",\n fromCache: false,\n capture,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note,\n fetchedAt: new Date(started).toISOString(),\n ...marks,\n },\n },\n delayMs,\n };\n }\n\n const header = headerOf(\n response.headers as Record<string, string | string[] | undefined>,\n );\n const ttfbMs = this.clock.now() - started;\n const classified = classify(response.statusCode, header, this.clock.now());\n let outcome: Outcome = classified.outcome;\n let note = classified.note;\n let body: Uint8Array | undefined;\n let blob: string | undefined;\n let truncated = false;\n let cut = false;\n let next: string | undefined;\n let result: HostResult | undefined;\n if (outcome === \"ok\") {\n try {\n const read = await readBody(response.body, header(\"content-encoding\"), maxBytes);\n body = read.bytes;\n truncated = read.truncated;\n cut = read.cut;\n if (read.cut)\n note = `the compressed body ended early, so the crawler keeps the ${String(read.bytes.length)} bytes before the end`;\n else if (truncated) note = `truncated at ${String(maxBytes)} bytes`;\n } catch (error) {\n if (this.closed) return { value: { fetched: closedResult(url) }, delayMs: 0 };\n outcome = \"error\";\n if (cancelled()) {\n note = \"cancelled by the caller\";\n result = \"neutral\";\n } else {\n const decode = error instanceof BodyError && error.kind === \"decode\";\n note = failedNote(decode ? \"undecodable body\" : \"body failed\", error);\n }\n }\n if (body !== undefined) {\n try {\n blob = (await this.options.blobs.put(body)).hash;\n } catch (error) {\n // a full disk is not a fault of the host\n outcome = \"error\";\n note = `blob store: ${messageOf(error)}`;\n body = undefined;\n result = \"neutral\";\n }\n }\n } else {\n try {\n await readBody(response.body, undefined, ERROR_BODY_BYTES);\n } catch {\n // the body of an error answer is not kept\n }\n }\n const elapsedMs = this.clock.now() - started;\n if (outcome === \"not-modified\") {\n if (stored !== undefined && usable?.blob) {\n body = stored;\n blob = usable.blob;\n } else note = \"not modified, but the crawler has no stored copy\";\n }\n let location: string | undefined;\n if (outcome === \"redirect\") {\n const raw = header(\"location\");\n if (raw === undefined) note = \"a redirect with no Location header\";\n else {\n try {\n const resolved = new URL(raw, target);\n next = resolved.href;\n location = redact(resolved, secretParams);\n note = `location ${location}`;\n } catch {\n outcome = \"error\";\n note = \"bad redirect target\";\n }\n }\n }\n\n const revisit = outcome === \"not-modified\" && usable !== undefined;\n // ── archived fallback: the signals of the site and of the archived headers apply to a copy ──\n const prefix = request_.originalHeaderPrefix?.toLowerCase();\n const archivedLines =\n prefix === undefined\n ? []\n : SIGNAL_HEADERS.flatMap((name) => {\n const value = header(`${prefix}${name}`);\n return value === undefined ? [] : [`${prefix}${name}: ${value}`];\n });\n let signals = readSignals(\n [...robotsLines, ...(request_.signalLines ?? []), ...archivedLines],\n header,\n pageSignals(body, header(\"content-type\")),\n );\n // ── end of archived fallback ──\n if (revisit) {\n // a 304 updates the stored answer and does not replace it (RFC 9111 §4.3.4)\n const merged: AiInput =\n signals.ai_input === \"no\" || usable.ai_input === \"no\"\n ? \"no\"\n : signals.ai_input !== \"unset\"\n ? signals.ai_input\n : usable.ai_input;\n signals = { ...signals, ai_input: merged };\n }\n const contentType =\n header(\"content-type\") ?? (revisit ? (usable.content_type ?? undefined) : undefined);\n // a web page in place of the data of an API (a login or an error page) is not the stored copy\n const unasked =\n request_.mode === \"api\" &&\n outcome === \"ok\" &&\n unaskedPage(request_.accept, contentType);\n if (unasked) {\n const said = `the server sent ${contentType ?? \"\"}, and the request accepts ${request_.accept ?? \"\"}, so the crawler keeps no stored copy of it`;\n note = note === undefined ? said : `${note}; ${said}`;\n }\n const etag = header(\"etag\") ?? (revisit ? usable.etag : null) ?? null;\n const lastModified =\n header(\"last-modified\") ?? (revisit ? usable.last_modified : null) ?? null;\n const fetchedAt = this.clock.now();\n const freshness = freshUntil(\n header,\n response.statusCode,\n fetchedAt,\n lastModified,\n policy.heuristicFreshnessCapMs,\n );\n const varyKey =\n header(\"vary\") !== undefined\n ? varyKeyOf(header(\"vary\"), sent)\n : revisit\n ? (usable.vary_key ?? null)\n : null;\n const cacheHeaders = Object.fromEntries(\n [\"cache-control\", \"expires\", \"age\", \"date\", \"vary\"].flatMap((name) => {\n const value = header(name);\n return value === undefined ? [] : [[name.replace(\"-\", \"_\"), value]];\n }),\n );\n this.emit({\n type: \"response\",\n url,\n host,\n status: response.statusCode,\n outcome,\n elapsedMs,\n });\n const capture = await this.record(\n revisit ? \"revisit\" : \"capture\",\n {\n ...base,\n url,\n status: response.statusCode,\n outcome,\n elapsed_ms: elapsedMs,\n ttfb_ms: ttfbMs,\n content_type: contentType ?? null,\n bytes: body?.length ?? 0,\n blob: blob ?? null,\n etag,\n last_modified: lastModified,\n ...(validators.length > 0 ? { validators } : {}),\n ...(revisit ? { revisit_of: usable.capture } : {}),\n robots: request_.mode === \"page\" ? \"allowed\" : \"not-checked\",\n ...(check !== undefined ? { robots_check: check } : {}),\n signals,\n ...(truncated ? { truncated: true } : {}),\n ...(cut ? { cut: true } : {}),\n ...(unasked ? { unasked: true } : {}),\n ...cacheHeaders,\n ...(freshness !== null ? { fresh_until: freshness } : {}),\n ...(varyKey !== null ? { vary_key: varyKey } : {}),\n ...(location !== undefined ? { location } : {}),\n ...(note !== undefined ? { note } : {}),\n ...(rangeText !== undefined ? { range: rangeText } : {}),\n ...signedFacts,\n },\n url,\n marks,\n );\n const fetchedIso = new Date(fetchedAt).toISOString();\n // a body that the server cut is not a copy to keep: the next request asks for the whole body\n // (a range is a part of the resource, so it is not the stored copy of the URL either)\n if (\n capture !== undefined &&\n (outcome === \"ok\" || outcome === \"not-modified\") &&\n blob !== undefined &&\n !cut &&\n !unasked &&\n range === undefined\n ) {\n this.store({\n url,\n capture,\n blob,\n // a truncated body keeps no validators, so a later request gets the whole body again\n etag: truncated ? null : etag,\n last_modified: truncated ? null : lastModified,\n content_type: contentType ?? null,\n fetched_at: fetchedIso,\n // an index that a replay built can hold 304 as the status of the stored body\n status: revisit ? (usable.status === 304 ? 200 : usable.status) : response.statusCode,\n ai_input: signals.ai_input,\n truncated: truncated ? true : undefined,\n fresh_until: freshness,\n vary_key: varyKey,\n });\n }\n if (\n capture !== undefined &&\n outcome === \"redirect\" &&\n next !== undefined &&\n (PERMANENT_REDIRECTS.has(response.statusCode) || freshness !== null)\n ) {\n this.store(\n {\n url,\n capture,\n blob: null,\n etag: null,\n last_modified: null,\n content_type: null,\n fetched_at: fetchedIso,\n status: response.statusCode,\n ai_input: \"unset\",\n location: location ?? null,\n fresh_until: freshness,\n vary_key: varyKey,\n },\n next,\n );\n }\n // the memo of the host learns which validator gets a 304 (a 200 with the stored body is a miss)\n const answer: ValidatorAnswer | undefined =\n validators.length === 0 || capture === undefined || robotsFile\n ? undefined\n : outcome === \"not-modified\"\n ? \"not-modified\"\n : outcome === \"ok\" && blob !== undefined && !cut\n ? blob === usable?.blob\n ? \"same\"\n : \"changed\"\n : undefined;\n if (answer !== undefined) {\n try {\n this.scheduler.store.noteValidators?.(\n host,\n {\n etag: headers[\"if-none-match\"] !== undefined,\n lastModified: headers[\"if-modified-since\"] !== undefined,\n answer,\n },\n fetchedAt,\n );\n } catch {\n // the memo is a hint: an error of its store does not fail the fetch\n }\n }\n const delayMs = this.settle(host, policy, {\n outcome,\n status: response.statusCode,\n retryAfterMs: classified.retryAfterMs,\n rest: rest(elapsedMs),\n robotsFile,\n ...(result !== undefined ? { result } : {}),\n });\n return {\n value: {\n fetched: {\n url,\n status: response.statusCode,\n outcome,\n fromCache: false,\n capture,\n blob,\n body,\n contentType,\n signals,\n note,\n ...(truncated ? { truncated: true } : {}),\n fetchedAt: fetchedIso,\n ...marks,\n },\n next,\n retryAfterMs: classified.retryAfterMs,\n },\n delayMs,\n };\n },\n this.paced(signal, request_.deadline),\n );\n } catch (error) {\n if (error instanceof HostStopped)\n return { fetched: await skip(stopNote(error.reason, error.until)) };\n if (error instanceof DeadlineReached)\n return { fetched: await skip(deadlineNote(error, request_.deadline)) };\n if (this.closed) return { fetched: closedResult(url) };\n if (request_.signal?.aborted === true)\n return { fetched: await skip(\"cancelled by the caller\") };\n throw error;\n }\n }\n\n /**\n * This method gives the function that decides each request of a rendered page. It refuses an\n * address that the egress policy refuses, and a URL of the page's host that robots.txt\n * disallows. For a different host, it uses a robots.txt that the crawler holds, when there is one.\n */\n private allowRequestFor(\n page: URL,\n robots: Robots,\n tokens: readonly string[],\n signal: AbortSignal,\n tally: { allowed: number; refused: number; readonly hosts: Set<string> },\n ): (url: string) => Promise<boolean> {\n const pageOrigin = originKey(page);\n const checked = new Map<string, Promise<string | undefined>>();\n const refuse = (): boolean => {\n tally.refused += 1;\n return false;\n };\n return async (raw: string): Promise<boolean> => {\n if (signal.aborted) return false;\n let target: URL;\n try {\n target = new URL(raw);\n } catch {\n return refuse();\n }\n if (\n target.protocol === \"data:\" ||\n target.protocol === \"blob:\" ||\n target.protocol === \"about:\"\n )\n return true;\n const web = WEB_PROTOCOLS.has(target.protocol);\n if (!web && target.protocol !== \"ws:\" && target.protocol !== \"wss:\") return refuse();\n let check = checked.get(target.hostname);\n if (check === undefined) {\n check = checkEgress(this.egress, target);\n checked.set(target.hostname, check);\n }\n if ((await check) !== undefined) return refuse();\n if (web) {\n const rules = originKey(target) === pageOrigin ? robots : this.knownRobots(target)?.robots;\n if (\n rules !== undefined &&\n !decide(rules, tokens, `${target.pathname}${target.search}`).allowed\n )\n return refuse();\n }\n tally.allowed += 1;\n if (tally.hosts.size < 50) tally.hosts.add(hostKey(target));\n return true;\n };\n }\n\n /**\n * This method tells why the browser did not go on to a URL that the page moved to: the egress\n * policy or robots.txt. For a different site, it uses a robots.txt that the crawler holds. It\n * gives undefined for the page itself, or for a URL that it cannot read.\n */\n private async movedRefusal(\n raw: string,\n page: URL,\n robots: Robots,\n tokens: readonly string[],\n ): Promise<{ readonly note: string; readonly url: string } | undefined> {\n let target: URL;\n try {\n target = new URL(raw);\n } catch {\n return undefined;\n }\n target.hash = \"\";\n if (target.href === page.href) return undefined;\n const url = redact(target);\n const moved = `the page moved to ${url}`;\n if (!WEB_PROTOCOLS.has(target.protocol))\n return { note: `${moved}, which is not an http(s) URL`, url };\n const egress = await checkEgress(this.egress, target);\n if (egress !== undefined) return { note: `${moved}: ${egress}`, url };\n const rules = originKey(target) === originKey(page) ? robots : this.knownRobots(target)?.robots;\n if (rules !== undefined) {\n const decision = decide(rules, tokens, `${target.pathname}${target.search}`);\n if (!decision.allowed)\n return {\n note: `${moved}: ${robotsRefusal(rules, decision.by ?? this.options.token)}`,\n url,\n };\n }\n return undefined;\n }\n\n /**\n * This method renders a page in the headless browser and captures what the page shows. The rules\n * of a fetch apply:\n *\n * - the egress policy and robots.txt, for the page, its final URL and each request of the page\n * - one request at a time for each host\n * - the backoff after a 429 or a 503\n * - a capture for each decision\n *\n * A render costs the site more than a fetch, so the host rests longer after one. A render that\n * the browser refuses (`RenderRefused`) does not count against the host.\n */\n async render(request_: RenderRequest): Promise<Fetched> {\n if (this.closed) return closedResult(request_.url);\n return this.track(\n this.renderOnce(request_).catch((error: unknown) =>\n this.closed\n ? closedResult(request_.url)\n : {\n ...closedResult(request_.url),\n outcome: \"error\" as const,\n note: `crawler error: ${messageOf(error)}`,\n },\n ),\n );\n }\n\n private async renderOnce(request_: RenderRequest): Promise<Fetched> {\n const base = {\n recipe: request_.recipe,\n mode: \"page\" as const,\n rendered: true,\n ...(request_.job ? { job: request_.job } : {}),\n };\n const marks = { scope: request_.scope, labels: request_.labels };\n const renderer = this.options.renderer;\n const admission = await this.admit(\n {\n transport: \"browser\",\n url: request_.url,\n mode: \"page\",\n base,\n secretParams: [],\n invalid: invalidNumber(\"maxAgeMs\", request_.maxAgeMs, true),\n onBehalfOf: request_.onBehalfOf,\n scope: request_.scope,\n labels: request_.labels,\n cached: async (url) => {\n const cached = await this.fromCache(url, \"rendered\", request_.maxAgeMs, {}, []);\n if (cached === undefined || cached.fetched.body === undefined) return undefined;\n return { fetched: { ...cached.fetched, url, contentType: RENDERED_TYPE } };\n },\n },\n async ({ target, skip }) => {\n if (renderer === undefined) return skip(\"no browser configured for rendering\");\n if (this.renderOff !== undefined) return skip(this.renderOff);\n // the browser does its own DNS, so the check here resolves the name too\n const resolved = await checkEgress(this.egress, target);\n return resolved === undefined ? undefined : skip(resolved);\n },\n );\n if (\"answer\" in admission) return admission.answer.fetched;\n const { target, url, host, policy, tokens, check, crawlDelaySeconds, skip } =\n admission.admitted;\n const verdict = admission.admitted.verdict!;\n const signal =\n request_.signal === undefined\n ? this.closing.signal\n : AbortSignal.any([request_.signal, this.closing.signal]);\n const tally = { allowed: 0, refused: 0, hosts: new Set<string>() };\n const allowRequest = this.allowRequestFor(target, verdict.robots, tokens, signal, tally);\n // ── signed requests: the browser signs each request of the page, or does not send it ──\n const sign =\n this.signer === undefined\n ? undefined\n : (raw: string): Readonly<Record<string, string>> | undefined => {\n let to: URL;\n try {\n to = new URL(raw);\n } catch {\n return undefined;\n }\n const signed = this.signFor(to);\n return signed === undefined || \"error\" in signed ? undefined : signed.headers;\n };\n const signedFacts =\n this.signer === undefined\n ? {}\n : { signed: { key_id: this.signer.keyId, agent: this.signer.signatureAgent } };\n // ── end of signed requests ──\n const rest = (elapsedMs: number): RestInput => ({\n kind: \"render\",\n elapsedMs,\n crawlDelaySeconds,\n });\n\n // the wait for a slot of the browser comes before the lease of the host, so the host does not wait for it\n let release: () => void;\n try {\n release = await this.renders.acquire(signal);\n } catch {\n if (this.closed) return closedResult(url);\n return skip(\"cancelled by the caller\");\n }\n let done: RenderDone;\n try {\n done = await this.scheduler.run(\n host,\n async (): Promise<Paced<RenderDone>> => {\n this.beforeDeadline(request_.deadline);\n const started = this.clock.now();\n this.emit({ type: \"request\", url, host, transport: \"browser\" });\n try {\n const rendered = await raceAbort(\n renderer!.render(target.href, {\n userAgent: this.options.userAgent,\n timeoutMs: policy.renderTimeoutMs,\n ...(this.options.from !== undefined\n ? { headers: { from: this.options.from } }\n : {}),\n allowRequest,\n maxRedirects: policy.maxRedirects,\n signal,\n ...(sign === undefined ? {} : { sign }),\n }),\n signal,\n );\n const header = (name: string): string | undefined =>\n rendered.headers[name.toLowerCase()];\n const classified: Classified =\n rendered.status === null\n ? { outcome: \"ok\", retryAfterMs: undefined, note: undefined }\n : classify(rendered.status, header, this.clock.now());\n const elapsedMs = this.clock.now() - started;\n const delayMs = this.settle(host, policy, {\n outcome: classified.outcome,\n status: rendered.status ?? 200,\n retryAfterMs: classified.retryAfterMs,\n rest: rest(elapsedMs),\n robotsFile: false,\n });\n return { value: { kind: \"rendered\", rendered, classified, elapsedMs }, delayMs };\n } catch (error) {\n const elapsedMs = this.clock.now() - started;\n if (this.closed) return { value: { kind: \"closed\" }, delayMs: 0 };\n if (request_.signal?.aborted === true) {\n return {\n value: { kind: \"cancelled\", elapsedMs },\n delayMs: restAfter(policy, rest(0)).ms,\n };\n }\n // a renderer of an application that has no browser is no fault of the host either\n const refused =\n error instanceof RenderRefused\n ? error\n : missingBrowser(messageOf(error))\n ? new RenderRefused(\"browser\", `rendering needs a browser: ${messageOf(error)}`)\n : undefined;\n if (refused !== undefined) {\n if (refused.reason === \"browser\") this.renderOff = refused.message;\n // a refused navigation, a file, a scheme or no browser is no fault of the host\n const delayMs = this.settle(host, policy, {\n outcome: \"skipped\",\n status: 0,\n retryAfterMs: undefined,\n rest: rest(0),\n robotsFile: false,\n result: \"neutral\",\n });\n return {\n value: {\n kind: \"refused\",\n reason: refused.reason,\n note: refused.message,\n url: refused.url,\n elapsedMs,\n },\n delayMs,\n };\n }\n const delayMs = this.settle(host, policy, {\n outcome: \"error\",\n status: 0,\n retryAfterMs: undefined,\n rest: rest(elapsedMs),\n robotsFile: false,\n });\n return {\n value: { kind: \"failed\", note: `render failed: ${messageOf(error)}`, elapsedMs },\n delayMs,\n };\n }\n },\n this.paced(signal, request_.deadline),\n );\n } catch (error) {\n if (error instanceof HostStopped) return skip(stopNote(error.reason, error.until));\n if (error instanceof DeadlineReached) return skip(deadlineNote(error, request_.deadline));\n if (this.closed) return closedResult(url);\n if (request_.signal?.aborted === true) return skip(\"cancelled by the caller\");\n throw error;\n } finally {\n release();\n }\n\n if (done.kind === \"closed\" || this.closed) return closedResult(url);\n const subrequests = { allowed: tally.allowed, refused: tally.refused, hosts: [...tally.hosts] };\n if (done.kind === \"refused\") {\n this.emit({\n type: \"response\",\n url,\n host,\n status: 0,\n outcome: done.reason === \"download\" ? \"not-html\" : \"skipped\",\n elapsedMs: done.elapsedMs,\n });\n if (done.reason !== \"download\") {\n const moved =\n done.url === undefined\n ? undefined\n : await this.movedRefusal(done.url, target, verdict.robots, tokens);\n return skip(moved?.note ?? done.note, {\n robots_check: check,\n subrequests,\n ...(moved === undefined ? {} : { final_url: moved.url }),\n });\n }\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n status: 0,\n outcome: \"not-html\",\n note: done.note,\n elapsed_ms: done.elapsedMs,\n robots_check: check,\n subrequests,\n ...signedFacts,\n },\n url,\n marks,\n );\n return { ...closedResult(url), outcome: \"not-html\", capture, note: done.note, ...marks };\n }\n if (done.kind === \"failed\" || done.kind === \"cancelled\") {\n const note = done.kind === \"failed\" ? done.note : \"cancelled by the caller\";\n this.emit({\n type: \"response\",\n url,\n host,\n status: 0,\n outcome: \"error\",\n elapsedMs: done.elapsedMs,\n });\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n status: 0,\n outcome: \"error\",\n note,\n elapsed_ms: done.elapsedMs,\n robots_check: check,\n subrequests,\n ...signedFacts,\n },\n url,\n marks,\n );\n return {\n url,\n status: 0,\n outcome: \"error\",\n fromCache: false,\n capture,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note,\n ...marks,\n };\n }\n\n const { rendered, classified, elapsedMs } = done;\n const header = (name: string): string | undefined => rendered.headers[name.toLowerCase()];\n // the page can move to a different URL: the same rules apply to it\n let finalTarget = target;\n let finalVerdict = verdict;\n try {\n finalTarget = new URL(rendered.finalUrl);\n finalTarget.hash = \"\";\n } catch {\n finalTarget = target;\n }\n const finalUrl = redact(finalTarget);\n const finalPath = `${finalTarget.pathname}${finalTarget.search}`;\n if (finalTarget.href !== target.href) {\n const moved = `the page moved to ${finalUrl}`;\n if (!WEB_PROTOCOLS.has(finalTarget.protocol))\n return skip(`${moved}, which is not an http(s) URL`, { final_url: finalUrl });\n const refusal = await checkEgress(this.egress, finalTarget);\n if (refusal !== undefined) return skip(`${moved}: ${refusal}`, { final_url: finalUrl });\n if (originKey(finalTarget) !== originKey(target)) {\n finalVerdict = await this.robotsFor(finalTarget, this.policyOf(finalTarget), request_);\n }\n const finalDecision = decide(finalVerdict.robots, tokens, finalPath);\n if (!finalDecision.allowed) {\n return skip(\n `${moved}: ${robotsRefusal(finalVerdict.robots, finalDecision.by ?? this.options.token)}`,\n {\n final_url: finalUrl,\n robots_check: robotsCheck(finalVerdict, tokens, finalDecision),\n },\n );\n }\n }\n let outcome: Outcome = classified.outcome;\n let note = classified.note;\n let body: Uint8Array | undefined;\n let blob: string | undefined;\n if (outcome === \"ok\") {\n body = new TextEncoder().encode(rendered.html);\n try {\n blob = (await this.options.blobs.put(body)).hash;\n } catch (error) {\n outcome = \"error\";\n note = `blob store: ${messageOf(error)}`;\n body = undefined;\n }\n }\n const signals = readSignals(\n signalsFor(finalVerdict.robots, tokens, finalPath),\n header,\n metaSignals(rendered.html),\n );\n const status = rendered.status ?? 200;\n const fetchedAt = this.clock.now();\n const freshness = freshUntil(\n header,\n status,\n fetchedAt,\n header(\"last-modified\"),\n policy.heuristicFreshnessCapMs,\n );\n this.emit({ type: \"response\", url, host, status, outcome, elapsedMs });\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n final_url: finalUrl,\n status,\n outcome,\n elapsed_ms: elapsedMs,\n content_type: RENDERED_TYPE,\n bytes: body?.length ?? 0,\n blob: blob ?? null,\n robots: \"allowed\",\n robots_check: check,\n signals,\n subrequests,\n ...(freshness !== null ? { fresh_until: freshness } : {}),\n ...(note !== undefined ? { note } : {}),\n ...signedFacts,\n },\n url,\n marks,\n );\n const fetchedIso = new Date(fetchedAt).toISOString();\n if (capture !== undefined && blob !== undefined) {\n this.store({\n url,\n variant: \"rendered\",\n capture,\n blob,\n etag: null,\n last_modified: null,\n content_type: RENDERED_TYPE,\n fetched_at: fetchedIso,\n status,\n ai_input: signals.ai_input,\n fresh_until: freshness,\n });\n }\n return {\n url,\n status,\n outcome,\n fromCache: false,\n capture,\n blob,\n body,\n contentType: RENDERED_TYPE,\n signals,\n note,\n fetchedAt: fetchedIso,\n ...marks,\n };\n }\n\n /**\n * This method removes the stops and the failure counts of one host, or of each host. It also\n * forgets the robots.txt verdicts that came from an unreachable robots.txt. The pace of each host\n * stays. A process that runs for a long time uses it, for example after the network comes back.\n */\n resetHosts(host?: string): void {\n const name = host?.toLowerCase().replace(/\\.$/, \"\");\n this.scheduler.reset(name);\n for (const key of this.robots.keys()) {\n const entry = this.robots.peek(key);\n const unreachable =\n entry?.value?.robots.unreachable !== undefined || entry?.value?.stale !== undefined;\n if (unreachable && (name === undefined || key.endsWith(`//${name}`))) this.robots.delete(key);\n }\n }\n\n // ── watch: the stop of a host ─────────────────────────────────────────────\n\n /**\n * This method gives the stop of the host of a URL, or undefined when the host is not stopped. A\n * watch uses it, so a page of a stopped host is due again at the end of the stop.\n */\n hostStop(url: string): ReturnType<HostStateStore[\"stopped\"]> {\n try {\n return this.scheduler.store.stopped(hostKey(new URL(url)), this.clock.now());\n } catch {\n return undefined;\n }\n }\n\n // ── end of watch ──────────────────────────────────────────────────────────\n\n /**\n * This method closes the crawler. Work in the queue ends as `skipped` with the note\n * `crawler closed`, and the crawler writes no capture after the close. Then it closes the\n * browser, the log and its own dispatcher. A second close waits for the first.\n */\n close(): Promise<void> {\n if (this.closingDone === undefined) {\n this.closed = true;\n this.closing.abort(new Error(\"crawler closed\"));\n this.closingDone = (async () => {\n await Promise.allSettled(this.running);\n await this.options.renderer?.close();\n await this.options.log.close?.();\n if (this.ownsDispatcher) await this.dispatcher.close();\n })();\n }\n return this.closingDone;\n }\n}\n","/**\n * This module makes two forms of a text equal when a reader thinks that they are equal. It has two\n * modes:\n *\n * - `quote`: for the verification of a quote. It applies NFKC (a ligature `fi` is `fi`, a\n * fullwidth `A` is `A`, an ellipsis `…` is `...`) to each character and its combining marks,\n * thus NFC and NFD are equal. It removes soft hyphens and zero-width characters. It makes\n * typographic quotes, dashes and the minus sign ASCII, makes each run of spaces one space, and\n * makes the letters lowercase. Accents stay, so \"résumé\" is not \"resume\".\n * - `search`: for the search index. It also removes the accents and the other nonspacing marks,\n * and it folds the case fully (`ß` is `ss`). It puts a space around each Chinese, Japanese and\n * Thai character, because these scripts write no space between words.\n *\n * Each character of the folded text knows the start and the end of its source in the original\n * text. Thus a match in the folded text gives exact positions in the original text.\n */\n\nexport type FoldMode = \"quote\" | \"search\";\n\n/** A folded text, with the position in the original text of each UTF-16 unit. */\nexport interface Folded {\n readonly text: string;\n /** This value is the start, in the original text, of the source of each unit. */\n readonly start: Int32Array;\n /** This value is the end, in the original text, of the source of each unit. */\n readonly end: Int32Array;\n}\n\nconst MARK = /\\p{M}/u;\nconst NONSPACING = /[\\p{Mn}\\p{Me}]/gu;\nconst IGNORABLE = /^\\p{Default_Ignorable_Code_Point}$/u;\nconst SPACE = /^\\s$/u;\nconst SEGMENTED =\n /[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Thai}\\p{Script=Lao}\\p{Script=Khmer}\\p{Script=Myanmar}]/u;\n/** These characters become ASCII: typographic quotes, dashes and the minus sign. */\nconst TYPOGRAPHIC: ReadonlyMap<number, number> = new Map([\n [0x2018, 0x27],\n [0x2019, 0x27],\n [0x201a, 0x27],\n [0x201b, 0x27],\n [0x2032, 0x27],\n [0x201c, 0x22],\n [0x201d, 0x22],\n [0x201e, 0x22],\n [0x201f, 0x22],\n [0x2033, 0x22],\n [0x2010, 0x2d],\n [0x2011, 0x2d],\n [0x2012, 0x2d],\n [0x2013, 0x2d],\n [0x2014, 0x2d],\n [0x2015, 0x2d],\n [0x2212, 0x2d],\n [0xfe58, 0x2d],\n [0xfe63, 0x2d],\n]);\n\nconst isAsciiSpace = (code: number): boolean => code === 0x20 || (code >= 0x09 && code <= 0x0d);\n\n/**\n * This function folds a text, and it keeps the position in the original text of each unit of the\n * result. Its time is linear. An ASCII character with no combining mark after it takes a short path.\n */\nexport const foldText = (text: string, mode: FoldMode): Folded => {\n let codes = new Uint16Array(Math.max(16, text.length + 16));\n let starts = new Int32Array(codes.length);\n let ends = new Int32Array(codes.length);\n let size = 0;\n let space = true;\n const grow = (need: number): void => {\n if (size + need <= codes.length) return;\n const length = Math.max(codes.length * 2, size + need);\n const nextCodes = new Uint16Array(length);\n nextCodes.set(codes.subarray(0, size));\n const nextStarts = new Int32Array(length);\n nextStarts.set(starts.subarray(0, size));\n const nextEnds = new Int32Array(length);\n nextEnds.set(ends.subarray(0, size));\n codes = nextCodes;\n starts = nextStarts;\n ends = nextEnds;\n };\n const push = (code: number, from: number, to: number): void => {\n grow(1);\n codes[size] = code;\n starts[size] = from;\n ends[size] = to;\n size += 1;\n };\n const pushSpace = (from: number, to: number): void => {\n if (space) return;\n push(0x20, from, to);\n space = true;\n };\n const length = text.length;\n let index = 0;\n while (index < length) {\n const code = text.charCodeAt(index);\n const next = index + 1 < length ? text.charCodeAt(index + 1) : 0;\n // the short path: an ASCII character that no combining mark follows\n if (code < 0x80 && next < 0x300) {\n if (isAsciiSpace(code)) pushSpace(index, index + 1);\n else {\n push(code >= 0x41 && code <= 0x5a ? code + 32 : code, index, index + 1);\n space = false;\n }\n index += 1;\n continue;\n }\n // a cluster: one code point and the combining marks after it\n const point = text.codePointAt(index)!;\n let end = index + (point > 0xffff ? 2 : 1);\n while (end < length) {\n const mark = text.codePointAt(end)!;\n if (!MARK.test(String.fromCodePoint(mark))) break;\n end += mark > 0xffff ? 2 : 1;\n }\n const cluster = text.slice(index, end);\n const first = String.fromCodePoint(point);\n if (IGNORABLE.test(first) && end === index + first.length) {\n index = end;\n continue;\n }\n if (SPACE.test(first)) {\n pushSpace(index, end);\n index = end;\n continue;\n }\n let folded =\n mode === \"quote\"\n ? cluster.normalize(\"NFKC\").toLowerCase()\n : cluster\n .normalize(\"NFKD\")\n .replace(NONSPACING, \"\")\n .normalize(\"NFC\")\n .toUpperCase()\n .toLowerCase();\n if (mode === \"search\" && SEGMENTED.test(first)) folded = ` ${folded} `;\n for (let at = 0; at < folded.length; at += 1) {\n let unit = folded.charCodeAt(at);\n unit = TYPOGRAPHIC.get(unit) ?? unit;\n if (unit === 0x20 || (unit < 0x80 && isAsciiSpace(unit)) || unit === 0xa0) {\n pushSpace(index, end);\n continue;\n }\n push(unit, index, end);\n space = false;\n }\n index = end;\n }\n if (size > 0 && codes[size - 1] === 0x20) size -= 1;\n let result = \"\";\n for (let at = 0; at < size; at += 8192) {\n result += String.fromCharCode(...codes.subarray(at, Math.min(size, at + 8192)));\n }\n return { text: result, start: starts.subarray(0, size), end: ends.subarray(0, size) };\n};\n\n/** This function gives the folded text alone, for a text with no need of positions. */\nexport const fold = (text: string, mode: FoldMode): string => foldText(text, mode).text;\n\n/** @beta */\nexport interface Verified {\n readonly found: boolean;\n /** These values are the character positions in the text of the document, when the quote is there. */\n readonly start: number | null;\n readonly end: number | null;\n /** This text is the exact quote as stored, with some text before and after it. */\n readonly excerpt: string | null;\n /** This value is the number of places where the quote is, with no overlap. */\n readonly count: number;\n}\n\nconst NOT_FOUND: Verified = { found: false, start: null, end: null, excerpt: null, count: 0 };\n\n/** This function moves a position off the second half of a surrogate pair, so a slice does not cut a character. */\nconst atBoundary = (text: string, position: number, forward: boolean): number => {\n const code = text.charCodeAt(position);\n if (position > 0 && position < text.length && code >= 0xdc00 && code <= 0xdfff) {\n return forward ? position + 1 : position - 1;\n }\n return position;\n};\n\n/**\n * This function finds a quote in a text, in the `quote` mode of `foldText`. It ignores case, runs\n * of spaces, typographic punctuation, soft hyphens, zero-width characters and the differences of\n * NFKC, NFC and NFD. The positions are in the original text.\n *\n * @beta\n */\nexport const locate = (text: string, quote: string): Verified => {\n const needle = fold(quote, \"quote\");\n if (needle.length === 0) return NOT_FOUND;\n const haystack = foldText(text, \"quote\");\n const hit = haystack.text.indexOf(needle);\n if (hit === -1) return NOT_FOUND;\n let count = 0;\n for (let at = hit; at !== -1; at = haystack.text.indexOf(needle, at + needle.length)) count += 1;\n const start = haystack.start[hit]!;\n const end = haystack.end[hit + needle.length - 1]!;\n const from = atBoundary(text, Math.max(0, start - 80), false);\n const to = atBoundary(text, Math.min(text.length, end + 80), true);\n return {\n found: true,\n start,\n end,\n excerpt: `${from > 0 ? \"…\" : \"\"}${text.slice(from, to)}${to < text.length ? \"…\" : \"\"}`,\n count,\n };\n};\n\n/** A token character of the search index: a letter, a digit, a private-use character or a spacing mark. */\nconst TOKEN_SPLIT = /[^\\p{L}\\p{N}\\p{Co}\\p{Mc}]+/u;\n\n/** This function gives the tokens of a text, as the search index reads them. */\nexport const tokensOf = (text: string): string[] =>\n fold(text, \"search\")\n .split(TOKEN_SPLIT)\n .filter((token) => token.length > 0);\n\n/** A query for the search index, in the two forms that `DocIndex.find` uses. */\nexport interface FtsQuery {\n /** Each part must match: the parts joined with AND. */\n readonly all: string;\n /** One part is enough: the parts joined with OR. */\n readonly any: string;\n /** These are the parts of `any`, each as FTS5 syntax. */\n readonly anyParts: readonly string[];\n /** These are the first tokens of the parts. A snippet marks them. */\n readonly terms: readonly string[];\n /** This value is the number of parts. */\n readonly parts: number;\n}\n\nconst MAX_PARTS = 16;\n\n/** This function removes the stars at the end of a word. It uses no regular expression, so its time is linear. */\nconst withoutStars = (word: string): string => {\n let end = word.length;\n while (end > 0 && word.charCodeAt(end - 1) === 0x2a) end -= 1;\n return word.slice(0, end);\n};\n\n/**\n * These English words are in almost every document. In the form that needs one part, they match\n * each document and cost the most time. Thus that form leaves them out when other words remain.\n */\nconst STOP_WORDS: ReadonlySet<string> = new Set([\n \"a\",\n \"an\",\n \"and\",\n \"are\",\n \"as\",\n \"at\",\n \"be\",\n \"by\",\n \"for\",\n \"from\",\n \"how\",\n \"in\",\n \"is\",\n \"it\",\n \"of\",\n \"on\",\n \"or\",\n \"that\",\n \"the\",\n \"this\",\n \"to\",\n \"was\",\n \"what\",\n \"when\",\n \"where\",\n \"which\",\n \"who\",\n \"with\",\n]);\n\n/**\n * This function reads a search query. The syntax:\n *\n * - A word is a term. The index folds case, accents and Unicode forms, so `cafe` finds `Café`. It\n * reduces English words to their stems, so `baking` finds `bake`.\n * - Text in double quotes is a phrase: its words must be next to each other.\n * - A `*` at the end of a word asks for a prefix: `bake*` finds `bakery`.\n * - A word of Chinese, Japanese or Thai is a phrase of its characters, so it matches in the text.\n * - Other punctuation is not syntax. The words `AND`, `OR`, `NOT` and `NEAR` are words.\n *\n * The function gives no query for a text with no word. It reads 16 parts or fewer. Each part is\n * quoted, so the text of a user cannot become FTS5 syntax. The form that needs one part (`any`)\n * leaves out the common English words (`STOP_WORDS`) when other words remain.\n */\nexport const parseQuery = (q: string): FtsQuery | undefined => {\n const parts: Array<{ readonly text: string; readonly term: string; readonly stop: boolean }> = [];\n for (const match of q.matchAll(/\"([^\"]*)\"(\\*?)|([^\\s\"]+)/g)) {\n if (parts.length >= MAX_PARTS) break;\n const phrase = match[1] !== undefined;\n const raw = phrase ? match[1]! : match[3]!;\n const prefix = phrase ? match[2] === \"*\" : raw.endsWith(\"*\");\n const tokens = tokensOf(phrase ? raw : withoutStars(raw));\n if (tokens.length === 0) continue;\n parts.push({\n text: `\"${tokens.join(\" \")}\"${prefix ? \" *\" : \"\"}`,\n term: tokens[0]!,\n stop: !phrase && !prefix && tokens.length === 1 && STOP_WORDS.has(tokens[0]!),\n });\n }\n if (parts.length === 0) return undefined;\n const words = parts.filter((part) => !part.stop);\n const any = words.length > 0 ? words : parts;\n return {\n all: parts.map((part) => part.text).join(\" AND \"),\n any: any.map((part) => part.text).join(\" OR \"),\n anyParts: any.map((part) => part.text),\n terms: any.map((part) => part.term),\n parts: parts.length,\n };\n};\n\n/** This function gives the FTS5 query that matches one part or more of a query. It is the form of version 0.1. */\nexport const ftsQuery = (q: string): string | undefined => parseQuery(q)?.any;\n\n/**\n * This function gives a short part of a text around the first match of a term, with the match in\n * square brackets. With no match, it gives the start of the text.\n */\nexport const snippetOf = (text: string, terms: readonly string[], width = 160): string => {\n if (text.length === 0) return \"\";\n let start = -1;\n let end = -1;\n const lower = text.toLowerCase();\n if (lower.length === text.length) {\n for (const term of terms) {\n const at = lower.indexOf(term);\n if (at !== -1 && (start === -1 || at < start)) {\n start = at;\n end = at + term.length;\n }\n }\n }\n if (start === -1) {\n const folded = foldText(text, \"search\");\n for (const term of terms) {\n const at = folded.text.indexOf(term);\n if (at !== -1) {\n const from = folded.start[at]!;\n if (start === -1 || from < start) {\n start = from;\n end = folded.end[at + term.length - 1]!;\n }\n }\n }\n }\n if (start === -1) {\n const cut = atBoundary(text, Math.min(text.length, width), false);\n return `${text.slice(0, cut)}${cut < text.length ? \"…\" : \"\"}`;\n }\n const from = atBoundary(text, Math.max(0, start - Math.floor(width / 3)), false);\n const to = atBoundary(text, Math.min(text.length, end + Math.ceil((width * 2) / 3)), true);\n const clean = (part: string): string => part.replace(/\\s+/g, \" \");\n return `${from > 0 ? \"…\" : \"\"}${clean(text.slice(from, start))}[${clean(text.slice(start, end))}]${clean(text.slice(end, to))}${to < text.length ? \"…\" : \"\"}`;\n};\n","import { constants, inflateSync } from \"node:zlib\";\nimport { getDocumentProxy, getMeta } from \"unpdf\";\n\n/**\n * The crawler reads the text of a PDF. Restaurant menus are often PDFs: 29 of 231 surveyed\n * restaurant sites link one. A menu that the crawler reads one time stays searchable. A paper in a\n * PDF reads the same way. A scanned PDF has no text, so the result is empty, not a guess.\n *\n * A small file can cost much work: a stream of 150 KB can inflate to 100 MB. Thus the reader:\n *\n * - Inflates the compressed streams first, with a limit, and refuses a file that inflates past it.\n * - Reads one page at a time, and lets other work run between the pages.\n * - Stops at a page limit, a text limit and a time limit, and says so (`truncated`). The time limit\n * stops a page in the middle too, and the text limit cuts the text of a page.\n *\n * @beta\n */\nexport interface PdfText {\n /** This is the title in the file, or null when the file names no real title (\"Microsoft Word - menu.docx\"). */\n readonly title: string | null;\n readonly text: string;\n readonly pages: number;\n /** This value is true when a limit stopped the reader before the last page. */\n readonly truncated: boolean;\n /** This note tells why the reader stopped or refused the file, or it is null. */\n readonly note: string | null;\n}\n\n/**\n * These are the limits of one read.\n *\n * @beta\n */\nexport interface PdfOptions {\n /** The reader reads no more pages than this. The default is 500. */\n readonly maxPages?: number | undefined;\n /** The reader stops when the text has this many characters, in the middle of a page too. The default is 200,000. */\n readonly maxChars?: number | undefined;\n /** The reader stops after this time in milliseconds, in the middle of a page too. The default is 20,000. */\n readonly timeBudgetMs?: number | undefined;\n /** The reader refuses a file whose streams inflate to more bytes than this. The default is 8 MB. */\n readonly maxInflatedBytes?: number | undefined;\n}\n\nconst MAX_PAGES = 500;\nconst MAX_CHARS = 200_000;\nconst TIME_BUDGET_MS = 20_000;\n/**\n * The text streams of 35 stored menus and papers inflate to 0.6 MB or less. The parser of pdf.js\n * reads a page with no pause. Thus a page of 8 MB of text operators stops the process for\n * approximately one second.\n */\nconst MAX_INFLATED_BYTES = 8 * 1024 * 1024;\n\n/**\n * This function finds a PDF from its first bytes or its content type. Some servers put bytes before `%PDF-`.\n *\n * @beta\n */\nexport const isPdf = (body: Uint8Array, contentType: string | null): boolean => {\n if (/\\bpdf\\b/i.test(contentType ?? \"\")) return true;\n const end = Math.min(body.length - 4, 1024);\n for (let index = 0; index < end; index += 1) {\n if (\n body[index] === 0x25 &&\n body[index + 1] === 0x50 &&\n body[index + 2] === 0x44 &&\n body[index + 3] === 0x46 &&\n body[index + 4] === 0x2d\n ) {\n return true;\n }\n }\n return false;\n};\n\n/** This function gives one line of text for each line of the PDF. It removes the empty lines. */\nconst tidy = (page: string): string =>\n page\n .split(/\\r?\\n/)\n .map((line) => line.replace(/\\s+/g, \" \").trim())\n .filter((line) => line.length > 0)\n .join(\"\\n\");\n\n/** A word processor writes the name of its file as the title (\"Microsoft Word - menu-final-v3.docx\"). */\nconst JUNK_TITLE =\n /^(?:microsoft (?:word|powerpoint|excel) - |untitled\\b|document\\d*$|presentation\\d*$)|\\.(?:docx?|pdf|pptx?|xlsx?|indd|ai|psd|rtf|odt|pages|txt)$/i;\n\nconst latin1 = (bytes: Uint8Array, start: number, end: number): string => {\n let out = \"\";\n for (let index = Math.max(0, start); index < end && index < bytes.length; index += 1) {\n out += String.fromCharCode(bytes[index]!);\n }\n return out;\n};\n\nconst indexOf = (bytes: Uint8Array, needle: string, from: number): number => {\n const first = needle.charCodeAt(0);\n for (\n let index = bytes.indexOf(first, from);\n index !== -1;\n index = bytes.indexOf(first, index + 1)\n ) {\n let match = true;\n for (let at = 1; at < needle.length; at += 1) {\n if (bytes[index + at] !== needle.charCodeAt(at)) {\n match = false;\n break;\n }\n }\n if (match) return index;\n }\n return -1;\n};\n\n/**\n * This function adds up the bytes that the compressed streams of a file inflate to, and stops at\n * the limit. It tries each stream: a stream in another format fails at its first bytes. An image\n * stream does not count, because the text reader does not decode it. The function gives true when\n * the file stays inside the limit.\n */\nconst inflatesWithin = (bytes: Uint8Array, limit: number): boolean => {\n let total = 0;\n let from = 0;\n for (;;) {\n const keyword = indexOf(bytes, \"stream\", from);\n if (keyword === -1) return true;\n from = keyword + 6;\n // \"endstream\" holds \"stream\" too\n if (latin1(bytes, keyword - 3, keyword) === \"end\") continue;\n const next = bytes[keyword + 6];\n if (next !== 0x0a && next !== 0x0d) continue;\n const dictionary = latin1(bytes, keyword - 1024, keyword);\n const head = dictionary.slice(Math.max(0, dictionary.lastIndexOf(\"<<\")));\n if (/\\/Subtype\\s*\\/Image\\b/.test(head)) continue;\n let start = keyword + 7;\n if (next === 0x0d && bytes[start] === 0x0a) start += 1;\n const declared = /\\/Length\\s+(\\d+)(?!\\s+\\d+\\s+R)/.exec(head);\n const stop = indexOf(bytes, \"endstream\", start);\n const end =\n declared !== null && start + Number(declared[1]) <= bytes.length\n ? start + Number(declared[1])\n : stop === -1\n ? bytes.length\n : stop;\n try {\n total += inflateSync(bytes.subarray(start, end), {\n maxOutputLength: Math.max(1, limit - total + 1),\n finishFlush: constants.Z_SYNC_FLUSH,\n }).length;\n } catch (error) {\n // a stream past the limit, or damaged data that pdf.js reads as far as it can\n if (error instanceof RangeError) return false;\n }\n if (total > limit) return false;\n from = Math.max(from, end);\n }\n};\n\n/** A page of pdf.js gives its text as items. A line ends where an item says so. */\ninterface TextItem {\n readonly str?: string;\n readonly hasEOL?: boolean;\n}\n\n/** This function lets the other work of the process run, for example another request. */\nconst pause = (): Promise<void> => new Promise((resolve) => setImmediate(resolve));\n\n/** This function gives the value of a promise, or undefined when the time ends first. */\nconst within = async <T>(promise: Promise<T>, ms: number): Promise<T | undefined> => {\n let timer: ReturnType<typeof setTimeout> | undefined;\n const late = new Promise<undefined>((resolve) => {\n timer = setTimeout(() => resolve(undefined), Math.max(0, ms));\n });\n try {\n return await Promise.race([promise, late]);\n } finally {\n clearTimeout(timer);\n }\n};\n\n/** This function keeps the first characters of a text, and does not cut a surrogate pair. */\nconst cutText = (text: string, length: number): string => {\n let end = Math.max(0, length);\n const code = text.charCodeAt(end);\n if (end > 0 && end < text.length && code >= 0xdc00 && code <= 0xdfff) end -= 1;\n return text.slice(0, end).trimEnd();\n};\n\n/** @beta */\nexport const readPdf = async (bytes: Uint8Array, options: PdfOptions = {}): Promise<PdfText> => {\n const maxPages = options.maxPages ?? MAX_PAGES;\n const maxChars = options.maxChars ?? MAX_CHARS;\n const budgetMs = options.timeBudgetMs ?? TIME_BUDGET_MS;\n const maxInflated = options.maxInflatedBytes ?? MAX_INFLATED_BYTES;\n if (!inflatesWithin(bytes, maxInflated)) {\n return {\n title: null,\n text: \"\",\n pages: 0,\n truncated: true,\n note: `the streams of the file inflate past ${String(Math.round(maxInflated / 1024 / 1024))} MB`,\n };\n }\n // pdf.js may take over the buffer it is handed, and this one is the caller's\n const pdf = await getDocumentProxy(new Uint8Array(bytes));\n try {\n const started = Date.now();\n const texts: string[] = [];\n let chars = 0;\n let note: string | null = null;\n const last = Math.min(pdf.numPages, maxPages);\n if (pdf.numPages > maxPages) note = `the reader stops after ${String(maxPages)} pages`;\n const late = (number: number): string =>\n `the reader stops after ${String(Math.round(budgetMs / 1000))} seconds, at page ${String(number - 1)}`;\n for (let number = 1; number <= last; number += 1) {\n if (number > 1) {\n await pause();\n if (Date.now() - started > budgetMs) {\n note = late(number);\n break;\n }\n }\n const page = await pdf.getPage(number);\n // pdf.js reads a content stream with no pause, so this limit stops only a page that waits (for\n // a font, for example). The limit of the inflated bytes keeps one page near one second\n const content = await within(page.getTextContent(), budgetMs - (Date.now() - started));\n if (content === undefined) {\n note = `the reader stops after ${String(Math.round(budgetMs / 1000))} seconds, in page ${String(number)}`;\n break;\n }\n const text = tidy(\n (content.items as readonly TextItem[])\n .filter((item) => item.str !== undefined)\n .map((item) => `${item.str ?? \"\"}${item.hasEOL === true ? \"\\n\" : \"\"}`)\n .join(\"\"),\n );\n page.cleanup();\n if (text.length > 0) {\n // the text and the blank lines between the pages so far, and the room that is left\n if (text.length > maxChars - chars) {\n texts.push(cutText(text, maxChars - chars));\n note = `the reader stops at ${String(maxChars)} characters, at page ${String(number)}`;\n break;\n }\n texts.push(text);\n chars += text.length + 2;\n }\n if (chars >= maxChars && number < pdf.numPages) {\n note = `the reader stops at ${String(maxChars)} characters, at page ${String(number)}`;\n break;\n }\n }\n const info = (await getMeta(pdf).catch(() => undefined))?.info;\n const named: unknown = info?.[\"Title\"];\n const title = typeof named === \"string\" ? named.replace(/\\s+/g, \" \").trim() : \"\";\n return {\n title: title.length > 0 && !JUNK_TITLE.test(title) ? title : null,\n text: texts.join(\"\\n\\n\"),\n pages: pdf.numPages,\n truncated: note !== null,\n note,\n };\n } finally {\n // the loading task owns the document and its worker\n await pdf.loadingTask.destroy();\n }\n};\n","import { brotliDecompressSync, gunzipSync, inflateRawSync, inflateSync } from \"node:zlib\";\nimport type { Fetched, FetchRequest } from \"./crawler.ts\";\nimport { redact } from \"./crawler.ts\";\nimport type { Doc, ExtractedDoc } from \"./docs.ts\";\nimport type { CrawlRuntime, Extractor, Indexed, PageOptions, PageResult } from \"./runtime.ts\";\nimport { HELD_BACK, fetchAndIndex, isHeldBack, pageExtractor } from \"./runtime.ts\";\nimport { readSignals } from \"./signals.ts\";\nimport { canonicalUrl } from \"./urls.ts\";\n\n/**\n * The archived fallback. When a site refuses a page, or is down, the crawler can give a copy from a\n * public web archive: the Wayback Machine, then Common Crawl. The crawler does not show the copy as a fresh fetch:\n *\n * - It is a document of its own (`archive:<url>`), of the kind `archived`. Its metadata and each\n * result that shows it have `archived`: the archive, the date of the copy and its URL there.\n * - The result keeps the outcome of the live fetch (`blocked`, for example), and its note names the\n * archive and the date.\n *\n * The fallback is opt-in (`fallback: \"archive\"`). The archives are third-party APIs with their own\n * politeness, so each request to them is a fetch of the same crawler. Their robots.txt, the pace of\n * each host, the egress policy and the identity of the crawler apply, and the capture log records\n * each request.\n *\n * By default, the crawler gives no archived copy of a page that the current robots.txt of its site\n * disallows for this agent. Only a stop, an error or a dead host gets the fallback.\n * `archiveDespiteRobots` widens it. The usage signals of the copy apply as for a live page.\n */\n\n/** The public web archives of the fallback. */\nexport type WebArchive = \"wayback\" | \"commoncrawl\";\n\n/** An archived copy: the archive, the time when the archive captured the page, and where the copy is. */\nexport interface Archived {\n readonly source: WebArchive;\n /** This time (ISO 8601) is when the archive captured the page. It is not the time of the fetch from the archive. */\n readonly timestamp: string;\n /** This URL is the copy in the archive: a Wayback Machine page, or the Common Crawl WARC file. */\n readonly url: string;\n /** This URL is the page that the archive captured. */\n readonly original: string;\n /** This id is the `WARC-Record-ID` of a Common Crawl copy in its WARC file. */\n readonly record?: string | undefined;\n}\n\n/** The recipe of the request for an archived copy. Its extractor reads a Wayback Machine copy or a Common Crawl WARC record. */\nexport const ARCHIVE_COPY_RECIPE = \"archive-copy\";\n\n/** The recipe of the requests to the indexes of the archives. They give no document. */\nexport const ARCHIVE_INDEX_RECIPE = \"archive-index\";\n\n/** The kind of the document of an archived copy. */\nexport const ARCHIVED_KIND = \"archived\";\n\n/**\n * This function gives the document id of the archived copy of a page.\n *\n * @beta\n */\nexport const archivedDocId = (url: string): string => `archive:${canonicalUrl(url)}`;\n\n/** The addresses of the archives. */\nconst WAYBACK_AVAILABLE = \"https://archive.org/wayback/available\";\nconst WAYBACK = \"https://web.archive.org/web/\";\nconst CC_INDEX = \"https://index.commoncrawl.org/\";\nconst CC_DATA = \"https://data.commoncrawl.org/\";\n\n/** The Common Crawl indexes that the fallback asks, the newest first. */\nconst CC_INDEXES = 3;\n/** The largest compressed WARC record that the fallback asks for. */\nconst MAX_RECORD_BYTES = 5 * 1024 * 1024;\n/** The largest decompressed WARC record or body that the fallback reads. */\nconst MAX_INFLATED_BYTES = 20 * 1024 * 1024;\n\n/** The Wayback Machine path of a raw copy: the 14 digits of the time, `id_`, and the URL of the page. */\nconst WAYBACK_RAW = /^\\/web\\/(\\d{14})id_\\/(https?:\\/\\/.+)$/s;\nconst HTML_ACCEPT = \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.5\";\nconst JSON_ACCEPT = \"application/json\";\n\n/** These are the reasons of a refusal that can get an archived copy. */\nexport type FallbackReason =\n | \"robots\"\n | \"unreachable\"\n | \"stopped\"\n | \"blocked\"\n | \"challenge\"\n | \"payment-required\"\n | \"unavailable\"\n | \"error\";\n\n// The notes of the crawler that tell why it did not send a request. The tests make sure that they agree.\nconst ROBOTS_DISALLOW = \"robots.txt disallows it\";\nconst ROBOTS_UNREACHABLE = \"robots.txt could not be read\";\nconst HOST_STOPPED = \"host stopped\";\n\n/**\n * This function tells if the result of a live fetch can get an archived copy, and why. These\n * refusals can: robots.txt, a stop of the host, a 401, a 402, a 403 and a challenge. A 502, 503 or\n * 504, a 5xx error and a dead host can too. A 404, a 429, a cancel and a refusal of the egress\n * policy cannot.\n */\nexport const fallbackReason = (indexed: Indexed): FallbackReason | undefined => {\n const { fetched } = indexed;\n if (indexed.blocked !== undefined) return \"challenge\";\n const note = fetched.note ?? \"\";\n switch (fetched.outcome) {\n case \"skipped\":\n if (note.startsWith(ROBOTS_DISALLOW)) return \"robots\";\n if (note.startsWith(ROBOTS_UNREACHABLE)) return \"unreachable\";\n if (note.startsWith(HOST_STOPPED)) return \"stopped\";\n return undefined;\n case \"blocked\":\n case \"challenge\":\n case \"payment-required\":\n case \"unavailable\":\n return fetched.outcome;\n case \"error\":\n if (fetched.status === 0 && note.startsWith(\"no response\")) return \"unreachable\";\n return fetched.status >= 500 ? \"error\" : undefined;\n default:\n return undefined;\n }\n};\n\n/** This function gives the `archived` of the metadata of a document, when the document is an archived copy. */\nexport const archivedOf = (meta: Readonly<Record<string, unknown>>): Archived | undefined => {\n const value = meta[\"archived\"];\n if (typeof value !== \"object\" || value === null) return undefined;\n const { source, timestamp, url, original, record } = value as Record<string, unknown>;\n if (\n (source !== \"wayback\" && source !== \"commoncrawl\") ||\n typeof timestamp !== \"string\" ||\n typeof url !== \"string\" ||\n typeof original !== \"string\"\n )\n return undefined;\n return { source, timestamp, url, original, ...(typeof record === \"string\" ? { record } : {}) };\n};\n\nconst ARCHIVE_NAMES: Readonly<Record<WebArchive, string>> = {\n wayback: \"the Wayback Machine\",\n commoncrawl: \"Common Crawl\",\n};\n\n/** This function gives the note of an archived copy: the archive and the date, and that it is not a fresh fetch. */\nexport const archivedNote = (archived: Archived): string =>\n `an archived copy from ${ARCHIVE_NAMES[archived.source]}, captured ${archived.timestamp}, not a fresh fetch of the page`;\n\n/** This function gives an ISO 8601 time with no milliseconds, or undefined for a time that is not valid. */\nconst isoSeconds = (ms: number): string | undefined =>\n Number.isFinite(ms) ? new Date(ms).toISOString().replace(/\\.\\d{3}Z$/, \"Z\") : undefined;\n\n/** This function reads the 14 digits of a Wayback Machine time (`20240131235959`) as an ISO 8601 time. */\nconst waybackTime = (digits: string): string | undefined => {\n const match = /^(\\d{4})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})$/.exec(digits);\n if (match === null) return undefined;\n const [, year, month, day, hour, minute, second] = match;\n const ms = Date.UTC(\n Number(year),\n Number(month) - 1,\n Number(day),\n Number(hour),\n Number(minute),\n Number(second),\n );\n const iso = isoSeconds(ms);\n // a date such as 20240231 rolls over; it is not the date that the archive wrote\n return iso !== undefined && iso.replace(/\\D/g, \"\").startsWith(digits) ? iso : undefined;\n};\n\nconst httpUrlOf = (text: string): string | undefined => {\n try {\n const url = new URL(text);\n return url.protocol === \"http:\" || url.protocol === \"https:\" ? url.href : undefined;\n } catch {\n return undefined;\n }\n};\n\n// ── the reader of a WARC record (Common Crawl) ──\n\ninterface WarcResponse {\n readonly target: string;\n readonly date: string;\n readonly record: string | undefined;\n readonly status: number;\n readonly header: (name: string) => string | undefined;\n readonly body: Uint8Array;\n}\n\nconst CRLF2 = Buffer.from(\"\\r\\n\\r\\n\");\n\n/** This function reads the header lines of a block as a map. A name can be in any case, and two lines of one name join with \", \". */\nconst headerMap = (text: string): Map<string, string> => {\n const out = new Map<string, string>();\n for (const line of text.split(/\\r?\\n/)) {\n const colon = line.indexOf(\":\");\n if (colon <= 0) continue;\n const name = line.slice(0, colon).trim().toLowerCase();\n const value = line.slice(colon + 1).trim();\n const known = out.get(name);\n out.set(name, known === undefined ? value : `${known}, ${value}`);\n }\n return out;\n};\n\n/** This function joins the chunks of a body with `Transfer-Encoding: chunked`. A body that is not in that form gives undefined. */\nconst dechunk = (body: Buffer): Buffer | undefined => {\n const parts: Buffer[] = [];\n let at = 0;\n let total = 0;\n for (;;) {\n const end = body.indexOf(\"\\r\\n\", at);\n if (end === -1) return undefined;\n const size = Number.parseInt(\n body.subarray(at, end).toString(\"latin1\").split(\";\")[0]!.trim(),\n 16,\n );\n if (!Number.isFinite(size) || size < 0) return undefined;\n if (size === 0) return Buffer.concat(parts);\n const start = end + 2;\n if (start + size > body.length) return undefined;\n total += size;\n if (total > MAX_INFLATED_BYTES) return undefined;\n parts.push(body.subarray(start, start + size));\n at = start + size + 2;\n }\n};\n\n/** This function decodes the content coding of an archived body, with a limit on the output. */\nconst decodeCoding = (body: Buffer, coding: string | undefined): Buffer | undefined => {\n const options = { maxOutputLength: MAX_INFLATED_BYTES };\n let out = body;\n for (const name of (coding ?? \"\")\n .split(\",\")\n .map((part) => part.trim().toLowerCase())\n .filter((part) => part.length > 0 && part !== \"identity\")\n .reverse()) {\n if (name === \"gzip\" || name === \"x-gzip\") out = gunzipSync(out, options);\n else if (name === \"br\") out = brotliDecompressSync(out, options);\n else if (name === \"deflate\") {\n try {\n out = inflateSync(out, options);\n } catch {\n out = inflateRawSync(out, options);\n }\n } else return undefined;\n }\n return out;\n};\n\n/**\n * This function reads one WARC record of the type `response`, as Common Crawl gives it. The\n * record is a gzip member with the WARC headers, the HTTP headers and the body. It gives undefined for a different\n * record, or for bytes that do not decode. Common Crawl keeps the decoded body and renames\n * `Content-Encoding` to `X-Crawler-Content-Encoding`. A record with the original codings is decoded too.\n */\nexport const readWarcResponse = (bytes: Uint8Array): WarcResponse | undefined => {\n if (bytes.length < 2 || bytes[0] !== 0x1f || bytes[1] !== 0x8b) return undefined;\n let record: Buffer;\n try {\n record = gunzipSync(bytes, { maxOutputLength: MAX_INFLATED_BYTES });\n } catch {\n return undefined;\n }\n if (!record.subarray(0, 8).toString(\"latin1\").startsWith(\"WARC/1.\")) return undefined;\n const warcEnd = record.indexOf(CRLF2);\n if (warcEnd === -1) return undefined;\n const warc = headerMap(record.subarray(0, warcEnd).toString(\"utf8\"));\n if (warc.get(\"warc-type\")?.toLowerCase() !== \"response\") return undefined;\n const target = httpUrlOf(warc.get(\"warc-target-uri\")?.replace(/^<|>$/g, \"\") ?? \"\");\n const date = isoSeconds(Date.parse(warc.get(\"warc-date\") ?? \"\"));\n if (target === undefined || date === undefined) return undefined;\n const length = Number(warc.get(\"content-length\") ?? Number.NaN);\n const start = warcEnd + 4;\n const block = record.subarray(\n start,\n Number.isInteger(length) && length >= 0\n ? Math.min(record.length, start + length)\n : record.length,\n );\n const httpEnd = block.indexOf(CRLF2);\n if (httpEnd === -1) return undefined;\n const head = block.subarray(0, httpEnd).toString(\"latin1\");\n const statusLine = /^HTTP\\/\\d(?:\\.\\d)? (\\d{3})/.exec(head);\n if (statusLine === null) return undefined;\n const headers = headerMap(head.slice(head.indexOf(\"\\n\") + 1));\n let body: Buffer | undefined = block.subarray(httpEnd + 4);\n try {\n if (/\\bchunked\\b/i.test(headers.get(\"transfer-encoding\") ?? \"\")) body = dechunk(body);\n if (body !== undefined) body = decodeCoding(body, headers.get(\"content-encoding\"));\n } catch {\n body = undefined;\n }\n if (body === undefined) return undefined;\n return {\n target,\n date,\n record: warc.get(\"warc-record-id\")?.replace(/^<|>$/g, \"\"),\n status: Number(statusLine[1]),\n header: (name) => headers.get(name.toLowerCase()),\n body: new Uint8Array(body),\n };\n};\n\n// ── the extractor of the recipe `archive-copy` ──\n\n/** This function changes the documents of an archived body into the documents of the copy. A bot check is not the content, so it gives no document. */\nconst asArchived = (\n docs: ReadonlyArray<ExtractedDoc>,\n archived: Archived,\n aiInput: \"no\" | undefined,\n): ExtractedDoc[] =>\n docs\n .filter((doc) => typeof doc.meta[\"blocked\"] !== \"string\" || doc.meta[\"blocked\"] === \"\")\n .map((doc) => ({\n ...doc,\n doc: archivedDocId(archived.original),\n kind: ARCHIVED_KIND,\n url: archived.url,\n meta: { ...doc.meta, archived_kind: doc.kind, archived },\n ...(aiInput === \"no\" ? { ai_input: \"no\" as const } : {}),\n }));\n\n/**\n * This extractor reads the captures of the recipe `archive-copy`. It has no side effects, so\n * `reindex` replays them. It reads two forms:\n *\n * - A Wayback Machine copy in the raw form (`/web/<time>id_/<url>`). The body is the body of the page.\n * - A Common Crawl WARC record (a gzip member). It reads the HTTP headers and the body from the record.\n *\n * It reads the body with the page reader, and gives a document of the kind `archived` with\n * `archived` in its metadata. The usage signals of the HTTP headers in a WARC record apply too.\n *\n * @beta\n */\nexport const archiveExtractor: Extractor = async (body, capture) => {\n let url: URL;\n try {\n url = new URL(capture.url);\n } catch {\n return [];\n }\n const raw = WAYBACK_RAW.exec(`${url.pathname}${url.search}`);\n if (raw !== null) {\n const timestamp = waybackTime(raw[1]!);\n const original = httpUrlOf(raw[2]!);\n if (timestamp === undefined || original === undefined) return [];\n const archived: Archived = {\n source: \"wayback\",\n timestamp,\n url: `${url.origin}/web/${raw[1]!}/${original}`,\n original,\n };\n return asArchived(\n await pageExtractor(body, { url: original, contentType: capture.contentType }),\n archived,\n undefined,\n );\n }\n const response = readWarcResponse(body);\n if (response === undefined || response.status < 200 || response.status >= 300) return [];\n const archived: Archived = {\n source: \"commoncrawl\",\n timestamp: response.date,\n url: capture.url,\n original: response.target,\n ...(response.record === undefined ? {} : { record: response.record }),\n };\n const signals = readSignals([], response.header);\n return asArchived(\n await pageExtractor(response.body, {\n url: response.target,\n contentType: response.header(\"content-type\") ?? null,\n }),\n archived,\n signals.ai_input === \"no\" ? \"no\" : undefined,\n );\n};\n\n// ── the fallback ──\n\n/** The result of `archivedCopy`. */\nexport interface ArchiveAnswer {\n /** This document is the archived copy, when an archive had one. */\n readonly doc?: Doc | undefined;\n readonly archived?: Archived | undefined;\n /** This value is true when the store answered, with no request. */\n readonly fromCache: boolean;\n /** These notes tell what each archive answered, and why no copy came. */\n readonly notes: readonly string[];\n}\n\n/** The parts of each request to an archive: the identity of the request, its budget and its labels. */\nconst common = (\n options: PageOptions,\n): Pick<\n FetchRequest,\n \"maxAgeMs\" | \"onBehalfOf\" | \"job\" | \"signal\" | \"scope\" | \"labels\" | \"deadline\"\n> => ({\n maxAgeMs: options.maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n job: options.job,\n signal: options.signal,\n scope: options.scope,\n labels: options.labels,\n deadline: options.deadline,\n});\n\nconst got = (fetched: Fetched): boolean =>\n (fetched.outcome === \"ok\" || fetched.outcome === \"not-modified\") && fetched.body !== undefined;\n\nconst said = (fetched: Fetched): string =>\n `${fetched.outcome}${fetched.status > 0 ? ` ${String(fetched.status)}` : \"\"}${fetched.note === undefined ? \"\" : `: ${fetched.note}`}`;\n\nconst jsonOf = (fetched: Fetched): unknown => {\n try {\n return JSON.parse(new TextDecoder().decode(fetched.body)) as unknown;\n } catch {\n return undefined;\n }\n};\n\n/** This function gives the archived document of a fetch of a copy, or a note that tells why there is none. */\nconst copyOf = (\n indexed: Indexed,\n name: string,\n): { readonly doc: Doc; readonly archived: Archived } | { readonly note: string } => {\n const doc = indexed.docs.find((one) => one.kind === ARCHIVED_KIND);\n const archived = doc === undefined ? undefined : archivedOf(doc.meta);\n if (doc !== undefined && archived !== undefined) return { doc, archived };\n if (!got(indexed.fetched))\n return { note: `${name}: the copy did not come (${said(indexed.fetched)})` };\n return {\n note: `${name}: the copy gave no text${indexed.error === undefined ? \" (a bot check or an empty page)\" : ` (${indexed.error})`}`,\n };\n};\n\n/** This function asks the Wayback Machine for its newest copy of a page, and fetches the raw copy. */\nconst fromWayback = async (\n runtime: CrawlRuntime,\n url: string,\n signalLines: readonly string[],\n options: PageOptions,\n): Promise<\n | { readonly doc: Doc; readonly archived: Archived; readonly fromCache: boolean }\n | { readonly note: string }\n> => {\n const name = ARCHIVE_NAMES.wayback;\n const lookup = await runtime.crawler.fetch({\n url: `${WAYBACK_AVAILABLE}?url=${encodeURIComponent(url)}`,\n mode: \"page\",\n recipe: ARCHIVE_INDEX_RECIPE,\n accept: JSON_ACCEPT,\n ...common(options),\n });\n if (!got(lookup)) return { note: `${name}: the index did not answer (${said(lookup)})` };\n const closest = (\n jsonOf(lookup) as { archived_snapshots?: { closest?: Record<string, unknown> } } | undefined\n )?.archived_snapshots?.closest;\n if (closest === undefined || closest[\"available\"] === false) return { note: `${name}: no copy` };\n const status = closest[\"status\"];\n if (status !== undefined && status !== \"200\")\n return {\n note: `${name}: the newest copy has the status ${JSON.stringify(status) ?? \"unknown\"}, not 200`,\n };\n const digits = typeof closest[\"timestamp\"] === \"string\" ? closest[\"timestamp\"] : \"\";\n const shown = typeof closest[\"url\"] === \"string\" ? closest[\"url\"] : \"\";\n const original = httpUrlOf(/\\/web\\/\\d{14}[a-z_]*\\/(.+)$/s.exec(shown)?.[1] ?? \"\");\n if (waybackTime(digits) === undefined || original === undefined)\n return { note: `${name}: the index gave no usable time or URL` };\n const copy = await fetchAndIndex(runtime, {\n url: `${WAYBACK}${digits}id_/${original}`,\n mode: \"page\",\n recipe: ARCHIVE_COPY_RECIPE,\n accept: HTML_ACCEPT,\n originalHeaderPrefix: \"x-archive-orig-\",\n signalLines,\n ...common(options),\n });\n const found = copyOf(copy, name);\n return \"note\" in found\n ? found\n : { ...found, fromCache: lookup.fromCache && copy.fetched.fromCache };\n};\n\ninterface CdxEntry {\n readonly timestamp: string;\n readonly filename: string;\n readonly offset: number;\n readonly length: number;\n}\n\n/** This function reads the JSON lines of a Common Crawl index answer, and gives the newest copy with the status 200. */\nconst newestOf = (fetched: Fetched): CdxEntry | undefined => {\n const text = new TextDecoder().decode(fetched.body);\n let best: CdxEntry | undefined;\n for (const line of text.split(\"\\n\")) {\n if (line.trim() === \"\") continue;\n let row: Record<string, unknown>;\n try {\n row = JSON.parse(line) as Record<string, unknown>;\n } catch {\n continue;\n }\n const { timestamp, filename, status } = row;\n const offset = Number(row[\"offset\"]);\n const length = Number(row[\"length\"]);\n // the file name comes from a third party: only a path of the crawl data, on the host of the data\n if (\n status !== \"200\" ||\n typeof timestamp !== \"string\" ||\n !/^\\d{14}$/.test(timestamp) ||\n typeof filename !== \"string\" ||\n !/^crawl-data\\/[\\w\\-./]+\\.warc\\.gz$/.test(filename) ||\n filename.includes(\"..\") ||\n !Number.isSafeInteger(offset) ||\n offset < 0 ||\n !Number.isSafeInteger(length) ||\n length < 1 ||\n length > MAX_RECORD_BYTES\n )\n continue;\n if (best === undefined || timestamp > best.timestamp)\n best = { timestamp, filename, offset, length };\n }\n return best;\n};\n\n/** This function asks the newest Common Crawl indexes for a page, and fetches its WARC record with a range request. */\nconst fromCommonCrawl = async (\n runtime: CrawlRuntime,\n url: string,\n signalLines: readonly string[],\n options: PageOptions,\n): Promise<\n | { readonly doc: Doc; readonly archived: Archived; readonly fromCache: boolean }\n | { readonly note: string }\n> => {\n const name = ARCHIVE_NAMES.commoncrawl;\n const info = await runtime.crawler.fetch({\n url: `${CC_INDEX}collinfo.json`,\n mode: \"page\",\n recipe: ARCHIVE_INDEX_RECIPE,\n accept: JSON_ACCEPT,\n ...common(options),\n });\n if (!got(info)) return { note: `${name}: the list of indexes did not come (${said(info)})` };\n const listed = jsonOf(info);\n const apis = (Array.isArray(listed) ? listed : [])\n .map((entry: unknown) =>\n typeof entry === \"object\" && entry !== null\n ? (entry as Record<string, unknown>)[\"cdx-api\"]\n : undefined,\n )\n .filter((api): api is string => typeof api === \"string\" && api.startsWith(CC_INDEX))\n .slice(0, CC_INDEXES);\n if (apis.length === 0) return { note: `${name}: the list of indexes is empty` };\n let entry: CdxEntry | undefined;\n let fromCache = info.fromCache;\n const misses: string[] = [];\n for (const api of apis) {\n const query = await runtime.crawler.fetch({\n url: `${api}?url=${encodeURIComponent(url)}&output=json`,\n mode: \"page\",\n recipe: ARCHIVE_INDEX_RECIPE,\n accept: JSON_ACCEPT,\n ...common(options),\n });\n fromCache &&= query.fromCache;\n if (got(query)) entry = newestOf(query);\n else if (query.outcome !== \"not-found\") misses.push(said(query));\n if (entry !== undefined) break;\n if (query.outcome === \"skipped\") break;\n }\n if (entry === undefined)\n return {\n note: `${name}: no copy in the ${String(apis.length)} newest indexes${misses.length > 0 ? ` (${misses.join(\"; \")})` : \"\"}`,\n };\n const copy = await fetchAndIndex(runtime, {\n url: `${CC_DATA}${entry.filename}`,\n mode: \"page\",\n recipe: ARCHIVE_COPY_RECIPE,\n accept: \"*/*\",\n range: { offset: entry.offset, length: entry.length },\n signalLines,\n ...common(options),\n });\n if (got(copy.fetched) && copy.fetched.status !== 206)\n return {\n note: `${name}: the server sent the status ${String(copy.fetched.status)}, not the part of the file`,\n };\n const found = copyOf(copy, name);\n return \"note\" in found ? found : { ...found, fromCache: false };\n};\n\n/**\n * This function gives an archived copy of a page after a refusal. It applies the rule for\n * robots.txt first: with no `archiveDespiteRobots`, a page that the current robots.txt of its site\n * disallows gets no copy. A stored copy younger than `maxAgeMs` answers with no request. Then it asks\n * the Wayback Machine, then Common Crawl, each one with polite fetches of the crawler.\n */\nexport const archivedCopy = async (\n runtime: CrawlRuntime,\n url: string,\n reason: FallbackReason,\n options: PageOptions,\n): Promise<ArchiveAnswer> => {\n let target: URL;\n try {\n target = new URL(url);\n } catch {\n return { fromCache: false, notes: [] };\n }\n target.hash = \"\";\n // an archive is a third party: a URL with a password or a token does not go to it\n if (redact(target) !== target.href)\n return {\n fromCache: false,\n notes: [\"the URL holds a secret, so the crawler does not send it to a web archive\"],\n };\n const despite = options.archiveDespiteRobots === true;\n const refused =\n \"the current robots.txt of the site disallows the page for this agent, so the crawler gives no archived copy (archiveDespiteRobots permits one)\";\n if (reason === \"robots\" && !despite) return { fromCache: false, notes: [refused] };\n // the rules now: a stop does not tell them, and the content signals of the site apply to the copy\n let rules: Awaited<ReturnType<CrawlRuntime[\"crawler\"][\"robotsAllow\"]>>;\n try {\n rules = await runtime.crawler.robotsAllow(target.href, {\n onBehalfOf: options.onBehalfOf,\n scope: options.scope,\n labels: options.labels,\n signal: options.signal,\n });\n } catch {\n // a cancel during the check: the result of the live fetch stays, with no copy\n return { fromCache: false, notes: [\"the fallback stopped before it read robots.txt\"] };\n }\n if (!rules.allowed && (rules.note ?? \"\").startsWith(ROBOTS_DISALLOW) && !despite)\n return { fromCache: false, notes: [refused] };\n const signalLines = rules.signals ?? [];\n // the store answers first, as for a live page\n const known = runtime.index.get(archivedDocId(target.href));\n const archivedKnown = known === undefined ? undefined : archivedOf(known.meta);\n if (\n known !== undefined &&\n archivedKnown !== undefined &&\n typeof options.maxAgeMs === \"number\" &&\n Date.parse(runtime.now()) - Date.parse(known.fetched_at) < options.maxAgeMs\n )\n return { doc: known, archived: archivedKnown, fromCache: true, notes: [] };\n const notes: string[] = [];\n for (const ask of [fromWayback, fromCommonCrawl]) {\n if (options.signal?.aborted === true) break;\n const answer = await ask(runtime, target.href, signalLines, options);\n if (\"note\" in answer) notes.push(answer.note);\n else return { doc: answer.doc, archived: answer.archived, fromCache: answer.fromCache, notes };\n }\n return { fromCache: false, notes: [`no archived copy: ${notes.join(\"; \")}`] };\n};\n\n/**\n * This function gives the result of a page after a refusal, with an archived copy when an archive\n * had one. The result keeps the outcome and the status of the live fetch. It gives undefined when\n * the refusal gets no copy, and `notes` tell why.\n */\nexport const archivedPage = async (\n runtime: CrawlRuntime,\n url: string,\n live: Indexed,\n options: PageOptions,\n): Promise<{ readonly page?: PageResult | undefined; readonly notes: readonly string[] }> => {\n const reason = fallbackReason(live);\n if (reason === undefined) return { notes: [] };\n const answer = await archivedCopy(runtime, url, reason, options);\n if (answer.doc === undefined || answer.archived === undefined) return { notes: answer.notes };\n const doc = runtime.index.get(answer.doc.doc) ?? answer.doc;\n const archived = archivedOf(doc.meta) ?? answer.archived;\n const held = isHeldBack(doc);\n const types = doc.meta[\"jsonld_types\"];\n const links = doc.meta[\"links\"];\n const note = [\n live.fetched.note,\n live.error,\n ...answer.notes,\n archivedNote(archived),\n ...(held ? [HELD_BACK] : []),\n ].filter((part): part is string => part !== undefined && part.length > 0);\n return {\n page: {\n url,\n doc: doc.doc,\n outcome: live.fetched.outcome,\n status: live.fetched.status,\n title: held ? null : doc.title,\n chars: doc.text.length,\n rendered: false,\n structured: Array.isArray(types) ? (types as string[]) : [],\n links: Array.isArray(links) ? links.length : 0,\n captures: [...doc.captures],\n capture: doc.capture,\n capture_count: doc.capture_count,\n fetched_at: doc.fetched_at,\n from_cache: answer.fromCache,\n held_back: held,\n archived,\n note: note.join(\"; \"),\n },\n notes: [],\n };\n};\n","import type { BlobStore } from \"./blobs.ts\";\nimport type { Crawler, FetchRequest, Fetched, RenderRequest } from \"./crawler.ts\";\nimport type {\n Doc,\n DocInput,\n ExtractedDoc,\n Found,\n Latest,\n RankMode,\n ReadingIndex,\n Variant,\n VersionAt,\n} from \"./docs.ts\";\nimport { locate } from \"./docs.ts\";\nimport type { PageExtract } from \"./html.ts\";\nimport { extractPage, typesOf } from \"./html.ts\";\nimport type { CaptureLog, CaptureReader } from \"./log.ts\";\nimport { isPdf, readPdf } from \"./pdf.ts\";\nimport type { CaptureRecord } from \"./record.ts\";\nimport { makeRecord } from \"./record.ts\";\nimport type { AiInput } from \"./signals.ts\";\nimport { decodeBody } from \"./text.ts\";\nimport { canonicalUrl } from \"./urls.ts\";\n// ── archived fallback ──\nimport type { Archived } from \"./fallback.ts\";\nimport { archiveExtractor, archivedNote, archivedOf, archivedPage } from \"./fallback.ts\";\n// ── end of archived fallback ──\n\n/**\n * An extractor changes one captured body into the documents that it holds. It has no side effects,\n * so `reindex` can replay each capture through it and build the index again with no request. It can\n * answer later, because the crawler reads a PDF asynchronously.\n *\n * An extractor that throws does not stop a fetch, a render or a rebuild. The runtime records the\n * error and gives no document for that capture.\n */\nexport type Extractor = (\n body: Uint8Array,\n capture: { readonly url: string; readonly contentType: string | null },\n) => ReadonlyArray<ExtractedDoc> | Promise<ReadonlyArray<ExtractedDoc>>;\n\n/**\n * This function gives the document id of a web page. Each page has one id.\n *\n * This export is internal: the root entry point does not have it since 0.2.0.\n */\nexport const pageDocId = (url: string): string => `url:${canonicalUrl(url)}`;\n\n/** The JSON-LD that a document keeps, in characters of JSON. */\nconst MAX_JSONLD_CHARS = 30_000;\n/** These properties make a JSON-LD record large, and they say little about the place: they go first when a record is too large. */\nconst BULKY = [\n \"review\",\n \"reviews\",\n \"comment\",\n \"hasMenu\",\n \"hasMenuSection\",\n \"hasMenuItem\",\n \"menu\",\n \"photo\",\n \"image\",\n \"subjectOf\",\n \"makesOffer\",\n \"hasOfferCatalog\",\n];\n/** These properties say where and when a place is open. A record with one of them is kept first. */\nconst PLACE_KEYS = [\"openingHours\", \"openingHoursSpecification\", \"address\", \"geo\", \"telephone\"];\n\n/**\n * This function keeps the JSON-LD records that fit the limit. A record with hours, an address or a\n * position goes first. A large record loses its reviews and its menu before it is left out. A\n * record that does not fit does not stop the records after it.\n */\nconst keepJsonLd = (\n records: readonly Record<string, unknown>[],\n): { readonly kept: Record<string, unknown>[]; readonly dropped: number } => {\n const place = (record: Record<string, unknown>): boolean =>\n PLACE_KEYS.some((key) => record[key] !== undefined);\n const ordered = [...records.filter(place), ...records.filter((record) => !place(record))];\n const kept: Record<string, unknown>[] = [];\n let dropped = 0;\n let room = MAX_JSONLD_CHARS;\n for (const record of ordered) {\n let size = JSON.stringify(record).length;\n let candidate = record;\n if (size > room) {\n candidate = Object.fromEntries(\n Object.entries(record).filter(([key]) => !BULKY.includes(key)),\n );\n size = JSON.stringify(candidate).length;\n }\n if (size > room) {\n dropped += 1;\n continue;\n }\n kept.push(candidate);\n room -= size;\n }\n return { kept, dropped };\n};\n\n/**\n * This extractor is the built-in recipe for a web page, plain or rendered. It keeps the visible\n * text, the JSON-LD (whole objects, up to a limit) and the links. It decodes the body in the\n * encoding that the page declares, with `decodeBody`. It does not throw: a page that it cannot read\n * gives a document that names the error. When the reader cut the text, the document keeps the whole\n * text in `full_text`, so `verify` finds a quote past the cut.\n *\n * This export is internal: the root entry point does not have it since 0.2.0.\n */\nexport const pageExtractor: Extractor = (body, capture) => {\n if (isPdf(body, capture.contentType)) return pdfDocs(body, capture.url);\n if (capture.contentType !== null && !/html|xml/i.test(capture.contentType)) return [];\n const doc = pageDocId(capture.url);\n let page: PageExtract;\n try {\n page = extractPage(decodeBody(body, capture.contentType), capture.url);\n } catch (error) {\n // one page that breaks the reader must not break the fetch, or each later reindex\n return [\n {\n doc,\n kind: \"page\",\n title: capture.url,\n text: capture.url,\n url: capture.url,\n meta: {\n extract_error: error instanceof Error ? error.message.slice(0, 300) : String(error),\n },\n },\n ];\n }\n const jsonld = keepJsonLd(page.jsonld);\n return [\n {\n doc,\n kind: \"page\",\n title: page.title,\n text: page.text.length > 0 ? page.text : page.title,\n url: capture.url,\n meta: {\n description: page.description,\n lang: page.lang,\n jsonld_types: [...new Set(page.jsonld.flatMap(typesOf))],\n jsonld: jsonld.kept,\n ...(jsonld.dropped > 0 ? { jsonld_dropped: jsonld.dropped } : {}),\n links: page.links.slice(0, 100),\n visible_chars: page.visibleChars,\n embedded: page.embedded,\n needs_render: page.needsRender,\n render_reason: page.renderReason,\n canonical: page.canonical,\n ...(page.feeds !== undefined && page.feeds.length > 0 ? { feeds: page.feeds } : {}),\n redirect: page.redirect,\n blocked: page.blocked,\n ...(page.notes.length > 0 ? { notes: page.notes } : {}),\n },\n ...(page.fullText !== undefined ? { full_text: page.fullText } : {}),\n },\n ];\n};\n\nconst MAX_PDF_TEXT_CHARS = 200_000;\n\n/**\n * This function changes a PDF into one document of its text. A PDF with no text, a scan for\n * example, gives a document of its title only, and the document says that it needs OCR. A damaged\n * file gives no document.\n */\nconst pdfDocs = async (body: Uint8Array, url: string): Promise<ReadonlyArray<ExtractedDoc>> => {\n let read;\n try {\n read = await readPdf(body, { maxChars: MAX_PDF_TEXT_CHARS });\n } catch {\n return [];\n }\n const file = new URL(url).pathname.split(\"/\").pop() ?? \"\";\n let named = file;\n try {\n named = decodeURIComponent(file);\n } catch {\n // a malformed escape: the name as written\n }\n const title = read.title ?? (named || url);\n const notes = {\n ...(read.truncated ? { truncated: true } : {}),\n ...(read.note !== null ? { note: read.note } : {}),\n };\n if (read.text.length === 0) {\n return [\n {\n doc: pageDocId(url),\n kind: \"pdf\",\n title,\n text: title,\n url,\n meta: {\n format: \"pdf\",\n pages: read.pages,\n visible_chars: 0,\n // a file that a budget stopped is not a scan\n ocr_needed: read.note === null,\n ...notes,\n },\n },\n ];\n }\n return [\n {\n doc: pageDocId(url),\n kind: \"pdf\",\n title,\n text: read.text.slice(0, MAX_PDF_TEXT_CHARS),\n url,\n meta: { format: \"pdf\", pages: read.pages, visible_chars: read.text.length, ...notes },\n },\n ];\n};\n\n/** @beta */\nexport class Extractors {\n // the archived fallback reads its copies with the built-in recipe `archive-copy`\n private readonly byRecipe = new Map<string, Extractor>([\n [\"page\", pageExtractor],\n [\"archive-copy\", archiveExtractor],\n ]);\n\n // 0.1.1 let an application register `archive-copy`. Thus an application can replace that built-in once.\n private readonly replaceable = new Set([\"archive-copy\"]);\n\n register(recipe: string, extractor: Extractor): void {\n if (this.byRecipe.has(recipe) && !this.replaceable.delete(recipe)) {\n throw new Error(`recipe ${recipe} already has an extractor`);\n }\n this.byRecipe.set(recipe, extractor);\n }\n\n get(recipe: string): Extractor | undefined {\n return this.byRecipe.get(recipe);\n }\n\n /** This method gives the names of the recipes that have an extractor. */\n recipes(): string[] {\n return [...this.byRecipe.keys()];\n }\n}\n\n/** @beta */\nexport interface CrawlRuntime {\n readonly crawler: Crawler;\n /** This index keeps the documents and the cache entries. `DocIndex` is the SQLite index. */\n readonly index: ReadingIndex;\n readonly blobs: BlobStore;\n /** This reader is the read side of the capture log. The function `reindex` replays it. */\n readonly captures: CaptureReader;\n /** This log is the write side of the capture log. */\n readonly log: CaptureLog;\n readonly extractors: Extractors;\n readonly now: () => string;\n}\n\n/**\n * This is the payload of a capture record, as the crawler writes it.\n *\n * @beta\n */\nexport interface CapturePayload {\n readonly url: string;\n readonly recipe: string;\n readonly mode: \"api\" | \"page\";\n readonly status?: number;\n readonly outcome?: string;\n readonly blob?: string | null;\n readonly content_type?: string | null;\n readonly etag?: string | null;\n readonly last_modified?: string | null;\n readonly fetched_at: string;\n readonly signals?: { readonly ai_input?: AiInput };\n /** This value is true when the headless browser made the capture, not a plain request. */\n readonly rendered?: boolean;\n /** This value is true when the body is only the first part of the response. */\n readonly truncated?: boolean;\n /** This value is true when the server cut a compressed body. The capture is not a copy to keep. */\n readonly cut?: boolean;\n /** This value is true when an API answered with a web page that the request did not accept. The capture is not a copy to keep. */\n readonly unasked?: boolean;\n /** This time (ISO 8601) is the end of the freshness that the server gave. Captures before version 0.2 do not have it. */\n readonly fresh_until?: string | null;\n /** This text holds the request headers that the response varies on. Captures before version 0.2 do not have it. */\n readonly vary_key?: string | null;\n /** This URL is the target of a redirect, with its secrets as `REDACTED`. */\n readonly location?: string | null;\n readonly note?: string;\n /** This text is the byte range of a range request (`bytes=0-99`). The capture is a part of the resource, not its stored copy. */\n readonly range?: string;\n}\n\n/** The structural fields of the metadata of a page: they hold no text of the page. A held-back page shows only these. */\nexport const STRUCTURAL_META: readonly string[] = [\n \"jsonld_types\",\n \"jsonld_dropped\",\n \"visible_chars\",\n \"lang\",\n \"embedded\",\n \"needs_render\",\n \"render_reason\",\n \"canonical\",\n \"redirect\",\n \"blocked\",\n \"format\",\n \"pages\",\n \"ocr_needed\",\n \"truncated\",\n // the label of an archived copy holds no text of the page\n \"archived\",\n \"archived_kind\",\n];\n\n/**\n * The note of a document that is held back.\n *\n * @beta\n */\nexport const HELD_BACK =\n \"This page asked not to be used as AI input, so its text is held back. Open the URL to read it, or read again with userProvided when a person gave this URL directly (the crawler logs that).\";\n\n/**\n * This function tells if a document is held back: the page asked not to be AI input, and no person\n * gave its URL directly. The only way to read a held-back document is `userProvided`.\n *\n * @beta\n */\nexport const isHeldBack = (doc: { readonly ai_input: AiInput }, userProvided = false): boolean =>\n doc.ai_input === \"no\" && !userProvided;\n\n/**\n * This function gives the metadata that a held-back document shows: only the structural fields, which hold no text of the page.\n *\n * @beta\n */\nexport const heldBackMeta = (meta: Readonly<Record<string, unknown>>): Record<string, unknown> =>\n Object.fromEntries(Object.entries(meta).filter(([key]) => STRUCTURAL_META.includes(key)));\n\nconst messageOf = (error: unknown): string =>\n error instanceof Error ? (error.message.split(/\\r?\\n/)[0] ?? \"\") : String(error);\n\n/** This function tells which bot check or firewall sent its page in place of the content, or gives undefined. */\nconst blockedBy = (doc: ExtractedDoc): string | undefined => {\n const blocked = doc.meta[\"blocked\"];\n return typeof blocked === \"string\" && blocked.length > 0 ? blocked : undefined;\n};\n\n/**\n * The result of `fetchAndIndex` and `renderAndIndex`.\n *\n * @beta\n */\nexport interface Indexed {\n readonly fetched: Fetched;\n readonly docs: readonly Doc[];\n /** This text tells why the capture gave no document: the error of the extractor, or a bot check. */\n readonly error?: string | undefined;\n /** This value names the bot check or the firewall that sent its page in place of the content. */\n readonly blocked?: string | undefined;\n}\n\n/** This function reads one capture with its extractor, and adds each document to the index as a version of the capture. */\nconst indexCapture = async (\n runtime: CrawlRuntime,\n fetched: Fetched,\n recipe: string,\n rendered: boolean,\n): Promise<Omit<Indexed, \"fetched\">> => {\n const capture = fetched.capture;\n if (fetched.body === undefined || capture === undefined) return { docs: [] };\n // a stored capture that the index already read gives its documents with no new reading\n if (fetched.fromCache) {\n const known = runtime.index.docsOfCapture(capture);\n if (known.length > 0) return { docs: known };\n }\n const extractor = runtime.extractors.get(recipe);\n if (extractor === undefined) return { docs: [] };\n let extracted: ReadonlyArray<ExtractedDoc>;\n try {\n extracted = await extractor(fetched.body, {\n url: fetched.url,\n contentType: rendered ? \"text/html; charset=utf-8\" : (fetched.contentType ?? null),\n });\n } catch (error) {\n return { docs: [], error: `extract failed (${recipe}): ${messageOf(error)}` };\n }\n const blocked = extracted.map(blockedBy).find((name) => name !== undefined);\n if (blocked !== undefined) {\n // a bot check is not the content: the index keeps the earlier text, and the host stops\n runtime.crawler.reportBlocked(fetched.url, blocked);\n return {\n docs: [],\n blocked,\n error: `the page is a bot check (${blocked}), not the content, so the crawler keeps nothing of it and stops the host`,\n };\n }\n const now = runtime.now();\n const docs = extracted.map((doc) =>\n runtime.index.upsert(\n {\n ...doc,\n captures: [capture],\n capture,\n fetched_at: fetched.fetchedAt ?? now,\n blob: fetched.blob ?? null,\n recipe,\n rendered,\n // an extractor can find a \"no\" in the body (the headers of an archived response), never a \"yes\"\n ai_input: doc.ai_input === \"no\" ? \"no\" : fetched.signals.ai_input,\n ...(fetched.scope !== undefined ? { scope: fetched.scope } : {}),\n ...(fetched.labels !== undefined ? { labels: fetched.labels } : {}),\n },\n now,\n ),\n );\n return { docs };\n};\n\n/**\n * This function fetches one URL and adds its documents to the index. Each recipe uses it. A fetch\n * from the cache gives the documents that the index has for that capture. Thus a repeated\n * question sends no request and reads nothing again. An extractor that throws gives no document and an\n * `error`, and the fetch still succeeds.\n *\n * @beta\n */\nexport const fetchAndIndex = async (\n runtime: CrawlRuntime,\n request: FetchRequest,\n): Promise<Indexed> => {\n const fetched = await runtime.crawler.fetch(request);\n if (fetched.outcome !== \"ok\" && fetched.outcome !== \"not-modified\") return { fetched, docs: [] };\n return { fetched, ...(await indexCapture(runtime, fetched, request.recipe, false)) };\n};\n\n/**\n * This function renders a page in the headless browser and adds what the page shows to the index.\n *\n * This export is internal: the root entry point does not have it since 0.2.0.\n */\nexport const renderAndIndex = async (\n runtime: CrawlRuntime,\n request: RenderRequest,\n): Promise<Indexed> => {\n const fetched = await runtime.crawler.render(request);\n if (fetched.outcome !== \"ok\") return { fetched, docs: [] };\n return { fetched, ...(await indexCapture(runtime, fetched, request.recipe, true)) };\n};\n\nexport interface PageResult {\n readonly url: string;\n readonly doc: string | null;\n readonly outcome: string;\n readonly status: number;\n /** This value is the title of the page. A held-back page shows no title. */\n readonly title: string | null;\n readonly chars: number;\n readonly rendered: boolean;\n /** These are the schema.org types that the JSON-LD of the page declares. */\n readonly structured: readonly string[];\n readonly links: number;\n /** These ids are the captures behind the current text, the newest first, 10 or fewer. */\n readonly captures: readonly string[];\n /** This id is the capture of the text that the document shows. A citation names it. */\n readonly capture: string | null;\n /** This value is the number of captures of the document. */\n readonly capture_count: number;\n /** This time (ISO 8601) is the fetch time of the text that the document shows. */\n readonly fetched_at: string | null;\n /** This value is true when the stored copy answered, and the crawler sent no request. */\n readonly from_cache: boolean;\n /** This value is true when the page asked not to be AI input. Its title and text are then held back. */\n readonly held_back: boolean;\n readonly note: string | null;\n /**\n * This value tells that the document is an archived copy, not a fresh fetch: the archive, the date\n * of the copy and its URL there. The outcome and the status stay those of the live fetch.\n */\n readonly archived?: Archived | undefined;\n}\n\n/**\n * These options control one page.\n *\n * @beta\n */\nexport interface PageOptions {\n readonly render: \"auto\" | \"never\" | \"always\";\n readonly maxAgeMs?: number | \"auto\" | undefined;\n readonly onBehalfOf?: readonly string[] | undefined;\n readonly job?: string | undefined;\n /** When this signal aborts, the crawler stops the fetch and the render. */\n readonly signal?: AbortSignal | undefined;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n /** This time (milliseconds since the epoch) is the end of a budget. No request of the page starts at or after it (`FetchRequest.deadline`). */\n readonly deadline?: number | undefined;\n // ── archived fallback ──\n /** The value `archive` gives a copy from a public web archive when the site refuses the page, or is down. The default is `none`. */\n readonly fallback?: \"archive\" | \"none\" | undefined;\n /** The value true also gives an archived copy of a page that the current robots.txt of its site disallows. The default is false. */\n readonly archiveDespiteRobots?: boolean | undefined;\n // ── end of archived fallback ──\n}\n\nconst HTML_TYPE = /html|xml/i;\n\n/**\n * This function reads one web page. It first sends a plain request. It opens the browser only when\n * the page shows almost nothing without JavaScript, or when the caller asks for it. It renders\n * only an HTML page: not a PDF, a different file or a bot check. A page that sends the reader on\n * with a `<meta http-equiv=\"refresh\">` gets one more plain fetch, of its target.\n *\n * @beta\n */\nexport const fetchPage = async (\n runtime: CrawlRuntime,\n url: string,\n options: PageOptions,\n): Promise<PageResult> => {\n const plainRequest = (target: string): FetchRequest => ({\n url: target,\n mode: \"page\",\n recipe: \"page\",\n accept: \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.5\",\n maxAgeMs: options.maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n job: options.job,\n signal: options.signal,\n scope: options.scope,\n labels: options.labels,\n deadline: options.deadline,\n });\n let plain = await fetchAndIndex(runtime, plainRequest(url));\n const notes: string[] = [];\n if (plain.error !== undefined) notes.push(plain.error);\n const target = plain.docs[0]?.meta[\"redirect\"];\n if (typeof target === \"string\" && target.length > 0 && plain.blocked === undefined) {\n const followed = await fetchAndIndex(runtime, plainRequest(target));\n notes.push(\n `the page sends the reader on to ${followed.fetched.url}, so the crawler fetched that page`,\n );\n if (followed.error !== undefined) notes.push(followed.error);\n plain = followed;\n }\n let docs = plain.docs;\n let rendered = false;\n let fromCache = plain.fetched.fromCache;\n const reachable = plain.fetched.outcome === \"ok\" || plain.fetched.outcome === \"not-modified\";\n const first = docs[0];\n const html =\n first?.kind === \"page\" &&\n (plain.fetched.contentType === undefined || HTML_TYPE.test(plain.fetched.contentType));\n const wanted =\n options.render === \"always\" ||\n (options.render === \"auto\" && first?.meta[\"needs_render\"] === true);\n if (reachable && wanted) {\n if (plain.blocked !== undefined) notes.push(\"the crawler does not render a bot check\");\n else if (first === undefined || !html) {\n notes.push(\"the crawler renders only an HTML page, not a PDF or a different file\");\n } else {\n // a render of an earlier body is not the page now: the render is fresh when the plain body changed\n const current = runtime.index.get(first.doc);\n const stale = current !== undefined && !current.rendered && current.capture_count > 1;\n const shown = await renderAndIndex(runtime, {\n url: plain.fetched.url,\n recipe: \"page\",\n maxAgeMs: stale ? 0 : options.maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n job: options.job,\n signal: options.signal,\n scope: options.scope,\n labels: options.labels,\n deadline: options.deadline,\n });\n if (shown.docs.length > 0) {\n docs = shown.docs;\n rendered = true;\n fromCache = fromCache && shown.fetched.fromCache;\n } else {\n notes.push(shown.error ?? shown.fetched.note ?? \"the render gave no document\");\n }\n }\n }\n // ── archived fallback: a refusal can get a copy from a web archive, clearly marked ──\n if (options.fallback === \"archive\" && docs.length === 0) {\n const fallback = await archivedPage(runtime, url, plain, options);\n if (fallback.page !== undefined) return fallback.page;\n notes.push(...fallback.notes);\n }\n // ── end of archived fallback ──\n const doc = docs[0] === undefined ? undefined : (runtime.index.get(docs[0].doc) ?? docs[0]);\n const held = doc !== undefined && isHeldBack(doc);\n const types = doc?.meta[\"jsonld_types\"];\n const links = doc?.meta[\"links\"];\n if (held) notes.push(HELD_BACK);\n const note = [plain.fetched.note, ...notes].filter(\n (part): part is string => part !== undefined && part.length > 0,\n );\n return {\n url,\n doc: doc?.doc ?? null,\n outcome: plain.fetched.outcome,\n status: plain.fetched.status,\n title: doc === undefined || held ? null : doc.title,\n chars: doc?.text.length ?? 0,\n rendered: doc?.rendered ?? rendered,\n structured: Array.isArray(types) ? (types as string[]) : [],\n links: Array.isArray(links) ? links.length : 0,\n captures: doc === undefined ? [] : [...doc.captures],\n capture: doc?.capture ?? null,\n capture_count: doc?.capture_count ?? 0,\n fetched_at: doc?.fetched_at ?? plain.fetched.fetchedAt ?? null,\n from_cache: fromCache,\n held_back: held,\n note: note.length > 0 ? note.join(\"; \") : null,\n };\n};\n\n// ── reading, verifying and finding ──────────────────────────────────────────\n\n/** This result is a part of one stored document. */\nexport interface ReadResult {\n readonly doc: string;\n readonly kind: string;\n /** This value is the title of the document. A held-back document shows no title. */\n readonly title: string;\n readonly url: string | null;\n /** This value is the metadata of the document. A held-back document shows only the structural fields. */\n readonly meta: Readonly<Record<string, unknown>>;\n readonly from: number;\n readonly to: number;\n readonly total: number;\n readonly text: string;\n readonly captures: readonly string[];\n /** This id is the capture of the text that the result reads. */\n readonly capture: string;\n /** This time (ISO 8601) is the fetch time of the text that the result reads. */\n readonly fetched_at: string;\n readonly ai_input: AiInput;\n /** This value is true when the page asked not to be AI input. The text is then empty. */\n readonly held_back: boolean;\n readonly note: string | null;\n /** This time (ISO 8601) is the `at` of the read. The result has it only when the read gave `at`. */\n readonly at?: string | undefined;\n /**\n * This value is true when `at` is before the first capture of the document. Then no version\n * answers: the text, the title, `capture` and `fetched_at` are empty, and the note tells why.\n */\n readonly before_first?: boolean | undefined;\n /** This value tells that the text is an archived copy, with the archive and the date of the copy. */\n readonly archived?: Archived | undefined;\n}\n\n/** This result tells if a quote is in a stored document, and where. */\nexport interface VerifyResult {\n readonly doc: string;\n /** This value is true when the quote is in the text. A held-back document gives false. */\n readonly found: boolean;\n /** These values are the positions of the quote in the text of `capture`. */\n readonly start: number | null;\n readonly end: number | null;\n readonly excerpt: string | null;\n readonly url: string | null;\n readonly captures: readonly string[];\n /** This id is the capture whose text holds the quote, or the capture that the result examined. */\n readonly capture: string | null;\n /** This hash is the blob of that capture. */\n readonly blob: string | null;\n /** This time (ISO 8601) is the fetch time of that capture. */\n readonly fetched_at: string | null;\n /** This value is true when only an earlier version of the document holds the quote. */\n readonly stale: boolean;\n /** This value is true when the quote is in the whole text of a page that the reader cut. The positions are then in that whole text. */\n readonly in_full_text: boolean;\n /** This value is false when the examined text is not complete (a PDF that the reader stopped). A quote that is not found can then be in the file. */\n readonly complete: boolean;\n /** This value is the number of places where the quote is. */\n readonly count: number;\n readonly held_back: boolean;\n readonly note: string | null;\n /** This time (ISO 8601) is the `at` of the verification. The result has it only when the verification gave `at`. */\n readonly at?: string | undefined;\n /** This value is true when `at` is before the first capture of the document. Then no version holds the quote, and the note tells why. */\n readonly before_first?: boolean | undefined;\n /**\n * This value tells that the examined text is an archived copy, with the archive and the date of\n * the copy. A citation of it names the archive and that date, not a date of a fresh fetch.\n */\n readonly archived?: Archived | undefined;\n}\n\n/** These options control a read. */\nexport interface ReadOptions {\n /** This value is the first character to read. The default is 0. */\n readonly from?: number | undefined;\n /** This value is the number of characters to read. The default is 4000. */\n readonly length?: number | undefined;\n /** This value is true when a person gave this URL directly. It is the only way to read a held-back document. The crawler logs it. */\n readonly userProvided?: boolean | undefined;\n /** This id reads the version of one capture, not the current version. */\n readonly capture?: string | undefined;\n /**\n * This time reads the version that was current at that time: an ISO 8601 text or a `Date`. A\n * date with no time is the start of that day in UTC. Give `capture` or `at`, not the two.\n */\n readonly at?: string | Date | undefined;\n}\n\n/** These options control a verification. */\nexport interface VerifyOptions {\n /** This id examines the version of one capture only. A citation of that capture thus stays true after a new fetch. */\n readonly capture?: string | undefined;\n /** This value is true when a person gave this URL directly. It is the only way to verify a quote in a held-back document. */\n readonly userProvided?: boolean | undefined;\n /** This time examines only the version that was current at that time (`ReadOptions.at`). Give `capture` or `at`, not the two. */\n readonly at?: string | Date | undefined;\n}\n\nconst knownDoc = (runtime: CrawlRuntime, id: string): Doc => {\n const doc = runtime.index.get(id);\n if (doc === undefined)\n throw new Error(`no stored document ${id}: find one with find(), or fetch its page`);\n return doc;\n};\n\n// ── time travel ─────────────────────────────────────────────────────────────\n\n/**\n * This function gives the ISO 8601 form of a time. A text that is not a time throws a `RangeError`.\n * A date with no time is the start of that day in UTC.\n */\nexport const isoTime = (name: string, value: string | Date): string => {\n const ms =\n value instanceof Date ? value.getTime() : typeof value === \"string\" ? Date.parse(value) : NaN;\n if (!Number.isFinite(ms)) {\n throw new RangeError(\n `${name} must be a time, such as 2026-10-01T12:00:00Z, not ${JSON.stringify(String(value))}`,\n );\n }\n return new Date(ms).toISOString();\n};\n\n/**\n * This function gives the version of a document that was current at a time (ISO 8601). It uses\n * `ReadingIndex.versionAt` when the index has it. Else it takes the newest version at or before\n * the time.\n *\n * @beta\n */\nexport const versionAtTime = (runtime: CrawlRuntime, id: string, at: string): VersionAt => {\n const index = runtime.index;\n if (index.versionAt !== undefined) return index.versionAt(id, at);\n const all = index.versions(id, { limit: 1_000_000 });\n const pick = all.find((version) => version.fetched_at <= at);\n return {\n version: pick === undefined ? undefined : index.versionText(id, pick.capture),\n first: all.at(-1)?.fetched_at,\n };\n};\n\n/** This function gives the note of a time before the first version of a document. */\nconst beforeFirstNote = (id: string, at: string, first: string | undefined): string =>\n `the document ${id} has no version at ${at} or before${first === undefined ? \"\" : `: its first capture is from ${first}`}`;\n\n/** This function gives the time of `at`, or undefined. It refuses `capture` and `at` together. */\nconst atOf = (options: {\n readonly capture?: string | undefined;\n readonly at?: string | Date | undefined;\n}): string | undefined => {\n if (options.at === undefined) return undefined;\n if (options.capture !== undefined)\n throw new RangeError(\"give capture or at, not the two: each one chooses a version\");\n return isoTime(\"at\", options.at);\n};\n\n// ── end of time travel ──────────────────────────────────────────────────────\n\nconst wholeNumber = (name: string, value: number | undefined, fallback: number): number => {\n if (value === undefined) return fallback;\n if (!Number.isFinite(value) || value < 0) {\n throw new RangeError(`${name} must be a number of 0 or more, not ${String(value)}`);\n }\n return Math.floor(value);\n};\n\n/** This function moves a position off the second half of a surrogate pair, so a slice does not cut a character. */\nconst onBoundary = (text: string, position: number): number => {\n const code = text.charCodeAt(position);\n return position > 0 && position < text.length && code >= 0xdc00 && code <= 0xdfff\n ? position - 1\n : position;\n};\n\n/**\n * This function writes an `override` record to the capture log. A person gave the URL of a\n * held-back document directly, and the hold stops for one action (`read` or `verify`). The record\n * takes the scope and the labels of `marks`, or else those of the document, as its captures do.\n *\n * @beta\n */\nexport const recordOverride = async (\n runtime: CrawlRuntime,\n doc: {\n readonly doc: string;\n readonly url: string | null;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n },\n action: string,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n): Promise<void> => {\n const at = runtime.now();\n const scope = marks.scope ?? doc.scope;\n const labels = marks.labels ?? doc.labels;\n await runtime.log.append(\n makeRecord({\n source: \"fetch\",\n kind: \"override\",\n payload: { doc: doc.doc, url: doc.url, action, reason: \"a person provided this URL\", at },\n received_at: at,\n external_id: doc.url ?? doc.doc,\n ...(scope !== undefined ? { scope } : {}),\n ...(labels !== undefined ? { labels } : {}),\n }),\n );\n};\n\n/**\n * This function reads a range of characters of one stored document. It applies the rule for a\n * held-back document: no title, no text, and only the structural metadata, unless `userProvided`.\n * The positions do not cut a character in two.\n *\n * @beta\n */\nexport const readDocument = async (\n runtime: CrawlRuntime,\n id: string,\n options: ReadOptions = {},\n): Promise<ReadResult> => {\n const from = wholeNumber(\"from\", options.from, 0);\n const length = wholeNumber(\"length\", options.length, 4000);\n const at = atOf(options);\n const doc = knownDoc(runtime, id);\n let version =\n options.capture === undefined ? undefined : runtime.index.versionText(id, options.capture);\n if (options.capture !== undefined && version === undefined) {\n throw new Error(`the document ${id} has no version from the capture ${options.capture}`);\n }\n const timed = at === undefined ? {} : { at, before_first: false };\n if (at !== undefined) {\n const found = versionAtTime(runtime, id, at);\n if (found.version === undefined) {\n // no version answers, so there is no text to hold back\n return {\n doc: doc.doc,\n kind: doc.kind,\n title: \"\",\n url: doc.url,\n meta: {},\n from: 0,\n to: 0,\n total: 0,\n text: \"\",\n captures: doc.captures,\n capture: \"\",\n fetched_at: \"\",\n ai_input: doc.ai_input,\n held_back: false,\n note: beforeFirstNote(id, at, found.first),\n at,\n before_first: true,\n };\n }\n version = found.version;\n }\n const aiInput: AiInput =\n doc.ai_input === \"no\" || version?.ai_input === \"no\"\n ? \"no\"\n : (version?.ai_input ?? doc.ai_input);\n const userProvided = options.userProvided ?? false;\n const held = isHeldBack({ ai_input: aiInput }, userProvided);\n if (aiInput === \"no\" && userProvided) await recordOverride(runtime, doc, \"read\");\n const text = version?.text ?? doc.text;\n const start = onBoundary(text, Math.min(from, text.length));\n const end = held ? start : onBoundary(text, Math.min(text.length, start + length));\n const meta = version?.meta ?? doc.meta;\n // ── archived fallback: an archived copy says so in each read ──\n const archived = archivedOf(meta);\n const notes = [...(held ? [HELD_BACK] : []), ...(archived ? [archivedNote(archived)] : [])];\n return {\n doc: doc.doc,\n kind: version?.kind ?? doc.kind,\n title: held ? \"\" : (version?.title ?? doc.title),\n url: version?.url ?? doc.url,\n meta: held ? heldBackMeta(meta) : meta,\n from: start,\n to: end,\n total: text.length,\n text: held ? \"\" : text.slice(start, end),\n captures: doc.captures,\n capture: version?.capture ?? doc.capture,\n fetched_at: version?.fetched_at ?? doc.fetched_at,\n ai_input: aiInput,\n held_back: held,\n note: notes.length > 0 ? notes.join(\" \") : null,\n ...timed,\n ...(archived !== undefined ? { archived } : {}),\n };\n};\n\n// ── archived fallback: the labels of an archived copy ──\n\n/** This function gives the `archived` field of a result, for the metadata of an archived copy. */\nconst archivedField = (\n meta: Readonly<Record<string, unknown>>,\n): { readonly archived?: Archived } => {\n const archived = archivedOf(meta);\n return archived === undefined ? {} : { archived };\n};\n\n/** This function gives the `archived` field and the note of a result: the note of an archived copy goes after the other note. */\nconst withArchived = (\n meta: Readonly<Record<string, unknown>>,\n note: string | null,\n): { readonly note: string | null; readonly archived?: Archived } => {\n const archived = archivedOf(meta);\n if (archived === undefined) return { note };\n const said = archivedNote(archived);\n return { note: note === null ? said : `${note}; ${said}`, archived };\n};\n\n// ── end of archived fallback ──\n\n/**\n * This function examines the versions of a document for a quote, and writes nothing. It also tells\n * if `userProvided` opened a held-back version, so that `verifyQuote` logs the override.\n */\nconst examineQuote = (\n runtime: CrawlRuntime,\n id: string,\n quote: string,\n options: VerifyOptions,\n): { readonly result: VerifyResult; readonly override: boolean } => {\n const at = atOf(options);\n const doc = knownDoc(runtime, id);\n let pinned =\n options.capture === undefined ? undefined : runtime.index.versionText(id, options.capture);\n if (options.capture !== undefined && pinned === undefined) {\n throw new Error(`the document ${id} has no version from the capture ${options.capture}`);\n }\n const userProvided = options.userProvided ?? false;\n const base = { doc: doc.doc, url: doc.url, captures: doc.captures };\n const timed = at === undefined ? {} : { at, before_first: false };\n const nothing = {\n found: false,\n start: null,\n end: null,\n excerpt: null,\n capture: null,\n blob: null,\n fetched_at: null,\n stale: false,\n in_full_text: false,\n count: 0,\n } as const;\n if (at !== undefined) {\n // the version that was current at the time is the only version that the check examines\n const found = versionAtTime(runtime, id, at);\n if (found.version === undefined) {\n const result: VerifyResult = {\n ...base,\n ...nothing,\n complete: true,\n held_back: false,\n note: beforeFirstNote(id, at, found.first),\n at,\n before_first: true,\n };\n return { result, override: false };\n }\n pinned = found.version;\n }\n const aiInput: AiInput = doc.ai_input === \"no\" || pinned?.ai_input === \"no\" ? \"no\" : doc.ai_input;\n if (isHeldBack({ ai_input: aiInput }, userProvided)) {\n const result: VerifyResult = {\n ...base,\n ...nothing,\n complete: false,\n held_back: true,\n note: HELD_BACK,\n ...timed,\n ...archivedField(pinned?.meta ?? doc.meta),\n };\n return { result, override: false };\n }\n const current = pinned ?? runtime.index.versionText(id);\n const candidates =\n pinned !== undefined\n ? [pinned]\n : [...(current === undefined ? [] : [current]), ...runtime.index.earlierTexts(id)];\n // a version that asked not to be AI input stays held back after the page drops the signal\n const held = candidates.filter((version) => version.ai_input === \"no\").length;\n const override = userProvided && (aiInput === \"no\" || held > 0);\n for (const [index, version] of candidates.entries()) {\n if (isHeldBack(version, userProvided)) continue;\n let hit = locate(version.text, quote);\n let inFull = false;\n if (!hit.found && version.full_text !== undefined) {\n hit = locate(version.full_text, quote);\n inFull = hit.found;\n }\n if (!hit.found) continue;\n const stale = pinned === undefined && index > 0;\n const result: VerifyResult = {\n ...base,\n found: true,\n start: hit.start,\n end: hit.end,\n excerpt: hit.excerpt,\n capture: version.capture,\n blob: version.blob,\n fetched_at: version.fetched_at,\n stale,\n in_full_text: inFull,\n complete: true,\n count: hit.count,\n held_back: false,\n ...withArchived(\n version.meta,\n stale ? \"only an earlier capture of the document holds the quote\" : null,\n ),\n ...timed,\n };\n return { result, override };\n }\n const incomplete =\n current !== undefined && current.meta[\"truncated\"] === true && current.full_text === undefined;\n // the note does not depend on the quote, so it tells nothing about the held-back text\n const notes = [\n ...(incomplete\n ? [\n \"the reader stopped before the end of the file, so the quote can be in the part that it did not read\",\n ]\n : []),\n ...(held > 0 && !userProvided\n ? [`the check did not examine the versions that asked not to be AI input (${String(held)})`]\n : []),\n ];\n const result: VerifyResult = {\n ...base,\n found: false,\n start: null,\n end: null,\n excerpt: null,\n capture: current?.capture ?? null,\n blob: current?.blob ?? null,\n fetched_at: current?.fetched_at ?? null,\n stale: false,\n in_full_text: false,\n complete: !incomplete,\n count: 0,\n held_back: false,\n ...withArchived(current?.meta ?? doc.meta, notes.length > 0 ? notes.join(\"; \") : null),\n ...timed,\n };\n return { result, override };\n};\n\n/**\n * This function finds a quote in a stored document, with `locate`, and writes nothing. It is the\n * synchronous part of `verifyQuote`. A held-back document gives `found: false` and no positions,\n * unless `userProvided`, and this function does not log the override. With no capture, it does not\n * examine a version that asked not to be AI input, also when a later capture dropped the signal.\n */\nexport const checkQuote = (\n runtime: CrawlRuntime,\n id: string,\n quote: string,\n options: VerifyOptions = {},\n): VerifyResult => examineQuote(runtime, id, quote, options).result;\n\n/**\n * This function finds a quote in a stored document, with `locate`. It tells which capture holds the\n * quote, with its blob, its fetch time and the positions:\n *\n * - With `capture`, it examines the version of that capture only.\n * - With no capture, it examines the current text first. When the quote is not there, it examines\n * the earlier texts, the newest first. A match there has `stale: true`.\n * - When the reader cut a text, it also examines the whole text.\n *\n * A held-back document gives `found: false` and no positions, so a quote cannot test its text,\n * unless `userProvided`. The same rule applies to each version. A version that asked not to be AI\n * input stays held back after a later capture drops the signal. The crawler logs an override when\n * `userProvided` opens a held-back version.\n *\n * @beta\n */\nexport const verifyQuote = async (\n runtime: CrawlRuntime,\n id: string,\n quote: string,\n options: VerifyOptions = {},\n): Promise<VerifyResult> => {\n const { result, override } = examineQuote(runtime, id, quote, options);\n if (override) await recordOverride(runtime, knownDoc(runtime, id), \"verify\");\n return result;\n};\n\n/**\n * This function searches the stored documents. The syntax of the query is the syntax of\n * `parseQuery`. A held-back document is not in the results. The option `rank` chooses the\n * ranking (`RankMode`), and the default is `exact`.\n *\n * @beta\n */\nexport const findDocuments = async (\n runtime: CrawlRuntime,\n q: string,\n options: {\n readonly limit?: number | undefined;\n readonly kind?: string | null | undefined;\n readonly rank?: RankMode | undefined;\n } = {},\n): Promise<Found[]> => {\n await Promise.resolve();\n const found = runtime.index.find(q, {\n limit: options.limit ?? 10,\n kind: options.kind ?? null,\n ...(options.rank !== undefined ? { rank: options.rank } : {}),\n });\n // ── archived fallback: a hit of an archived copy says so ──\n return found.map((hit) => {\n if (hit.kind !== \"archived\") return hit;\n const meta = runtime.index.get(hit.doc)?.meta;\n return meta === undefined ? hit : { ...hit, ...archivedField(meta) };\n });\n};\n\n// ── the rebuild ─────────────────────────────────────────────────────────────\n\n/** The counts of a rebuild. */\nexport interface ReindexResult {\n /** This value is the number of captures that the rebuild read again. */\n readonly captures: number;\n readonly docs: number;\n readonly urls: number;\n /** This value is the number of captures whose extractor threw. */\n readonly failed: number;\n /** This value is the number of captures whose blob is missing or damaged. */\n readonly missing: number;\n /** This value is the number of damaged lines of the log that the rebuild skipped. */\n readonly damaged: number;\n /** This value is the number of documents of recipes with no extractor, kept from the old index. */\n readonly carried: number;\n /** These recipes have captures in the log and no extractor in this runtime. */\n readonly unknown_recipes: readonly string[];\n /** These are the first errors of the extractors. */\n readonly errors: ReadonlyArray<{ readonly capture: string; readonly error: string }>;\n}\n\nconst RENDERED_MARK = \"#rendered\";\nconst MAX_ERRORS = 20;\n/**\n * The crawler reads the captures of these recipes for itself. They are robots.txt, the sitemaps\n * and feeds of `discover`, and the indexes of the web archives. They give no document.\n */\nconst CRAWLER_RECIPES: ReadonlySet<string> = new Set([\n \"robots.txt\",\n \"sitemap\",\n \"feed\",\n \"archive-index\",\n]);\n\n/** This function gives the URL of a capture and its variant. A URL of version 0.1 can end in `#rendered`. */\nconst captureTarget = (\n payload: CapturePayload,\n): { readonly url: string; readonly variant: Variant } => {\n if (payload.url.endsWith(RENDERED_MARK)) {\n return { url: payload.url.slice(0, -RENDERED_MARK.length), variant: \"rendered\" };\n }\n return { url: payload.url, variant: payload.rendered === true ? \"rendered\" : \"plain\" };\n};\n\n/** A replay writes into a new generation of the index (a rebuild), or into the index itself (the tail of the log). */\ninterface ReplayTarget {\n upsert(input: DocInput, now: string): unknown;\n setLatest(entry: Latest): void;\n latest(url: string, variant?: Variant): Latest | undefined;\n}\n\n/** These are the counts of a replay. */\ninterface ReplayTally {\n captures: number;\n failed: number;\n missing: number;\n readonly unknown: Set<string>;\n readonly errors: Array<{ readonly capture: string; readonly error: string }>;\n}\n\nconst newTally = (): ReplayTally => ({\n captures: 0,\n failed: 0,\n missing: 0,\n unknown: new Set(),\n errors: [],\n});\n\n/** This function replays the cache entry of one capture, as the crawler made it at the time of the fetch. */\nconst replayLatest = (\n build: ReplayTarget,\n envelope: CaptureRecord,\n payload: CapturePayload,\n): void => {\n const { url, variant } = captureTarget(payload);\n const fetchedAt = payload.fetched_at ?? envelope.received_at;\n if (payload.outcome === \"redirect\") {\n const location =\n payload.location ??\n (typeof payload.note === \"string\" && payload.note.startsWith(\"location \")\n ? payload.note.slice(9)\n : null);\n const permanent = payload.status === 301 || payload.status === 308;\n if (location !== null && (permanent || typeof payload.fresh_until === \"string\")) {\n build.setLatest({\n url,\n variant,\n capture: envelope.id,\n blob: null,\n etag: null,\n last_modified: null,\n content_type: null,\n fetched_at: fetchedAt,\n status: payload.status ?? 301,\n ai_input: \"unset\",\n location,\n fresh_until: payload.fresh_until ?? null,\n vary_key: payload.vary_key ?? null,\n });\n }\n return;\n }\n // a body that the server cut, a web page in place of the data of an API, or a part of a resource\n // from a range request was never the stored copy\n if (payload.cut === true || payload.unasked === true || payload.range !== undefined) return;\n const previous = build.latest(url, variant);\n const revisit = envelope.kind === \"revisit\" || payload.outcome === \"not-modified\";\n const truncated = payload.truncated === true;\n // a 304 updates the stored answer: its status stays, and its validators fill the gaps\n const etag = truncated ? null : (payload.etag ?? (revisit ? (previous?.etag ?? null) : null));\n const lastModified = truncated\n ? null\n : (payload.last_modified ?? (revisit ? (previous?.last_modified ?? null) : null));\n const kept = revisit ? (previous?.content_type ?? undefined) : undefined;\n const contentType =\n payload.content_type ?? kept ?? (variant === \"rendered\" ? \"text/html; charset=utf-8\" : null);\n const status = revisit\n ? previous === undefined || previous.status < 200 || previous.status >= 300\n ? 200\n : previous.status\n : (payload.status ?? 200);\n build.setLatest({\n url,\n variant,\n capture: envelope.id,\n blob: payload.blob ?? null,\n etag,\n last_modified: lastModified,\n content_type: contentType,\n fetched_at: fetchedAt,\n status,\n ai_input: payload.signals?.ai_input ?? \"unset\",\n ...(truncated ? { truncated: true } : {}),\n fresh_until: payload.fresh_until ?? null,\n vary_key: payload.vary_key ?? (revisit ? (previous?.vary_key ?? null) : null),\n });\n};\n\n/** This function replays one record of the log: its cache entry, and the documents that its extractor gives. */\nconst replayRecord = async (\n runtime: CrawlRuntime,\n target: ReplayTarget,\n envelope: CaptureRecord,\n tally: ReplayTally,\n): Promise<void> => {\n if (envelope.kind !== \"capture\" && envelope.kind !== \"revisit\") return;\n const payload = envelope.payload as CapturePayload | null;\n if (payload === null || typeof payload !== \"object\" || typeof payload.url !== \"string\") return;\n if (payload.outcome === \"redirect\") {\n replayLatest(target, envelope, payload);\n return;\n }\n if (typeof payload.blob !== \"string\") return;\n if (payload.outcome !== \"ok\" && payload.outcome !== \"not-modified\") return;\n tally.captures += 1;\n replayLatest(target, envelope, payload);\n const extractor = runtime.extractors.get(payload.recipe);\n if (extractor === undefined) {\n // the crawler reads robots.txt, the sitemaps and the feeds for itself; they give no document\n if (!CRAWLER_RECIPES.has(payload.recipe)) tally.unknown.add(payload.recipe);\n return;\n }\n const body = await runtime.blobs.get(payload.blob);\n if (body === undefined) {\n tally.missing += 1;\n return;\n }\n const { url, variant } = captureTarget(payload);\n const latest = target.latest(url, variant);\n let extracted: ReadonlyArray<ExtractedDoc>;\n try {\n extracted = await extractor(body, {\n url,\n contentType:\n variant === \"rendered\"\n ? \"text/html; charset=utf-8\"\n : (payload.content_type ?? latest?.content_type ?? null),\n });\n } catch (error) {\n tally.failed += 1;\n if (tally.errors.length < MAX_ERRORS)\n tally.errors.push({ capture: envelope.id, error: messageOf(error) });\n return;\n }\n // a bot check is not the content, so it is not a version of the document\n if (extracted.some((doc) => blockedBy(doc) !== undefined)) return;\n for (const doc of extracted) {\n target.upsert(\n {\n ...doc,\n captures: [envelope.id],\n capture: envelope.id,\n fetched_at: payload.fetched_at ?? envelope.received_at,\n blob: payload.blob,\n recipe: payload.recipe,\n rendered: variant === \"rendered\",\n ai_input: doc.ai_input === \"no\" ? \"no\" : (payload.signals?.ai_input ?? \"unset\"),\n ...(envelope.scope !== undefined ? { scope: envelope.scope } : {}),\n ...(envelope.labels !== undefined ? { labels: envelope.labels } : {}),\n },\n envelope.received_at,\n );\n }\n};\n\n/**\n * This function builds the reading index again from the capture log and the blobs. The index is a\n * projection, and this function is the proof that its deletion loses nothing. These are its rules:\n *\n * - It builds a new generation of the tables and commits it in batches. The old index answers\n * until the end, and a failed rebuild leaves it as it was.\n * - It reads the log as a stream, so its memory does not grow with the log.\n * - It counts and skips a damaged line, a missing blob and an extractor that throws.\n * - The documents of a recipe that has no extractor in this runtime stay, from the old index.\n * - A capture that a different writer adds during the rebuild goes into the new index.\n *\n * @beta\n */\nexport const reindex = async (\n runtime: CrawlRuntime,\n options: { readonly batch?: number | undefined } = {},\n): Promise<ReindexResult> => {\n const since = runtime.now();\n const build = runtime.index.beginRebuild(\n options.batch === undefined ? {} : { batch: options.batch },\n );\n const tally = newTally();\n let damaged = 0;\n try {\n const scan = runtime.captures.scan(\n { source: \"fetch\" },\n {\n onDamaged: () => {\n damaged += 1;\n },\n },\n );\n for await (const envelope of scan) await replayRecord(runtime, build, envelope, tally);\n const carried = build.carry(new Set(runtime.extractors.recipes()));\n const counts = build.commit(since);\n return {\n captures: tally.captures,\n ...counts,\n failed: tally.failed,\n missing: tally.missing,\n damaged,\n carried,\n unknown_recipes: [...tally.unknown].sort(),\n errors: tally.errors,\n };\n } catch (error) {\n build.abort();\n throw error;\n }\n};\n\n/**\n * The result of `catchUpIndex`.\n *\n * @beta\n */\nexport interface CatchUpResult extends ReindexResult {\n /**\n * The value `rebuild` tells that the index was stale, and the function built it again from the\n * whole log. The value `tail` tells that it read only the captures that the index missed.\n */\n readonly mode: \"rebuild\" | \"tail\";\n}\n\n/** The tail starts this long before the newest cache entry. Two writers and their clocks can differ. */\nconst TAIL_SLACK_MS = 10 * 60_000;\n\n/** This function tells if the index missed a record of the log: no cache entry as new as it, or no version of its documents. */\nconst missed = (runtime: CrawlRuntime, envelope: CaptureRecord): boolean => {\n if (envelope.kind !== \"capture\" && envelope.kind !== \"revisit\") return false;\n const payload = envelope.payload as CapturePayload | null;\n if (payload === null || typeof payload !== \"object\" || typeof payload.url !== \"string\")\n return false;\n const fetchedAt = payload.fetched_at ?? envelope.received_at;\n const { url, variant } = captureTarget(payload);\n const entry = runtime.index.latest(url, variant);\n if (payload.outcome === \"redirect\") return entry === undefined || entry.fetched_at < fetchedAt;\n if (\n typeof payload.blob !== \"string\" ||\n (payload.outcome !== \"ok\" && payload.outcome !== \"not-modified\")\n )\n return false;\n // a range request leaves no cache entry: only its documents tell if the index has it\n if (payload.range !== undefined)\n return (\n runtime.extractors.get(payload.recipe) !== undefined && !runtime.index.hasVersion(envelope.id)\n );\n if (entry === undefined || entry.fetched_at < fetchedAt) return true;\n return (\n runtime.extractors.get(payload.recipe) !== undefined && !runtime.index.hasVersion(envelope.id)\n );\n};\n\n/**\n * This function brings the reading index up to the capture log, with no request. `openCrawler`\n * uses it at open. An application that uses `createRuntime` uses it the same way:\n *\n * - A stale index (a new schema, or a deleted file) is built again with `reindex`.\n * - An index that is behind the log gets the captures that it missed. A process can stop between\n * the log and the index, and SQLite can lose its last transactions after a power cut. The\n * function reads the log from a short time before the newest cache entry.\n *\n * It gives the counts, or undefined when the index missed nothing.\n *\n * @beta\n */\nexport const catchUpIndex = async (runtime: CrawlRuntime): Promise<CatchUpResult | undefined> => {\n if (runtime.index.stale) return { mode: \"rebuild\", ...(await reindex(runtime)) };\n const newest = runtime.index.newestFetch();\n const since =\n newest === undefined ? undefined : new Date(Date.parse(newest) - TAIL_SLACK_MS).toISOString();\n const tally = newTally();\n let damaged = 0;\n const scan = runtime.captures.scan(\n { source: \"fetch\", ...(since === undefined ? {} : { since }) },\n {\n onDamaged: () => {\n damaged += 1;\n },\n },\n );\n for await (const envelope of scan) {\n if (missed(runtime, envelope)) await replayRecord(runtime, runtime.index, envelope, tally);\n }\n if (tally.captures === 0) return undefined;\n return {\n mode: \"tail\",\n captures: tally.captures,\n ...runtime.index.counts(),\n failed: tally.failed,\n missing: tally.missing,\n damaged,\n carried: 0,\n unknown_recipes: [...tally.unknown].sort(),\n errors: tally.errors,\n };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAmCA,IAAa,gBAAb,cAAmC,MAAM;CAG5B;CACA;CAHX,OAAwB;CACxB,YACE,SACA,MACA;EACA,MAAM,WAAW,SAAS,IAAI,CAAC;EAHtB,KAAA,UAAA;EACA,KAAA,OAAA;EAGT,KAAK,OAAO;CACd;AACF;;AAGA,MAAa,cAAc,SAAiB,SAC1C,kBAAkB,QAAQ,MAAM,YAAY,KAAK,IAAI,IAAI,OAAO,IAAI,GAAG,KAAK;;;;;;AAS9E,MAAa,aAAa,SAAoC;CAC5D,MAAM,QAAQ,KAAK,MAAM,GAAG;CAC5B,IAAI,MAAM,SAAS,KAAK,MAAM,MAAM,SAAS,OAAO,IAAI,MAAM,IAAI;CAClE,IAAI,MAAM,WAAW,KAAK,MAAM,SAAS,GAAG,OAAO,KAAA;CACnD,MAAM,UAAoB,CAAC;CAC3B,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI;EACJ,IAAI,sBAAsB,KAAK,IAAI,GACjC,QAAQ,KAAK,WAAW,IAAI,IAAI,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE;OAC9D,IAAI,YAAY,KAAK,IAAI,GAAG,QAAQ,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC;OACpE,IAAI,oBAAoB,KAAK,IAAI,GAAG,QAAQ,OAAO,IAAI;OACvD,OAAO,KAAA;EACZ,QAAQ,KAAK,KAAK;CACpB;CACA,MAAM,OAAO,QAAQ,IAAI;CACzB,IAAI,QAAQ,MAAM,UAAU,QAAQ,GAAG,GAAG,OAAO,KAAA;CACjD,IAAI,QAAQ,QAAQ,IAAI,QAAQ,SAAS,OAAO,KAAA;CAChD,IAAI,OAAO;CACX,MAAM,OAAiB,CAAC;CACxB,KAAK,IAAI,QAAQ,QAAQ,QAAQ,QAAQ,GAAG,SAAS,GAAG;EACtD,KAAK,QAAQ,OAAO,GAAG;EACvB,OAAO,KAAK,MAAM,OAAO,GAAG;CAC9B;CACA,OAAO,CAAC,GAAG,SAAS,GAAG,IAAI;AAC7B;;AAGA,MAAa,aAAa,SAAoC;CAC5D,IAAI,QAAQ,KAAK,WAAW,GAAG,KAAK,KAAK,SAAS,GAAG,IAAI,KAAK,MAAM,GAAG,EAAE,IAAI;CAC7E,MAAM,OAAO,MAAM,QAAQ,GAAG;CAC9B,IAAI,SAAS,IAAI,QAAQ,MAAM,MAAM,GAAG,IAAI;CAC5C,IAAI,KAAK,KAAK,MAAM,GAAG,OAAO,KAAA;CAC9B,MAAM,YAAY,MAAM,YAAY,GAAG;CACvC,MAAM,OAAO,MAAM,MAAM,YAAY,CAAC;CACtC,IAAI,KAAK,SAAS,GAAG,GAAG;EACtB,MAAM,KAAK,UAAU,IAAI;EACzB,IAAI,OAAO,KAAA,GAAW,OAAO,KAAA;EAC7B,MAAM,OAAQ,GAAG,MAAO,IAAK,GAAG;EAChC,MAAM,MAAO,GAAG,MAAO,IAAK,GAAG;EAC/B,QAAQ,GAAG,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,KAAK,SAAS,EAAE,EAAE,GAAG,IAAI,SAAS,EAAE;CACjF;CACA,MAAM,CAAC,OAAO,IAAI,QAAQ,MAAM,MAAM,IAAI;CAC1C,MAAM,YAAY,SAChB,KAAK,WAAW,IAAI,CAAC,IAAI,KAAK,MAAM,GAAG,CAAC,CAAC,KAAK,UAAU,OAAO,SAAS,OAAO,EAAE,CAAC;CACpF,MAAM,QAAQ,SAAS,IAAI;CAC3B,MAAM,OAAO,SAAS,KAAA,IAAY,CAAC,IAAI,SAAS,IAAI;CACpD,MAAM,SAAS;EAAC,GAAG;EAAO,GAAG,MAAc,IAAI,MAAM,SAAS,KAAK,MAAM,CAAC,CAAC,KAAK,CAAC;EAAG,GAAG;CAAI;CAC3F,IAAI,OAAO,WAAW,GAAG,OAAO,KAAA;CAChC,OAAO,OAAO,SAAS,UAAU,CAAC,SAAS,GAAG,QAAQ,GAAI,CAAC;AAC7D;AAEA,MAAM,MAAM,SAAwB,UAAU,IAAI;AAClD,MAAM,MAAM,SAAwB,UAAU,IAAI;;AAGlD,MAAM,YAAY,SAAgB,QAAe,SAA0B;CACzE,IAAI,QAAQ,WAAW,OAAO,QAAQ,OAAO;CAC7C,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;EACtD,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,OAAO,QAAQ,CAAC,CAAC;EACtD,IAAI,SAAS,GAAG,OAAO;EACvB,MAAM,OAAQ,OAAS,IAAI,OAAS;EACpC,KAAK,QAAQ,SAAU,WAAW,OAAO,SAAU,OAAO,OAAO;CACnE;CACA,OAAO;AACT;AAGA,MAAM,YAA6D;CACjE;EAAC,GAAG,iBAAiB;EAAG;EAAI;CAAU;CACtC;EAAC,GAAG,eAAe;EAAG;EAAI;CAAU;CACpC;EAAC,GAAG,iBAAiB;EAAG;EAAI;CAAU;CACtC;EAAC,GAAG,SAAS;EAAG;EAAG;CAAa;CAChC;EAAC,GAAG,UAAU;EAAG;EAAG;CAAS;CAC7B;EAAC,GAAG,YAAY;EAAG;EAAI;CAAmB;CAC1C;EAAC,GAAG,WAAW;EAAG;EAAG;CAAU;CAC/B;EAAC,GAAG,aAAa;EAAG;EAAI;CAAY;CACpC;EAAC,GAAG,YAAY;EAAG;EAAI;CAAS;CAChC;EAAC,GAAG,WAAW;EAAG;EAAI;CAAU;CAChC;EAAC,GAAG,WAAW;EAAG;EAAI;CAAe;CACrC;EAAC,GAAG,aAAa;EAAG;EAAI;CAAS;CACjC;EAAC,GAAG,YAAY;EAAG;EAAI;CAAW;CAClC;EAAC,GAAG,cAAc;EAAG;EAAI;CAAe;CACxC;EAAC,GAAG,aAAa;EAAG;EAAI;CAAe;CACvC;EAAC,GAAG,WAAW;EAAG;EAAG;CAAW;CAChC;EAAC,GAAG,WAAW;EAAG;EAAG;CAAU;AACjC;AAEA,MAAM,YAA6D;CACjE;EAAC,GAAG,eAAe;EAAG;EAAK;CAAU;CACrC;EAAC,GAAG,IAAI;EAAG;EAAK;CAAa;CAC7B;EAAC,GAAG,KAAK;EAAG;EAAK;CAAU;CAC3B;EAAC,GAAG,aAAa;EAAG;EAAI;CAAU;CAClC;EAAC,GAAG,OAAO;EAAG;EAAI;CAAU;CAC5B;EAAC,GAAG,QAAQ;EAAG;EAAI;CAAU;CAC7B;EAAC,GAAG,YAAY;EAAG;EAAI;CAAe;CACtC;EAAC,GAAG,QAAQ;EAAG;EAAG;CAAc;CAChC;EAAC,GAAG,QAAQ;EAAG;EAAI;CAAY;CAC/B;EAAC,GAAG,QAAQ;EAAG;EAAI;CAAS;CAC5B;EAAC,GAAG,QAAQ;EAAG;EAAG;CAAW;AAC/B;AAEA,MAAM,YAAY,YAChB,UAAU,MAAM,CAAC,QAAQ,UAAU,SAAS,SAAS,QAAQ,IAAI,CAAC,CAAC,GAAG;AAExE,MAAM,YAAY,YAAuC;CACvD,MAAM,UAAU,MAAc,OAC5B,QAAQ,MAAM,MAAM,EAAE,CAAC,CAAC,OAAO,SAAS,SAAS,CAAC;CACpD,MAAM,YAAY,SAAqC,SAAS,QAAQ,MAAM,MAAM,OAAO,CAAC,CAAC;CAE7F,IAAI,OAAO,GAAG,EAAE,KAAK,QAAQ,QAAQ,OAAQ,QAAQ,QAAQ,KAAM,OAAO,SAAS,EAAE;CACrF,IAAI,SAAS,SAAS,GAAG,WAAW,GAAG,EAAE,GAAG,OAAO,SAAS,EAAE;CAC9D,IAAI,QAAQ,OAAO,MAAQ,QAAQ,OAAO,GAAM,OAAO,SAAS,CAAC;CACjE,MAAM,SAAS,UAAU,MAAM,CAAC,QAAQ,UAAU,SAAS,SAAS,QAAQ,IAAI,CAAC,CAAC,GAAG;CACrF,IAAI,WAAW,KAAA,GAAW,OAAO;CAEjC,IAAI,OAAO,GAAG,EAAE,GAAG,OAAO,SAAS,EAAE,KAAK;CAC1C,IAAI,QAAQ,OAAO,GAAG,OAAO;AAE/B;;;;;;AAOA,MAAa,eAAe,YAAwC;CAClE,MAAM,OAAO,UAAU,OAAO;CAC9B,IAAI,SAAS,KAAA,GAAW,OAAO,SAAS,IAAI;CAC5C,MAAM,MAAM,UAAU,OAAO;CAC7B,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,SAAS,GAAG;AACrD;AAEA,MAAM,UAAU,SAAyB,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;;AAGpF,MAAM,aAAuD;CAC3D,CAAC,oBAAoB,UAAU;CAC/B,CAAC,gCAAgC,UAAU;CAC3C,CAAC,oBAAoB,UAAU;AACjC;AAEA,MAAM,aAAa,SAAoC,UAAU,IAAI,KAAK,UAAU,IAAI;AAExF,MAAM,WAAW,OAAe,YAA4B;CAC1D,MAAM,QAAQ,MAAM,QAAQ,GAAG;CAC/B,MAAM,OAAO,UAAU,UAAU,KAAK,QAAQ,MAAM,MAAM,GAAG,KAAK,CAAC;CACnE,IAAI,SAAS,KAAA,KAAa,KAAK,WAAW,QAAQ,QAAQ,OAAO;CACjE,MAAM,OAAO,UAAU,KAAK,KAAK,SAAS,IAAI,OAAO,MAAM,MAAM,QAAQ,CAAC,CAAC;CAC3E,OACE,OAAO,UAAU,IAAI,KAAK,QAAQ,KAAK,QAAQ,KAAK,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI;AAElG;;AAGA,MAAM,aAAa,QAAuB,MAAc,YAAyC;CAC/F,IAAI,OAAO,iBAAiB,MAAM,OAAO;CACzC,MAAM,OAAO,OAAO,IAAI;CACxB,MAAM,QAAQ,YAAY,KAAA,IAAY,UAAU,IAAI,IAAI,UAAU,OAAO;CACzE,KAAK,MAAM,OAAO,OAAO,cAAc,CAAC,GAAG;EACzC,MAAM,QAAQ,OAAO,GAAG;EACxB,IAAI,UAAU,MAAM,OAAO;EAC3B,IAAI,MAAM,WAAW,IAAI,KAAK,KAAK,SAAS,MAAM,MAAM,CAAC,CAAC,GAAG,OAAO;EACpE,IAAI,UAAU,KAAA,KAAa,QAAQ,OAAO,KAAK,GAAG,OAAO;CAC3D;CACA,OAAO;AACT;;AAGA,MAAM,SAAS,SAAyB;CACtC,MAAM,QAAQ,UAAU,IAAI;CAC5B,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,MAAM,WAAW,GAAG,OAAO,MAAM,KAAK,GAAG;CAC7C,MAAM,SAAmB,CAAC;CAC1B,KAAK,IAAI,QAAQ,GAAG,QAAQ,IAAI,SAAS,GACvC,OAAO,MAAO,MAAM,UAAW,IAAK,MAAM,QAAQ,GAAA,CAAK,SAAS,EAAE,CAAC;CACrE,OAAO,OAAO,KAAK,GAAG,CAAC,CAAC,QAAQ,oBAAoB,IAAI;AAC1D;;;;;;AAOA,MAAa,iBAAiB,QAAuB,SAAqC;CACxF,IAAI,OAAO,iBAAiB,MAAM,OAAO,KAAA;CACzC,MAAM,OAAO,OAAO,IAAI;CAExB,IADgB,UAAU,IAChB,MAAM,KAAA,GAAW;EACzB,MAAM,OAAO,YAAY,IAAI;EAC7B,OAAO,SAAS,KAAA,KAAa,UAAU,QAAQ,MAAM,KAAA,CAAS,IAC1D,KAAA,IACA,WAAW,MAAM,IAAI,GAAG,IAAI;CAClC;CACA,KAAK,MAAM,CAAC,SAAS,SAAS,YAC5B,IAAI,QAAQ,KAAK,IAAI,KAAK,CAAC,UAAU,QAAQ,MAAM,KAAA,CAAS,GAAG,OAAO,WAAW,MAAM,IAAI;AAG/F;AAaA,MAAM,cAA6B,MAAM,SAAS,aAChDA,OAAa,MAAM,SAAS,QAAQ;;AAGtC,MAAM,UACJ,QACA,MACA,cACsF;CACtF,MAAM,SAA0B,CAAC;CACjC,IAAI;CACJ,KAAK,MAAM,SAAS,WAAW;EAC7B,MAAM,OAAO,YAAY,MAAM,OAAO;EACtC,IAAI,SAAS,KAAA,KAAa,UAAU,QAAQ,MAAM,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK;OAC9E,YAAY,IAAI,cAAc,MAAM,MAAM,OAAO,GAAG,IAAI;CAC/D;CACA,OAAO;EACL;EACA,SAAS,OAAO,WAAW,IAAK,WAAW,IAAI,cAAc,MAAM,YAAY,IAAK,KAAA;CACtF;AACF;;;;;;;;AASA,MAAa,gBACV,QAAuB,UAAyB,gBAChD,MAAc,SAAwB,aAAa;CAClD,QAAQ,MAAM;EAAE,GAAG;EAAS,KAAK;CAAK,IAAI,OAAO,cAAc;EAC7D,IAAI,UAAU,MAAM;GAClB,SAAS,OAAO,CAAC,CAAC;GAClB;EACF;EACA,MAAM,EAAE,QAAQ,YACd,OAAO,iBAAiB,OACpB;GAAE,QAAQ;GAAW,SAAS,KAAA;EAAU,IACxC,OAAO,QAAQ,MAAM,SAAS;EACpC,IAAI,YAAY,KAAA,GAAW;GACzB,SAAS,SAAS,CAAC,CAAC;GACpB;EACF;EACA,IAAI,QAAQ,QAAQ,MAAM,SAAS,MAAM,MAAM;OAC1C,SAAS,MAAM,OAAO,EAAE,CAAE,SAAS,OAAO,EAAE,CAAE,MAAM;CAC3D,CAAC;AACH;;;;;;;;;;AAWF,MAAa,cAAc,OACzB,QACA,KACA,UAAyB,eACO;CAChC,IAAI,OAAO,iBAAiB,MAAM,OAAO,KAAA;CACzC,MAAM,SAAS,cAAc,QAAQ,IAAI,QAAQ;CACjD,IAAI,WAAW,KAAA,GAAW,OAAO;CACjC,MAAM,OAAO,OAAO,IAAI,QAAQ;CAChC,IAAI,UAAU,IAAI,MAAM,KAAA,KAAa,UAAU,QAAQ,MAAM,KAAA,CAAS,GAAG,OAAO,KAAA;CAChF,MAAM,YAAY,MAAM,IAAI,SAAkC,WAC5D,QAAQ,MAAM,EAAE,KAAK,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS,KAAK,CAAC,CACvE;CACA,IAAI,qBAAqB,OAAO,OAAO,KAAA;CACvC,KAAK,MAAM,SAAS,WAAW;EAC7B,MAAM,OAAO,YAAY,MAAM,OAAO;EACtC,IAAI,SAAS,KAAA,KAAa,CAAC,UAAU,QAAQ,MAAM,MAAM,OAAO,GAC9D,OAAO,WAAW,MAAM,MAAM,OAAO,GAAG,IAAI;CAChD;AAEF;;AAGA,MAAa,eAAe,UAA8C;CACxE,KACE,IAAI,UAAU,OAAO,QAAQ,GAC7B,YAAY,KAAA,KAAa,YAAY,QAAQ,QAAQ,GACrD,SAAS,GACT;EACA,IAAI,mBAAmB,eAAe,OAAO;EAC7C,IAAI,mBAAmB,gBAAgB;GACrC,MAAM,QAAQ,QAAQ,OAAO,MAAM,SAAkB,gBAAgB,aAAa;GAElF,IAAI,UAAU,KAAA,GAAW,OAAO;EAClC;EACA,UAAW,QAAgC;CAC7C;AAEF;;;;;;;ACtTA,MAAa,mBAAmB,UAA2B;CACzD,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO,KAAK,UAAU,KAAK,KAAK;CAElC,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,IAAI,MAAM,KAAK,SAAS,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE;CAMlE,OAAO,IAJS,OAAO,QAAQ,KAAgC,CAAC,CAC7D,QAAQ,GAAG,UAAU,SAAS,KAAA,CAAS,CAAC,CACxC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,CACtC,KAAK,CAAC,KAAK,UAAU,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG,gBAAgB,IAAI,GACrD,CAAC,CAAC,KAAK,GAAG,EAAE;AAC/B;;AAGA,MAAa,UAAU,SACrB,WAAW,QAAQ,CAAC,CAAC,OAAO,MAAM,MAAM,CAAC,CAAC,OAAO,KAAK;;;;;;;;AASxD,MAAa,cAAc,UAAsC;CAC/D,MAAM,eAAe,OACnB,gBAAgB;EACd,QAAQ,MAAM;EACd,MAAM,MAAM;EACZ,aAAa,MAAM,eAAe;EAClC,SAAS,MAAM;CACjB,CAAC,CACH;CACA,OAAO;EACL,IAAI,GAAG,MAAM,OAAO,GAAG,aAAa,MAAM,GAAG,EAAE;EAC/C,QAAQ,MAAM;EACd,MAAM,MAAM;EACZ,aAAa,MAAM;EACnB,SAAS,MAAM;EACf;EACA,GAAI,MAAM,gBAAgB,KAAA,IAAY,EAAE,aAAa,MAAM,YAAY,IAAI,CAAC;EAC5E,GAAI,MAAM,gBAAgB,KAAA,IAAY,EAAE,aAAa,MAAM,YAAY,IAAI,CAAC;EAC5E,GAAI,MAAM,UAAU,KAAA,IAAY,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;EAC1D,GAAI,MAAM,WAAW,KAAA,KAAa,OAAO,KAAK,MAAM,MAAM,CAAC,CAAC,SAAS,IACjE,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,EAAE,IAC9B,CAAC;CACP;AACF;;;;;;;;AC4OA,MAAa,gBAAgB;;;;;;AAe7B,MAAa,iBAAiB;;AAG9B,MAAa,gBAAgB;;;;;;;AAQ7B,MAAa,wBAA2C;CACtD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,MAAM,mBAAmB;AACzB,MAAM,eAAe;AACrB,MAAM,eAAe;AACrB,MAAM,gCAAqC,IAAI,IAAI,CAAC,SAAS,QAAQ,CAAC;AACtE,MAAM,cAAc;AACpB,MAAM,sCAA2C,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC;AACnE,MAAM,qCAA0C,IAAI,IAAI;CACtD;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;AACzD,CAAC;;AAED,MAAM,mCAAwC,IAAI,IAAI;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;AAED,MAAM,qCAA0C,IAAI,IAAI;CACtD;CACA;CACA;AACF,CAAC;;AAED,MAAM,mCAAwC,IAAI,IAAI;CACpD;CACA;CACA;CAEA;CACA;CACA;AACF,CAAC;;AAGD,MAAM,oCAAyC,IAAI,IAAI;CACrD;CACA;CACA;AACF,CAAC;;AAID,MAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;AACF;;;;;AAMA,MAAM,mCAAwC,IAAI,IAAI,CAAC,YAAY,aAAa,CAAC;;;;;;AAOjF,MAAM,eAAe,QAA4B,gBAA6C;CAC5F,MAAM,QAAQ,eAAe,GAAA,CAAI,MAAM,GAAG,CAAC,CAAC,EAAE,CAAE,KAAK,CAAC,CAAC,YAAY;CACnE,IACE,WAAW,KAAA,KACX,OAAO,KAAK,MAAM,MACjB,SAAS,eAAe,SAAS,yBAElC,OAAO;CACT,OAAO,CAAC,OAAO,MAAM,GAAG,CAAC,CAAC,MAAM,UAAU;EACxC,MAAM,CAAC,QAAQ,IAAI,GAAG,UAAU,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC;EACxF,MAAM,IAAI,OAAO,MAAM,UAAU,MAAM,WAAW,IAAI,CAAC;EACvD,IAAI,MAAM,KAAA,KAAa,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,GAAG,OAAO;EACxD,OACE,UAAU,SACV,UAAU,QACT,MAAM,SAAS,IAAI,KAAK,KAAK,WAAW,MAAM,MAAM,GAAG,EAAE,CAAC;CAE/D,CAAC;AACH;;AAGA,MAAM,eAAe,MAA8B,gBACjD,SAAS,KAAA,KAAa,CAAC,QAAQ,KAAK,eAAe,EAAE,IACjD,CAAC,IACD,YAAY,IAAI,YAAY,QAAQ,CAAC,CAAC,OAAO,KAAK,SAAS,GAAG,KAAM,CAAC,CAAC;;AAG5E,IAAM,QAAN,MAAY;CAImB;CAH7B,OAAe;CACf,UAA8C,CAAC;CAE/C,YAAY,KAA8B;EAAb,KAAA,MAAA;CAAc;;CAG3C,MAAM,QAAQ,QAA0C;EACtD,IAAI,KAAK,OAAO,KAAK,KAAK;GACxB,KAAK,QAAQ;GACb,OAAO,KAAK,SAAS;EACvB;EACA,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,aAAmB;IACvB,MAAM,KAAK,KAAK,QAAQ,QAAQ,EAAE;IAClC,IAAI,OAAO,IAAI,KAAK,QAAQ,OAAO,IAAI,CAAC;IACxC,OAAO,OAAO,kBAAkB,QAAQ,OAAO,yBAAS,IAAI,MAAM,WAAW,CAAC;GAChF;GACA,MAAM,WAAiB;IACrB,OAAO,oBAAoB,SAAS,IAAI;IACxC,QAAQ;GACV;GACA,IAAI,OAAO,SAAS;IAClB,KAAK;IACL;GACF;GACA,KAAK,QAAQ,KAAK,EAAE;GACpB,OAAO,iBAAiB,SAAS,MAAM,EAAE,MAAM,KAAK,CAAC;EACvD,CAAC;EAED,OAAO,KAAK,SAAS;CACvB;CAEA,WAA+B;EAC7B,IAAI,QAAQ;EACZ,aAAa;GACX,IAAI,OAAO;GACX,QAAQ;GACR,MAAM,OAAO,KAAK,QAAQ,MAAM;GAChC,IAAI,SAAS,KAAA,GAAW,KAAK;QACxB,KAAK,QAAQ;EACpB;CACF;AACF;;AAGA,MAAM,kBAAkB,YACtB,QAAQ,WAAA,iQAAwB,KAChC,4EAA4E,KAAK,OAAO;;AAG1F,MAAM,iBAAiB,QAAgB,OACrC,OAAO,gBAAgB,KAAA,IACnB,+BAA+B,OAC/B,iCAAiC,OAAO,YAAY;AAE1D,MAAMC,eAAa,UACjB,iBAAiB,QAAS,MAAM,QAAQ,MAAM,OAAO,CAAC,CAAC,MAAM,KAAM,OAAO,KAAK;AAEjF,MAAM,SAAS,OACb,OAAO,SAAS,EAAE,KAAK,KAAK,IAAI,EAAE,IAAI,eAAe,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY,IAAI,KAAA;AAEpF,MAAM,YAAY,QAAgB,UAA0B;CAC1D,MAAM,MAAM,MAAM,KAAK;CACvB,OAAO,QAAQ,KAAA,IAAY,iBAAiB,WAAW,sBAAsB,IAAI,IAAI;AACvF;;AAGA,IAAM,kBAAN,cAA8B,MAAM;CAGvB;CAFX,YAEE,SACA;EACA,MAAM,aAAa;EAFV,KAAA,UAAA;CAGX;AACF;AAEA,MAAM,gBAAgB,OAAwB,aAC5C,GAAG,cAAc,wBAAwB,MAAM,MAAM,OAAO,KAAK,eAAe,sCAAsC,MAAM,YAAY,GAAU,KAAK;AAEzJ,MAAM,aAAa,iBACjB,IAAI,IAAI,CAAC,GAAG,uBAAuB,GAAG,YAAY,CAAC,CAAC,KAAK,SAAS,KAAK,YAAY,CAAC,CAAC;AAEvF,MAAM,eAAe,QAAwB;CAC3C,IAAI;EACF,OAAO,mBAAmB,IAAI,QAAQ,OAAO,GAAG,CAAC,CAAC,CAAC,YAAY;CACjE,QAAQ;EACN,OAAO,IAAI,YAAY;CACzB;AACF;;;;;;AAOA,MAAa,UAAU,KAAU,eAAkC,CAAC,MAAc;CAChF,MAAM,OAAO,IAAI,IAAI,IAAI,IAAI;CAC7B,KAAK,WAAW;CAChB,KAAK,WAAW;CAChB,IAAI,KAAK,OAAO,SAAS,GAAG;EAC1B,MAAM,UAAU,UAAU,YAAY;EAUtC,KAAK,SAAS,IATA,KAAK,OAChB,MAAM,CAAC,CAAC,CACR,MAAM,GAAG,CAAC,CACV,KAAK,SAAS;GACb,MAAM,SAAS,KAAK,QAAQ,GAAG;GAC/B,IAAI,WAAW,IAAI,OAAO;GAC1B,MAAM,OAAO,KAAK,MAAM,GAAG,MAAM;GACjC,OAAO,QAAQ,IAAI,YAAY,IAAI,CAAC,IAAI,GAAG,KAAK,aAAa;EAC/D,CACoB,CAAC,CAAC,KAAK,GAAG;CAClC;CACA,OAAO,KAAK;AACd;;AAGA,MAAM,gBAAgB,KAAU,iBAA6C;CAC3E,IAAI,IAAI,aAAa,MAAM,IAAI,aAAa,IAAI,OAAO;CACvD,MAAM,UAAU,UAAU,YAAY;CACtC,OAAO,IAAI,OACR,MAAM,CAAC,CAAC,CACR,MAAM,GAAG,CAAC,CACV,MACE,SAAS,KAAK,SAAS,GAAG,KAAK,QAAQ,IAAI,YAAY,KAAK,MAAM,GAAG,KAAK,QAAQ,GAAG,CAAC,CAAC,CAAC,CAC3F;AACJ;;AAGA,MAAM,WAAW,QACf,GAAG,IAAI,SAAS,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE,IAAI,IAAI,SAAS,KAAK,KAAK,IAAI,IAAI;AAEpF,MAAM,aAAa,QAAqB,GAAG,IAAI,SAAS,IAAI,QAAQ,GAAG;AAEvE,MAAM,YACH,aACA,SAAqC;CACpC,MAAM,QAAQ,QAAQ,KAAK,YAAY;CACvC,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,KAAK,IAAI,IAAI;AACnD;AAEF,IAAM,MAAN,MAAgB;CAEe;CAD7B,sBAAuB,IAAI,IAAU;CACrC,YAAY,KAA8B;EAAb,KAAA,MAAA;CAAc;CAE3C,IAAI,KAAuB;EACzB,MAAM,QAAQ,KAAK,IAAI,IAAI,GAAG;EAC9B,IAAI,UAAU,KAAA,GAAW;GACvB,KAAK,IAAI,OAAO,GAAG;GACnB,KAAK,IAAI,IAAI,KAAK,KAAK;EACzB;EACA,OAAO;CACT;CAEA,KAAK,KAAuB;EAC1B,OAAO,KAAK,IAAI,IAAI,GAAG;CACzB;CAEA,IAAI,KAAQ,OAAgB;EAC1B,KAAK,IAAI,OAAO,GAAG;EACnB,KAAK,IAAI,IAAI,KAAK,KAAK;EACvB,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,KAAK,IAAI,OAAO,KAAK,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAU;CACjF;CAEA,OAAO,KAAc;EACnB,KAAK,IAAI,OAAO,GAAG;CACrB;CAEA,OAAY;EACV,OAAO,CAAC,GAAG,KAAK,IAAI,KAAK,CAAC;CAC5B;CAEA,IAAI,OAAe;EACjB,OAAO,KAAK,IAAI;CAClB;AACF;;AAGA,IAAM,YAAN,cAAwB,MAAM;CAEjB;CADX,YACE,MACA,OACA;EACA,MAAMA,YAAU,KAAK,GAAG,EAAE,MAAM,CAAC;EAHxB,KAAA,OAAA;CAIX;AACF;;AAGA,MAAM,mBAA8B;CAClC,IAAI;CACJ,MAAM,QAAmB,IAAI,UAAU;EACrC,UAAU,OAAe,WAAW,MAAM;GACxC,IAAI,UAAU,KAAA,GAAW;IAKvB,QAHE,MAAM,UAAU,MACf,MAAM,KAAM,QAAU,MACrB,MAAM,MAAO,IAAK,MAAM,MAAO,OAAO,IAC3B,cAAc,IAAI,iBAAiB;IAClD,MAAM,GAAG,SAAS,SAAiB,MAAM,KAAK,IAAI,CAAC;IACnD,MAAM,GAAG,UAAU,UAAU,MAAM,QAAQ,KAAK,CAAC;GACnD;GACA,MAAM,MAAM,aAAa,KAAK,CAAC;EACjC;EACA,MAAM,MAAM;GACV,IAAI,UAAU,KAAA,GAAW;IACvB,KAAK;IACL;GACF;GACA,MAAM,KAAK,aAAa,KAAK,CAAC;GAC9B,MAAM,IAAI;EACZ;EACA,QAAQ,OAAO,UAAU;GACvB,OAAO,QAAQ;GACf,SAAS,KAAK;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAEA,MAAM,cAAc,WAA8B;CAChD,IAAI,WAAW,UAAU,WAAW,UAAU,OAAO,aAAa;CAClE,IAAI,WAAW,MAAM,OAAO,uBAAuB;CACnD,IAAI,WAAW,WAAW,OAAO,WAAW;CAC5C,MAAM,IAAI,UAAU,0BAAU,IAAI,MAAM,iCAAiC,OAAO,EAAE,CAAC;AACrF;;AAGA,MAAM,cAAc,UAClB,OAAO,UAAU,YACjB,UAAU,QACV,UAAU,UACT,MAAM,SAAS,iBAAiB,0BAA0B,KAAKA,YAAU,KAAK,CAAC;;;;;;;AAQlF,MAAM,WAAW,OACf,MACA,UACA,aACgG;CAChG,MAAM,WAAW,YAAY,GAAA,CAC1B,MAAM,GAAG,CAAC,CACV,KAAK,WAAW,OAAO,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAC5C,QAAQ,WAAW,OAAO,SAAS,KAAK,CAAC,iBAAiB,IAAI,MAAM,CAAC,CAAC,CACtE,QAAQ;CAEX,KAAK,GAAG,eAAe,KAAA,CAAS;CAChC,IAAI;CACJ,IAAI;EACF,WAAW,QAAQ,IAAI,UAAU;CACnC,SAAS,OAAO;EACd,KAAK,QAAQ;EACb,MAAM;CACR;CAEA,MAAM,SAAS,WAAW,IAAI;CAC9B,IAAI,WAAW;CACf,IAAI,SAAmB;CACvB,IAAI,SAAS,SAAS,GAAG;EACvB,KAAK,GAAG,SAAS,UAAkB;GACjC,YAAY,MAAM;EACpB,CAAC;EACD,KAAK,MAAM,WAAW,UAAU;GAC9B,MAAM,SAAS;GACf,OAAO,GAAG,UAAU,UAAU,QAAQ,QAAQ,KAAK,CAAC;GACpD,SAAS,OAAO,KAAK,OAAO;EAC9B;CACF;CACA,MAAM,SAAmB,CAAC;CAC1B,IAAI,OAAO;CACX,IAAI,YAAY;CAChB,IAAI,MAAM;CACV,IAAI;EACF,WAAW,MAAM,SAAS,QAAiC;GACzD,MAAM,OAAO,WAAW;GACxB,IAAI,MAAM,SAAS,MAAM;IAEvB,IAAI,OAAO,GAAG,OAAO,KAAK,MAAM,SAAS,GAAG,IAAI,CAAC;IACjD,QAAQ,KAAK,IAAI,GAAG,IAAI;IACxB,YAAY;IACZ;GACF;GACA,OAAO,KAAK,KAAK;GACjB,QAAQ,MAAM;GACd,IAAI,WAAW,QAAQ;IACrB,YAAY;IACZ;GACF;EACF;CACF,SAAS,OAAO;EACd,MAAM,YAAY,KAAK,YAAY,QAAQ,SAAS,WAAW;EAE/D,IAAI,CAAC,aAAa,OAAO,KAAK,WAAW,KAAK,GAAG;GAC/C,MAAM;GACN,YAAY;EACd,OACE,MAAM,IAAI,UAAU,YAAY,cAAc,UAAU,KAAK;CAEjE,UAAU;EACR,KAAK,QAAQ;EACb,KAAK,MAAM,WAAW,UAAU,QAAQ,QAAQ;CAClD;CAEA,MAAM,QAAQ,IAAI,WAAW,IAAI;CACjC,IAAI,KAAK;CACT,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,IAAI,OAAO,EAAE;EACnB,MAAM,MAAM;CACd;CACA,OAAO;EAAE;EAAO;EAAW;CAAI;AACjC;AAEA,MAAM,cAAc,UAAmD;CACrE,MAAM,sBAAM,IAAI,IAAoB;CACpC,KAAK,MAAM,SAAS,SAAS,GAAA,CAAI,MAAM,GAAG,GAAG;EAC3C,MAAM,SAAS,KAAK,QAAQ,GAAG;EAC/B,MAAM,QAAQ,WAAW,KAAK,OAAO,KAAK,MAAM,GAAG,MAAM,EAAA,CAAG,KAAK,CAAC,CAAC,YAAY;EAC/E,IAAI,KAAK,WAAW,GAAG;EACvB,MAAM,MAAM,WAAW,KAAK,KAAK,KAAK,MAAM,SAAS,CAAC,CAAC,CAAC,KAAK;EAC7D,IAAI,IACF,MACA,IAAI,UAAU,KAAK,IAAI,WAAW,IAAG,KAAK,IAAI,SAAS,IAAG,IAAI,IAAI,MAAM,GAAG,EAAE,IAAI,GACnF;CACF;CACA,OAAO;AACT;AAEA,MAAM,eAAe;;;;;;;AAQrB,MAAM,cACJ,QACA,QACA,WACA,cACA,UACkB;CAClB,MAAM,UAAU,WAAW,OAAO,eAAe,CAAC;CAClD,IAAI,QAAQ,IAAI,UAAU,KAAK,QAAQ,IAAI,UAAU,GAAG,OAAO,MAAM,SAAS,KAAK;CACnF,MAAM,UAAU,OAAO,KAAK,CAAC,EAAE,KAAK,KAAK;CACzC,MAAM,MAAM,QAAQ,KAAK,OAAO,IAAI,OAAO,OAAO,IAAI,MAAO;CAC7D,MAAM,OAAO,cAAc,OAAO,MAAM,CAAC,KAAK;CAC9C,IAAI;CACJ,MAAM,SAAS,QAAQ,IAAI,SAAS;CACpC,MAAM,UAAU,OAAO,SAAS;CAChC,IAAI,WAAW,KAAA,KAAa,QAAQ,KAAK,MAAM,GAAG,WAAW,OAAO,MAAM,IAAI;MACzE,IAAI,YAAY,KAAA,GAAW;EAE9B,MAAM,KAAK,cAAc,OAAO;EAChC,WAAW,OAAO,KAAA,IAAY,IAAI,KAAK;CACzC,OAAO,IAAI,mBAAmB,IAAI,MAAM,GAAG;EACzC,MAAM,WAAW,cAAc,gBAAgB,KAAA,CAAS;EACxD,IAAI,aAAa,KAAA,KAAa,WAAW,MACvC,WAAW,KAAK,IAAI,QAAQ,OAAO,YAAY,EAAE;OAC9C,IAAI,oBAAoB,IAAI,MAAM,GAAG,WAAW;CACvD;CACA,IAAI,aAAa,KAAA,GAAW,OAAO;CACnC,OAAO,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,UAAU,YAAY,IAAI,GAAG,CAAC,KAAK;AACnF;;AAGA,MAAM,aACJ,MACA,SACkB;CAClB,IAAI,SAAS,KAAA,GAAW,OAAO;CAC/B,MAAM,QAAQ,KACX,MAAM,GAAG,CAAC,CACV,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CACxC,QAAQ,SAAS,KAAK,SAAS,KAAK,CAAC,iBAAiB,IAAI,IAAI,CAAC;CAClE,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO;CAChC,IAAI,MAAM,WAAW,GAAG,OAAO;CAC/B,OAAO,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,CAAC,CACvB,KAAK,CAAC,CACN,KAAK,SAAS,GAAG,KAAK,GAAG,KAAK,SAAS,IAAI,CAAC,CAC5C,KAAK,IAAI;AACd;AAEA,MAAM,eACJ,QACA,SACY;CACZ,IAAI,WAAW,KAAA,KAAa,WAAW,MAAM,OAAO;CACpD,IAAI,WAAW,KAAK,OAAO;CAC3B,OAAO,OAAO,MAAM,IAAI,CAAC,CAAC,OAAO,SAAS;EACxC,MAAM,SAAS,KAAK,QAAQ,GAAG;EAC/B,QAAQ,KAAK,KAAK,MAAM,GAAG,MAAM,MAAM,QAAQ,KAAK,MAAM,SAAS,CAAC;CACtE,CAAC;AACH;AAEA,MAAM,sBACJ,YAEA,YAAY,KAAA,IACR,KAAA,IACA,OAAO,YACL,OAAO,QAAQ,OAAO,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,IAAI,KAAK,YAAY,CAAC,CAAC,CACxF;AAEN,MAAM,iBACJ,MACA,OACA,SACuB;CACvB,IAAI,UAAU,KAAA,KAAc,QAAQ,UAAU,QAAS,OAAO,KAAA;CAC9D,OAAO,OAAO,UAAU,YAAY,SAAS,IACzC,KAAA,IACA,mBAAmB,KAAK,MAAM,OAAO,KAAK;AAChD;AAEA,MAAM,gBAAgB,SAA0B;CAC9C;CACA,QAAQ;CACR,SAAS;CACT,WAAW;CACX,SAAS,KAAA;CACT,MAAM,KAAA;CACN,MAAM,KAAA;CACN,aAAa,KAAA;CACb,SAAS;CACT,MAAM;AACR;AAgGA,MAAM,eACJ,SACA,QACA,cAC6B;CAC7B,SAAS,QAAQ,WAAW;CAC5B;CACA,MACE,SAAS,SAAS,KAAA,IACd,OACA;EACE,MAAM,SAAS,KAAK,QAAQ;EAC5B,SAAS,SAAS,KAAK;EACvB,OAAO,SAAS,KAAK;CACvB;CACN,YAAY,SAAS,SAAS,EAAE,EAAE,QAAQ;CAC1C,GAAI,QAAQ,OAAO,gBAAgB,KAAA,IAAY,EAAE,aAAa,QAAQ,OAAO,YAAY,IAAI,CAAC;CAC9F,GAAI,QAAQ,UAAU,KAAA,IAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAChE;AAEA,MAAM,aAAgB,MAAkB,WAAoC;CAC1E,IAAI,OAAO,SAAS;EAClB,KAAK,YAAY,KAAA,CAAS;EAC1B,OAAO,QAAQ,OAAO,OAAO,MAAM;CACrC;CACA,OAAO,IAAI,SAAY,SAAS,WAAW;EACzC,MAAM,eAAqB,OAAO,OAAO,MAAM;EAC/C,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;EACvD,KAAK,MACF,UAAU;GACT,OAAO,oBAAoB,SAAS,MAAM;GAC1C,QAAQ,KAAK;EACf,IACC,UAAmB;GAClB,OAAO,oBAAoB,SAAS,MAAM;GAC1C,OAAO,KAAK;EACd,CACF;CACF,CAAC;AACH;;AAUA,IAAa,UAAb,MAAqB;CA+BU;CA9B7B;CACA;CACA;CACA;CACA;CACA;CACA,SAA0B,IAAI,IAAyB,IAAI;;CAE3D,OAAwB,IAAI,IAG1B,GAAM;CACR,2BAA4B,IAAI,IAA2B;CAC3D,0BAA2B,IAAI,IAAsB;CACrD,UAA2B,IAAI,gBAAgB;CAC/C,SAAiB;CACjB;;CAEA;;CAEA;;CAEA;;CAEA,4BAA6B,IAAI,QAA6B;;CAG9D;CAGA,YAAY,SAA0C;EAAzB,KAAA,UAAA;EAE3B,KAAK,SAAS,SAAS,QAAQ,MAAM;EACrC,IAAI,CAAC,eAAe,QAAQ,KAAK,GAC/B,MAAM,IAAI,MACR,cAAc,QAAQ,MAAM,mEAC9B;EAEF,KAAK,MAAM,CAAC,MAAM,UAAU,CAC1B,CAAC,aAAa,QAAQ,SAAS,GAC/B,CAAC,QAAQ,QAAQ,IAAI,CACvB,GACE,IAAI,UAAU,KAAA,KAAa,CAAC,YAAY,KAAK,KAAK,GAChD,MAAM,IAAI,MACR,GAAG,KAAK,IAAI,MAAM,uEACpB;EAGJ,YAAY,QAAQ,UAAU;EAC9B,KAAK,QAAQ,QAAQ,SAAS;EAC9B,KAAK,SAAS,QAAQ,UAAU,KAAK;EACrC,KAAK,SAAS,QAAQ,UAAU,CAAC;EAGjC,MAAM,QAAQ,KAAK;EACnB,MAAM,YAAY,KAAK;EACvB,KAAK,YAAY,IAAI,cACnB;GACE,WAAW,MAAM,IAAI;GACrB,QAAQ,IAAI,WAAW;IACrB,MAAM,WAAW,WAAW,KAAA,IAAY,KAAA,IAAY,UAAU,IAAI,MAAM;IACxE,MAAM,UAAU,MAAM,IAAI,IAAI;IAC9B,OAAO,aAAa,KAAA,KAAa,WAAW,WACxC,QAAQ,OAAO,IAAI,gBAAgB,OAAO,CAAC,IAC3C,MAAM,MAAM,IAAI,MAAM;GAC5B;EACF,GACA,QAAQ,aAAa,IACrB,EAAE,OAAO,QAAQ,UAAU,CAC7B;EAEA,KAAK,aACH,QAAQ,qBAAqB,SAC7B,QAAQ,MAAM,YAAY,MAAA,gBAAoB,YAAY,KAC1D,QAAQ,UAAU,SAAA,8CAAiB,IAC/B,CAAC,aAAa,IACd,CAAC;EACP,MAAM,aAAa,QAAQ,cAAc;EACzC,IAAI,CAAC,OAAO,UAAU,UAAU,KAAK,aAAa,GAChD,MAAM,IAAI,WAAW,mDAAmD,OAAO,UAAU,GAAG;EAE9F,KAAK,UAAU,IAAI,MAAM,UAAU;EACnC,KAAK,iBAAiB,QAAQ,eAAe,KAAA;EAC7C,KAAK,aACH,QAAQ,cACR,IAAI,MAAM;GAER,kBAAkB;GAClB,qBAAqB;GACrB,SAAS;GAET,SAAS;IAAE,SAAS;IAAQ,QAAQ,aAAa,KAAK,MAAM;GAAE;GAC9D,gBAAgB;GAChB,aAAa;EACf,CAAC;CACL;CAEA,MAAsB;EACpB,OAAO,IAAI,KAAK,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,YAAY;CAChD;CAEA,SAAiB,KAA4B;EAC3C,OAAO,UAAU,KAAK,QAAQ,YAAY,IAAI,QAAQ;CACxD;;;;;CAMA,MAAc,QAAqB,UAA2C;EAC5E,IAAI,aAAa,KAAA,GAAW,OAAO;EACnC,MAAM,MAAM,YAAY,IAAI,CAAC,MAAM,CAAC;EACpC,KAAK,UAAU,IAAI,KAAK,QAAQ;EAChC,OAAO;CACT;;CAGA,eAAuB,UAAoC;EACzD,MAAM,MAAM,KAAK,MAAM,IAAI;EAC3B,IAAI,aAAa,KAAA,KAAa,OAAO,UAAU,MAAM,IAAI,gBAAgB,GAAG;CAC9E;CAEA,KAAa,OAAyB;EACpC,IAAI;GACF,KAAK,QAAQ,UAAU,KAAK;EAC9B,QAAQ,CAER;CACF;CAEA,MAAiB,MAA8B;EAC7C,KAAK,QAAQ,IAAI,IAAI;EACrB,KAAU,WACF,KAAK,QAAQ,OAAO,IAAI,SACxB,KAAK,QAAQ,OAAO,IAAI,CAChC;EACA,OAAO;CACT;;CAGA,OAAe,MAAyC;EACtD,IAAI;GACF,OAAO,KAAK,UAAU,MAAM,aAAa,IAAI;EAC/C,QAAQ;GACN;EACF;CACF;CAEA,SAAiB,MAAc,OAAe,QAAsB;EAClE,KAAK,UAAU,MAAM,KAAK,MAAM,OAAO,MAAM;EAC7C,KAAK,KAAK;GAAE,MAAM;GAAQ;GAAM;GAAO;EAAO,CAAC;CACjD;CAEA,MAAc,OACZ,MACA,SACA,KACA,SAGI,CAAC,GACwB;EAE7B,IAAI,KAAK,QAAQ,OAAO,KAAA;EACxB,MAAM,KAAK,KAAK,IAAI;EACpB,MAAM,QAAQ,OAAO,SAAS,KAAK,QAAQ;EAC3C,MAAM,SAAS,WAAW;GACxB,QAAQ;GACR;GACA,SAAS;IAAE,QAAA;IAAwB,GAAG;IAAS,YAAY;GAAG;GAC9D,aAAa;GACb,aAAa;GACb,aAAa;GACb,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;GACvC,GAAI,OAAO,WAAW,KAAA,IAAY,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;EACjE,CAAC;EACD,MAAM,KAAK,QAAQ,IAAI,OAAO,MAAM;EACpC,OAAO,OAAO;CAChB;CAEA,MAAc,KACZ,MACA,KACA,MACA,QAAiC,CAAC,GAClC,SAGI,CAAC,GACa;EAClB,KAAK,KAAK;GAAE,MAAM;GAAQ;GAAK;EAAK,CAAC;EACrC,OAAO;GACL;GACA,QAAQ;GACR,SAAS;GACT,WAAW;GACX,SAAS,MAAM,KAAK,OAAO,WAAW;IAAE,GAAG;IAAM;IAAK;IAAM,GAAG;GAAM,GAAG,KAAK,MAAM;GACnF,MAAM,KAAA;GACN,MAAM,KAAA;GACN,aAAa,KAAA;GACb,SAAS;GACT;EACF;CACF;;CAGA,OAAe,KAAa,UAAmB,SAAiC;EAC9E,MAAM,MAAM,WAAW,GAAG;EAC1B,MAAM,MAAM,KAAK,QAAQ,MAAM,OAAO,KAAK,OAAO;EAClD,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;EAC9B,MAAM,OAAO,KAAK,KAAK,IAAI,GAAG,QAAQ,GAAG,KAAK;EAC9C,IAAI,SAAS,KAAA,KAAa,KAAK,MAAM,YAAY,IAAI,SAAS,OAAO;EACrE,MAAM,OAAO,OAAO,YAAY,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,WAAW,UAAU,KAAA,CAAS,CAAC;EAC9F,OAAO;GAAE,GAAG,KAAK;GAAO,GAAG;EAAK;CAClC;CAEA,MAAc,OAAmB,UAAyB;EACxD,MAAM,UAAU,MAAM,WAAW;EACjC,MAAM,QAAQ;GAAE,GAAG;GAAO,KAAK,WAAW,MAAM,GAAG;GAAG;EAAQ;EAC9D,KAAK,QAAQ,MAAM,UAAU,KAAK;EAClC,KAAK,KAAK,IAAI,GAAG,QAAQ,GAAG,MAAM,OAAO;GACvC,OAAO;GACP,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;EAC/C,CAAC;CACH;;CAGA,WAAmB,OAAmB,cAAqD;EACzF,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,MAAM,WAAW,QAAQ,GAAG,WAAW,MAAM,GAAG,GAAG;EAClF,IAAI,MAAM,aAAa,KAAA,KAAa,KAAK,MAAM,YAAY,MAAM,SAAS,OAAO,KAAK;EACtF,IAAI,MAAM,aAAa,KAAA,KAAa,MAAM,aAAa,MAAM,OAAO,KAAA;EACpE,IAAI;GACF,MAAM,SAAS,IAAI,IAAI,MAAM,QAAQ;GAErC,OAAO,aAAa,QAAQ,YAAY,IAAI,KAAA,IAAY,OAAO;EACjE,QAAQ;GACN;EACF;CACF;;CAGA,MAAc,UACZ,KACA,SACA,QACA,MACA,cAC2B;EAC3B,IAAI,WAAW,KAAA,GAAW,OAAO,KAAA;EACjC,MAAM,QAAQ,KAAK,OAAO,KAAK,OAAO;EACtC,IAAI,UAAU,KAAA,KAAa,CAAC,YAAY,MAAM,UAAU,IAAI,GAAG,OAAO,KAAA;EACtE,MAAM,MAAM,KAAK,MAAM,IAAI;EAC3B,MAAM,YAAY,KAAK,MAAM,MAAM,UAAU;EAC7C,IAAI,CAAC,OAAO,SAAS,SAAS,GAAG,OAAO,KAAA;EACxC,MAAM,MAAM,MAAM;EAClB,MAAM,QACJ,MAAM,gBAAgB,KAAA,KAAa,MAAM,gBAAgB,OACrD,MACA,KAAK,MAAM,MAAM,WAAW;EAClC,MAAM,QAAQ,OAAO,SAAS,KAAK,KAAK,QAAQ;EAChD,MAAM,WAAW,MAAM,UAAU,OAAO,MAAM,SAAS;EAQvD,IAAI,EALF,WAAW,SACP,QACA,YAAY,CAAC,oBAAoB,IAAI,MAAM,MAAM,IAC/C,SAAS,MAAM,SACf,MAAM,SACF,OAAO,KAAA;EACnB,IAAI,UAAU;GACZ,MAAM,OAAO,KAAK,WAAW,OAAO,YAAY;GAChD,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;GAC/B,OAAO;IACL,SAAS;KACP;KACA,QAAQ,MAAM;KACd,SAAS;KACT,WAAW;KACX,SAAS,MAAM;KACf,MAAM,KAAA;KACN,MAAM,KAAA;KACN,aAAa,KAAA;KACb,SAAS;KACT,MAAM,YAAY,MAAM,YAAY,OAAO,IAAI,IAAI,IAAI,GAAG,YAAY;KACtE,WAAW,MAAM;IACnB;IACA;GACF;EACF;EACA,IAAI,MAAM,SAAS,QAAQ,MAAM,SAAS,OAAO,MAAM,UAAU,KAAK,OAAO,KAAA;EAC7E,MAAM,OAAO,MAAM,KAAK,QAAQ,MAAM,IAAI,MAAM,IAAI;EACpD,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;EAC/B,OAAO,EACL,SAAS;GACP;GACA,QAAQ,MAAM;GACd,SAAS;GACT,WAAW;GACX,SAAS,MAAM;GACf,MAAM,MAAM;GACZ;GACA,aAAa,MAAM,gBAAgB,KAAA;GACnC,SAAS;IAAE,UAAU,MAAM,YAAY;IAAS,KAAK,CAAC;GAAE;GACxD,MAAM,MAAM,cAAc,OAAO,qBAAqB,KAAA;GACtD,GAAI,MAAM,cAAc,OAAO,EAAE,WAAW,KAAK,IAAI,CAAC;GACtD,WAAW,MAAM;EACnB,EACF;CACF;;;;;;CAQA,QAAgB,QAMF;EACZ,IAAI,KAAK,WAAW,KAAA,GAAW,OAAO,KAAA;EACtC,IAAI;EACJ,IAAI;GACF,QAAQ,KAAK,OAAO,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC;EACnD,SAAS,OAAO;GACd,OAAO,EAAE,OAAO,wCAAwCA,YAAU,KAAK,IAAI;EAC7E;EAEA,MAAM,UAAkC,CAAC;EACzC,KAAK,MAAM,CAAC,MAAM,UAAU,OAAO,QAAQ,KAAK,GAAG;GACjD,MAAM,QAAQ,KAAK,YAAY;GAC/B,IAAI,iBAAiB,IAAI,KAAK,GAAG;GACjC,IAAI,OAAO,UAAU,YAAY,CAAC,YAAY,KAAK,KAAK,GACtD,OAAO,EAAE,OAAO,qBAAqB,MAAM,+BAA+B;GAC5E,QAAQ,SAAS;EACnB;EACA,OAAO;GACL;GACA,OAAO,EAAE,QAAQ;IAAE,QAAQ,KAAK,OAAO;IAAO,OAAO,KAAK,OAAO;GAAe,EAAE;EACpF;CACF;CAGA,eAAuB,UAAgD;EACrE,MAAM,UAAkC,CAAC;EAEzC,KAAK,MAAM,CAAC,MAAM,UAAU,OAAO,QAAQ,SAAS,WAAW,CAAC,CAAC,GAAG;GAClE,MAAM,QAAQ,KAAK,YAAY;GAC/B,IAAI,iBAAiB,IAAI,KAAK,GAAG;GAEjC,IAAI,KAAK,WAAW,KAAA,KAAa,kBAAkB,IAAI,KAAK,GAAG;GAC/D,QAAQ,SAAS;EACnB;EACA,QAAQ,YAAY,SAAS,UAAU,QAAQ,aAAa;EAC5D,QAAQ,gBAAgB,KAAK,QAAQ;EACrC,QAAQ,qBAAqB;EAC7B,IAAI,KAAK,QAAQ,SAAS,KAAA,GAAW,QAAQ,UAAU,KAAK,QAAQ;EACpE,OAAO;CACT;;;;;;CAOA,OAAe,MAAc,QAA0B,MAAsB;EAC3E,MAAM,QAAQ,KAAK,UAAU;EAC7B,MAAM,MAAM,KAAK,MAAM,IAAI;EAE3B,MAAM,UADO,UAAU,QAAQ,KAAK,IACjB,CAAC,CAAC;EACrB,MAAM,UAAU,IAAY,WAA2B;GACrD,IAAI,KAAK,SAAS,KAAK,KAAK;IAAE,MAAM;IAAQ;IAAM;IAAI;GAAO,CAAC;GAC9D,OAAO;EACT;EACA,IAAI,KAAK,WAAW,KAAA,GAAW;GAC7B,MAAM,OAAO,MAAM,KAAK,MAAM;GAC9B,OAAO;EACT;EACA,MAAM,UAAU,KAAK;EACrB,IAAI,YAAY,kBAAkB,YAAY,eAAe;GAC3D,MAAM,EAAE,UAAU,MAAM,OAAO,MAAM,MAAM;GAC3C,MAAM,QAAQ,KAAK;GACnB,IAAI,UAAU,KAAA,KAAa,QAAQ,OAAO,iBAAiB;IAEzD,KAAK,SACH,MACA,MAAM,KAAK,IAAI,OAAO,WAAW,GACjC,iBAAiB,OAAO,KAAK,MAAM,QAAQ,GAAI,CAAC,EAAE,GACpD;IACA,OAAO;GACT;GACA,OAAO,OACL,KAAK,IACH,SACA,SAAS,cAAc,OAAO,OAAO,eAAe,OAAO,cAAc,KAAK,MAAM,CACtF,GACA,UAAU,KAAA,IAAY,YAAY,aACpC;EACF;EACA,IAAI,YAAY,SAAS;GACvB,IAAI,cAAc,KAAK,MAAM,GAAG;IAC9B,MAAM,OAAO,MAAM,SAAS;IAC5B,OAAO;GACT;GACA,MAAM,EAAE,aAAa,MAAM,OAAO,MAAM,SAAS;GACjD,IAAI,YAAY,OAAO,aACrB,KAAK,SAAS,MAAM,MAAM,OAAO,eAAe,GAAG,OAAO,QAAQ,EAAE,mBAAmB;GACzF,OAAO,OACL,KAAK,IACH,SACA,cAAc,UAAU,OAAO,eAAe,OAAO,cAAc,KAAK,MAAM,CAChF,GACA,SACF;EACF;EACA,IAAI,YAAY,aAAa,KAAK,SAAS,MAAM,MAAM,OAAO,iBAAiB,OAAO;OACjF,IAAI,YAAY,oBACnB,KAAK,SAAS,MAAM,MAAM,OAAO,eAAe,OAAO;OACpD,IAAI,YAAY,aAAa,CAAC,KAAK,YACtC,KAAK,SAAS,MAAM,MAAM,OAAO,eAAe,OAAO;EACzD,MAAM,OAAO,MAAM,SAAS;EAC5B,OAAO;CACT;;;;;CAMA,cAAc,KAAa,MAAoB;EAC7C,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,GAAG;EACtB,QAAQ;GACN;EACF;EACA,MAAM,SAAS,KAAK,SAAS,MAAM;EACnC,KAAK,SACH,QAAQ,MAAM,GACd,KAAK,MAAM,IAAI,IAAI,OAAO,iBAC1B,cAAc,KAAK,EACrB;CACF;;;;;;;CAQA,MAAc,MACZ,MACA,OAGsE;EACtE,MAAM,QAAQ;GAAE,OAAO,KAAK;GAAO,QAAQ,KAAK;EAAO;EACvD,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,KAAK,GAAG;EAC3B,QAAQ;GACN,OAAO,EACL,QAAQ,EAAE,SAAS,MAAM,KAAK,KAAK,KAAK,MAAM,KAAK,KAAK,mBAAmB,CAAC,GAAG,KAAK,EAAE,EACxF;EACF;EACA,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GACpC,OAAO,EACL,QAAQ,EACN,SAAS,MAAM,KAAK,KAClB,KAAK,MACL,OAAO,QAAQ,KAAK,YAAY,GAChC,sBACA,CAAC,GACD,KACF,EACF,EACF;EAGF,OAAO,OAAO;EACd,MAAM,MAAM,OAAO,QAAQ,KAAK,YAAY;EAC5C,MAAM,QAAQ,MAAc,WAC1B,KAAK,KAAK,KAAK,MAAM,KAAK,MAAM,QAAQ,KAAK;EAC/C,MAAM,SAAS,OAAO,MAAc,YAAsC,EACxE,QAAQ,EAAE,SAAS,MAAM,KAAK,MAAM,MAAM,EAAE,EAC9C;EACA,IAAI,KAAK,YAAY,KAAA,GAAW,OAAO,OAAO,KAAK,OAAO;EAC1D,MAAM,UAAU,cAAc,KAAK,QAAQ,OAAO,QAAQ;EAC1D,IAAI,YAAY,KAAA,GAAW,OAAO,OAAO,OAAO;EAChD,MAAM,SAAS;GAAC,KAAK,QAAQ;GAAO,GAAI,KAAK,cAAc,CAAC;GAAI,GAAG,KAAK;EAAU;EAClF,MAAM,WAAW,OAAO,MAAM,UAAU,CAAC,eAAe,KAAK,CAAC;EAC9D,IAAI,KAAK,SAAS,UAAU,aAAa,KAAA,GACvC,OAAO,OAAO,+BAA+B,SAAS,EAAE;EAC1D,MAAM,OAAO,QAAQ,MAAM;EAC3B,MAAM,SAAS,KAAK,SAAS,MAAM;EACnC,MAAM,OAAO,GAAG,OAAO,WAAW,OAAO;EAGzC,IAAI;EACJ,IAAI,KAAK,SAAS,QAAQ;GACxB,UACE,OAAO,SAAS,IAAI,MAAM,KAAK,UAAU,QAAQ,QAAQ,IAAI,IAAI,KAAK,YAAY,MAAM;GAC1F,IAAI,YAAY,KAAA,GAAW;IACzB,MAAM,WAAW,OAAO,QAAQ,QAAQ,QAAQ,IAAI;IACpD,IAAI,CAAC,SAAS,SACZ,OAAO,OAAO,cAAc,QAAQ,QAAQ,SAAS,MAAM,KAAK,QAAQ,KAAK,GAAG,EAC9E,cAAc,YAAY,SAAS,QAAQ,QAAQ,EACrD,CAAC;GAEL;EACF;EACA,MAAM,SAAS,MAAM,KAAK,OAAO,GAAG;EACpC,IAAI,WAAW,KAAA,GAAW,OAAO,EAAE,QAAQ,OAAO;EAElD,MAAM,OAAO,KAAK,UAAU,OAAO,IAAI;EACvC,IAAI,SAAS,KAAA,GAAW,OAAO,OAAO,SAAS,KAAK,QAAQ,KAAK,KAAK,CAAC;EACvE,MAAM,QAAQ,MAAM,QAAQ;GAAE;GAAQ;GAAK;GAAM;GAAQ;GAAQ;EAAK,CAAC;EACvE,IAAI,UAAU,KAAA,GAAW,OAAO,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE;EAE7D,IAAI,cAAiC,CAAC;EACtC,IAAI,oBAAoB;EACxB,IAAI;EACJ,IAAI,KAAK,SAAS,QAAQ;GACxB,YAAY,MAAM,KAAK,UAAU,QAAQ,QAAQ,IAAI;GACrD,MAAM,WAAW,OAAO,QAAQ,QAAQ,QAAQ,IAAI;GACpD,QAAQ,YAAY,SAAS,QAAQ,QAAQ;GAC7C,IAAI,CAAC,SAAS,SACZ,OAAO,OAAO,cAAc,QAAQ,QAAQ,SAAS,MAAM,KAAK,QAAQ,KAAK,GAAG,EAC9E,cAAc,MAChB,CAAC;GAEH,cAAc,WAAW,QAAQ,QAAQ,QAAQ,IAAI;GACrD,oBAAoB,SAAS,qBAAqB;EACpD;EACA,IAAI,KAAK,QAAQ,OAAO,EAAE,QAAQ,EAAE,SAAS,aAAa,GAAG,EAAE,EAAE;EACjE,OAAO,EACL,UAAU;GACR;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;EACF,EACF;CACF;;CAGA,MAAM,MAAM,UAA0C;EACpD,IAAI,KAAK,QAAQ,OAAO,aAAa,SAAS,GAAG;EACjD,OAAO,KAAK,MAAM,KAAK,OAAO,QAAQ,CAAC;CACzC;CAEA,MAAc,OAAO,UAA0C;EAC7D,MAAM,SACJ,SAAS,WAAW,KAAA,IAChB,KAAK,QAAQ,SACb,YAAY,IAAI,CAAC,SAAS,QAAQ,KAAK,QAAQ,MAAM,CAAC;EAC5D,IAAI,MAAM,SAAS;EACnB,IAAI,UAAU,SAAS;EACvB,IAAI;EACJ,IAAI,eAAe,eAAe;EAClC,IAAI;GACF,MAAM,QAAQ,IAAI,IAAI,GAAG;GACzB,SAAS,MAAM;GACf,eAAe,KAAK,SAAS,KAAK,CAAC,CAAC;EACtC,QAAQ,CAER;EACA,IAAI;GACF,KAAK,IAAI,MAAM,IAAK,OAAO,GAAG;IAC5B,MAAM,OAAO,MAAM,KAAK,IAAI;KAAE,GAAG;KAAU;KAAK;IAAQ,GAAG,MAAM;IACjE,IAAI,KAAK,SAAS,KAAA,GAAW,OAAO,KAAK;IACzC,IAAI,OAAO,cACT,OAAO;KACL,GAAG,KAAK;KACR,SAAS;KACT,MAAM,aAAa,OAAO,YAAY,EAAE;IAC1C;IAEF,MAAM,OAAO,IAAI,IAAI,KAAK,IAAI;IAE9B,IAAI,KAAK,WAAW,QAAQ,UAAU,mBAAmB,OAAO;IAChE,SAAS,KAAK;IACd,MAAM,KAAK;GACb;EACF,SAAS,OAAO;GACd,IAAI,KAAK,QAAQ,OAAO,aAAa,SAAS,GAAG;GACjD,OAAO;IACL,GAAG,aAAa,SAAS,GAAG;IAC5B,SAAS;IACT,MAAM,kBAAkBA,YAAU,KAAK;GACzC;EACF;CACF;;CAGA,IAAY,UAAwB,QAAoC;EAEtE,IAAI,SAAS,WAAW,KAAA,GAAW,OAAO,KAAK,UAAU,UAAU,MAAM;EACzE,MAAM,MAAM,KAAK,UAAU;GACzB,WAAW,SAAS,IAAI,MAAM,GAAG,CAAC,CAAC,MAAM,SAAS,GAAG;GACrD,SAAS;GACT,SAAS;GACT,SAAS,UAAU;GACnB,SAAS,WAAW;GACpB,SAAS,gBAAgB;GACzB,SAAS,iBAAiB;GAC1B,SAAS,YAAY;GACrB,SAAS,YAAY;GACrB,SAAS,cAAc;GACvB,SAAS,OAAO;GAChB,SAAS,SAAS;GAClB,SAAS,UAAU;GACnB,SAAS,YAAY;GAErB,SAAS,SAAS;GAClB,SAAS,eAAe;GACxB,SAAS,wBAAwB;EACnC,CAAC;EACD,MAAM,UAAU,KAAK,SAAS,IAAI,GAAG;EACrC,IAAI,YAAY,KAAA,GAAW,OAAO;EAClC,MAAM,OAAO,KAAK,UAAU,UAAU,MAAM,CAAC,CAAC,cAAc,KAAK,SAAS,OAAO,GAAG,CAAC;EACrF,KAAK,SAAS,IAAI,KAAK,IAAI;EAC3B,OAAO;CACT;CAEA,YAAoB,QAAwC;EAC1D,MAAM,QAAQ,KAAK,OAAO,KAAK,UAAU,MAAM,CAAC;EAChD,OAAO,UAAU,KAAA,KAAa,KAAK,MAAM,IAAI,IAAI,MAAM,KAAK,MAAM,QAC9D,MAAM,QACN,KAAA;CACN;;;;;;CAOA,UACE,QACA,QACA,QAGI,CAAC,GACmB;EACxB,MAAM,MAAM,UAAU,MAAM;EAC5B,MAAM,MAAM,KAAK,MAAM,IAAI;EAC3B,MAAM,SAAS,KAAK,OAAO,IAAI,GAAG;EAClC,IAAI,WAAW,KAAA,KAAa,MAAM,OAAO,KAAK,OAAO,OAAO;GAE1D,IAAI,OAAO,UAAU,KAAA,GAAW,KAAK,KAAK,OAAO,SAAS,KAAK;GAC/D,OAAO,OAAO;EAChB;EACA,MAAM,UAAmB;GAAE,OAAO,KAAA;GAAW,QAAQ,KAAA;EAAU;EAC/D,KAAK,KAAK,SAAS,KAAK;EACxB,MAAM,QAAqB;GACzB,IAAI;GACJ;GACA,OAAO,OAAO;GACd,SAAS,QAAQ,QAAQ;IAAE,QAAQ,kBAAkB,cAAc;IAAG,SAAS,KAAA;GAAU,CAAC;GAC1F,OAAO,KAAA;GACP,QAAQ,QAAQ,UAAU;GAC1B,MAAM,QAAQ;EAChB;EACA,MAAM,UAAU,KAAK,YAAY,QAAQ,QAAQ,OAAO,CAAC,CACtD,OAAO,WAAoB;GAC1B,QAAQ,kBAAkB,sBAAsBA,YAAU,KAAK,GAAG;GAClE,SAAS,KAAA;GACT,cAAc,KAAA;EAChB,EAAE,CAAC,CACF,KAAK,OAAO,SAAiC;GAC5C,IAAI,UAAyB;IAAE,QAAQ,KAAK;IAAQ,SAAS,KAAK;GAAQ;GAC1E,IAAI,KAAK,OAAO,gBAAgB,KAAA,GAAW;IACzC,MAAM,SAAS;IACf,MAAM,OAAO;KAAE;KAAS,IAAI,KAAK,MAAM,IAAI;IAAE;GAC/C,OAAO;IACL,MAAM,UAAU;IAChB,MAAM,UAAU,cACd,MAAM,QACN,OAAO,eACP,OAAO,kBACP,KAAK,MACP;IACA,MAAM,QAAQ,KAAK,IACjB,OAAO,kBACP,KAAK,IAAI,KAAK,gBAAgB,GAAG,OAAO,CAC1C;IACA,MAAM,OACJ,MAAM,SAAS,KAAA,KAAa,KAAK,MAAM,IAAI,IAAI,MAAM,KAAK,MAAM,OAAO,gBACnE,MAAM,KAAK,UACX,MAAM,KAAK,aAAa,QAAQ,MAAM;IAC5C,IAAI,SAAS,KAAA,GAAW,UAAU;KAAE,GAAG;KAAM,OAAO,KAAK,OAAO;IAAY;GAC9E;GACA,MAAM,QAAQ;GACd,OAAO;EACT,CAAC;EACH,KAAK,OAAO,IAAI,KAAK,KAAK;EAC1B,OAAO,MAAM;CACf;;;;;;CAOA,MAAM,WACJ,KACA,QAGI,CAAC,GAKJ;EACD,MAAM,SAAS,IAAI,IAAI,GAAG;EAC1B,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GACpC,OAAO;GAAE,QAAQ,kBAAkB,oBAAoB;GAAG,SAAS,KAAA;EAAU;EAC/E,OAAO,KAAK,UAAU,QAAQ,KAAK,SAAS,MAAM,GAAG,KAAK;CAC5D;;;;;;;CAQA,MAAM,YACJ,KACA,UAKI,CAAC,GAMJ;EACD,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,GAAG;EACtB,QAAQ;GACN,OAAO;IAAE,SAAS;IAAO,MAAM;GAAkB;EACnD;EACA,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GAAG,OAAO;GAAE,SAAS;GAAO,MAAM;EAAqB;EAC7F,IAAI,KAAK,QAAQ,OAAO;GAAE,SAAS;GAAO,MAAM;EAAiB;EACjE,MAAM,SAAS;GAAC,KAAK,QAAQ;GAAO,GAAI,QAAQ,cAAc,CAAC;GAAI,GAAG,KAAK;EAAU;EACrF,MAAM,WAAW,OAAO,MAAM,UAAU,CAAC,eAAe,KAAK,CAAC;EAC9D,IAAI,aAAa,KAAA,GACf,OAAO;GAAE,SAAS;GAAO,MAAM,+BAA+B,SAAS;EAAG;EAC5E,MAAM,OAAO,KAAK,UAAU,QAAQ,KAAK,SAAS,MAAM,GAAG,OAAO;EAClE,MAAM,UAAU,OAAO,QAAQ,WAAW,KAAA,IAAY,OAAO,UAAU,MAAM,QAAQ,MAAM;EAC3F,MAAM,OAAO,GAAG,OAAO,WAAW,OAAO;EACzC,MAAM,WAAW,OAAO,QAAQ,QAAQ,QAAQ,IAAI;EAGpD,MAAM,QAAQ,WAAW,QAAQ,QAAQ,QAAQ,IAAI;EACrD,MAAM,UAAU,MAAM,SAAS,IAAI,EAAE,SAAS,MAAM,IAAI,CAAC;EACzD,OAAO,SAAS,UACZ;GAAE,SAAS;GAAM,MAAM,KAAA;GAAW,GAAG;EAAQ,IAC7C;GACE,SAAS;GACT,MAAM,cAAc,QAAQ,QAAQ,SAAS,MAAM,KAAK,QAAQ,KAAK;GACrE,GAAG;EACL;CACN;;CAGA,MAAc,aACZ,QACA,QACoC;EACpC,IAAI;GACF,MAAM,MAAM,KAAK,OAAO,OAAO,IAAI,IAAI,eAAe,OAAO,MAAM,CAAC,CAAC;GACrE,IAAI,QAAQ,KAAA,KAAa,IAAI,SAAS,QAAQ,IAAI,SAAS,OAAO,IAAI,UAAU,KAC9E,OAAO,KAAA;GACT,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,MAAM,IAAI,UAAU,IAAI,OAAO,eAAe,OAAO,KAAA;GACjF,MAAM,OAAO,MAAM,KAAK,QAAQ,MAAM,IAAI,IAAI,IAAI;GAClD,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;GAC/B,MAAM,SAAS,mBAAmB,KAAK,IAAI,cAAc,IAAI;GAC7D,OAAO,OAAO,gBAAgB,KAAA,IAAY;IAAE;IAAQ,SAAS,IAAI;GAAQ,IAAI,KAAA;EAC/E,QAAQ;GACN;EACF;CACF;;CAGA,KACE,SACA,OAIM;EACN,MAAM,QAAQ,MAAM,SAAS,KAAK,QAAQ;EAC1C,MAAM,OAAO,KAAK,QAAQ;EAC1B,IAAI,QAAQ,UAAU,KAAA,GAAW,QAAQ,QAAQ;OAC5C,IAAI,UAAU,KAAA,KAAa,SAAS,KAAA,KAAa,KAAK,KAAK,IAAI,KAAK,QAAQ,KAAK,GACpF,QAAQ,QAAQ;EAClB,IAAI,MAAM,WAAW,KAAA,GAAW,QAAQ,SAAS;GAAE,GAAG,MAAM;GAAQ,GAAG,QAAQ;EAAO;CACxF;CAEA,MAAc,YACZ,QACA,QACA,SACqB;EACrB,IAAI,MAAM,IAAI,IAAI,eAAe,OAAO,MAAM;EAC9C,MAAM,eACJ,QACA,SACA,kBACgB;GAChB,QAAQ,kBAAkB,MAAM;GAChC;GACA;EACF;EACA,KAAK,IAAI,MAAM,GAAG,OAAO,OAAO,cAAc,OAAO,GAAG;GACtD,MAAM,OAAO,MAAM,KAAK,UACtB;IACE,KAAK,IAAI;IACT,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,eAAe,OAAO;IAEtB,IAAI,QAAQ;KACV,OAAO,QAAQ;IACjB;IACA,IAAI,SAAS;KACX,OAAO,QAAQ;IACjB;GACF,GACA,KAAK,QAAQ,MACf;GACA,MAAM,SAAS,KAAK;GACpB,IAAI,OAAO,YAAY,YAAY;IACjC,IAAI,KAAK,SAAS,KAAA,GAChB,OAAO,YAAY,OAAO,QAAQ,sCAAsC,OAAO,OAAO;IACxF,MAAM,IAAI,IAAI,KAAK,IAAI;IACvB;GACF;GACA,IAAI,OAAO,YAAY,aAAa,OAAO,WAAW,GACpD,OAAO,YAAY,OAAO,QAAQ,eAAe,OAAO,OAAO;GAEjE,IAAI,OAAO,YAAY,WAAW,OAAO,UAAU,OAAO,OAAO,SAAS,KACxE,OAAO,YAAY,OAAO,QAAQ,8BAA8B,OAAO,OAAO;GAEhF,IAAI,OAAO,MAAM,WAAW,iCAAiC,MAAM,MACjE,OAAO,YAAY,OAAO,MAAM,OAAO,OAAO;GAChD,IAAI,OAAO,YAAY,gBACrB,OAAO,OAAO,SAAS,KAAA,IACnB,YAAY,kCAAkC,OAAO,OAAO,IAC5D;IACE,QAAQ,mBAAmB,KAAK,OAAO,eAAe,MAAM,OAAO,IAAI;IACvE,SAAS,OAAO;IAChB,cAAc,KAAA;GAChB;GAUN,OAAO;IAAE,QAPP,OAAO,UAAU,OAAO,OAAO,SAAS,MACpC,mBACE,OAAO,QACP,OAAO,eAAe,MACtB,OAAO,wBAAQ,IAAI,WAAW,CAChC,IACA,iBAAiB,OAAO,QAAQ,KAAA,CAAS;IAC9B,SAAS,OAAO;IAAS,cAAc,KAAK;GAAa;EAC5E;EACA,OAAO,YAAY,aAAa,OAAO,OAAO,YAAY,EAAE,aAAa,KAAA,CAAS;CACpF;CAEA,MAAc,UAAU,UAAwB,QAAoC;EAClF,MAAM,OAAO;GACX,QAAQ,SAAS;GACjB,MAAM,SAAS;GACf,GAAI,SAAS,MAAM,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC;EAC9C;EACA,MAAM,eAAe,SAAS,gBAAgB,CAAC;EAC/C,MAAM,OAAO,KAAK,eAAe,QAAQ;EAEzC,MAAM,QAAQ;GACZ,IAAI,QAAQ;IACV,OAAO,SAAS;GAClB;GACA,IAAI,SAAS;IACX,OAAO,SAAS;GAClB;EACF;EAEA,MAAM,QAAQ,SAAS;EACvB,MAAM,WACJ,UAAU,KAAA,KACT,OAAO,cAAc,MAAM,MAAM,KAChC,MAAM,UAAU,KAChB,OAAO,cAAc,MAAM,MAAM,KACjC,MAAM,UAAU,IACd,KAAA,IACA,4BAA4B,KAAK,UAAU,KAAK;EAEtD,MAAM,YAAY,MAAM,KAAK,MAAM;GACjC,WAAW;GACX,KAAK,SAAS;GACd,MAAM,SAAS;GACf;GACA;GACA,SACE,cAAc,iBAAiB,SAAS,eAAe,KAAK,KAC5D,cAAc,YAAY,SAAS,UAAU,KAAK,KAClD,cAAc,YAAY,SAAS,UAAU,IAAI,KACjD;GACF,YAAY,SAAS;GACrB,OAAO,SAAS;GAChB,QAAQ,SAAS;GACjB,SAAS,QACP,UAAU,KAAA,IACN,QAAQ,QAAQ,KAAA,CAAS,IACzB,KAAK,UAAU,KAAK,SAAS,SAAS,UAAU,MAAM,YAAY;EAC1E,CAAC;EACD,IAAI,YAAY,WAAW,OAAO,UAAU;EAC5C,MAAM,EAAE,QAAQ,KAAK,MAAM,QAAQ,OAAO,aAAa,mBAAmB,SACxE,UAAU;EACZ,MAAM,WAAW,KAAK,IAAI,SAAS,YAAY,OAAO,UAAU,OAAO,UAAU,QAAQ;EACzF,MAAM,QAAQ,eAAkC;GAC9C,MAAM,SAAS;GACf;GACA;GACA,eAAe,SAAS;EAC1B;EACA,MAAM,aAAa,SAAS,WAAW;EAEvC,IAAI;GACF,OAAO,MAAM,KAAK,UAAU,IAC1B,MACA,YAAkC;IAEhC,MAAM,QACJ,UAAU,KAAA,IACN,MAAM,KAAK,UAAU,KAAK,SAAS,SAAS,UAAU,MAAM,YAAY,IACxE,KAAA;IACN,IAAI,UAAU,KAAA,GAAW,OAAO;KAAE,OAAO;KAAO,SAAS;IAAE;IAE3D,KAAK,eAAe,SAAS,QAAQ;IAErC,MAAM,SAAS,UAAU,KAAA,IAAY,KAAK,OAAO,GAAG,IAAI,KAAA;IAExD,MAAM,SACJ,WAAW,KAAA,KAAa,OAAO,cAAc,QAAQ,YAAY,OAAO,UAAU,IAAI,IAClF,SACA,KAAA;IACN,IAAI;IACJ,IAAI,QAAQ,SAAS,OAAO,QAAQ,OAAO,gBACzC,SAAS,MAAM,KAAK,QAAQ,MAAM,IAAI,OAAO,IAAI;IACnD,MAAM,UAAkC,EAAE,GAAG,KAAK;IAGlD,MAAM,OACJ,WAAW,KAAA,KAAa,WAAW,KAAA,IAC/B;KAAE,MAAM;KAAO,cAAc;IAAM,IACnC,iBACE,aAAa,KAAA,IAAY,KAAK,OAAO,IAAI,GACzC;KAAE,MAAM,QAAQ,OAAO,IAAI;KAAG,cAAc,QAAQ,OAAO,aAAa;IAAE,GAC1E,KAAK,MAAM,IAAI,CACjB;IACN,IAAI,KAAK,QAAQ,QAAQ,MAAM,QAAQ,mBAAmB,OAAO;IACjE,IAAI,KAAK,gBAAgB,QAAQ,eAC/B,QAAQ,uBAAuB,OAAO;IACxC,MAAM,aAAa,CAAC,iBAAiB,mBAAmB,CAAC,CAAC,QACvD,SAAS,QAAQ,UAAU,KAAA,CAC9B;IAEA,MAAM,YACJ,UAAU,KAAA,IACN,KAAA,IACA,SAAS,OAAO,MAAM,MAAM,EAAE,GAAG,OAAO,MAAM,SAAS,MAAM,SAAS,CAAC;IAC7E,IAAI,cAAc,KAAA,GAAW,QAAQ,WAAW;IAEhD,MAAM,SAAS,KAAK,QAAQ,MAAM;IAClC,IAAI,WAAW,KAAA,KAAa,WAAW,QACrC,OAAO;KAAE,OAAO,EAAE,SAAS,MAAM,KAAK,OAAO,KAAK,EAAE;KAAG,SAAS;IAAE;IACpE,IAAI,WAAW,KAAA,GAAW,OAAO,OAAO,SAAS,OAAO,OAAO;IAC/D,MAAM,cAAc,QAAQ,SAAS,CAAC;IAEtC,MAAM,UAAU,KAAK,MAAM,IAAI;IAC/B,MAAM,UAAU,YAAY,QAAQ,OAAO,gBAAgB;IAC3D,MAAM,QAAQ,YAAY,IAAI,CAAC,SAAS,MAAM,CAAC;IAC/C,MAAM,kBAA2B,SAAS,QAAQ,YAAY;IAC9D,MAAM,cAAc,QAAgB,UAClC,QAAQ,UACJ,GAAG,OAAO,oBAAoB,OAAO,KAAK,MAAM,OAAO,mBAAmB,GAAI,CAAC,EAAE,MACjF,GAAG,OAAO,IAAIA,YAAU,KAAK;IAEnC,KAAK,KAAK;KAAE,MAAM;KAAW;KAAK;KAAM,WAAW;IAAO,CAAC;IAC3D,IAAI;IACJ,IAAI;KACF,WAAW,MAAM,QAAQ,QAAQ;MAC/B,QAAQ;MACR;MACA,YAAY,KAAK;MACjB,QAAQ;KACV,CAAC;IACH,SAAS,OAAO;KACd,MAAM,SAAS,YAAY,KAAK;KAChC,IAAI,WAAW,KAAA,GACb,OAAO;MAAE,OAAO,EAAE,SAAS,MAAM,KAAK,OAAO,OAAO,EAAE;MAAG,SAAS;KAAE;KACtE,IAAI,KAAK,QAAQ,OAAO;MAAE,OAAO,EAAE,SAAS,aAAa,GAAG,EAAE;MAAG,SAAS;KAAE;KAC5E,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;KACrC,MAAM,OAAO,UAAU,IAAI,4BAA4B,WAAW,eAAe,KAAK;KACtF,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;MACxC,SAAS;MACT,QAAQ;MACR,cAAc,KAAA;MACd,MAAM,KAAK,SAAS;MACpB;MACA,GAAI,UAAU,IAAI,EAAE,QAAQ,UAAmB,IAAI,CAAC;KACtD,CAAC;KACD,KAAK,KAAK;MAAE,MAAM;MAAY;MAAK;MAAM,QAAQ;MAAG,SAAS;MAAS;KAAU,CAAC;KACjF,MAAM,UAAU,MAAM,KAAK,OACzB,WACA;MACE,GAAG;MACH;MACA,QAAQ;MACR,SAAS;MACT;MACA,YAAY;MACZ,GAAI,QAAQ,EAAE,cAAc,MAAM,IAAI,CAAC;MACvC,GAAI,cAAc,KAAA,IAAY,EAAE,OAAO,UAAU,IAAI,CAAC;MACtD,GAAG;KACL,GACA,KACA,KACF;KACA,OAAO;MACL,OAAO,EACL,SAAS;OACP;OACA,QAAQ;OACR,SAAS;OACT,WAAW;OACX;OACA,MAAM,KAAA;OACN,MAAM,KAAA;OACN,aAAa,KAAA;OACb,SAAS;OACT;OACA,WAAW,IAAI,KAAK,OAAO,CAAC,CAAC,YAAY;OACzC,GAAG;MACL,EACF;MACA;KACF;IACF;IAEA,MAAM,SAAS,SACb,SAAS,OACX;IACA,MAAM,SAAS,KAAK,MAAM,IAAI,IAAI;IAClC,MAAM,aAAa,SAAS,SAAS,YAAY,QAAQ,KAAK,MAAM,IAAI,CAAC;IACzE,IAAI,UAAmB,WAAW;IAClC,IAAI,OAAO,WAAW;IACtB,IAAI;IACJ,IAAI;IACJ,IAAI,YAAY;IAChB,IAAI,MAAM;IACV,IAAI;IACJ,IAAI;IACJ,IAAI,YAAY,MAAM;KACpB,IAAI;MACF,MAAM,OAAO,MAAM,SAAS,SAAS,MAAM,OAAO,kBAAkB,GAAG,QAAQ;MAC/E,OAAO,KAAK;MACZ,YAAY,KAAK;MACjB,MAAM,KAAK;MACX,IAAI,KAAK,KACP,OAAO,6DAA6D,OAAO,KAAK,MAAM,MAAM,EAAE;WAC3F,IAAI,WAAW,OAAO,gBAAgB,OAAO,QAAQ,EAAE;KAC9D,SAAS,OAAO;MACd,IAAI,KAAK,QAAQ,OAAO;OAAE,OAAO,EAAE,SAAS,aAAa,GAAG,EAAE;OAAG,SAAS;MAAE;MAC5E,UAAU;MACV,IAAI,UAAU,GAAG;OACf,OAAO;OACP,SAAS;MACX,OAEE,OAAO,WADQ,iBAAiB,aAAa,MAAM,SAAS,WACjC,qBAAqB,eAAe,KAAK;KAExE;KACA,IAAI,SAAS,KAAA,GACX,IAAI;MACF,QAAQ,MAAM,KAAK,QAAQ,MAAM,IAAI,IAAI,EAAA,CAAG;KAC9C,SAAS,OAAO;MAEd,UAAU;MACV,OAAO,eAAeA,YAAU,KAAK;MACrC,OAAO,KAAA;MACP,SAAS;KACX;IAEJ,OACE,IAAI;KACF,MAAM,SAAS,SAAS,MAAM,KAAA,GAAW,gBAAgB;IAC3D,QAAQ,CAER;IAEF,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;IACrC,IAAI,YAAY,gBAAgB;KAC9B,IAAI,WAAW,KAAA,KAAa,QAAQ,MAAM;MACxC,OAAO;MACP,OAAO,OAAO;KAChB,OAAO,OAAO;IAChB;IACA,IAAI;IACJ,IAAI,YAAY,YAAY;KAC1B,MAAM,MAAM,OAAO,UAAU;KAC7B,IAAI,QAAQ,KAAA,GAAW,OAAO;UAE5B,IAAI;MACF,MAAM,WAAW,IAAI,IAAI,KAAK,MAAM;MACpC,OAAO,SAAS;MAChB,WAAW,OAAO,UAAU,YAAY;MACxC,OAAO,YAAY;KACrB,QAAQ;MACN,UAAU;MACV,OAAO;KACT;IAEJ;IAEA,MAAM,UAAU,YAAY,kBAAkB,WAAW,KAAA;IAEzD,MAAM,SAAS,SAAS,sBAAsB,YAAY;IAC1D,MAAM,gBACJ,WAAW,KAAA,IACP,CAAC,IACD,eAAe,SAAS,SAAS;KAC/B,MAAM,QAAQ,OAAO,GAAG,SAAS,MAAM;KACvC,OAAO,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,GAAG,SAAS,KAAK,IAAI,OAAO;IACjE,CAAC;IACP,IAAI,UAAU,YACZ;KAAC,GAAG;KAAa,GAAI,SAAS,eAAe,CAAC;KAAI,GAAG;IAAa,GAClE,QACA,YAAY,MAAM,OAAO,cAAc,CAAC,CAC1C;IAEA,IAAI,SAAS;KAEX,MAAM,SACJ,QAAQ,aAAa,QAAQ,OAAO,aAAa,OAC7C,OACA,QAAQ,aAAa,UACnB,QAAQ,WACR,OAAO;KACf,UAAU;MAAE,GAAG;MAAS,UAAU;KAAO;IAC3C;IACA,MAAM,cACJ,OAAO,cAAc,MAAM,UAAW,OAAO,gBAAgB,KAAA,IAAa,KAAA;IAE5E,MAAM,UACJ,SAAS,SAAS,SAClB,YAAY,QACZ,YAAY,SAAS,QAAQ,WAAW;IAC1C,IAAI,SAAS;KACX,MAAM,OAAO,mBAAmB,eAAe,GAAG,4BAA4B,SAAS,UAAU,GAAG;KACpG,OAAO,SAAS,KAAA,IAAY,OAAO,GAAG,KAAK,IAAI;IACjD;IACA,MAAM,OAAO,OAAO,MAAM,MAAM,UAAU,OAAO,OAAO,SAAS;IACjE,MAAM,eACJ,OAAO,eAAe,MAAM,UAAU,OAAO,gBAAgB,SAAS;IACxE,MAAM,YAAY,KAAK,MAAM,IAAI;IACjC,MAAM,YAAY,WAChB,QACA,SAAS,YACT,WACA,cACA,OAAO,uBACT;IACA,MAAM,UACJ,OAAO,MAAM,MAAM,KAAA,IACf,UAAU,OAAO,MAAM,GAAG,IAAI,IAC9B,UACG,OAAO,YAAY,OACpB;IACR,MAAM,eAAe,OAAO,YAC1B;KAAC;KAAiB;KAAW;KAAO;KAAQ;IAAM,CAAC,CAAC,SAAS,SAAS;KACpE,MAAM,QAAQ,OAAO,IAAI;KACzB,OAAO,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,CAAC,KAAK,QAAQ,KAAK,GAAG,GAAG,KAAK,CAAC;IACpE,CAAC,CACH;IACA,KAAK,KAAK;KACR,MAAM;KACN;KACA;KACA,QAAQ,SAAS;KACjB;KACA;IACF,CAAC;IACD,MAAM,UAAU,MAAM,KAAK,OACzB,UAAU,YAAY,WACtB;KACE,GAAG;KACH;KACA,QAAQ,SAAS;KACjB;KACA,YAAY;KACZ,SAAS;KACT,cAAc,eAAe;KAC7B,OAAO,MAAM,UAAU;KACvB,MAAM,QAAQ;KACd;KACA,eAAe;KACf,GAAI,WAAW,SAAS,IAAI,EAAE,WAAW,IAAI,CAAC;KAC9C,GAAI,UAAU,EAAE,YAAY,OAAO,QAAQ,IAAI,CAAC;KAChD,QAAQ,SAAS,SAAS,SAAS,YAAY;KAC/C,GAAI,UAAU,KAAA,IAAY,EAAE,cAAc,MAAM,IAAI,CAAC;KACrD;KACA,GAAI,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;KACvC,GAAI,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;KAC3B,GAAI,UAAU,EAAE,SAAS,KAAK,IAAI,CAAC;KACnC,GAAG;KACH,GAAI,cAAc,OAAO,EAAE,aAAa,UAAU,IAAI,CAAC;KACvD,GAAI,YAAY,OAAO,EAAE,UAAU,QAAQ,IAAI,CAAC;KAChD,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;KAC7C,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;KACrC,GAAI,cAAc,KAAA,IAAY,EAAE,OAAO,UAAU,IAAI,CAAC;KACtD,GAAG;IACL,GACA,KACA,KACF;IACA,MAAM,aAAa,IAAI,KAAK,SAAS,CAAC,CAAC,YAAY;IAGnD,IACE,YAAY,KAAA,MACX,YAAY,QAAQ,YAAY,mBACjC,SAAS,KAAA,KACT,CAAC,OACD,CAAC,WACD,UAAU,KAAA,GAEV,KAAK,MAAM;KACT;KACA;KACA;KAEA,MAAM,YAAY,OAAO;KACzB,eAAe,YAAY,OAAO;KAClC,cAAc,eAAe;KAC7B,YAAY;KAEZ,QAAQ,UAAW,OAAO,WAAW,MAAM,MAAM,OAAO,SAAU,SAAS;KAC3E,UAAU,QAAQ;KAClB,WAAW,YAAY,OAAO,KAAA;KAC9B,aAAa;KACb,UAAU;IACZ,CAAC;IAEH,IACE,YAAY,KAAA,KACZ,YAAY,cACZ,SAAS,KAAA,MACR,oBAAoB,IAAI,SAAS,UAAU,KAAK,cAAc,OAE/D,KAAK,MACH;KACE;KACA;KACA,MAAM;KACN,MAAM;KACN,eAAe;KACf,cAAc;KACd,YAAY;KACZ,QAAQ,SAAS;KACjB,UAAU;KACV,UAAU,YAAY;KACtB,aAAa;KACb,UAAU;IACZ,GACA,IACF;IAGF,MAAM,SACJ,WAAW,WAAW,KAAK,YAAY,KAAA,KAAa,aAChD,KAAA,IACA,YAAY,iBACV,iBACA,YAAY,QAAQ,SAAS,KAAA,KAAa,CAAC,MACzC,SAAS,QAAQ,OACf,SACA,YACF,KAAA;IACV,IAAI,WAAW,KAAA,GACb,IAAI;KACF,KAAK,UAAU,MAAM,iBACnB,MACA;MACE,MAAM,QAAQ,qBAAqB,KAAA;MACnC,cAAc,QAAQ,yBAAyB,KAAA;MAC/C;KACF,GACA,SACF;IACF,QAAQ,CAER;IAEF,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;KACxC;KACA,QAAQ,SAAS;KACjB,cAAc,WAAW;KACzB,MAAM,KAAK,SAAS;KACpB;KACA,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;IAC3C,CAAC;IACD,OAAO;KACL,OAAO;MACL,SAAS;OACP;OACA,QAAQ,SAAS;OACjB;OACA,WAAW;OACX;OACA;OACA;OACA;OACA;OACA;OACA,GAAI,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;OACvC,WAAW;OACX,GAAG;MACL;MACA;MACA,cAAc,WAAW;KAC3B;KACA;IACF;GACF,GACA,KAAK,MAAM,QAAQ,SAAS,QAAQ,CACtC;EACF,SAAS,OAAO;GACd,IAAI,iBAAiB,aACnB,OAAO,EAAE,SAAS,MAAM,KAAK,SAAS,MAAM,QAAQ,MAAM,KAAK,CAAC,EAAE;GACpE,IAAI,iBAAiB,iBACnB,OAAO,EAAE,SAAS,MAAM,KAAK,aAAa,OAAO,SAAS,QAAQ,CAAC,EAAE;GACvE,IAAI,KAAK,QAAQ,OAAO,EAAE,SAAS,aAAa,GAAG,EAAE;GACrD,IAAI,SAAS,QAAQ,YAAY,MAC/B,OAAO,EAAE,SAAS,MAAM,KAAK,yBAAyB,EAAE;GAC1D,MAAM;EACR;CACF;;;;;;CAOA,gBACE,MACA,QACA,QACA,QACA,OACmC;EACnC,MAAM,aAAa,UAAU,IAAI;EACjC,MAAM,0BAAU,IAAI,IAAyC;EAC7D,MAAM,eAAwB;GAC5B,MAAM,WAAW;GACjB,OAAO;EACT;EACA,OAAO,OAAO,QAAkC;GAC9C,IAAI,OAAO,SAAS,OAAO;GAC3B,IAAI;GACJ,IAAI;IACF,SAAS,IAAI,IAAI,GAAG;GACtB,QAAQ;IACN,OAAO,OAAO;GAChB;GACA,IACE,OAAO,aAAa,WACpB,OAAO,aAAa,WACpB,OAAO,aAAa,UAEpB,OAAO;GACT,MAAM,MAAM,cAAc,IAAI,OAAO,QAAQ;GAC7C,IAAI,CAAC,OAAO,OAAO,aAAa,SAAS,OAAO,aAAa,QAAQ,OAAO,OAAO;GACnF,IAAI,QAAQ,QAAQ,IAAI,OAAO,QAAQ;GACvC,IAAI,UAAU,KAAA,GAAW;IACvB,QAAQ,YAAY,KAAK,QAAQ,MAAM;IACvC,QAAQ,IAAI,OAAO,UAAU,KAAK;GACpC;GACA,IAAK,MAAM,UAAW,KAAA,GAAW,OAAO,OAAO;GAC/C,IAAI,KAAK;IACP,MAAM,QAAQ,UAAU,MAAM,MAAM,aAAa,SAAS,KAAK,YAAY,MAAM,CAAC,EAAE;IACpF,IACE,UAAU,KAAA,KACV,CAAC,OAAO,OAAO,QAAQ,GAAG,OAAO,WAAW,OAAO,QAAQ,CAAC,CAAC,SAE7D,OAAO,OAAO;GAClB;GACA,MAAM,WAAW;GACjB,IAAI,MAAM,MAAM,OAAO,IAAI,MAAM,MAAM,IAAI,QAAQ,MAAM,CAAC;GAC1D,OAAO;EACT;CACF;;;;;;CAOA,MAAc,aACZ,KACA,MACA,QACA,QACsE;EACtE,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,GAAG;EACtB,QAAQ;GACN;EACF;EACA,OAAO,OAAO;EACd,IAAI,OAAO,SAAS,KAAK,MAAM,OAAO,KAAA;EACtC,MAAM,MAAM,OAAO,MAAM;EACzB,MAAM,QAAQ,qBAAqB;EACnC,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GACpC,OAAO;GAAE,MAAM,GAAG,MAAM;GAAgC;EAAI;EAC9D,MAAM,SAAS,MAAM,YAAY,KAAK,QAAQ,MAAM;EACpD,IAAI,WAAW,KAAA,GAAW,OAAO;GAAE,MAAM,GAAG,MAAM,IAAI;GAAU;EAAI;EACpE,MAAM,QAAQ,UAAU,MAAM,MAAM,UAAU,IAAI,IAAI,SAAS,KAAK,YAAY,MAAM,CAAC,EAAE;EACzF,IAAI,UAAU,KAAA,GAAW;GACvB,MAAM,WAAW,OAAO,OAAO,QAAQ,GAAG,OAAO,WAAW,OAAO,QAAQ;GAC3E,IAAI,CAAC,SAAS,SACZ,OAAO;IACL,MAAM,GAAG,MAAM,IAAI,cAAc,OAAO,SAAS,MAAM,KAAK,QAAQ,KAAK;IACzE;GACF;EACJ;CAEF;;;;;;;;;;;;;CAcA,MAAM,OAAO,UAA2C;EACtD,IAAI,KAAK,QAAQ,OAAO,aAAa,SAAS,GAAG;EACjD,OAAO,KAAK,MACV,KAAK,WAAW,QAAQ,CAAC,CAAC,OAAO,UAC/B,KAAK,SACD,aAAa,SAAS,GAAG,IACzB;GACE,GAAG,aAAa,SAAS,GAAG;GAC5B,SAAS;GACT,MAAM,kBAAkBA,YAAU,KAAK;EACzC,CACN,CACF;CACF;CAEA,MAAc,WAAW,UAA2C;EAClE,MAAM,OAAO;GACX,QAAQ,SAAS;GACjB,MAAM;GACN,UAAU;GACV,GAAI,SAAS,MAAM,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC;EAC9C;EACA,MAAM,QAAQ;GAAE,OAAO,SAAS;GAAO,QAAQ,SAAS;EAAO;EAC/D,MAAM,WAAW,KAAK,QAAQ;EAC9B,MAAM,YAAY,MAAM,KAAK,MAC3B;GACE,WAAW;GACX,KAAK,SAAS;GACd,MAAM;GACN;GACA,cAAc,CAAC;GACf,SAAS,cAAc,YAAY,SAAS,UAAU,IAAI;GAC1D,YAAY,SAAS;GACrB,OAAO,SAAS;GAChB,QAAQ,SAAS;GACjB,QAAQ,OAAO,QAAQ;IACrB,MAAM,SAAS,MAAM,KAAK,UAAU,KAAK,YAAY,SAAS,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,WAAW,KAAA,KAAa,OAAO,QAAQ,SAAS,KAAA,GAAW,OAAO,KAAA;IACtE,OAAO,EAAE,SAAS;KAAE,GAAG,OAAO;KAAS;KAAK,aAAa;IAAc,EAAE;GAC3E;EACF,GACA,OAAO,EAAE,QAAQ,WAAW;GAC1B,IAAI,aAAa,KAAA,GAAW,OAAO,KAAK,qCAAqC;GAC7E,IAAI,KAAK,cAAc,KAAA,GAAW,OAAO,KAAK,KAAK,SAAS;GAE5D,MAAM,WAAW,MAAM,YAAY,KAAK,QAAQ,MAAM;GACtD,OAAO,aAAa,KAAA,IAAY,KAAA,IAAY,KAAK,QAAQ;EAC3D,CACF;EACA,IAAI,YAAY,WAAW,OAAO,UAAU,OAAO;EACnD,MAAM,EAAE,QAAQ,KAAK,MAAM,QAAQ,QAAQ,OAAO,mBAAmB,SACnE,UAAU;EACZ,MAAM,UAAU,UAAU,SAAS;EACnC,MAAM,SACJ,SAAS,WAAW,KAAA,IAChB,KAAK,QAAQ,SACb,YAAY,IAAI,CAAC,SAAS,QAAQ,KAAK,QAAQ,MAAM,CAAC;EAC5D,MAAM,QAAQ;GAAE,SAAS;GAAG,SAAS;GAAG,uBAAO,IAAI,IAAY;EAAE;EACjE,MAAM,eAAe,KAAK,gBAAgB,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,KAAK;EAEvF,MAAM,OACJ,KAAK,WAAW,KAAA,IACZ,KAAA,KACC,QAA8D;GAC7D,IAAI;GACJ,IAAI;IACF,KAAK,IAAI,IAAI,GAAG;GAClB,QAAQ;IACN;GACF;GACA,MAAM,SAAS,KAAK,QAAQ,EAAE;GAC9B,OAAO,WAAW,KAAA,KAAa,WAAW,SAAS,KAAA,IAAY,OAAO;EACxE;EACN,MAAM,cACJ,KAAK,WAAW,KAAA,IACZ,CAAC,IACD,EAAE,QAAQ;GAAE,QAAQ,KAAK,OAAO;GAAO,OAAO,KAAK,OAAO;EAAe,EAAE;EAEjF,MAAM,QAAQ,eAAkC;GAC9C,MAAM;GACN;GACA;EACF;EAGA,IAAI;EACJ,IAAI;GACF,UAAU,MAAM,KAAK,QAAQ,QAAQ,MAAM;EAC7C,QAAQ;GACN,IAAI,KAAK,QAAQ,OAAO,aAAa,GAAG;GACxC,OAAO,KAAK,yBAAyB;EACvC;EACA,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,KAAK,UAAU,IAC1B,MACA,YAAwC;IACtC,KAAK,eAAe,SAAS,QAAQ;IACrC,MAAM,UAAU,KAAK,MAAM,IAAI;IAC/B,KAAK,KAAK;KAAE,MAAM;KAAW;KAAK;KAAM,WAAW;IAAU,CAAC;IAC9D,IAAI;KACF,MAAM,WAAW,MAAM,UACrB,SAAU,OAAO,OAAO,MAAM;MAC5B,WAAW,KAAK,QAAQ;MACxB,WAAW,OAAO;MAClB,GAAI,KAAK,QAAQ,SAAS,KAAA,IACtB,EAAE,SAAS,EAAE,MAAM,KAAK,QAAQ,KAAK,EAAE,IACvC,CAAC;MACL;MACA,cAAc,OAAO;MACrB;MACA,GAAI,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,KAAK;KACvC,CAAC,GACD,MACF;KACA,MAAM,UAAU,SACd,SAAS,QAAQ,KAAK,YAAY;KACpC,MAAM,aACJ,SAAS,WAAW,OAChB;MAAE,SAAS;MAAM,cAAc,KAAA;MAAW,MAAM,KAAA;KAAU,IAC1D,SAAS,SAAS,QAAQ,QAAQ,KAAK,MAAM,IAAI,CAAC;KACxD,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;KACrC,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;MACxC,SAAS,WAAW;MACpB,QAAQ,SAAS,UAAU;MAC3B,cAAc,WAAW;MACzB,MAAM,KAAK,SAAS;MACpB,YAAY;KACd,CAAC;KACD,OAAO;MAAE,OAAO;OAAE,MAAM;OAAY;OAAU;OAAY;MAAU;MAAG;KAAQ;IACjF,SAAS,OAAO;KACd,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;KACrC,IAAI,KAAK,QAAQ,OAAO;MAAE,OAAO,EAAE,MAAM,SAAS;MAAG,SAAS;KAAE;KAChE,IAAI,SAAS,QAAQ,YAAY,MAC/B,OAAO;MACL,OAAO;OAAE,MAAM;OAAa;MAAU;MACtC,SAAS,UAAU,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;KACtC;KAGF,MAAM,UACJ,iBAAiB,gBACb,QACA,eAAeA,YAAU,KAAK,CAAC,IAC7B,IAAI,cAAc,WAAW,8BAA8BA,YAAU,KAAK,GAAG,IAC7E,KAAA;KACR,IAAI,YAAY,KAAA,GAAW;MACzB,IAAI,QAAQ,WAAW,WAAW,KAAK,YAAY,QAAQ;MAE3D,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;OACxC,SAAS;OACT,QAAQ;OACR,cAAc,KAAA;OACd,MAAM,KAAK,CAAC;OACZ,YAAY;OACZ,QAAQ;MACV,CAAC;MACD,OAAO;OACL,OAAO;QACL,MAAM;QACN,QAAQ,QAAQ;QAChB,MAAM,QAAQ;QACd,KAAK,QAAQ;QACb;OACF;OACA;MACF;KACF;KACA,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;MACxC,SAAS;MACT,QAAQ;MACR,cAAc,KAAA;MACd,MAAM,KAAK,SAAS;MACpB,YAAY;KACd,CAAC;KACD,OAAO;MACL,OAAO;OAAE,MAAM;OAAU,MAAM,kBAAkBA,YAAU,KAAK;OAAK;MAAU;MAC/E;KACF;IACF;GACF,GACA,KAAK,MAAM,QAAQ,SAAS,QAAQ,CACtC;EACF,SAAS,OAAO;GACd,IAAI,iBAAiB,aAAa,OAAO,KAAK,SAAS,MAAM,QAAQ,MAAM,KAAK,CAAC;GACjF,IAAI,iBAAiB,iBAAiB,OAAO,KAAK,aAAa,OAAO,SAAS,QAAQ,CAAC;GACxF,IAAI,KAAK,QAAQ,OAAO,aAAa,GAAG;GACxC,IAAI,SAAS,QAAQ,YAAY,MAAM,OAAO,KAAK,yBAAyB;GAC5E,MAAM;EACR,UAAU;GACR,QAAQ;EACV;EAEA,IAAI,KAAK,SAAS,YAAY,KAAK,QAAQ,OAAO,aAAa,GAAG;EAClE,MAAM,cAAc;GAAE,SAAS,MAAM;GAAS,SAAS,MAAM;GAAS,OAAO,CAAC,GAAG,MAAM,KAAK;EAAE;EAC9F,IAAI,KAAK,SAAS,WAAW;GAC3B,KAAK,KAAK;IACR,MAAM;IACN;IACA;IACA,QAAQ;IACR,SAAS,KAAK,WAAW,aAAa,aAAa;IACnD,WAAW,KAAK;GAClB,CAAC;GACD,IAAI,KAAK,WAAW,YAAY;IAC9B,MAAM,QACJ,KAAK,QAAQ,KAAA,IACT,KAAA,IACA,MAAM,KAAK,aAAa,KAAK,KAAK,QAAQ,QAAQ,QAAQ,MAAM;IACtE,OAAO,KAAK,OAAO,QAAQ,KAAK,MAAM;KACpC,cAAc;KACd;KACA,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,MAAM,IAAI;IACxD,CAAC;GACH;GACA,MAAM,UAAU,MAAM,KAAK,OACzB,WACA;IACE,GAAG;IACH;IACA,QAAQ;IACR,SAAS;IACT,MAAM,KAAK;IACX,YAAY,KAAK;IACjB,cAAc;IACd;IACA,GAAG;GACL,GACA,KACA,KACF;GACA,OAAO;IAAE,GAAG,aAAa,GAAG;IAAG,SAAS;IAAY;IAAS,MAAM,KAAK;IAAM,GAAG;GAAM;EACzF;EACA,IAAI,KAAK,SAAS,YAAY,KAAK,SAAS,aAAa;GACvD,MAAM,OAAO,KAAK,SAAS,WAAW,KAAK,OAAO;GAClD,KAAK,KAAK;IACR,MAAM;IACN;IACA;IACA,QAAQ;IACR,SAAS;IACT,WAAW,KAAK;GAClB,CAAC;GAiBD,OAAO;IACL;IACA,QAAQ;IACR,SAAS;IACT,WAAW;IACX,SAAA,MArBoB,KAAK,OACzB,WACA;KACE,GAAG;KACH;KACA,QAAQ;KACR,SAAS;KACT;KACA,YAAY,KAAK;KACjB,cAAc;KACd;KACA,GAAG;IACL,GACA,KACA,KACF;IAOE,MAAM,KAAA;IACN,MAAM,KAAA;IACN,aAAa,KAAA;IACb,SAAS;IACT;IACA,GAAG;GACL;EACF;EAEA,MAAM,EAAE,UAAU,YAAY,cAAc;EAC5C,MAAM,UAAU,SAAqC,SAAS,QAAQ,KAAK,YAAY;EAEvF,IAAI,cAAc;EAClB,IAAI,eAAe;EACnB,IAAI;GACF,cAAc,IAAI,IAAI,SAAS,QAAQ;GACvC,YAAY,OAAO;EACrB,QAAQ;GACN,cAAc;EAChB;EACA,MAAM,WAAW,OAAO,WAAW;EACnC,MAAM,YAAY,GAAG,YAAY,WAAW,YAAY;EACxD,IAAI,YAAY,SAAS,OAAO,MAAM;GACpC,MAAM,QAAQ,qBAAqB;GACnC,IAAI,CAAC,cAAc,IAAI,YAAY,QAAQ,GACzC,OAAO,KAAK,GAAG,MAAM,gCAAgC,EAAE,WAAW,SAAS,CAAC;GAC9E,MAAM,UAAU,MAAM,YAAY,KAAK,QAAQ,WAAW;GAC1D,IAAI,YAAY,KAAA,GAAW,OAAO,KAAK,GAAG,MAAM,IAAI,WAAW,EAAE,WAAW,SAAS,CAAC;GACtF,IAAI,UAAU,WAAW,MAAM,UAAU,MAAM,GAC7C,eAAe,MAAM,KAAK,UAAU,aAAa,KAAK,SAAS,WAAW,GAAG,QAAQ;GAEvF,MAAM,gBAAgB,OAAO,aAAa,QAAQ,QAAQ,SAAS;GACnE,IAAI,CAAC,cAAc,SACjB,OAAO,KACL,GAAG,MAAM,IAAI,cAAc,aAAa,QAAQ,cAAc,MAAM,KAAK,QAAQ,KAAK,KACtF;IACE,WAAW;IACX,cAAc,YAAY,cAAc,QAAQ,aAAa;GAC/D,CACF;EAEJ;EACA,IAAI,UAAmB,WAAW;EAClC,IAAI,OAAO,WAAW;EACtB,IAAI;EACJ,IAAI;EACJ,IAAI,YAAY,MAAM;GACpB,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,SAAS,IAAI;GAC7C,IAAI;IACF,QAAQ,MAAM,KAAK,QAAQ,MAAM,IAAI,IAAI,EAAA,CAAG;GAC9C,SAAS,OAAO;IACd,UAAU;IACV,OAAO,eAAeA,YAAU,KAAK;IACrC,OAAO,KAAA;GACT;EACF;EACA,MAAM,UAAU,YACd,WAAW,aAAa,QAAQ,QAAQ,SAAS,GACjD,QACA,YAAY,SAAS,IAAI,CAC3B;EACA,MAAM,SAAS,SAAS,UAAU;EAClC,MAAM,YAAY,KAAK,MAAM,IAAI;EACjC,MAAM,YAAY,WAChB,QACA,QACA,WACA,OAAO,eAAe,GACtB,OAAO,uBACT;EACA,KAAK,KAAK;GAAE,MAAM;GAAY;GAAK;GAAM;GAAQ;GAAS;EAAU,CAAC;EACrE,MAAM,UAAU,MAAM,KAAK,OACzB,WACA;GACE,GAAG;GACH;GACA,WAAW;GACX;GACA;GACA,YAAY;GACZ,cAAc;GACd,OAAO,MAAM,UAAU;GACvB,MAAM,QAAQ;GACd,QAAQ;GACR,cAAc;GACd;GACA;GACA,GAAI,cAAc,OAAO,EAAE,aAAa,UAAU,IAAI,CAAC;GACvD,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;GACrC,GAAG;EACL,GACA,KACA,KACF;EACA,MAAM,aAAa,IAAI,KAAK,SAAS,CAAC,CAAC,YAAY;EACnD,IAAI,YAAY,KAAA,KAAa,SAAS,KAAA,GACpC,KAAK,MAAM;GACT;GACA,SAAS;GACT;GACA;GACA,MAAM;GACN,eAAe;GACf,cAAc;GACd,YAAY;GACZ;GACA,UAAU,QAAQ;GAClB,aAAa;EACf,CAAC;EAEH,OAAO;GACL;GACA;GACA;GACA,WAAW;GACX;GACA;GACA;GACA,aAAa;GACb;GACA;GACA,WAAW;GACX,GAAG;EACL;CACF;;;;;;CAOA,WAAW,MAAqB;EAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;EAClD,KAAK,UAAU,MAAM,IAAI;EACzB,KAAK,MAAM,OAAO,KAAK,OAAO,KAAK,GAAG;GACpC,MAAM,QAAQ,KAAK,OAAO,KAAK,GAAG;GAGlC,KADE,OAAO,OAAO,OAAO,gBAAgB,KAAA,KAAa,OAAO,OAAO,UAAU,KAAA,OACxD,SAAS,KAAA,KAAa,IAAI,SAAS,KAAK,MAAM,IAAI,KAAK,OAAO,OAAO,GAAG;EAC9F;CACF;;;;;CAQA,SAAS,KAAoD;EAC3D,IAAI;GACF,OAAO,KAAK,UAAU,MAAM,QAAQ,QAAQ,IAAI,IAAI,GAAG,CAAC,GAAG,KAAK,MAAM,IAAI,CAAC;EAC7E,QAAQ;GACN;EACF;CACF;;;;;;CASA,QAAuB;EACrB,IAAI,KAAK,gBAAgB,KAAA,GAAW;GAClC,KAAK,SAAS;GACd,KAAK,QAAQ,sBAAM,IAAI,MAAM,gBAAgB,CAAC;GAC9C,KAAK,eAAe,YAAY;IAC9B,MAAM,QAAQ,WAAW,KAAK,OAAO;IACrC,MAAM,KAAK,QAAQ,UAAU,MAAM;IACnC,MAAM,KAAK,QAAQ,IAAI,QAAQ;IAC/B,IAAI,KAAK,gBAAgB,MAAM,KAAK,WAAW,MAAM;GACvD,EAAA,CAAG;EACL;EACA,OAAO,KAAK;CACd;AACF;;;ACr4FA,MAAM,OAAO;AACb,MAAM,aAAa;AACnB,MAAM,YAAY;AAClB,MAAM,QAAQ;AACd,MAAM,YACJ;;AAEF,MAAM,8BAA2C,IAAI,IAAI;CACvD,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,OAAQ,EAAI;CACb,CAAC,OAAQ,EAAI;AACf,CAAC;AAED,MAAM,gBAAgB,SAA0B,SAAS,MAAS,QAAQ,KAAQ,QAAQ;;;;;AAM1F,MAAa,YAAY,MAAc,SAA2B;CAChE,IAAI,QAAQ,IAAI,YAAY,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;CAC1D,IAAI,SAAS,IAAI,WAAW,MAAM,MAAM;CACxC,IAAI,OAAO,IAAI,WAAW,MAAM,MAAM;CACtC,IAAI,OAAO;CACX,IAAI,QAAQ;CACZ,MAAM,QAAQ,SAAuB;EACnC,IAAI,OAAO,QAAQ,MAAM,QAAQ;EACjC,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,GAAG,OAAO,IAAI;EACrD,MAAM,YAAY,IAAI,YAAY,MAAM;EACxC,UAAU,IAAI,MAAM,SAAS,GAAG,IAAI,CAAC;EACrC,MAAM,aAAa,IAAI,WAAW,MAAM;EACxC,WAAW,IAAI,OAAO,SAAS,GAAG,IAAI,CAAC;EACvC,MAAM,WAAW,IAAI,WAAW,MAAM;EACtC,SAAS,IAAI,KAAK,SAAS,GAAG,IAAI,CAAC;EACnC,QAAQ;EACR,SAAS;EACT,OAAO;CACT;CACA,MAAM,QAAQ,MAAc,MAAc,OAAqB;EAC7D,KAAK,CAAC;EACN,MAAM,QAAQ;EACd,OAAO,QAAQ;EACf,KAAK,QAAQ;EACb,QAAQ;CACV;CACA,MAAM,aAAa,MAAc,OAAqB;EACpD,IAAI,OAAO;EACX,KAAK,IAAM,MAAM,EAAE;EACnB,QAAQ;CACV;CACA,MAAM,SAAS,KAAK;CACpB,IAAI,QAAQ;CACZ,OAAO,QAAQ,QAAQ;EACrB,MAAM,OAAO,KAAK,WAAW,KAAK;EAClC,MAAM,OAAO,QAAQ,IAAI,SAAS,KAAK,WAAW,QAAQ,CAAC,IAAI;EAE/D,IAAI,OAAO,OAAQ,OAAO,KAAO;GAC/B,IAAI,aAAa,IAAI,GAAG,UAAU,OAAO,QAAQ,CAAC;QAC7C;IACH,KAAK,QAAQ,MAAQ,QAAQ,KAAO,OAAO,KAAK,MAAM,OAAO,QAAQ,CAAC;IACtE,QAAQ;GACV;GACA,SAAS;GACT;EACF;EAEA,MAAM,QAAQ,KAAK,YAAY,KAAK;EACpC,IAAI,MAAM,SAAS,QAAQ,QAAS,IAAI;EACxC,OAAO,MAAM,QAAQ;GACnB,MAAM,OAAO,KAAK,YAAY,GAAG;GACjC,IAAI,CAAC,KAAK,KAAK,OAAO,cAAc,IAAI,CAAC,GAAG;GAC5C,OAAO,OAAO,QAAS,IAAI;EAC7B;EACA,MAAM,UAAU,KAAK,MAAM,OAAO,GAAG;EACrC,MAAM,QAAQ,OAAO,cAAc,KAAK;EACxC,IAAI,UAAU,KAAK,KAAK,KAAK,QAAQ,QAAQ,MAAM,QAAQ;GACzD,QAAQ;GACR;EACF;EACA,IAAI,MAAM,KAAK,KAAK,GAAG;GACrB,UAAU,OAAO,GAAG;GACpB,QAAQ;GACR;EACF;EACA,IAAI,SACF,SAAS,UACL,QAAQ,UAAU,MAAM,CAAC,CAAC,YAAY,IACtC,QACG,UAAU,MAAM,CAAC,CACjB,QAAQ,YAAY,EAAE,CAAC,CACvB,UAAU,KAAK,CAAC,CAChB,YAAY,CAAC,CACb,YAAY;EACrB,IAAI,SAAS,YAAY,UAAU,KAAK,KAAK,GAAG,SAAS,IAAI,OAAO;EACpE,KAAK,IAAI,KAAK,GAAG,KAAK,OAAO,QAAQ,MAAM,GAAG;GAC5C,IAAI,OAAO,OAAO,WAAW,EAAE;GAC/B,OAAO,YAAY,IAAI,IAAI,KAAK;GAChC,IAAI,SAAS,MAAS,OAAO,OAAQ,aAAa,IAAI,KAAM,SAAS,KAAM;IACzE,UAAU,OAAO,GAAG;IACpB;GACF;GACA,KAAK,MAAM,OAAO,GAAG;GACrB,QAAQ;EACV;EACA,QAAQ;CACV;CACA,IAAI,OAAO,KAAK,MAAM,OAAO,OAAO,IAAM,QAAQ;CAClD,IAAI,SAAS;CACb,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,MAAM,MAChC,UAAU,OAAO,aAAa,GAAG,MAAM,SAAS,IAAI,KAAK,IAAI,MAAM,KAAK,IAAI,CAAC,CAAC;CAEhF,OAAO;EAAE,MAAM;EAAQ,OAAO,OAAO,SAAS,GAAG,IAAI;EAAG,KAAK,KAAK,SAAS,GAAG,IAAI;CAAE;AACtF;;AAGA,MAAa,QAAQ,MAAc,SAA2B,SAAS,MAAM,IAAI,CAAC,CAAC;AAcnF,MAAM,YAAsB;CAAE,OAAO;CAAO,OAAO;CAAM,KAAK;CAAM,SAAS;CAAM,OAAO;AAAE;;AAG5F,MAAM,cAAc,MAAc,UAAkB,YAA6B;CAC/E,MAAM,OAAO,KAAK,WAAW,QAAQ;CACrC,IAAI,WAAW,KAAK,WAAW,KAAK,UAAU,QAAQ,SAAU,QAAQ,OACtE,OAAO,UAAU,WAAW,IAAI,WAAW;CAE7C,OAAO;AACT;;;;;;;;AASA,MAAa,UAAU,MAAc,UAA4B;CAC/D,MAAM,SAAS,KAAK,OAAO,OAAO;CAClC,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,WAAW,SAAS,MAAM,OAAO;CACvC,MAAM,MAAM,SAAS,KAAK,QAAQ,MAAM;CACxC,IAAI,QAAQ,IAAI,OAAO;CACvB,IAAI,QAAQ;CACZ,KAAK,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,SAAS,KAAK,QAAQ,QAAQ,KAAK,OAAO,MAAM,GAAG,SAAS;CAC/F,MAAM,QAAQ,SAAS,MAAM;CAC7B,MAAM,MAAM,SAAS,IAAI,MAAM,OAAO,SAAS;CAC/C,MAAM,OAAO,WAAW,MAAM,KAAK,IAAI,GAAG,QAAQ,EAAE,GAAG,KAAK;CAC5D,MAAM,KAAK,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,MAAM,EAAE,GAAG,IAAI;CACjE,OAAO;EACL,OAAO;EACP;EACA;EACA,SAAS,GAAG,OAAO,IAAI,MAAM,KAAK,KAAK,MAAM,MAAM,EAAE,IAAI,KAAK,KAAK,SAAS,MAAM;EAClF;CACF;AACF;;AAGA,MAAM,cAAc;;AAGpB,MAAa,YAAY,SACvB,KAAK,MAAM,QAAQ,CAAC,CACjB,MAAM,WAAW,CAAC,CAClB,QAAQ,UAAU,MAAM,SAAS,CAAC;AAgBvC,MAAM,YAAY;;AAGlB,MAAM,gBAAgB,SAAyB;CAC7C,IAAI,MAAM,KAAK;CACf,OAAO,MAAM,KAAK,KAAK,WAAW,MAAM,CAAC,MAAM,IAAM,OAAO;CAC5D,OAAO,KAAK,MAAM,GAAG,GAAG;AAC1B;;;;;AAMA,MAAM,6BAAkC,IAAI,IAAI;CAC9C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;;;;;;;;;;AAgBD,MAAa,cAAc,MAAoC;CAC7D,MAAM,QAAyF,CAAC;CAChG,KAAK,MAAM,SAAS,EAAE,SAAS,2BAA2B,GAAG;EAC3D,IAAI,MAAM,UAAU,WAAW;EAC/B,MAAM,SAAS,MAAM,OAAO,KAAA;EAC5B,MAAM,MAAM,SAAS,MAAM,KAAM,MAAM;EACvC,MAAM,SAAS,SAAS,MAAM,OAAO,MAAM,IAAI,SAAS,GAAG;EAC3D,MAAM,SAAS,SAAS,SAAS,MAAM,aAAa,GAAG,CAAC;EACxD,IAAI,OAAO,WAAW,GAAG;EACzB,MAAM,KAAK;GACT,MAAM,IAAI,OAAO,KAAK,GAAG,EAAE,GAAG,SAAS,OAAO;GAC9C,MAAM,OAAO;GACb,MAAM,CAAC,UAAU,CAAC,UAAU,OAAO,WAAW,KAAK,WAAW,IAAI,OAAO,EAAG;EAC9E,CAAC;CACH;CACA,IAAI,MAAM,WAAW,GAAG,OAAO,KAAA;CAC/B,MAAM,QAAQ,MAAM,QAAQ,SAAS,CAAC,KAAK,IAAI;CAC/C,MAAM,MAAM,MAAM,SAAS,IAAI,QAAQ;CACvC,OAAO;EACL,KAAK,MAAM,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,OAAO;EAChD,KAAK,IAAI,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,MAAM;EAC7C,UAAU,IAAI,KAAK,SAAS,KAAK,IAAI;EACrC,OAAO,IAAI,KAAK,SAAS,KAAK,IAAI;EAClC,OAAO,MAAM;CACf;AACF;;;;;AASA,MAAa,aAAa,MAAc,OAA0B,QAAQ,QAAgB;CACxF,IAAI,KAAK,WAAW,GAAG,OAAO;CAC9B,IAAI,QAAQ;CACZ,IAAI,MAAM;CACV,MAAM,QAAQ,KAAK,YAAY;CAC/B,IAAI,MAAM,WAAW,KAAK,QACxB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,KAAK,MAAM,QAAQ,IAAI;EAC7B,IAAI,OAAO,OAAO,UAAU,MAAM,KAAK,QAAQ;GAC7C,QAAQ;GACR,MAAM,KAAK,KAAK;EAClB;CACF;CAEF,IAAI,UAAU,IAAI;EAChB,MAAM,SAAS,SAAS,MAAM,QAAQ;EACtC,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,KAAK,OAAO,KAAK,QAAQ,IAAI;GACnC,IAAI,OAAO,IAAI;IACb,MAAM,OAAO,OAAO,MAAM;IAC1B,IAAI,UAAU,MAAM,OAAO,OAAO;KAChC,QAAQ;KACR,MAAM,OAAO,IAAI,KAAK,KAAK,SAAS;IACtC;GACF;EACF;CACF;CACA,IAAI,UAAU,IAAI;EAChB,MAAM,MAAM,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,KAAK,GAAG,KAAK;EAChE,OAAO,GAAG,KAAK,MAAM,GAAG,GAAG,IAAI,MAAM,KAAK,SAAS,MAAM;CAC3D;CACA,MAAM,OAAO,WAAW,MAAM,KAAK,IAAI,GAAG,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC,GAAG,KAAK;CAC/E,MAAM,KAAK,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,MAAM,KAAK,KAAM,QAAQ,IAAK,CAAC,CAAC,GAAG,IAAI;CACzF,MAAM,SAAS,SAAyB,KAAK,QAAQ,QAAQ,GAAG;CAChE,OAAO,GAAG,OAAO,IAAI,MAAM,KAAK,MAAM,KAAK,MAAM,MAAM,KAAK,CAAC,EAAE,GAAG,MAAM,KAAK,MAAM,OAAO,GAAG,CAAC,EAAE,GAAG,MAAM,KAAK,MAAM,KAAK,EAAE,CAAC,IAAI,KAAK,KAAK,SAAS,MAAM;AAC3J;;;AC9TA,MAAM,YAAY;AAClB,MAAM,YAAY;AAClB,MAAM,iBAAiB;;;;;;AAMvB,MAAMC,uBAAqB;;;;;;AAO3B,MAAa,SAAS,MAAkB,gBAAwC;CAC9E,IAAI,WAAW,KAAK,eAAe,EAAE,GAAG,OAAO;CAC/C,MAAM,MAAM,KAAK,IAAI,KAAK,SAAS,GAAG,IAAI;CAC1C,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,SAAS,GACxC,IACE,KAAK,WAAW,MAChB,KAAK,QAAQ,OAAO,MACpB,KAAK,QAAQ,OAAO,MACpB,KAAK,QAAQ,OAAO,MACpB,KAAK,QAAQ,OAAO,IAEpB,OAAO;CAGX,OAAO;AACT;;AAGA,MAAM,QAAQ,SACZ,KACG,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAC/C,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,IAAI;;AAGd,MAAM,aACJ;AAEF,MAAM,UAAU,OAAmB,OAAe,QAAwB;CACxE,IAAI,MAAM;CACV,KAAK,IAAI,QAAQ,KAAK,IAAI,GAAG,KAAK,GAAG,QAAQ,OAAO,QAAQ,MAAM,QAAQ,SAAS,GACjF,OAAO,OAAO,aAAa,MAAM,MAAO;CAE1C,OAAO;AACT;AAEA,MAAM,WAAW,OAAmB,QAAgB,SAAyB;CAC3E,MAAM,QAAQ,OAAO,WAAW,CAAC;CACjC,KACE,IAAI,QAAQ,MAAM,QAAQ,OAAO,IAAI,GACrC,UAAU,IACV,QAAQ,MAAM,QAAQ,OAAO,QAAQ,CAAC,GACtC;EACA,IAAI,QAAQ;EACZ,KAAK,IAAI,KAAK,GAAG,KAAK,OAAO,QAAQ,MAAM,GACzC,IAAI,MAAM,QAAQ,QAAQ,OAAO,WAAW,EAAE,GAAG;GAC/C,QAAQ;GACR;EACF;EAEF,IAAI,OAAO,OAAO;CACpB;CACA,OAAO;AACT;;;;;;;AAQA,MAAM,kBAAkB,OAAmB,UAA2B;CACpE,IAAI,QAAQ;CACZ,IAAI,OAAO;CACX,SAAS;EACP,MAAM,UAAU,QAAQ,OAAO,UAAU,IAAI;EAC7C,IAAI,YAAY,IAAI,OAAO;EAC3B,OAAO,UAAU;EAEjB,IAAI,OAAO,OAAO,UAAU,GAAG,OAAO,MAAM,OAAO;EACnD,MAAM,OAAO,MAAM,UAAU;EAC7B,IAAI,SAAS,MAAQ,SAAS,IAAM;EACpC,MAAM,aAAa,OAAO,OAAO,UAAU,MAAM,OAAO;EACxD,MAAM,OAAO,WAAW,MAAM,KAAK,IAAI,GAAG,WAAW,YAAY,IAAI,CAAC,CAAC;EACvE,IAAI,wBAAwB,KAAK,IAAI,GAAG;EACxC,IAAI,QAAQ,UAAU;EACtB,IAAI,SAAS,MAAQ,MAAM,WAAW,IAAM,SAAS;EACrD,MAAM,WAAW,iCAAiC,KAAK,IAAI;EAC3D,MAAM,OAAO,QAAQ,OAAO,aAAa,KAAK;EAC9C,MAAM,MACJ,aAAa,QAAQ,QAAQ,OAAO,SAAS,EAAE,KAAK,MAAM,SACtD,QAAQ,OAAO,SAAS,EAAE,IAC1B,SAAS,KACP,MAAM,SACN;EACR,IAAI;GACF,SAAS,YAAY,MAAM,SAAS,OAAO,GAAG,GAAG;IAC/C,iBAAiB,KAAK,IAAI,GAAG,QAAQ,QAAQ,CAAC;IAC9C,aAAa,UAAU;GACzB,CAAC,CAAC,CAAC;EACL,SAAS,OAAO;GAEd,IAAI,iBAAiB,YAAY,OAAO;EAC1C;EACA,IAAI,QAAQ,OAAO,OAAO;EAC1B,OAAO,KAAK,IAAI,MAAM,GAAG;CAC3B;AACF;;AASA,MAAM,cAA6B,IAAI,SAAS,YAAY,aAAa,OAAO,CAAC;;AAGjF,MAAM,SAAS,OAAU,SAAqB,OAAuC;CACnF,IAAI;CACJ,MAAM,OAAO,IAAI,SAAoB,YAAY;EAC/C,QAAQ,iBAAiB,QAAQ,KAAA,CAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAC;CAC9D,CAAC;CACD,IAAI;EACF,OAAO,MAAM,QAAQ,KAAK,CAAC,SAAS,IAAI,CAAC;CAC3C,UAAU;EACR,aAAa,KAAK;CACpB;AACF;;AAGA,MAAM,WAAW,MAAc,WAA2B;CACxD,IAAI,MAAM,KAAK,IAAI,GAAG,MAAM;CAC5B,MAAM,OAAO,KAAK,WAAW,GAAG;CAChC,IAAI,MAAM,KAAK,MAAM,KAAK,UAAU,QAAQ,SAAU,QAAQ,OAAQ,OAAO;CAC7E,OAAO,KAAK,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ;AACpC;;AAGA,MAAa,UAAU,OAAO,OAAmB,UAAsB,CAAC,MAAwB;CAC9F,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,WAAW,QAAQ,gBAAgB;CACzC,MAAM,cAAc,QAAQ,oBAAoBA;CAChD,IAAI,CAAC,eAAe,OAAO,WAAW,GACpC,OAAO;EACL,OAAO;EACP,MAAM;EACN,OAAO;EACP,WAAW;EACX,MAAM,wCAAwC,OAAO,KAAK,MAAM,cAAc,OAAO,IAAI,CAAC,EAAE;CAC9F;CAGF,MAAM,MAAM,MAAM,iBAAiB,IAAI,WAAW,KAAK,CAAC;CACxD,IAAI;EACF,MAAM,UAAU,KAAK,IAAI;EACzB,MAAM,QAAkB,CAAC;EACzB,IAAI,QAAQ;EACZ,IAAI,OAAsB;EAC1B,MAAM,OAAO,KAAK,IAAI,IAAI,UAAU,QAAQ;EAC5C,IAAI,IAAI,WAAW,UAAU,OAAO,0BAA0B,OAAO,QAAQ,EAAE;EAC/E,MAAM,QAAQ,WACZ,0BAA0B,OAAO,KAAK,MAAM,WAAW,GAAI,CAAC,EAAE,oBAAoB,OAAO,SAAS,CAAC;EACrG,KAAK,IAAI,SAAS,GAAG,UAAU,MAAM,UAAU,GAAG;GAChD,IAAI,SAAS,GAAG;IACd,MAAM,MAAM;IACZ,IAAI,KAAK,IAAI,IAAI,UAAU,UAAU;KACnC,OAAO,KAAK,MAAM;KAClB;IACF;GACF;GACA,MAAM,OAAO,MAAM,IAAI,QAAQ,MAAM;GAGrC,MAAM,UAAU,MAAM,OAAO,KAAK,eAAe,GAAG,YAAY,KAAK,IAAI,IAAI,QAAQ;GACrF,IAAI,YAAY,KAAA,GAAW;IACzB,OAAO,0BAA0B,OAAO,KAAK,MAAM,WAAW,GAAI,CAAC,EAAE,oBAAoB,OAAO,MAAM;IACtG;GACF;GACA,MAAM,OAAO,KACV,QAAQ,MACN,QAAQ,SAAS,KAAK,QAAQ,KAAA,CAAS,CAAC,CACxC,KAAK,SAAS,GAAG,KAAK,OAAO,KAAK,KAAK,WAAW,OAAO,OAAO,IAAI,CAAC,CACrE,KAAK,EAAE,CACZ;GACA,KAAK,QAAQ;GACb,IAAI,KAAK,SAAS,GAAG;IAEnB,IAAI,KAAK,SAAS,WAAW,OAAO;KAClC,MAAM,KAAK,QAAQ,MAAM,WAAW,KAAK,CAAC;KAC1C,OAAO,uBAAuB,OAAO,QAAQ,EAAE,uBAAuB,OAAO,MAAM;KACnF;IACF;IACA,MAAM,KAAK,IAAI;IACf,SAAS,KAAK,SAAS;GACzB;GACA,IAAI,SAAS,YAAY,SAAS,IAAI,UAAU;IAC9C,OAAO,uBAAuB,OAAO,QAAQ,EAAE,uBAAuB,OAAO,MAAM;IACnF;GACF;EACF;EAEA,MAAM,UADQ,MAAM,QAAQ,GAAG,CAAC,CAAC,YAAY,KAAA,CAAS,EAAA,EAAI,KAAA,GAC5B;EAC9B,MAAM,QAAQ,OAAO,UAAU,WAAW,MAAM,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,IAAI;EAC9E,OAAO;GACL,OAAO,MAAM,SAAS,KAAK,CAAC,WAAW,KAAK,KAAK,IAAI,QAAQ;GAC7D,MAAM,MAAM,KAAK,MAAM;GACvB,OAAO,IAAI;GACX,WAAW,SAAS;GACpB;EACF;CACF,UAAU;EAER,MAAM,IAAI,YAAY,QAAQ;CAChC;AACF;;;;AC9NA,MAAa,sBAAsB;;AAGnC,MAAa,uBAAuB;;AAGpC,MAAa,gBAAgB;;;;;;AAO7B,MAAa,iBAAiB,QAAwB,WAAW,aAAa,GAAG;;AAGjF,MAAM,oBAAoB;AAC1B,MAAM,UAAU;AAChB,MAAM,WAAW;AACjB,MAAM,UAAU;;AAGhB,MAAM,aAAa;;AAEnB,MAAM,mBAAmB;;AAEzB,MAAM,qBAAqB;;AAG3B,MAAM,cAAc;AACpB,MAAM,cAAc;AACpB,MAAM,cAAc;AAcpB,MAAM,kBAAkB;AACxB,MAAM,qBAAqB;AAC3B,MAAM,eAAe;;;;;;;AAQrB,MAAa,kBAAkB,YAAiD;CAC9E,MAAM,EAAE,YAAY;CACpB,IAAI,QAAQ,YAAY,KAAA,GAAW,OAAO;CAC1C,MAAM,OAAO,QAAQ,QAAQ;CAC7B,QAAQ,QAAQ,SAAhB;EACE,KAAK;GACH,IAAI,KAAK,WAAW,eAAe,GAAG,OAAO;GAC7C,IAAI,KAAK,WAAW,kBAAkB,GAAG,OAAO;GAChD,IAAI,KAAK,WAAW,YAAY,GAAG,OAAO;GAC1C;EACF,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,eACH,OAAO,QAAQ;EACjB,KAAK;GACH,IAAI,QAAQ,WAAW,KAAK,KAAK,WAAW,aAAa,GAAG,OAAO;GACnE,OAAO,QAAQ,UAAU,MAAM,UAAU,KAAA;EAC3C,SACE;CACJ;AACF;;AAGA,MAAa,cAAc,SAAkE;CAC3F,MAAM,QAAQ,KAAK;CACnB,IAAI,OAAO,UAAU,YAAY,UAAU,MAAM,OAAO,KAAA;CACxD,MAAM,EAAE,QAAQ,WAAW,KAAK,UAAU,WAAW;CACrD,IACG,WAAW,aAAa,WAAW,iBACpC,OAAO,cAAc,YACrB,OAAO,QAAQ,YACf,OAAO,aAAa,UAEpB,OAAO,KAAA;CACT,OAAO;EAAE;EAAQ;EAAW;EAAK;EAAU,GAAI,OAAO,WAAW,WAAW,EAAE,OAAO,IAAI,CAAC;CAAG;AAC/F;AAEA,MAAM,gBAAsD;CAC1D,SAAS;CACT,aAAa;AACf;;AAGA,MAAa,gBAAgB,aAC3B,yBAAyB,cAAc,SAAS,QAAQ,aAAa,SAAS,UAAU;;AAG1F,MAAM,cAAc,OAClB,OAAO,SAAS,EAAE,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,aAAa,GAAG,IAAI,KAAA;;AAG/E,MAAM,eAAe,WAAuC;CAC1D,MAAM,QAAQ,+CAA+C,KAAK,MAAM;CACxE,IAAI,UAAU,MAAM,OAAO,KAAA;CAC3B,MAAM,GAAG,MAAM,OAAO,KAAK,MAAM,QAAQ,UAAU;CACnD,MAAM,KAAK,KAAK,IACd,OAAO,IAAI,GACX,OAAO,KAAK,IAAI,GAChB,OAAO,GAAG,GACV,OAAO,IAAI,GACX,OAAO,MAAM,GACb,OAAO,MAAM,CACf;CACA,MAAM,MAAM,WAAW,EAAE;CAEzB,OAAO,QAAQ,KAAA,KAAa,IAAI,QAAQ,OAAO,EAAE,CAAC,CAAC,WAAW,MAAM,IAAI,MAAM,KAAA;AAChF;AAEA,MAAM,aAAa,SAAqC;CACtD,IAAI;EACF,MAAM,MAAM,IAAI,IAAI,IAAI;EACxB,OAAO,IAAI,aAAa,WAAW,IAAI,aAAa,WAAW,IAAI,OAAO,KAAA;CAC5E,QAAQ;EACN;CACF;AACF;AAaA,MAAM,QAAQ,OAAO,KAAK,UAAU;;AAGpC,MAAM,aAAa,SAAsC;CACvD,MAAM,sBAAM,IAAI,IAAoB;CACpC,KAAK,MAAM,QAAQ,KAAK,MAAM,OAAO,GAAG;EACtC,MAAM,QAAQ,KAAK,QAAQ,GAAG;EAC9B,IAAI,SAAS,GAAG;EAChB,MAAM,OAAO,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,YAAY;EACrD,MAAM,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC,CAAC,KAAK;EACzC,MAAM,QAAQ,IAAI,IAAI,IAAI;EAC1B,IAAI,IAAI,MAAM,UAAU,KAAA,IAAY,QAAQ,GAAG,MAAM,IAAI,OAAO;CAClE;CACA,OAAO;AACT;;AAGA,MAAM,WAAW,SAAqC;CACpD,MAAM,QAAkB,CAAC;CACzB,IAAI,KAAK;CACT,IAAI,QAAQ;CACZ,SAAS;EACP,MAAM,MAAM,KAAK,QAAQ,QAAQ,EAAE;EACnC,IAAI,QAAQ,IAAI,OAAO,KAAA;EACvB,MAAM,OAAO,OAAO,SAClB,KAAK,SAAS,IAAI,GAAG,CAAC,CAAC,SAAS,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAE,KAAK,GAC9D,EACF;EACA,IAAI,CAAC,OAAO,SAAS,IAAI,KAAK,OAAO,GAAG,OAAO,KAAA;EAC/C,IAAI,SAAS,GAAG,OAAO,OAAO,OAAO,KAAK;EAC1C,MAAM,QAAQ,MAAM;EACpB,IAAI,QAAQ,OAAO,KAAK,QAAQ,OAAO,KAAA;EACvC,SAAS;EACT,IAAI,QAAQ,oBAAoB,OAAO,KAAA;EACvC,MAAM,KAAK,KAAK,SAAS,OAAO,QAAQ,IAAI,CAAC;EAC7C,KAAK,QAAQ,OAAO;CACtB;AACF;;AAGA,MAAM,gBAAgB,MAAc,WAAmD;CACrF,MAAM,UAAU,EAAE,iBAAiB,mBAAmB;CACtD,IAAI,MAAM;CACV,KAAK,MAAM,SAAS,UAAU,GAAA,CAC3B,MAAM,GAAG,CAAC,CACV,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CACxC,QAAQ,SAAS,KAAK,SAAS,KAAK,SAAS,UAAU,CAAC,CACxD,QAAQ,GACT,IAAI,SAAS,UAAU,SAAS,UAAU,MAAM,WAAW,KAAK,OAAO;MAClE,IAAI,SAAS,MAAM,MAAM,qBAAqB,KAAK,OAAO;MAC1D,IAAI,SAAS,WAChB,IAAI;EACF,MAAM,YAAY,KAAK,OAAO;CAChC,QAAQ;EACN,MAAM,eAAe,KAAK,OAAO;CACnC;MACK,OAAO,KAAA;CAEhB,OAAO;AACT;;;;;;;AAQA,MAAa,oBAAoB,UAAgD;CAC/E,IAAI,MAAM,SAAS,KAAK,MAAM,OAAO,MAAQ,MAAM,OAAO,KAAM,OAAO,KAAA;CACvE,IAAI;CACJ,IAAI;EACF,SAAS,WAAW,OAAO,EAAE,iBAAiB,mBAAmB,CAAC;CACpE,QAAQ;EACN;CACF;CACA,IAAI,CAAC,OAAO,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,QAAQ,CAAC,CAAC,WAAW,SAAS,GAAG,OAAO,KAAA;CAC5E,MAAM,UAAU,OAAO,QAAQ,KAAK;CACpC,IAAI,YAAY,IAAI,OAAO,KAAA;CAC3B,MAAM,OAAO,UAAU,OAAO,SAAS,GAAG,OAAO,CAAC,CAAC,SAAS,MAAM,CAAC;CACnE,IAAI,KAAK,IAAI,WAAW,CAAC,EAAE,YAAY,MAAM,YAAY,OAAO,KAAA;CAChE,MAAM,SAAS,UAAU,KAAK,IAAI,iBAAiB,CAAC,EAAE,QAAQ,UAAU,EAAE,KAAK,EAAE;CACjF,MAAM,OAAO,WAAW,KAAK,MAAM,KAAK,IAAI,WAAW,KAAK,EAAE,CAAC;CAC/D,IAAI,WAAW,KAAA,KAAa,SAAS,KAAA,GAAW,OAAO,KAAA;CACvD,MAAM,SAAS,OAAO,KAAK,IAAI,gBAAgB,KAAK,GAAU;CAC9D,MAAM,QAAQ,UAAU;CACxB,MAAM,QAAQ,OAAO,SACnB,OACA,OAAO,UAAU,MAAM,KAAK,UAAU,IAClC,KAAK,IAAI,OAAO,QAAQ,QAAQ,MAAM,IACtC,OAAO,MACb;CACA,MAAM,UAAU,MAAM,QAAQ,KAAK;CACnC,IAAI,YAAY,IAAI,OAAO,KAAA;CAC3B,MAAM,OAAO,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,SAAS,QAAQ;CACzD,MAAM,aAAa,6BAA6B,KAAK,IAAI;CACzD,IAAI,eAAe,MAAM,OAAO,KAAA;CAChC,MAAM,UAAU,UAAU,KAAK,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC;CAC5D,IAAI,OAA2B,MAAM,SAAS,UAAU,CAAC;CACzD,IAAI;EACF,IAAI,eAAe,KAAK,QAAQ,IAAI,mBAAmB,KAAK,EAAE,GAAG,OAAO,QAAQ,IAAI;EACpF,IAAI,SAAS,KAAA,GAAW,OAAO,aAAa,MAAM,QAAQ,IAAI,kBAAkB,CAAC;CACnF,QAAQ;EACN,OAAO,KAAA;CACT;CACA,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;CAC/B,OAAO;EACL;EACA;EACA,QAAQ,KAAK,IAAI,gBAAgB,CAAC,EAAE,QAAQ,UAAU,EAAE;EACxD,QAAQ,OAAO,WAAW,EAAE;EAC5B,SAAS,SAAS,QAAQ,IAAI,KAAK,YAAY,CAAC;EAChD,MAAM,IAAI,WAAW,IAAI;CAC3B;AACF;;AAKA,MAAM,cACJ,MACA,UACA,YAEA,KACG,QAAQ,QAAQ,OAAO,IAAI,KAAK,eAAe,YAAY,IAAI,KAAK,eAAe,EAAE,CAAC,CACtF,KAAK,SAAS;CACb,GAAG;CACH,KAAK,cAAc,SAAS,QAAQ;CACpC,MAAM;CACN,KAAK,SAAS;CACd,MAAM;EAAE,GAAG,IAAI;EAAM,eAAe,IAAI;EAAM;CAAS;CACvD,GAAI,YAAY,OAAO,EAAE,UAAU,KAAc,IAAI,CAAC;AACxD,EAAE;;;;;;;;;;;;;AAcN,MAAa,mBAA8B,OAAO,MAAM,YAAY;CAClE,IAAI;CACJ,IAAI;EACF,MAAM,IAAI,IAAI,QAAQ,GAAG;CAC3B,QAAQ;EACN,OAAO,CAAC;CACV;CACA,MAAM,MAAM,YAAY,KAAK,GAAG,IAAI,WAAW,IAAI,QAAQ;CAC3D,IAAI,QAAQ,MAAM;EAChB,MAAM,YAAY,YAAY,IAAI,EAAG;EACrC,MAAM,WAAW,UAAU,IAAI,EAAG;EAClC,IAAI,cAAc,KAAA,KAAa,aAAa,KAAA,GAAW,OAAO,CAAC;EAC/D,MAAM,WAAqB;GACzB,QAAQ;GACR;GACA,KAAK,GAAG,IAAI,OAAO,OAAO,IAAI,GAAI,GAAG;GACrC;EACF;EACA,OAAO,WACL,MAAM,cAAc,MAAM;GAAE,KAAK;GAAU,aAAa,QAAQ;EAAY,CAAC,GAC7E,UACA,KAAA,CACF;CACF;CACA,MAAM,WAAW,iBAAiB,IAAI;CACtC,IAAI,aAAa,KAAA,KAAa,SAAS,SAAS,OAAO,SAAS,UAAU,KAAK,OAAO,CAAC;CACvF,MAAM,WAAqB;EACzB,QAAQ;EACR,WAAW,SAAS;EACpB,KAAK,QAAQ;EACb,UAAU,SAAS;EACnB,GAAI,SAAS,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,SAAS,OAAO;CACrE;CACA,MAAM,UAAU,YAAY,CAAC,GAAG,SAAS,MAAM;CAC/C,OAAO,WACL,MAAM,cAAc,SAAS,MAAM;EACjC,KAAK,SAAS;EACd,aAAa,SAAS,OAAO,cAAc,KAAK;CAClD,CAAC,GACD,UACA,QAAQ,aAAa,OAAO,OAAO,KAAA,CACrC;AACF;;AAgBA,MAAM,UACJ,aAII;CACJ,UAAU,QAAQ;CAClB,YAAY,QAAQ;CACpB,KAAK,QAAQ;CACb,QAAQ,QAAQ;CAChB,OAAO,QAAQ;CACf,QAAQ,QAAQ;CAChB,UAAU,QAAQ;AACpB;AAEA,MAAM,OAAO,aACV,QAAQ,YAAY,QAAQ,QAAQ,YAAY,mBAAmB,QAAQ,SAAS,KAAA;AAEvF,MAAM,QAAQ,YACZ,GAAG,QAAQ,UAAU,QAAQ,SAAS,IAAI,IAAI,OAAO,QAAQ,MAAM,MAAM,KAAK,QAAQ,SAAS,KAAA,IAAY,KAAK,KAAK,QAAQ;AAE/H,MAAM,UAAU,YAA8B;CAC5C,IAAI;EACF,OAAO,KAAK,MAAM,IAAI,YAAY,CAAC,CAAC,OAAO,QAAQ,IAAI,CAAC;CAC1D,QAAQ;EACN;CACF;AACF;;AAGA,MAAM,UACJ,SACA,SACmF;CACnF,MAAM,MAAM,QAAQ,KAAK,MAAM,QAAQ,IAAI,SAAS,aAAa;CACjE,MAAM,WAAW,QAAQ,KAAA,IAAY,KAAA,IAAY,WAAW,IAAI,IAAI;CACpE,IAAI,QAAQ,KAAA,KAAa,aAAa,KAAA,GAAW,OAAO;EAAE;EAAK;CAAS;CACxE,IAAI,CAAC,IAAI,QAAQ,OAAO,GACtB,OAAO,EAAE,MAAM,GAAG,KAAK,2BAA2B,KAAK,QAAQ,OAAO,EAAE,GAAG;CAC7E,OAAO,EACL,MAAM,GAAG,KAAK,yBAAyB,QAAQ,UAAU,KAAA,IAAY,oCAAoC,KAAK,QAAQ,MAAM,KAC9H;AACF;;AAGA,MAAM,cAAc,OAClB,SACA,KACA,aACA,YAIG;CACH,MAAM,OAAO,cAAc;CAC3B,MAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM;EACzC,KAAK,GAAG,kBAAkB,OAAO,mBAAmB,GAAG;EACvD,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,GAAG,OAAO,OAAO;CACnB,CAAC;CACD,IAAI,CAAC,IAAI,MAAM,GAAG,OAAO,EAAE,MAAM,GAAG,KAAK,8BAA8B,KAAK,MAAM,EAAE,GAAG;CACvF,MAAM,UACJ,OAAO,MAAM,CAAC,EACb,oBAAoB;CACvB,IAAI,YAAY,KAAA,KAAa,QAAQ,iBAAiB,OAAO,OAAO,EAAE,MAAM,GAAG,KAAK,WAAW;CAC/F,MAAM,SAAS,QAAQ;CACvB,IAAI,WAAW,KAAA,KAAa,WAAW,OACrC,OAAO,EACL,MAAM,GAAG,KAAK,mCAAmC,KAAK,UAAU,MAAM,KAAK,UAAU,WACvF;CACF,MAAM,SAAS,OAAO,QAAQ,iBAAiB,WAAW,QAAQ,eAAe;CACjF,MAAM,QAAQ,OAAO,QAAQ,WAAW,WAAW,QAAQ,SAAS;CACpE,MAAM,WAAW,UAAU,+BAA+B,KAAK,KAAK,CAAC,GAAG,MAAM,EAAE;CAChF,IAAI,YAAY,MAAM,MAAM,KAAA,KAAa,aAAa,KAAA,GACpD,OAAO,EAAE,MAAM,GAAG,KAAK,wCAAwC;CACjE,MAAM,OAAO,MAAM,cAAc,SAAS;EACxC,KAAK,GAAG,UAAU,OAAO,MAAM;EAC/B,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,sBAAsB;EACtB;EACA,GAAG,OAAO,OAAO;CACnB,CAAC;CACD,MAAM,QAAQ,OAAO,MAAM,IAAI;CAC/B,OAAO,UAAU,QACb,QACA;EAAE,GAAG;EAAO,WAAW,OAAO,aAAa,KAAK,QAAQ;CAAU;AACxE;;AAUA,MAAM,YAAY,YAA2C;CAC3D,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,QAAQ,IAAI;CAClD,IAAI;CACJ,KAAK,MAAM,QAAQ,KAAK,MAAM,IAAI,GAAG;EACnC,IAAI,KAAK,KAAK,MAAM,IAAI;EACxB,IAAI;EACJ,IAAI;GACF,MAAM,KAAK,MAAM,IAAI;EACvB,QAAQ;GACN;EACF;EACA,MAAM,EAAE,WAAW,UAAU,WAAW;EACxC,MAAM,SAAS,OAAO,IAAI,SAAS;EACnC,MAAM,SAAS,OAAO,IAAI,SAAS;EAEnC,IACE,WAAW,SACX,OAAO,cAAc,YACrB,CAAC,WAAW,KAAK,SAAS,KAC1B,OAAO,aAAa,YACpB,CAAC,oCAAoC,KAAK,QAAQ,KAClD,SAAS,SAAS,IAAI,KACtB,CAAC,OAAO,cAAc,MAAM,KAC5B,SAAS,KACT,CAAC,OAAO,cAAc,MAAM,KAC5B,SAAS,KACT,SAAS,kBAET;EACF,IAAI,SAAS,KAAA,KAAa,YAAY,KAAK,WACzC,OAAO;GAAE;GAAW;GAAU;GAAQ;EAAO;CACjD;CACA,OAAO;AACT;;AAGA,MAAM,kBAAkB,OACtB,SACA,KACA,aACA,YAIG;CACH,MAAM,OAAO,cAAc;CAC3B,MAAM,OAAO,MAAM,QAAQ,QAAQ,MAAM;EACvC,KAAK,GAAG,SAAS;EACjB,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,GAAG,OAAO,OAAO;CACnB,CAAC;CACD,IAAI,CAAC,IAAI,IAAI,GAAG,OAAO,EAAE,MAAM,GAAG,KAAK,sCAAsC,KAAK,IAAI,EAAE,GAAG;CAC3F,MAAM,SAAS,OAAO,IAAI;CAC1B,MAAM,QAAQ,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,EAAA,CAC7C,KAAK,UACJ,OAAO,UAAU,YAAY,UAAU,OAClC,MAAkC,aACnC,KAAA,CACN,CAAC,CACA,QAAQ,QAAuB,OAAO,QAAQ,YAAY,IAAI,WAAW,QAAQ,CAAC,CAAC,CACnF,MAAM,GAAG,UAAU;CACtB,IAAI,KAAK,WAAW,GAAG,OAAO,EAAE,MAAM,GAAG,KAAK,gCAAgC;CAC9E,IAAI;CACJ,IAAI,YAAY,KAAK;CACrB,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,MAAM,QAAQ,QAAQ,MAAM;GACxC,KAAK,GAAG,IAAI,OAAO,mBAAmB,GAAG,EAAE;GAC3C,MAAM;GACN,QAAQ;GACR,QAAQ;GACR,GAAG,OAAO,OAAO;EACnB,CAAC;EACD,cAAc,MAAM;EACpB,IAAI,IAAI,KAAK,GAAG,QAAQ,SAAS,KAAK;OACjC,IAAI,MAAM,YAAY,aAAa,OAAO,KAAK,KAAK,KAAK,CAAC;EAC/D,IAAI,UAAU,KAAA,GAAW;EACzB,IAAI,MAAM,YAAY,WAAW;CACnC;CACA,IAAI,UAAU,KAAA,GACZ,OAAO,EACL,MAAM,GAAG,KAAK,mBAAmB,OAAO,KAAK,MAAM,EAAE,iBAAiB,OAAO,SAAS,IAAI,KAAK,OAAO,KAAK,IAAI,EAAE,KAAK,KACxH;CACF,MAAM,OAAO,MAAM,cAAc,SAAS;EACxC,KAAK,GAAG,UAAU,MAAM;EACxB,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,OAAO;GAAE,QAAQ,MAAM;GAAQ,QAAQ,MAAM;EAAO;EACpD;EACA,GAAG,OAAO,OAAO;CACnB,CAAC;CACD,IAAI,IAAI,KAAK,OAAO,KAAK,KAAK,QAAQ,WAAW,KAC/C,OAAO,EACL,MAAM,GAAG,KAAK,+BAA+B,OAAO,KAAK,QAAQ,MAAM,EAAE,4BAC3E;CACF,MAAM,QAAQ,OAAO,MAAM,IAAI;CAC/B,OAAO,UAAU,QAAQ,QAAQ;EAAE,GAAG;EAAO,WAAW;CAAM;AAChE;;;;;;;AAQA,MAAa,eAAe,OAC1B,SACA,KACA,QACA,YAC2B;CAC3B,IAAI;CACJ,IAAI;EACF,SAAS,IAAI,IAAI,GAAG;CACtB,QAAQ;EACN,OAAO;GAAE,WAAW;GAAO,OAAO,CAAC;EAAE;CACvC;CACA,OAAO,OAAO;CAEd,IAAI,OAAO,MAAM,MAAM,OAAO,MAC5B,OAAO;EACL,WAAW;EACX,OAAO,CAAC,0EAA0E;CACpF;CACF,MAAM,UAAU,QAAQ,yBAAyB;CACjD,MAAM,UACJ;CACF,IAAI,WAAW,YAAY,CAAC,SAAS,OAAO;EAAE,WAAW;EAAO,OAAO,CAAC,OAAO;CAAE;CAEjF,IAAI;CACJ,IAAI;EACF,QAAQ,MAAM,QAAQ,QAAQ,YAAY,OAAO,MAAM;GACrD,YAAY,QAAQ;GACpB,OAAO,QAAQ;GACf,QAAQ,QAAQ;GAChB,QAAQ,QAAQ;EAClB,CAAC;CACH,QAAQ;EAEN,OAAO;GAAE,WAAW;GAAO,OAAO,CAAC,gDAAgD;EAAE;CACvF;CACA,IAAI,CAAC,MAAM,YAAY,MAAM,QAAQ,GAAA,CAAI,WAAW,eAAe,KAAK,CAAC,SACvE,OAAO;EAAE,WAAW;EAAO,OAAO,CAAC,OAAO;CAAE;CAC9C,MAAM,cAAc,MAAM,WAAW,CAAC;CAEtC,MAAM,QAAQ,QAAQ,MAAM,IAAI,cAAc,OAAO,IAAI,CAAC;CAC1D,MAAM,gBAAgB,UAAU,KAAA,IAAY,KAAA,IAAY,WAAW,MAAM,IAAI;CAC7E,IACE,UAAU,KAAA,KACV,kBAAkB,KAAA,KAClB,OAAO,QAAQ,aAAa,YAC5B,KAAK,MAAM,QAAQ,IAAI,CAAC,IAAI,KAAK,MAAM,MAAM,UAAU,IAAI,QAAQ,UAEnE,OAAO;EAAE,KAAK;EAAO,UAAU;EAAe,WAAW;EAAM,OAAO,CAAC;CAAE;CAC3E,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,OAAO,CAAC,aAAa,eAAe,GAAG;EAChD,IAAI,QAAQ,QAAQ,YAAY,MAAM;EACtC,MAAM,SAAS,MAAM,IAAI,SAAS,OAAO,MAAM,aAAa,OAAO;EACnE,IAAI,UAAU,QAAQ,MAAM,KAAK,OAAO,IAAI;OACvC,OAAO;GAAE,KAAK,OAAO;GAAK,UAAU,OAAO;GAAU,WAAW,OAAO;GAAW;EAAM;CAC/F;CACA,OAAO;EAAE,WAAW;EAAO,OAAO,CAAC,qBAAqB,MAAM,KAAK,IAAI,GAAG;CAAE;AAC9E;;;;;;AAOA,MAAa,eAAe,OAC1B,SACA,KACA,MACA,YAC2F;CAC3F,MAAM,SAAS,eAAe,IAAI;CAClC,IAAI,WAAW,KAAA,GAAW,OAAO,EAAE,OAAO,CAAC,EAAE;CAC7C,MAAM,SAAS,MAAM,aAAa,SAAS,KAAK,QAAQ,OAAO;CAC/D,IAAI,OAAO,QAAQ,KAAA,KAAa,OAAO,aAAa,KAAA,GAAW,OAAO,EAAE,OAAO,OAAO,MAAM;CAC5F,MAAM,MAAM,QAAQ,MAAM,IAAI,OAAO,IAAI,GAAG,KAAK,OAAO;CACxD,MAAM,WAAW,WAAW,IAAI,IAAI,KAAK,OAAO;CAChD,MAAM,OAAO,WAAW,GAAG;CAC3B,MAAM,QAAQ,IAAI,KAAK;CACvB,MAAM,QAAQ,IAAI,KAAK;CACvB,MAAM,OAAO;EACX,KAAK,QAAQ;EACb,KAAK;EACL,GAAG,OAAO;EACV,aAAa,QAAQ;EACrB,GAAI,OAAO,CAAC,SAAS,IAAI,CAAC;CAC5B,CAAC,CAAC,QAAQ,SAAyB,SAAS,KAAA,KAAa,KAAK,SAAS,CAAC;CACxE,OAAO;EACL,MAAM;GACJ;GACA,KAAK,IAAI;GACT,SAAS,KAAK,QAAQ;GACtB,QAAQ,KAAK,QAAQ;GACrB,OAAO,OAAO,OAAO,IAAI;GACzB,OAAO,IAAI,KAAK;GAChB,UAAU;GACV,YAAY,MAAM,QAAQ,KAAK,IAAK,QAAqB,CAAC;GAC1D,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,SAAS;GAC7C,UAAU,CAAC,GAAG,IAAI,QAAQ;GAC1B,SAAS,IAAI;GACb,eAAe,IAAI;GACnB,YAAY,IAAI;GAChB,YAAY,OAAO;GACnB,WAAW;GACX;GACA,MAAM,KAAK,KAAK,IAAI;EACtB;EACA,OAAO,CAAC;CACV;AACF;;;;;;;;ACvpBA,MAAa,aAAa,QAAwB,OAAO,aAAa,GAAG;;AAGzE,MAAM,mBAAmB;;AAEzB,MAAM,QAAQ;CACZ;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;AAEA,MAAM,aAAa;CAAC;CAAgB;CAA6B;CAAW;CAAO;AAAW;;;;;;AAO9F,MAAM,cACJ,YAC2E;CAC3E,MAAM,SAAS,WACb,WAAW,MAAM,QAAQ,OAAO,SAAS,KAAA,CAAS;CACpD,MAAM,UAAU,CAAC,GAAG,QAAQ,OAAO,KAAK,GAAG,GAAG,QAAQ,QAAQ,WAAW,CAAC,MAAM,MAAM,CAAC,CAAC;CACxF,MAAM,OAAkC,CAAC;CACzC,IAAI,UAAU;CACd,IAAI,OAAO;CACX,KAAK,MAAM,UAAU,SAAS;EAC5B,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,CAAC;EAClC,IAAI,YAAY;EAChB,IAAI,OAAO,MAAM;GACf,YAAY,OAAO,YACjB,OAAO,QAAQ,MAAM,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,SAAS,GAAG,CAAC,CAC/D;GACA,OAAO,KAAK,UAAU,SAAS,CAAC,CAAC;EACnC;EACA,IAAI,OAAO,MAAM;GACf,WAAW;GACX;EACF;EACA,KAAK,KAAK,SAAS;EACnB,QAAQ;CACV;CACA,OAAO;EAAE;EAAM;CAAQ;AACzB;;;;;;;;;;AAWA,MAAa,iBAA4B,MAAM,YAAY;CACzD,IAAI,MAAM,MAAM,QAAQ,WAAW,GAAG,OAAO,QAAQ,MAAM,QAAQ,GAAG;CACtE,IAAI,QAAQ,gBAAgB,QAAQ,CAAC,YAAY,KAAK,QAAQ,WAAW,GAAG,OAAO,CAAC;CACpF,MAAM,MAAM,UAAU,QAAQ,GAAG;CACjC,IAAI;CACJ,IAAI;EACF,OAAO,YAAY,WAAW,MAAM,QAAQ,WAAW,GAAG,QAAQ,GAAG;CACvE,SAAS,OAAO;EAEd,OAAO,CACL;GACE;GACA,MAAM;GACN,OAAO,QAAQ;GACf,MAAM,QAAQ;GACd,KAAK,QAAQ;GACb,MAAM,EACJ,eAAe,iBAAiB,QAAQ,MAAM,QAAQ,MAAM,GAAG,GAAG,IAAI,OAAO,KAAK,EACpF;EACF,CACF;CACF;CACA,MAAM,SAAS,WAAW,KAAK,MAAM;CACrC,OAAO,CACL;EACE;EACA,MAAM;EACN,OAAO,KAAK;EACZ,MAAM,KAAK,KAAK,SAAS,IAAI,KAAK,OAAO,KAAK;EAC9C,KAAK,QAAQ;EACb,MAAM;GACJ,aAAa,KAAK;GAClB,MAAM,KAAK;GACX,cAAc,CAAC,GAAG,IAAI,IAAI,KAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;GACvD,QAAQ,OAAO;GACf,GAAI,OAAO,UAAU,IAAI,EAAE,gBAAgB,OAAO,QAAQ,IAAI,CAAC;GAC/D,OAAO,KAAK,MAAM,MAAM,GAAG,GAAG;GAC9B,eAAe,KAAK;GACpB,UAAU,KAAK;GACf,cAAc,KAAK;GACnB,eAAe,KAAK;GACpB,WAAW,KAAK;GAChB,GAAI,KAAK,UAAU,KAAA,KAAa,KAAK,MAAM,SAAS,IAAI,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;GACjF,UAAU,KAAK;GACf,SAAS,KAAK;GACd,GAAI,KAAK,MAAM,SAAS,IAAI,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;EACvD;EACA,GAAI,KAAK,aAAa,KAAA,IAAY,EAAE,WAAW,KAAK,SAAS,IAAI,CAAC;CACpE,CACF;AACF;AAEA,MAAM,qBAAqB;;;;;;AAO3B,MAAM,UAAU,OAAO,MAAkB,QAAsD;CAC7F,IAAI;CACJ,IAAI;EACF,OAAO,MAAM,QAAQ,MAAM,EAAE,UAAU,mBAAmB,CAAC;CAC7D,QAAQ;EACN,OAAO,CAAC;CACV;CACA,MAAM,OAAO,IAAI,IAAI,GAAG,CAAC,CAAC,SAAS,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;CACvD,IAAI,QAAQ;CACZ,IAAI;EACF,QAAQ,mBAAmB,IAAI;CACjC,QAAQ,CAER;CACA,MAAM,QAAQ,KAAK,UAAU,SAAS;CACtC,MAAM,QAAQ;EACZ,GAAI,KAAK,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;EAC5C,GAAI,KAAK,SAAS,OAAO,EAAE,MAAM,KAAK,KAAK,IAAI,CAAC;CAClD;CACA,IAAI,KAAK,KAAK,WAAW,GACvB,OAAO,CACL;EACE,KAAK,UAAU,GAAG;EAClB,MAAM;EACN;EACA,MAAM;EACN;EACA,MAAM;GACJ,QAAQ;GACR,OAAO,KAAK;GACZ,eAAe;GAEf,YAAY,KAAK,SAAS;GAC1B,GAAG;EACL;CACF,CACF;CAEF,OAAO,CACL;EACE,KAAK,UAAU,GAAG;EAClB,MAAM;EACN;EACA,MAAM,KAAK,KAAK,MAAM,GAAG,kBAAkB;EAC3C;EACA,MAAM;GAAE,QAAQ;GAAO,OAAO,KAAK;GAAO,eAAe,KAAK,KAAK;GAAQ,GAAG;EAAM;CACtF,CACF;AACF;;AAGA,IAAa,aAAb,MAAwB;CAEtB,2BAA4B,IAAI,IAAuB,CACrD,CAAC,QAAQ,aAAa,GACtB,CAAC,gBAAgB,gBAAgB,CACnC,CAAC;CAGD,8BAA+B,IAAI,IAAI,CAAC,cAAc,CAAC;CAEvD,SAAS,QAAgB,WAA4B;EACnD,IAAI,KAAK,SAAS,IAAI,MAAM,KAAK,CAAC,KAAK,YAAY,OAAO,MAAM,GAC9D,MAAM,IAAI,MAAM,UAAU,OAAO,0BAA0B;EAE7D,KAAK,SAAS,IAAI,QAAQ,SAAS;CACrC;CAEA,IAAI,QAAuC;EACzC,OAAO,KAAK,SAAS,IAAI,MAAM;CACjC;;CAGA,UAAoB;EAClB,OAAO,CAAC,GAAG,KAAK,SAAS,KAAK,CAAC;CACjC;AACF;;AAqDA,MAAa,kBAAqC;CAChD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;AACF;;;;;;AAOA,MAAa,YACX;;;;;;;AAQF,MAAa,cAAc,KAAqC,eAAe,UAC7E,IAAI,aAAa,QAAQ,CAAC;;;;;;AAO5B,MAAa,gBAAgB,SAC3B,OAAO,YAAY,OAAO,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,gBAAgB,SAAS,GAAG,CAAC,CAAC;AAE1F,MAAM,aAAa,UACjB,iBAAiB,QAAS,MAAM,QAAQ,MAAM,OAAO,CAAC,CAAC,MAAM,KAAM,OAAO,KAAK;;AAGjF,MAAM,aAAa,QAA0C;CAC3D,MAAM,UAAU,IAAI,KAAK;CACzB,OAAO,OAAO,YAAY,YAAY,QAAQ,SAAS,IAAI,UAAU,KAAA;AACvE;;AAiBA,MAAM,eAAe,OACnB,SACA,SACA,QACA,aACsC;CACtC,MAAM,UAAU,QAAQ;CACxB,IAAI,QAAQ,SAAS,KAAA,KAAa,YAAY,KAAA,GAAW,OAAO,EAAE,MAAM,CAAC,EAAE;CAE3E,IAAI,QAAQ,WAAW;EACrB,MAAM,QAAQ,QAAQ,MAAM,cAAc,OAAO;EACjD,IAAI,MAAM,SAAS,GAAG,OAAO,EAAE,MAAM,MAAM;CAC7C;CACA,MAAM,YAAY,QAAQ,WAAW,IAAI,MAAM;CAC/C,IAAI,cAAc,KAAA,GAAW,OAAO,EAAE,MAAM,CAAC,EAAE;CAC/C,IAAI;CACJ,IAAI;EACF,YAAY,MAAM,UAAU,QAAQ,MAAM;GACxC,KAAK,QAAQ;GACb,aAAa,WAAW,6BAA8B,QAAQ,eAAe;EAC/E,CAAC;CACH,SAAS,OAAO;EACd,OAAO;GAAE,MAAM,CAAC;GAAG,OAAO,mBAAmB,OAAO,KAAK,UAAU,KAAK;EAAI;CAC9E;CACA,MAAM,UAAU,UAAU,IAAI,SAAS,CAAC,CAAC,MAAM,SAAS,SAAS,KAAA,CAAS;CAC1E,IAAI,YAAY,KAAA,GAAW;EAEzB,QAAQ,QAAQ,cAAc,QAAQ,KAAK,OAAO;EAClD,OAAO;GACL,MAAM,CAAC;GACP;GACA,OAAO,4BAA4B,QAAQ;EAC7C;CACF;CACA,MAAM,MAAM,QAAQ,IAAI;CAmBxB,OAAO,EAAE,MAlBI,UAAU,KAAK,QAC1B,QAAQ,MAAM,OACZ;EACE,GAAG;EACH,UAAU,CAAC,OAAO;EAClB;EACA,YAAY,QAAQ,aAAa;EACjC,MAAM,QAAQ,QAAQ;EACtB;EACA;EAEA,UAAU,IAAI,aAAa,OAAO,OAAO,QAAQ,QAAQ;EACzD,GAAI,QAAQ,UAAU,KAAA,IAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;EAC9D,GAAI,QAAQ,WAAW,KAAA,IAAY,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;CACnE,GACA,GACF,CAEU,EAAE;AAChB;;;;;;;;;AAUA,MAAa,gBAAgB,OAC3B,SACA,YACqB;CACrB,MAAM,UAAU,MAAM,QAAQ,QAAQ,MAAM,OAAO;CACnD,IAAI,QAAQ,YAAY,QAAQ,QAAQ,YAAY,gBAAgB,OAAO;EAAE;EAAS,MAAM,CAAC;CAAE;CAC/F,OAAO;EAAE;EAAS,GAAI,MAAM,aAAa,SAAS,SAAS,QAAQ,QAAQ,KAAK;CAAG;AACrF;;;;;;AAOA,MAAa,iBAAiB,OAC5B,SACA,YACqB;CACrB,MAAM,UAAU,MAAM,QAAQ,QAAQ,OAAO,OAAO;CACpD,IAAI,QAAQ,YAAY,MAAM,OAAO;EAAE;EAAS,MAAM,CAAC;CAAE;CACzD,OAAO;EAAE;EAAS,GAAI,MAAM,aAAa,SAAS,SAAS,QAAQ,QAAQ,IAAI;CAAG;AACpF;AA0DA,MAAM,YAAY;;;;;;;;;AAUlB,MAAa,YAAY,OACvB,SACA,KACA,YACwB;CACxB,MAAM,gBAAgB,YAAkC;EACtD,KAAK;EACL,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,UAAU,QAAQ;EAClB,YAAY,QAAQ;EACpB,KAAK,QAAQ;EACb,QAAQ,QAAQ;EAChB,OAAO,QAAQ;EACf,QAAQ,QAAQ;EAChB,UAAU,QAAQ;CACpB;CACA,IAAI,QAAQ,MAAM,cAAc,SAAS,aAAa,GAAG,CAAC;CAC1D,MAAM,QAAkB,CAAC;CACzB,IAAI,MAAM,UAAU,KAAA,GAAW,MAAM,KAAK,MAAM,KAAK;CACrD,MAAM,SAAS,MAAM,KAAK,EAAE,EAAE,KAAK;CACnC,IAAI,OAAO,WAAW,YAAY,OAAO,SAAS,KAAK,MAAM,YAAY,KAAA,GAAW;EAClF,MAAM,WAAW,MAAM,cAAc,SAAS,aAAa,MAAM,CAAC;EAClE,MAAM,KACJ,mCAAmC,SAAS,QAAQ,IAAI,mCAC1D;EACA,IAAI,SAAS,UAAU,KAAA,GAAW,MAAM,KAAK,SAAS,KAAK;EAC3D,QAAQ;CACV;CACA,IAAI,OAAO,MAAM;CACjB,IAAI,WAAW;CACf,IAAI,YAAY,MAAM,QAAQ;CAC9B,MAAM,YAAY,MAAM,QAAQ,YAAY,QAAQ,MAAM,QAAQ,YAAY;CAC9E,MAAM,QAAQ,KAAK;CACnB,MAAM,OACJ,OAAO,SAAS,WACf,MAAM,QAAQ,gBAAgB,KAAA,KAAa,UAAU,KAAK,MAAM,QAAQ,WAAW;CACtF,MAAM,SACJ,QAAQ,WAAW,YAClB,QAAQ,WAAW,UAAU,OAAO,KAAK,oBAAoB;CAChE,IAAI,aAAa,QAAQ;EACvB,IAAI,MAAM,YAAY,KAAA,GAAW,MAAM,KAAK,yCAAyC;OAChF,IAAI,UAAU,KAAA,KAAa,CAAC,MAC/B,MAAM,KAAK,sEAAsE;OAC5E;GAEL,MAAM,UAAU,QAAQ,MAAM,IAAI,MAAM,GAAG;GAC3C,MAAM,QAAQ,YAAY,KAAA,KAAa,CAAC,QAAQ,YAAY,QAAQ,gBAAgB;GACpF,MAAM,QAAQ,MAAM,eAAe,SAAS;IAC1C,KAAK,MAAM,QAAQ;IACnB,QAAQ;IACR,UAAU,QAAQ,IAAI,QAAQ;IAC9B,YAAY,QAAQ;IACpB,KAAK,QAAQ;IACb,QAAQ,QAAQ;IAChB,OAAO,QAAQ;IACf,QAAQ,QAAQ;IAChB,UAAU,QAAQ;GACpB,CAAC;GACD,IAAI,MAAM,KAAK,SAAS,GAAG;IACzB,OAAO,MAAM;IACb,WAAW;IACX,YAAY,aAAa,MAAM,QAAQ;GACzC,OACE,MAAM,KAAK,MAAM,SAAS,MAAM,QAAQ,QAAQ,6BAA6B;EAEjF;CACF;CAEA,IAAI,QAAQ,aAAa,aAAa,KAAK,WAAW,GAAG;EACvD,MAAM,WAAW,MAAM,aAAa,SAAS,KAAK,OAAO,OAAO;EAChE,IAAI,SAAS,SAAS,KAAA,GAAW,OAAO,SAAS;EACjD,MAAM,KAAK,GAAG,SAAS,KAAK;CAC9B;CAEA,MAAM,MAAM,KAAK,OAAO,KAAA,IAAY,KAAA,IAAa,QAAQ,MAAM,IAAI,KAAK,EAAE,CAAC,GAAG,KAAK,KAAK;CACxF,MAAM,OAAO,QAAQ,KAAA,KAAa,WAAW,GAAG;CAChD,MAAM,QAAQ,KAAK,KAAK;CACxB,MAAM,QAAQ,KAAK,KAAK;CACxB,IAAI,MAAM,MAAM,KAAK,SAAS;CAC9B,MAAM,OAAO,CAAC,MAAM,QAAQ,MAAM,GAAG,KAAK,CAAC,CAAC,QACzC,SAAyB,SAAS,KAAA,KAAa,KAAK,SAAS,CAChE;CACA,OAAO;EACL;EACA,KAAK,KAAK,OAAO;EACjB,SAAS,MAAM,QAAQ;EACvB,QAAQ,MAAM,QAAQ;EACtB,OAAO,QAAQ,KAAA,KAAa,OAAO,OAAO,IAAI;EAC9C,OAAO,KAAK,KAAK,UAAU;EAC3B,UAAU,KAAK,YAAY;EAC3B,YAAY,MAAM,QAAQ,KAAK,IAAK,QAAqB,CAAC;EAC1D,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,SAAS;EAC7C,UAAU,QAAQ,KAAA,IAAY,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ;EACnD,SAAS,KAAK,WAAW;EACzB,eAAe,KAAK,iBAAiB;EACrC,YAAY,KAAK,cAAc,MAAM,QAAQ,aAAa;EAC1D,YAAY;EACZ,WAAW;EACX,MAAM,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI;CAC5C;AACF;AAsGA,MAAM,YAAY,SAAuB,OAAoB;CAC3D,MAAM,MAAM,QAAQ,MAAM,IAAI,EAAE;CAChC,IAAI,QAAQ,KAAA,GACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,0CAA0C;CACrF,OAAO;AACT;;;;;AAQA,MAAa,WAAW,MAAc,UAAiC;CACrE,MAAM,KACJ,iBAAiB,OAAO,MAAM,QAAQ,IAAI,OAAO,UAAU,WAAW,KAAK,MAAM,KAAK,IAAI;CAC5F,IAAI,CAAC,OAAO,SAAS,EAAE,GACrB,MAAM,IAAI,WACR,GAAG,KAAK,qDAAqD,KAAK,UAAU,OAAO,KAAK,CAAC,GAC3F;CAEF,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY;AAClC;;;;;;;;AASA,MAAa,iBAAiB,SAAuB,IAAY,OAA0B;CACzF,MAAM,QAAQ,QAAQ;CACtB,IAAI,MAAM,cAAc,KAAA,GAAW,OAAO,MAAM,UAAU,IAAI,EAAE;CAChE,MAAM,MAAM,MAAM,SAAS,IAAI,EAAE,OAAO,IAAU,CAAC;CACnD,MAAM,OAAO,IAAI,MAAM,YAAY,QAAQ,cAAc,EAAE;CAC3D,OAAO;EACL,SAAS,SAAS,KAAA,IAAY,KAAA,IAAY,MAAM,YAAY,IAAI,KAAK,OAAO;EAC5E,OAAO,IAAI,GAAG,EAAE,CAAC,EAAE;CACrB;AACF;;AAGA,MAAM,mBAAmB,IAAY,IAAY,UAC/C,gBAAgB,GAAG,qBAAqB,GAAG,YAAY,UAAU,KAAA,IAAY,KAAK,+BAA+B;;AAGnH,MAAM,QAAQ,YAGY;CACxB,IAAI,QAAQ,OAAO,KAAA,GAAW,OAAO,KAAA;CACrC,IAAI,QAAQ,YAAY,KAAA,GACtB,MAAM,IAAI,WAAW,6DAA6D;CACpF,OAAO,QAAQ,MAAM,QAAQ,EAAE;AACjC;AAIA,MAAM,eAAe,MAAc,OAA2B,aAA6B;CACzF,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GACrC,MAAM,IAAI,WAAW,GAAG,KAAK,sCAAsC,OAAO,KAAK,GAAG;CAEpF,OAAO,KAAK,MAAM,KAAK;AACzB;;AAGA,MAAM,cAAc,MAAc,aAA6B;CAC7D,MAAM,OAAO,KAAK,WAAW,QAAQ;CACrC,OAAO,WAAW,KAAK,WAAW,KAAK,UAAU,QAAQ,SAAU,QAAQ,QACvE,WAAW,IACX;AACN;;;;;;;;AASA,MAAa,iBAAiB,OAC5B,SACA,KAMA,QACA,QAGI,CAAC,MACa;CAClB,MAAM,KAAK,QAAQ,IAAI;CACvB,MAAM,QAAQ,MAAM,SAAS,IAAI;CACjC,MAAM,SAAS,MAAM,UAAU,IAAI;CACnC,MAAM,QAAQ,IAAI,OAChB,WAAW;EACT,QAAQ;EACR,MAAM;EACN,SAAS;GAAE,KAAK,IAAI;GAAK,KAAK,IAAI;GAAK;GAAQ,QAAQ;GAA8B;EAAG;EACxF,aAAa;EACb,aAAa,IAAI,OAAO,IAAI;EAC5B,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;EACvC,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CAC3C,CAAC,CACH;AACF;;;;;;;;AASA,MAAa,eAAe,OAC1B,SACA,IACA,UAAuB,CAAC,MACA;CACxB,MAAM,OAAO,YAAY,QAAQ,QAAQ,MAAM,CAAC;CAChD,MAAM,SAAS,YAAY,UAAU,QAAQ,QAAQ,GAAI;CACzD,MAAM,KAAK,KAAK,OAAO;CACvB,MAAM,MAAM,SAAS,SAAS,EAAE;CAChC,IAAI,UACF,QAAQ,YAAY,KAAA,IAAY,KAAA,IAAY,QAAQ,MAAM,YAAY,IAAI,QAAQ,OAAO;CAC3F,IAAI,QAAQ,YAAY,KAAA,KAAa,YAAY,KAAA,GAC/C,MAAM,IAAI,MAAM,gBAAgB,GAAG,mCAAmC,QAAQ,SAAS;CAEzF,MAAM,QAAQ,OAAO,KAAA,IAAY,CAAC,IAAI;EAAE;EAAI,cAAc;CAAM;CAChE,IAAI,OAAO,KAAA,GAAW;EACpB,MAAM,QAAQ,cAAc,SAAS,IAAI,EAAE;EAC3C,IAAI,MAAM,YAAY,KAAA,GAEpB,OAAO;GACL,KAAK,IAAI;GACT,MAAM,IAAI;GACV,OAAO;GACP,KAAK,IAAI;GACT,MAAM,CAAC;GACP,MAAM;GACN,IAAI;GACJ,OAAO;GACP,MAAM;GACN,UAAU,IAAI;GACd,SAAS;GACT,YAAY;GACZ,UAAU,IAAI;GACd,WAAW;GACX,MAAM,gBAAgB,IAAI,IAAI,MAAM,KAAK;GACzC;GACA,cAAc;EAChB;EAEF,UAAU,MAAM;CAClB;CACA,MAAM,UACJ,IAAI,aAAa,QAAQ,SAAS,aAAa,OAC3C,OACC,SAAS,YAAY,IAAI;CAChC,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,OAAO,WAAW,EAAE,UAAU,QAAQ,GAAG,YAAY;CAC3D,IAAI,YAAY,QAAQ,cAAc,MAAM,eAAe,SAAS,KAAK,MAAM;CAC/E,MAAM,OAAO,SAAS,QAAQ,IAAI;CAClC,MAAM,QAAQ,WAAW,MAAM,KAAK,IAAI,MAAM,KAAK,MAAM,CAAC;CAC1D,MAAM,MAAM,OAAO,QAAQ,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,QAAQ,MAAM,CAAC;CACjF,MAAM,OAAO,SAAS,QAAQ,IAAI;CAElC,MAAM,WAAW,WAAW,IAAI;CAChC,MAAM,QAAQ,CAAC,GAAI,OAAO,CAAC,SAAS,IAAI,CAAC,GAAI,GAAI,WAAW,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,CAAE;CAC1F,OAAO;EACL,KAAK,IAAI;EACT,MAAM,SAAS,QAAQ,IAAI;EAC3B,OAAO,OAAO,KAAM,SAAS,SAAS,IAAI;EAC1C,KAAK,SAAS,OAAO,IAAI;EACzB,MAAM,OAAO,aAAa,IAAI,IAAI;EAClC,MAAM;EACN,IAAI;EACJ,OAAO,KAAK;EACZ,MAAM,OAAO,KAAK,KAAK,MAAM,OAAO,GAAG;EACvC,UAAU,IAAI;EACd,SAAS,SAAS,WAAW,IAAI;EACjC,YAAY,SAAS,cAAc,IAAI;EACvC,UAAU;EACV,WAAW;EACX,MAAM,MAAM,SAAS,IAAI,MAAM,KAAK,GAAG,IAAI;EAC3C,GAAG;EACH,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;CAC/C;AACF;;AAKA,MAAM,iBACJ,SACqC;CACrC,MAAM,WAAW,WAAW,IAAI;CAChC,OAAO,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS;AAClD;;AAGA,MAAM,gBACJ,MACA,SACmE;CACnE,MAAM,WAAW,WAAW,IAAI;CAChC,IAAI,aAAa,KAAA,GAAW,OAAO,EAAE,KAAK;CAC1C,MAAM,OAAO,aAAa,QAAQ;CAClC,OAAO;EAAE,MAAM,SAAS,OAAO,OAAO,GAAG,KAAK,IAAI;EAAQ;CAAS;AACrE;;;;;AAQA,MAAM,gBACJ,SACA,IACA,OACA,YACkE;CAClE,MAAM,KAAK,KAAK,OAAO;CACvB,MAAM,MAAM,SAAS,SAAS,EAAE;CAChC,IAAI,SACF,QAAQ,YAAY,KAAA,IAAY,KAAA,IAAY,QAAQ,MAAM,YAAY,IAAI,QAAQ,OAAO;CAC3F,IAAI,QAAQ,YAAY,KAAA,KAAa,WAAW,KAAA,GAC9C,MAAM,IAAI,MAAM,gBAAgB,GAAG,mCAAmC,QAAQ,SAAS;CAEzF,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,OAAO;EAAE,KAAK,IAAI;EAAK,KAAK,IAAI;EAAK,UAAU,IAAI;CAAS;CAClE,MAAM,QAAQ,OAAO,KAAA,IAAY,CAAC,IAAI;EAAE;EAAI,cAAc;CAAM;CAChE,MAAM,UAAU;EACd,OAAO;EACP,OAAO;EACP,KAAK;EACL,SAAS;EACT,SAAS;EACT,MAAM;EACN,YAAY;EACZ,OAAO;EACP,cAAc;EACd,OAAO;CACT;CACA,IAAI,OAAO,KAAA,GAAW;EAEpB,MAAM,QAAQ,cAAc,SAAS,IAAI,EAAE;EAC3C,IAAI,MAAM,YAAY,KAAA,GAUpB,OAAO;GAAE,QAAA;IARP,GAAG;IACH,GAAG;IACH,UAAU;IACV,WAAW;IACX,MAAM,gBAAgB,IAAI,IAAI,MAAM,KAAK;IACzC;IACA,cAAc;GAEF;GAAG,UAAU;EAAM;EAEnC,SAAS,MAAM;CACjB;CACA,MAAM,UAAmB,IAAI,aAAa,QAAQ,QAAQ,aAAa,OAAO,OAAO,IAAI;CACzF,IAAI,WAAW,EAAE,UAAU,QAAQ,GAAG,YAAY,GAUhD,OAAO;EAAE,QAAA;GARP,GAAG;GACH,GAAG;GACH,UAAU;GACV,WAAW;GACX,MAAM;GACN,GAAG;GACH,GAAG,cAAc,QAAQ,QAAQ,IAAI,IAAI;EAE7B;EAAG,UAAU;CAAM;CAEnC,MAAM,UAAU,UAAU,QAAQ,MAAM,YAAY,EAAE;CACtD,MAAM,aACJ,WAAW,KAAA,IACP,CAAC,MAAM,IACP,CAAC,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,CAAC,OAAO,GAAI,GAAG,QAAQ,MAAM,aAAa,EAAE,CAAC;CAErF,MAAM,OAAO,WAAW,QAAQ,YAAY,QAAQ,aAAa,IAAI,CAAC,CAAC;CACvE,MAAM,WAAW,iBAAiB,YAAY,QAAQ,OAAO;CAC7D,KAAK,MAAM,CAAC,OAAO,YAAY,WAAW,QAAQ,GAAG;EACnD,IAAI,WAAW,SAAS,YAAY,GAAG;EACvC,IAAI,MAAM,OAAO,QAAQ,MAAM,KAAK;EACpC,IAAI,SAAS;EACb,IAAI,CAAC,IAAI,SAAS,QAAQ,cAAc,KAAA,GAAW;GACjD,MAAM,OAAO,QAAQ,WAAW,KAAK;GACrC,SAAS,IAAI;EACf;EACA,IAAI,CAAC,IAAI,OAAO;EAChB,MAAM,QAAQ,WAAW,KAAA,KAAa,QAAQ;EAqB9C,OAAO;GAAE,QAAA;IAnBP,GAAG;IACH,OAAO;IACP,OAAO,IAAI;IACX,KAAK,IAAI;IACT,SAAS,IAAI;IACb,SAAS,QAAQ;IACjB,MAAM,QAAQ;IACd,YAAY,QAAQ;IACpB;IACA,cAAc;IACd,UAAU;IACV,OAAO,IAAI;IACX,WAAW;IACX,GAAG,aACD,QAAQ,MACR,QAAQ,4DAA4D,IACtE;IACA,GAAG;GAES;GAAG;EAAS;CAC5B;CACA,MAAM,aACJ,YAAY,KAAA,KAAa,QAAQ,KAAK,iBAAiB,QAAQ,QAAQ,cAAc,KAAA;CAEvF,MAAM,QAAQ,CACZ,GAAI,aACA,CACE,qGACF,IACA,CAAC,GACL,GAAI,OAAO,KAAK,CAAC,eACb,CAAC,yEAAyE,OAAO,IAAI,EAAE,EAAE,IACzF,CAAC,CACP;CAkBA,OAAO;EAAE,QAAA;GAhBP,GAAG;GACH,OAAO;GACP,OAAO;GACP,KAAK;GACL,SAAS;GACT,SAAS,SAAS,WAAW;GAC7B,MAAM,SAAS,QAAQ;GACvB,YAAY,SAAS,cAAc;GACnC,OAAO;GACP,cAAc;GACd,UAAU,CAAC;GACX,OAAO;GACP,WAAW;GACX,GAAG,aAAa,SAAS,QAAQ,IAAI,MAAM,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,IAAI;GACrF,GAAG;EAES;EAAG;CAAS;AAC5B;;;;;;;;;;;;;;;;;AA+BA,MAAa,cAAc,OACzB,SACA,IACA,OACA,UAAyB,CAAC,MACA;CAC1B,MAAM,EAAE,QAAQ,aAAa,aAAa,SAAS,IAAI,OAAO,OAAO;CACrE,IAAI,UAAU,MAAM,eAAe,SAAS,SAAS,SAAS,EAAE,GAAG,QAAQ;CAC3E,OAAO;AACT;;;;;;;;AASA,MAAa,gBAAgB,OAC3B,SACA,GACA,UAII,CAAC,MACgB;CACrB,MAAM,QAAQ,QAAQ;CAOtB,OANc,QAAQ,MAAM,KAAK,GAAG;EAClC,OAAO,QAAQ,SAAS;EACxB,MAAM,QAAQ,QAAQ;EACtB,GAAI,QAAQ,SAAS,KAAA,IAAY,EAAE,MAAM,QAAQ,KAAK,IAAI,CAAC;CAC7D,CAEW,CAAC,CAAC,KAAK,QAAQ;EACxB,IAAI,IAAI,SAAS,YAAY,OAAO;EACpC,MAAM,OAAO,QAAQ,MAAM,IAAI,IAAI,GAAG,CAAC,EAAE;EACzC,OAAO,SAAS,KAAA,IAAY,MAAM;GAAE,GAAG;GAAK,GAAG,cAAc,IAAI;EAAE;CACrE,CAAC;AACH;AAwBA,MAAM,gBAAgB;AACtB,MAAM,aAAa;;;;;AAKnB,MAAM,kCAAuC,IAAI,IAAI;CACnD;CACA;CACA;CACA;AACF,CAAC;;AAGD,MAAM,iBACJ,YACwD;CACxD,IAAI,QAAQ,IAAI,SAAS,aAAa,GACpC,OAAO;EAAE,KAAK,QAAQ,IAAI,MAAM,GAAG,EAAqB;EAAG,SAAS;CAAW;CAEjF,OAAO;EAAE,KAAK,QAAQ;EAAK,SAAS,QAAQ,aAAa,OAAO,aAAa;CAAQ;AACvF;AAkBA,MAAM,kBAA+B;CACnC,UAAU;CACV,QAAQ;CACR,SAAS;CACT,yBAAS,IAAI,IAAI;CACjB,QAAQ,CAAC;AACX;;AAGA,MAAM,gBACJ,OACA,UACA,YACS;CACT,MAAM,EAAE,KAAK,YAAY,cAAc,OAAO;CAC9C,MAAM,YAAY,QAAQ,cAAc,SAAS;CACjD,IAAI,QAAQ,YAAY,YAAY;EAClC,MAAM,WACJ,QAAQ,aACP,OAAO,QAAQ,SAAS,YAAY,QAAQ,KAAK,WAAW,WAAW,IACpE,QAAQ,KAAK,MAAM,CAAC,IACpB;EACN,MAAM,YAAY,QAAQ,WAAW,OAAO,QAAQ,WAAW;EAC/D,IAAI,aAAa,SAAS,aAAa,OAAO,QAAQ,gBAAgB,WACpE,MAAM,UAAU;GACd;GACA;GACA,SAAS,SAAS;GAClB,MAAM;GACN,MAAM;GACN,eAAe;GACf,cAAc;GACd,YAAY;GACZ,QAAQ,QAAQ,UAAU;GAC1B,UAAU;GACV;GACA,aAAa,QAAQ,eAAe;GACpC,UAAU,QAAQ,YAAY;EAChC,CAAC;EAEH;CACF;CAGA,IAAI,QAAQ,QAAQ,QAAQ,QAAQ,YAAY,QAAQ,QAAQ,UAAU,KAAA,GAAW;CACrF,MAAM,WAAW,MAAM,OAAO,KAAK,OAAO;CAC1C,MAAM,UAAU,SAAS,SAAS,aAAa,QAAQ,YAAY;CACnE,MAAM,YAAY,QAAQ,cAAc;CAExC,MAAM,OAAO,YAAY,OAAQ,QAAQ,SAAS,UAAW,UAAU,QAAQ,OAAQ;CACvF,MAAM,eAAe,YACjB,OACC,QAAQ,kBAAkB,UAAW,UAAU,iBAAiB,OAAQ;CAC7E,MAAM,OAAO,UAAW,UAAU,gBAAgB,KAAA,IAAa,KAAA;CAC/D,MAAM,cACJ,QAAQ,gBAAgB,SAAS,YAAY,aAAa,6BAA6B;CACzF,MAAM,SAAS,UACX,aAAa,KAAA,KAAa,SAAS,SAAS,OAAO,SAAS,UAAU,MACpE,MACA,SAAS,SACV,QAAQ,UAAU;CACvB,MAAM,UAAU;EACd;EACA;EACA,SAAS,SAAS;EAClB,MAAM,QAAQ,QAAQ;EACtB;EACA,eAAe;EACf,cAAc;EACd,YAAY;EACZ;EACA,UAAU,QAAQ,SAAS,YAAY;EACvC,GAAI,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;EACvC,aAAa,QAAQ,eAAe;EACpC,UAAU,QAAQ,aAAa,UAAW,UAAU,YAAY,OAAQ;CAC1E,CAAC;AACH;;AAGA,MAAM,eAAe,OACnB,SACA,QACA,UACA,UACkB;CAClB,IAAI,SAAS,SAAS,aAAa,SAAS,SAAS,WAAW;CAChE,MAAM,UAAU,SAAS;CACzB,IAAI,YAAY,QAAQ,OAAO,YAAY,YAAY,OAAO,QAAQ,QAAQ,UAAU;CACxF,IAAI,QAAQ,YAAY,YAAY;EAClC,aAAa,QAAQ,UAAU,OAAO;EACtC;CACF;CACA,IAAI,OAAO,QAAQ,SAAS,UAAU;CACtC,IAAI,QAAQ,YAAY,QAAQ,QAAQ,YAAY,gBAAgB;CACpE,MAAM,YAAY;CAClB,aAAa,QAAQ,UAAU,OAAO;CACtC,MAAM,YAAY,QAAQ,WAAW,IAAI,QAAQ,MAAM;CACvD,IAAI,cAAc,KAAA,GAAW;EAE3B,IAAI,CAAC,gBAAgB,IAAI,QAAQ,MAAM,GAAG,MAAM,QAAQ,IAAI,QAAQ,MAAM;EAC1E;CACF;CACA,MAAM,OAAO,MAAM,QAAQ,MAAM,IAAI,QAAQ,IAAI;CACjD,IAAI,SAAS,KAAA,GAAW;EACtB,MAAM,WAAW;EACjB;CACF;CACA,MAAM,EAAE,KAAK,YAAY,cAAc,OAAO;CAC9C,MAAM,SAAS,OAAO,OAAO,KAAK,OAAO;CACzC,IAAI;CACJ,IAAI;EACF,YAAY,MAAM,UAAU,MAAM;GAChC;GACA,aACE,YAAY,aACR,6BACC,QAAQ,gBAAgB,QAAQ,gBAAgB;EACzD,CAAC;CACH,SAAS,OAAO;EACd,MAAM,UAAU;EAChB,IAAI,MAAM,OAAO,SAAS,YACxB,MAAM,OAAO,KAAK;GAAE,SAAS,SAAS;GAAI,OAAO,UAAU,KAAK;EAAE,CAAC;EACrE;CACF;CAEA,IAAI,UAAU,MAAM,QAAQ,UAAU,GAAG,MAAM,KAAA,CAAS,GAAG;CAC3D,KAAK,MAAM,OAAO,WAChB,OAAO,OACL;EACE,GAAG;EACH,UAAU,CAAC,SAAS,EAAE;EACtB,SAAS,SAAS;EAClB,YAAY,QAAQ,cAAc,SAAS;EAC3C,MAAM,QAAQ;EACd,QAAQ,QAAQ;EAChB,UAAU,YAAY;EACtB,UAAU,IAAI,aAAa,OAAO,OAAQ,QAAQ,SAAS,YAAY;EACvE,GAAI,SAAS,UAAU,KAAA,IAAY,EAAE,OAAO,SAAS,MAAM,IAAI,CAAC;EAChE,GAAI,SAAS,WAAW,KAAA,IAAY,EAAE,QAAQ,SAAS,OAAO,IAAI,CAAC;CACrE,GACA,SAAS,WACX;AAEJ;;;;;;;;;;;;;;AAeA,MAAa,UAAU,OACrB,SACA,UAAmD,CAAC,MACzB;CAC3B,MAAM,QAAQ,QAAQ,IAAI;CAC1B,MAAM,QAAQ,QAAQ,MAAM,aAC1B,QAAQ,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,QAAQ,MAAM,CAC5D;CACA,MAAM,QAAQ,SAAS;CACvB,IAAI,UAAU;CACd,IAAI;EACF,MAAM,OAAO,QAAQ,SAAS,KAC5B,EAAE,QAAQ,QAAQ,GAClB,EACE,iBAAiB;GACf,WAAW;EACb,EACF,CACF;EACA,WAAW,MAAM,YAAY,MAAM,MAAM,aAAa,SAAS,OAAO,UAAU,KAAK;EACrF,MAAM,UAAU,MAAM,MAAM,IAAI,IAAI,QAAQ,WAAW,QAAQ,CAAC,CAAC;EACjE,MAAM,SAAS,MAAM,OAAO,KAAK;EACjC,OAAO;GACL,UAAU,MAAM;GAChB,GAAG;GACH,QAAQ,MAAM;GACd,SAAS,MAAM;GACf;GACA;GACA,iBAAiB,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,KAAK;GACzC,QAAQ,MAAM;EAChB;CACF,SAAS,OAAO;EACd,MAAM,MAAM;EACZ,MAAM;CACR;AACF;;AAgBA,MAAM,gBAAgB;;AAGtB,MAAM,UAAU,SAAuB,aAAqC;CAC1E,IAAI,SAAS,SAAS,aAAa,SAAS,SAAS,WAAW,OAAO;CACvE,MAAM,UAAU,SAAS;CACzB,IAAI,YAAY,QAAQ,OAAO,YAAY,YAAY,OAAO,QAAQ,QAAQ,UAC5E,OAAO;CACT,MAAM,YAAY,QAAQ,cAAc,SAAS;CACjD,MAAM,EAAE,KAAK,YAAY,cAAc,OAAO;CAC9C,MAAM,QAAQ,QAAQ,MAAM,OAAO,KAAK,OAAO;CAC/C,IAAI,QAAQ,YAAY,YAAY,OAAO,UAAU,KAAA,KAAa,MAAM,aAAa;CACrF,IACE,OAAO,QAAQ,SAAS,YACvB,QAAQ,YAAY,QAAQ,QAAQ,YAAY,gBAEjD,OAAO;CAET,IAAI,QAAQ,UAAU,KAAA,GACpB,OACE,QAAQ,WAAW,IAAI,QAAQ,MAAM,MAAM,KAAA,KAAa,CAAC,QAAQ,MAAM,WAAW,SAAS,EAAE;CAEjG,IAAI,UAAU,KAAA,KAAa,MAAM,aAAa,WAAW,OAAO;CAChE,OACE,QAAQ,WAAW,IAAI,QAAQ,MAAM,MAAM,KAAA,KAAa,CAAC,QAAQ,MAAM,WAAW,SAAS,EAAE;AAEjG;;;;;;;;;;;;;;AAeA,MAAa,eAAe,OAAO,YAA8D;CAC/F,IAAI,QAAQ,MAAM,OAAO,OAAO;EAAE,MAAM;EAAW,GAAI,MAAM,QAAQ,OAAO;CAAG;CAC/E,MAAM,SAAS,QAAQ,MAAM,YAAY;CACzC,MAAM,QACJ,WAAW,KAAA,IAAY,KAAA,qBAAY,IAAI,KAAK,KAAK,MAAM,MAAM,IAAI,aAAa,EAAA,CAAE,YAAY;CAC9F,MAAM,QAAQ,SAAS;CACvB,IAAI,UAAU;CACd,MAAM,OAAO,QAAQ,SAAS,KAC5B;EAAE,QAAQ;EAAS,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM;CAAG,GAC7D,EACE,iBAAiB;EACf,WAAW;CACb,EACF,CACF;CACA,WAAW,MAAM,YAAY,MAC3B,IAAI,OAAO,SAAS,QAAQ,GAAG,MAAM,aAAa,SAAS,QAAQ,OAAO,UAAU,KAAK;CAE3F,IAAI,MAAM,aAAa,GAAG,OAAO,KAAA;CACjC,OAAO;EACL,MAAM;EACN,UAAU,MAAM;EAChB,GAAG,QAAQ,MAAM,OAAO;EACxB,QAAQ,MAAM;EACd,SAAS,MAAM;EACf;EACA,SAAS;EACT,iBAAiB,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,KAAK;EACzC,QAAQ,MAAM;CAChB;AACF"}
|
|
1
|
+
{"version":3,"file":"runtime-p5WN2UAA.mjs","names":["systemLookup","messageOf","MAX_INFLATED_BYTES"],"sources":["../src/egress.ts","../src/record.ts","../src/crawler.ts","../src/fold.ts","../src/pdf.ts","../src/fallback.ts","../src/runtime.ts"],"sourcesContent":["import type { LookupAddress, LookupAllOptions, LookupOptions } from \"node:dns\";\nimport { lookup as systemLookup } from \"node:dns\";\nimport type { LookupFunction } from \"node:net\";\nimport { isIP } from \"node:net\";\n\n/**\n * The egress policy keeps the crawler on the public internet. A page or an agent can ask for an\n * internal address. Examples are a loopback service, a router on the LAN and the metadata\n * service of a cloud host. The crawler refuses these addresses by default.\n *\n * - The crawler fetches only `http:` and `https:` URLs.\n * - The check uses the address after DNS. The `lookup` hook of the connection checks the address\n * that the socket then uses, so a DNS answer that changes cannot pass the check.\n * - The crawler examines a literal IP host directly, in each spelling that the URL parser reads.\n * - The crawler examines each redirect hop again.\n *\n * A refusal is a skipped fetch with a clear note. It does not throw an error.\n */\n\n/** These options change the egress policy. With no options, the crawler fetches only public addresses. */\nexport interface EgressOptions {\n /** The value true lets the crawler fetch loopback, private and link-local addresses. The default is false. */\n readonly allowPrivate?: boolean | undefined;\n /**\n * These hosts are permitted although their address is not public. An entry is a host name, a\n * name pattern (`*.corp.example`), an IP address or a CIDR range (`10.0.0.0/8`).\n */\n readonly allowHosts?: readonly string[] | undefined;\n}\n\n/**\n * The connection hook gives this error when the policy refuses each address of a host.\n *\n * @beta\n */\nexport class EgressRefused extends Error {\n readonly code: string = \"EGRESS_REFUSED\";\n constructor(\n readonly address: string,\n readonly kind: string,\n ) {\n super(egressNote(address, kind));\n this.name = \"EgressRefused\";\n }\n}\n\n/** This function gives the note of a refusal, with the address and the kind of address. */\nexport const egressNote = (address: string, kind: string): string =>\n `egress policy: ${address} is ${/^[aeiou]/i.test(kind) ? \"an\" : \"a\"} ${kind} address; set allowPrivate to fetch it`;\n\ntype Bytes = readonly number[];\n\n/**\n * This function reads an IPv4 address the way the URL parser reads a host. The address has one to\n * four parts, and each part is decimal, octal (`0177`) or hexadecimal (`0x7f`). Thus\n * `2130706433` and `0x7f.1` both give 127.0.0.1. A text that is not an IPv4 address gives no value.\n */\nexport const parseIPv4 = (text: string): Bytes | undefined => {\n const parts = text.split(\".\");\n if (parts.length > 1 && parts[parts.length - 1] === \"\") parts.pop();\n if (parts.length === 0 || parts.length > 4) return undefined;\n const numbers: number[] = [];\n for (const part of parts) {\n let value: number;\n if (/^0[xX][0-9a-fA-F]*$/.test(part))\n value = part.length === 2 ? 0 : Number.parseInt(part.slice(2), 16);\n else if (/^0[0-7]+$/.test(part)) value = Number.parseInt(part.slice(1), 8);\n else if (/^(0|[1-9][0-9]*)$/.test(part)) value = Number(part);\n else return undefined;\n numbers.push(value);\n }\n const last = numbers.pop()!;\n if (numbers.some((value) => value > 255)) return undefined;\n if (last >= 256 ** (4 - numbers.length)) return undefined;\n let rest = last;\n const tail: number[] = [];\n for (let index = numbers.length; index < 4; index += 1) {\n tail.unshift(rest % 256);\n rest = Math.floor(rest / 256);\n }\n return [...numbers, ...tail];\n};\n\n/** This function reads an IPv6 address, with or without brackets and a zone. It gives 16 bytes. */\nexport const parseIPv6 = (text: string): Bytes | undefined => {\n let value = text.startsWith(\"[\") && text.endsWith(\"]\") ? text.slice(1, -1) : text;\n const zone = value.indexOf(\"%\");\n if (zone !== -1) value = value.slice(0, zone);\n if (isIP(value) !== 6) return undefined;\n const lastColon = value.lastIndexOf(\":\");\n const tail = value.slice(lastColon + 1);\n if (tail.includes(\".\")) {\n const v4 = parseIPv4(tail);\n if (v4 === undefined) return undefined;\n const high = (v4[0]! << 8) | v4[1]!;\n const low = (v4[2]! << 8) | v4[3]!;\n value = `${value.slice(0, lastColon + 1)}${high.toString(16)}:${low.toString(16)}`;\n }\n const [head = \"\", rest] = value.split(\"::\");\n const groupsOf = (part: string): number[] =>\n part.length === 0 ? [] : part.split(\":\").map((group) => Number.parseInt(group, 16));\n const front = groupsOf(head);\n const back = rest === undefined ? [] : groupsOf(rest);\n const groups = [...front, ...Array<number>(8 - front.length - back.length).fill(0), ...back];\n if (groups.length !== 8) return undefined;\n return groups.flatMap((group) => [group >> 8, group & 0xff]);\n};\n\nconst v4 = (text: string): Bytes => parseIPv4(text)!;\nconst v6 = (text: string): Bytes => parseIPv6(text)!;\n\n/** This function tells if the first `bits` bits of two addresses agree. */\nconst inPrefix = (address: Bytes, prefix: Bytes, bits: number): boolean => {\n if (address.length !== prefix.length) return false;\n for (let index = 0; index < address.length; index += 1) {\n const take = Math.max(0, Math.min(8, bits - index * 8));\n if (take === 0) return true;\n const mask = (0xff << (8 - take)) & 0xff;\n if ((address[index]! & mask) !== (prefix[index]! & mask)) return false;\n }\n return true;\n};\n\n// the more specific ranges come first, so a metadata address does not read as link-local\nconst V4_RANGES: ReadonlyArray<readonly [Bytes, number, string]> = [\n [v4(\"169.254.169.254\"), 32, \"metadata\"],\n [v4(\"168.63.129.16\"), 32, \"metadata\"],\n [v4(\"100.100.100.200\"), 32, \"metadata\"],\n [v4(\"0.0.0.0\"), 8, \"unspecified\"],\n [v4(\"10.0.0.0\"), 8, \"private\"],\n [v4(\"100.64.0.0\"), 10, \"carrier-grade NAT\"],\n [v4(\"127.0.0.0\"), 8, \"loopback\"],\n [v4(\"169.254.0.0\"), 16, \"link-local\"],\n [v4(\"172.16.0.0\"), 12, \"private\"],\n [v4(\"192.0.0.0\"), 24, \"reserved\"],\n [v4(\"192.0.2.0\"), 24, \"documentation\"],\n [v4(\"192.168.0.0\"), 16, \"private\"],\n [v4(\"198.18.0.0\"), 15, \"benchmark\"],\n [v4(\"198.51.100.0\"), 24, \"documentation\"],\n [v4(\"203.0.113.0\"), 24, \"documentation\"],\n [v4(\"224.0.0.0\"), 4, \"multicast\"],\n [v4(\"240.0.0.0\"), 4, \"reserved\"],\n];\n\nconst V6_RANGES: ReadonlyArray<readonly [Bytes, number, string]> = [\n [v6(\"fd00:ec2::254\"), 128, \"metadata\"],\n [v6(\"::\"), 128, \"unspecified\"],\n [v6(\"::1\"), 128, \"loopback\"],\n [v6(\"64:ff9b:1::\"), 48, \"reserved\"],\n [v6(\"100::\"), 64, \"reserved\"],\n [v6(\"2001::\"), 32, \"reserved\"],\n [v6(\"2001:db8::\"), 32, \"documentation\"],\n [v6(\"fc00::\"), 7, \"unique local\"],\n [v6(\"fe80::\"), 10, \"link-local\"],\n [v6(\"fec0::\"), 10, \"private\"],\n [v6(\"ff00::\"), 8, \"multicast\"],\n];\n\nconst kindOfV4 = (address: Bytes): string | undefined =>\n V4_RANGES.find(([prefix, bits]) => inPrefix(address, prefix, bits))?.[2];\n\nconst kindOfV6 = (address: Bytes): string | undefined => {\n const zeroes = (from: number, to: number): boolean =>\n address.slice(from, to).every((byte) => byte === 0);\n const embedded = (from: number): string | undefined => kindOfV4(address.slice(from, from + 4));\n // an IPv4 address in an IPv6 form has the kind of the IPv4 address\n if (zeroes(0, 10) && address[10] === 0xff && address[11] === 0xff) return embedded(12);\n if (inPrefix(address, v6(\"64:ff9b::\"), 96)) return embedded(12);\n if (address[0] === 0x20 && address[1] === 0x02) return embedded(2);\n const listed = V6_RANGES.find(([prefix, bits]) => inPrefix(address, prefix, bits))?.[2];\n if (listed !== undefined) return listed;\n // the old IPv4-compatible form (::a.b.c.d) and the rest of ::/8 are not public\n if (zeroes(0, 12)) return embedded(12) ?? \"reserved\";\n if (address[0] === 0) return \"reserved\";\n return undefined;\n};\n\n/**\n * This function gives the kind of an address that is not public, for example `loopback` or\n * `private`. It reads each IPv4 spelling and each IPv6 form. A public address, or a text that is not\n * an address, gives no value.\n */\nexport const addressKind = (address: string): string | undefined => {\n const four = parseIPv4(address);\n if (four !== undefined) return kindOfV4(four);\n const six = parseIPv6(address);\n return six === undefined ? undefined : kindOfV6(six);\n};\n\nconst nameOf = (host: string): string => host.trim().toLowerCase().replace(/\\.$/, \"\");\n\n/** These names have a fixed meaning. They do not go to DNS. */\nconst NAME_KINDS: ReadonlyArray<readonly [RegExp, string]> = [\n [/(^|\\.)localhost$/, \"loopback\"],\n [/^metadata\\.google\\.internal$/, \"metadata\"],\n [/^metadata\\.goog$/, \"metadata\"],\n];\n\nconst literalOf = (host: string): Bytes | undefined => parseIPv6(host) ?? parseIPv4(host);\n\nconst inRange = (entry: string, address: Bytes): boolean => {\n const slash = entry.indexOf(\"/\");\n const base = literalOf(slash === -1 ? entry : entry.slice(0, slash));\n if (base === undefined || base.length !== address.length) return false;\n const bits = slash === -1 ? base.length * 8 : Number(entry.slice(slash + 1));\n return (\n Number.isInteger(bits) && bits >= 0 && bits <= base.length * 8 && inPrefix(address, base, bits)\n );\n};\n\n/** This function tells if an entry of `allowHosts` permits this host name or this address. */\nconst permitted = (policy: EgressOptions, host: string, address: string | undefined): boolean => {\n if (policy.allowPrivate === true) return true;\n const name = nameOf(host);\n const bytes = address === undefined ? literalOf(name) : literalOf(address);\n for (const raw of policy.allowHosts ?? []) {\n const entry = nameOf(raw);\n if (entry === name) return true;\n if (entry.startsWith(\"*.\") && name.endsWith(entry.slice(1))) return true;\n if (bytes !== undefined && inRange(entry, bytes)) return true;\n }\n return false;\n};\n\n/** This function gives a short form of an address for a note: IPv6 in brackets, IPv4 with dots. */\nconst shown = (host: string): string => {\n const bytes = literalOf(host);\n if (bytes === undefined) return host;\n if (bytes.length === 4) return bytes.join(\".\");\n const groups: string[] = [];\n for (let index = 0; index < 16; index += 2)\n groups.push(((bytes[index]! << 8) | bytes[index + 1]!).toString(16));\n return groups.join(\":\").replace(/(^|:)0(:0)+(:|$)/, \"::\");\n};\n\n/**\n * This function examines a host before DNS: a literal IP address in any spelling, and the names\n * with a fixed meaning (`localhost`). It gives the note of a refusal, or no value. A host name that\n * needs DNS gets its check in the connection hook.\n */\nexport const egressRefusal = (policy: EgressOptions, host: string): string | undefined => {\n if (policy.allowPrivate === true) return undefined;\n const name = nameOf(host);\n const literal = literalOf(name);\n if (literal !== undefined) {\n const kind = addressKind(name);\n return kind === undefined || permitted(policy, name, undefined)\n ? undefined\n : egressNote(shown(name), kind);\n }\n for (const [pattern, kind] of NAME_KINDS) {\n if (pattern.test(name) && !permitted(policy, name, undefined)) return egressNote(name, kind);\n }\n return undefined;\n};\n\n/**\n * This is the form of `dns.lookup` that the egress hook uses. Tests give a resolver of their own.\n *\n * @beta\n */\nexport type EgressResolve = (\n host: string,\n options: LookupAllOptions,\n callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void,\n) => void;\n\nconst resolveAll: EgressResolve = (host, options, callback) =>\n systemLookup(host, options, callback);\n\n/** This function keeps the addresses that the policy permits. When it keeps none, it gives the refusal. */\nconst screen = (\n policy: EgressOptions,\n host: string,\n addresses: readonly LookupAddress[],\n): { readonly usable: LookupAddress[]; readonly refusal: EgressRefused | undefined } => {\n const usable: LookupAddress[] = [];\n let refusal: EgressRefused | undefined;\n for (const entry of addresses) {\n const kind = addressKind(entry.address);\n if (kind === undefined || permitted(policy, host, entry.address)) usable.push(entry);\n else refusal ??= new EgressRefused(shown(entry.address), kind);\n }\n return {\n usable,\n refusal: usable.length === 0 ? (refusal ?? new EgressRefused(host, \"unresolved\")) : undefined,\n };\n};\n\n/**\n * This function gives a `lookup` hook for the connection options of an undici `Agent`. The hook\n * resolves the name, removes each address that the policy refuses, and gives the rest to the\n * socket. When no address stays, the connection fails with `EgressRefused`.\n *\n * @beta\n */\nexport const egressLookup =\n (policy: EgressOptions, resolve: EgressResolve = resolveAll): LookupFunction =>\n (host: string, options: LookupOptions, callback) => {\n resolve(host, { ...options, all: true }, (error, addresses) => {\n if (error !== null) {\n callback(error, []);\n return;\n }\n const { usable, refusal } =\n policy.allowPrivate === true\n ? { usable: addresses, refusal: undefined }\n : screen(policy, host, addresses);\n if (refusal !== undefined) {\n callback(refusal, []);\n return;\n }\n if (options.all === true) callback(null, usable);\n else callback(null, usable[0]!.address, usable[0]!.family);\n });\n };\n\n/**\n * This function examines a URL with DNS, for a client that does its own DNS, for example the\n * browser. It refuses a host when one of its addresses is not public. A name that DNS cannot\n * resolve gets no refusal, because the client cannot connect to it either. The function gives\n * the note of a refusal, or no value.\n *\n * The client resolves the name again, so a DNS answer that changes between the two lookups can\n * pass. The connection hook (`egressLookup`) does not have this gap.\n */\nexport const checkEgress = async (\n policy: EgressOptions,\n url: URL,\n resolve: EgressResolve = resolveAll,\n): Promise<string | undefined> => {\n if (policy.allowPrivate === true) return undefined;\n const before = egressRefusal(policy, url.hostname);\n if (before !== undefined) return before;\n const name = nameOf(url.hostname);\n if (literalOf(name) !== undefined || permitted(policy, name, undefined)) return undefined;\n const addresses = await new Promise<LookupAddress[] | Error>((settle) =>\n resolve(name, { all: true }, (error, found) => settle(error ?? found)),\n );\n if (addresses instanceof Error) return undefined;\n for (const entry of addresses) {\n const kind = addressKind(entry.address);\n if (kind !== undefined && !permitted(policy, name, entry.address))\n return egressNote(shown(entry.address), kind);\n }\n return undefined;\n};\n\n/** This function finds an `EgressRefused` in an error, its cause or its group of errors. */\nexport const egressError = (error: unknown): EgressRefused | undefined => {\n for (\n let current = error, depth = 0;\n current !== undefined && current !== null && depth < 5;\n depth += 1\n ) {\n if (current instanceof EgressRefused) return current;\n if (current instanceof AggregateError) {\n const inner = current.errors.find((item: unknown) => item instanceof EgressRefused) as\n EgressRefused | undefined;\n if (inner !== undefined) return inner;\n }\n current = (current as { cause?: unknown }).cause;\n }\n return undefined;\n};\n","import { createHash } from \"node:crypto\";\n\n/**\n * A capture record is one line of the capture log. Each fetch decision makes one record, and a\n * fetch that the crawler declines makes one too.\n *\n * The record id comes from a hash of its content, so the same decision always gets the same id.\n * A citation names this id.\n *\n * The id has 48 bits: the source and the first 12 hexadecimal digits of the hash. Approximately\n * 1 million records give a chance of a collision of 0.2 percent. The package keeps this length on purpose,\n * because the ids of the earlier capture logs and their citations must stay the same. A later\n * version can make the ids of new records longer behind a record version. It must still read a\n * 12-digit id.\n */\nexport interface CaptureRecord {\n readonly id: string;\n /** The source of the record. The crawler writes `fetch`, and an application can add its own sources. */\n readonly source: string;\n readonly kind: string;\n readonly external_id?: string | undefined;\n readonly occurred_at?: string | undefined;\n readonly received_at: string;\n readonly payload: unknown;\n readonly content_hash: string;\n /** This label comes from the application. The crawler writes it as it is and does not read it. */\n readonly scope?: string | undefined;\n /**\n * These labels come from the application, for example the lineage of a request. The crawler writes\n * them as they are and does not read them. Like `scope`, they are not part of the hash.\n */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/**\n * These are the fields of a new record. The function `makeRecord` calculates the id and the hash.\n *\n * @beta\n */\nexport interface RecordInput {\n readonly source: string;\n readonly kind: string;\n readonly payload: unknown;\n readonly received_at: string;\n readonly external_id?: string | undefined;\n readonly occurred_at?: string | undefined;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/**\n * This function writes a value as JSON with the keys of each object in sorted order. Equal values\n * thus always give equal text, and equal text gives an equal hash.\n */\nexport const stableStringify = (value: unknown): string => {\n if (value === null || typeof value !== \"object\") {\n return JSON.stringify(value) ?? \"null\";\n }\n if (Array.isArray(value)) {\n return `[${value.map((item) => stableStringify(item)).join(\",\")}]`;\n }\n const entries = Object.entries(value as Record<string, unknown>)\n .filter(([, item]) => item !== undefined)\n .sort(([a], [b]) => a.localeCompare(b))\n .map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`);\n return `{${entries.join(\",\")}}`;\n};\n\n/** This function gives the SHA-256 hash of a text as hexadecimal digits. */\nexport const sha256 = (text: string): string =>\n createHash(\"sha256\").update(text, \"utf8\").digest(\"hex\");\n\n/**\n * This function makes a capture record. The hash covers the source, the kind, the external id\n * and the payload. The id is the source and the first 12 digits of the hash. The times, `scope`\n * and `labels` are not part of the hash.\n *\n * @beta\n */\nexport const makeRecord = (input: RecordInput): CaptureRecord => {\n const content_hash = sha256(\n stableStringify({\n source: input.source,\n kind: input.kind,\n external_id: input.external_id ?? null,\n payload: input.payload,\n }),\n );\n return {\n id: `${input.source}.${content_hash.slice(0, 12)}`,\n source: input.source,\n kind: input.kind,\n received_at: input.received_at,\n payload: input.payload,\n content_hash,\n ...(input.external_id !== undefined ? { external_id: input.external_id } : {}),\n ...(input.occurred_at !== undefined ? { occurred_at: input.occurred_at } : {}),\n ...(input.scope !== undefined ? { scope: input.scope } : {}),\n ...(input.labels !== undefined && Object.keys(input.labels).length > 0\n ? { labels: { ...input.labels } }\n : {}),\n };\n};\n","import type { Readable } from \"node:stream\";\nimport { Transform } from \"node:stream\";\nimport { createBrotliDecompress, createGunzip, createInflate, createInflateRaw } from \"node:zlib\";\nimport type { Dispatcher } from \"undici\";\nimport { Agent, request } from \"undici\";\nimport type { BlobStore } from \"./blobs.ts\";\nimport type { Classified, Outcome } from \"./classify.ts\";\nimport { classify, isClientError, parseHttpDate } from \"./classify.ts\";\nimport type { Variant } from \"./docs.ts\";\nimport type { EgressOptions } from \"./egress.ts\";\nimport { checkEgress, egressError, egressLookup, egressRefusal } from \"./egress.ts\";\nimport type {\n Clock,\n HostResult,\n HostStateStore,\n Paced,\n PolicyOptions,\n PolitenessPolicy,\n RestInput,\n ValidatorAnswer,\n ValidatorMemo,\n} from \"./politeness.ts\";\nimport {\n DEFAULT_POLICY,\n HostScheduler,\n MAX_STOP_MS,\n HostStopped,\n checkPolicy,\n equalJitterMs,\n policyFor,\n realClock,\n restAfter,\n validatorsToSend,\n} from \"./politeness.ts\";\nimport type { CaptureRecord } from \"./record.ts\";\nimport { makeRecord } from \"./record.ts\";\nimport type { Rendered, Renderer } from \"./render.ts\";\nimport type { RefusedReason } from \"./render.ts\";\nimport { NO_PLAYWRIGHT, RenderRefused } from \"./render.ts\";\nimport type { Robots, RobotsDecision } from \"./robots.ts\";\nimport {\n decide,\n isProductToken,\n robotsFromResponse,\n robotsFromStatus,\n signalsFor,\n unreachableRobots,\n} from \"./robots.ts\";\nimport type { AiInput, Signals } from \"./signals.ts\";\nimport { NO_SIGNALS, metaSignals, readSignals } from \"./signals.ts\";\nimport { BOT_PAGE, DEFAULT_TOKEN } from \"./version.ts\";\nimport { cacheKeyOf } from \"./urls.ts\";\n// ── signed requests (Web Bot Auth) ──\nimport type { RequestSigner, WebBotAuthOptions } from \"./webbotauth.ts\";\nimport { signerOf } from \"./webbotauth.ts\";\n// ── end of signed requests ──\n\n/**\n * The crawler is a polite fetcher. It sends one request at a time to each host, and it obeys\n * robots.txt for web pages. Its `User-Agent` header tells what it is. It asks again with the\n * validators of the stored copy, and it stops when a site refuses. Each decision goes into the\n * capture log, and a fetch that the crawler declines goes there too.\n *\n * There are two kinds of request. A documented public API (`api`) has its own terms and rate\n * limits. A web page (`page`) obeys robots.txt. The settings come from measurements:\n *\n * - The `request` function of undici on one `Agent`.\n * - A keep-alive time longer than the delay for each host. With the default of 4 seconds, each\n * polite request opens a new connection.\n * - A backoff with jitter in the queue, not the retry handler of undici. That handler limits\n * Retry-After to 30 seconds and adds no jitter.\n * - A maximum size for each body, after decompression. The crawler keeps the first part of a\n * longer body and marks it as truncated.\n *\n * A fetch and a render go through one admission. It examines the URL, the egress policy, the\n * product tokens, robots.txt, the cache and the stops of the host. Only the transport is\n * different: an HTTP request or the headless browser.\n *\n * The egress policy (`EgressOptions`) keeps the crawler on public addresses by default. The\n * politeness policy (`PolitenessPolicy`) holds each limit, and a host can have its own values.\n * The fetch and the render do not throw an error: each problem is an outcome with a note.\n */\n\n/**\n * The outcome of a fetch or a render. A `skipped` request did not go out. A render of a URL that\n * gives a file and not a page is `not-html`.\n */\nexport type FetchOutcome = Outcome | \"skipped\" | \"not-html\";\n\n/** @beta */\nexport interface FetchRequest {\n readonly url: string;\n /** The value `api` is a documented API with its own terms. The value `page` is a web page, and robots.txt applies. */\n readonly mode: \"api\" | \"page\";\n /** This recipe reads the capture again. The function `reindex` uses it to replay the capture. */\n readonly recipe: string;\n readonly accept?: string | undefined;\n /**\n * These are more request headers. The crawler does not write them into the capture. They\n * cannot replace `User-Agent`, `From`, `Accept-Encoding` or the validators. A redirect to a\n * different origin removes `Authorization`, `Cookie` and `Proxy-Authorization`.\n */\n readonly headers?: Readonly<Record<string, string>> | undefined;\n /**\n * These query parameters contain secrets. Each capture shows `REDACTED` in their place. The\n * names in `DEFAULT_SECRET_PARAMS` get the same treatment with no list.\n */\n readonly secretParams?: readonly string[] | undefined;\n /** This value is the documented pace of the source: the shortest time between two of its requests. */\n readonly minIntervalMs?: number | undefined;\n /**\n * A stored capture younger than this age answers, and the crawler sends no request. The value\n * `auto` uses the freshness that the server gave (RFC 9111): `Cache-Control`, `Expires`, or 10\n * percent of the age of `Last-Modified`.\n */\n readonly maxAgeMs?: number | \"auto\" | undefined;\n readonly maxBytes?: number | undefined;\n /**\n * These are the product tokens of an agent that the fetch acts for, for example `Claude-User`.\n * The robots.txt rules for these tokens apply too.\n */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** This name connects the captures to the job that asked for them. */\n readonly job?: string | undefined;\n /** When this signal aborts, the crawler stops the fetch. The result is then `skipped` or `error`, with a note. */\n readonly signal?: AbortSignal | undefined;\n /** This label goes into each capture record of the request. It replaces the `scope` of the crawler. */\n readonly scope?: string | undefined;\n /** These labels go into each capture record of the request, for example its lineage. They are not part of the capture id. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n /**\n * This time (milliseconds since the epoch, on the clock of the crawler) is the end of a budget.\n * The request does not start at or after it. When the host is not free before it, the result is\n * `skipped`, and its note starts with `DEADLINE_NOTE`.\n */\n readonly deadline?: number | undefined;\n // ── archived fallback: the parts of a request to a web archive ──\n /**\n * This byte range is the part of the resource to fetch (a `Range` request). The crawler then sends\n * no validators, keeps no stored copy for the URL, and reads `length` bytes at most. The capture\n * records the range.\n */\n readonly range?: { readonly offset: number; readonly length: number } | undefined;\n /** These lines of usage signals apply to the response too. A copy from a web archive gets the content signals of the robots.txt of its site. */\n readonly signalLines?: readonly string[] | undefined;\n /**\n * This prefix names the response headers that hold the headers of an archived response, for\n * example `x-archive-orig-` of the Wayback Machine. The usage signals of those headers apply too.\n */\n readonly originalHeaderPrefix?: string | undefined;\n // ── end of archived fallback ──\n}\n\n/** @beta */\nexport interface Fetched {\n /** This URL is the URL of the capture. Its secrets show as `REDACTED`. */\n readonly url: string;\n readonly status: number;\n readonly outcome: FetchOutcome;\n /** This value is true when a stored capture answered, with no request. */\n readonly fromCache: boolean;\n /** This id is the capture record behind the body. A citation names it. */\n readonly capture: string | undefined;\n readonly blob: string | undefined;\n readonly body: Uint8Array | undefined;\n readonly contentType: string | undefined;\n readonly signals: Signals;\n readonly note: string | undefined;\n /** This value is true when the body was longer than the maximum, so the crawler kept only its first part. */\n readonly truncated?: boolean | undefined;\n /** This time (ISO 8601) is the time of the fetch of the body. For a stored copy, it is the time of that copy. */\n readonly fetchedAt?: string | undefined;\n /** This label is the scope of the capture. */\n readonly scope?: string | undefined;\n /** These labels are the labels of the capture. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/**\n * The crawler writes each record to a capture writer. The class `CaptureLog` is one.\n *\n * @beta\n */\nexport interface CaptureWriter {\n append(record: CaptureRecord): Promise<void>;\n close?(): Promise<void>;\n}\n\n/**\n * A cache entry is the last answer for a URL: its capture, its body and its validators.\n *\n * @beta\n */\nexport interface CacheEntry {\n /** This text is the cache key: the canonical URL, with its secrets as `REDACTED`. */\n readonly url: string;\n readonly capture: string;\n readonly blob: string | null;\n readonly etag: string | null;\n readonly last_modified: string | null;\n readonly content_type: string | null;\n readonly fetched_at: string;\n readonly status: number;\n /** This value is what the page said about AI input at the time of the fetch. */\n readonly ai_input: AiInput;\n /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */\n readonly location?: string | null | undefined;\n /** This value is true when the stored body is only the first part of the response. */\n readonly truncated?: boolean | undefined;\n /** This time (ISO 8601) is the end of the freshness that the server gave. */\n readonly fresh_until?: string | null | undefined;\n /** This text holds the request headers that the response varies on (`Vary`), with their values. */\n readonly vary_key?: string | null | undefined;\n /** This value tells a plain response from a page that the browser rendered. The default is `plain`. */\n readonly variant?: Variant | undefined;\n}\n\n/**\n * The cache index holds the last answer for each URL. The crawler reads it before a request, and\n * it writes it after an answer. The class `DocIndex` is one. An index that keeps only the\n * necessary fields of `CacheEntry` works, and the crawler keeps the other fields in memory.\n *\n * @beta\n */\nexport interface CacheIndex {\n latest(url: string, variant?: Variant): CacheEntry | undefined;\n setLatest(entry: CacheEntry): void;\n}\n\n/** An event tells a listener what the crawler does. The listener cannot change it. */\nexport type CrawlEvent =\n | {\n readonly type: \"request\";\n readonly url: string;\n readonly host: string;\n readonly transport: \"http\" | \"browser\";\n }\n | {\n readonly type: \"response\";\n readonly url: string;\n readonly host: string;\n readonly status: number;\n readonly outcome: FetchOutcome;\n readonly elapsedMs: number;\n }\n | { readonly type: \"wait\"; readonly host: string; readonly ms: number; readonly reason: string }\n | {\n readonly type: \"stop\";\n readonly host: string;\n readonly until: number;\n readonly reason: string;\n }\n | { readonly type: \"skip\"; readonly url: string; readonly note: string };\n\n/** @beta */\nexport interface CrawlerOptions {\n /** This product token is the name that robots.txt rules match. It contains only letters, `_` and `-` (RFC 9309). */\n readonly token: string;\n /** This text is the full `User-Agent` header. */\n readonly userAgent: string;\n /** This contact goes into the `From` header. Without it, the crawler sends no `From` header. */\n readonly from?: string | undefined;\n readonly log: CaptureWriter;\n readonly blobs: BlobStore;\n readonly index: CacheIndex;\n readonly clock?: Clock | undefined;\n readonly random?: (() => number) | undefined;\n /** This value is the number of hosts that get requests at one time. The default is 16. */\n readonly maxActive?: number | undefined;\n /**\n * This value tells if the crawler also obeys the robots.txt group of `PoliteCrawler`. That applies\n * when the `User-Agent` links the bot page of polite-crawler (`BOT_PAGE`) and the token is different.\n * Site owners read that page, and they write their rules for `PoliteCrawler`. The most restrictive\n * of the groups applies, as for `onBehalfOf`. The default is true. Set false for a bot page of your own.\n */\n readonly obeyDefaultToken?: boolean | undefined;\n /**\n * This value is the number of pages that the browser renders at one time. The default is 2. Each\n * render is a browser context of its own, which holds much memory. A render waits for a slot before\n * the lease of its host, so the wait does not count toward the time of the host.\n */\n readonly maxRenders?: number | undefined;\n /**\n * This undici dispatcher sends the requests. With a dispatcher of your own, the crawler checks\n * only literal addresses. Put `egressLookup` in its connection options to check the address after DNS.\n */\n readonly dispatcher?: Dispatcher | undefined;\n /** This label goes into each capture record as it is. */\n readonly scope?: string | undefined;\n /**\n * This function ranks the scopes of the requests, the most private scope highest. One fetch of a\n * robots.txt can serve requests of different scopes. Its record then takes the scope with the\n * highest rank, and the labels of each request. With no function, the first request gives the scope.\n */\n readonly scopeRank?: ((scope: string) => number) | undefined;\n /** This browser renders the pages that need JavaScript. Without it, the crawler renders no page. */\n readonly renderer?: Renderer | undefined;\n /** This policy controls the addresses that the crawler fetches. By default, the crawler fetches only public addresses. */\n readonly egress?: EgressOptions | undefined;\n /** These values change the limits of the politeness policy, for each host or for some hosts. */\n readonly politeness?: PolicyOptions | undefined;\n /** This store keeps the pace, the counts and the stops of each host. The default keeps them in memory. */\n readonly hostState?: HostStateStore | undefined;\n /** This function gets an event for each request, response, wait, stop and skip. An error in it does not stop the crawler. */\n readonly onEvent?: ((event: CrawlEvent) => void) | undefined;\n // ── signed requests (Web Bot Auth) ──\n /**\n * This signer signs each request, robots.txt and the requests of a rendered page too. Give the\n * options of `createWebBotAuthSigner`, or a `RequestSigner`. Without it, the crawler signs nothing.\n * A signature adds an identity: the `User-Agent` and `From` headers stay as they are.\n */\n readonly signer?: RequestSigner | WebBotAuthOptions | undefined;\n // ── end of signed requests ──\n}\n\n/** @beta */\nexport interface RenderRequest {\n readonly url: string;\n readonly recipe: string;\n /** A stored render younger than this age answers. The value `auto` uses the freshness that the server gave. */\n readonly maxAgeMs?: number | \"auto\" | undefined;\n readonly onBehalfOf?: readonly string[] | undefined;\n readonly job?: string | undefined;\n /** When this signal aborts, the crawler stops the render, and the browser stops the page. */\n readonly signal?: AbortSignal | undefined;\n /** This label goes into the capture record of the render. It replaces the `scope` of the crawler. */\n readonly scope?: string | undefined;\n /** These labels go into the capture record of the render. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n /** This time (milliseconds since the epoch) is the end of a budget. The render does not start at or after it (`FetchRequest.deadline`). */\n readonly deadline?: number | undefined;\n}\n\n/**\n * The note of a request starts with this text when its host was not free before its deadline, so the request did not start.\n *\n * @beta\n */\nexport const DEADLINE_NOTE = \"deadline reached\";\n\n/**\n * This function gives the key of the cache entry of a rendered page in the form of version 0.1.\n * The index keeps the variant in a column of its own, and it reads this form too.\n *\n * This export is internal: the root entry point does not have it since 0.2.0.\n */\nexport const renderedKey = (url: string): string => `${url}#rendered`;\n\n/**\n * This value is the version of the payload of the records that the crawler writes. A record with no `schema` is version 1.\n *\n * @beta\n */\nexport const CAPTURE_SCHEMA = 2;\n\n/** The content type of a rendered page: the browser gives its text as UTF-8. */\nexport const RENDERED_TYPE = \"text/html; charset=utf-8\";\n\n/**\n * These query parameters hold secrets in many APIs. The crawler writes `REDACTED` in place of\n * their values in each capture, in addition to the names of `secretParams`.\n *\n * @beta\n */\nexport const DEFAULT_SECRET_PARAMS: readonly string[] = [\n \"access_token\",\n \"api-key\",\n \"api_key\",\n \"apikey\",\n \"auth\",\n \"auth_token\",\n \"client_secret\",\n \"id_token\",\n \"passwd\",\n \"password\",\n \"pwd\",\n \"refresh_token\",\n \"secret\",\n \"session\",\n \"session_id\",\n \"sessionid\",\n \"sig\",\n \"signature\",\n \"token\",\n \"x-amz-credential\",\n \"x-amz-security-token\",\n \"x-amz-signature\",\n \"x-goog-credential\",\n \"x-goog-signature\",\n];\n\nconst ERROR_BODY_BYTES = 64 * 1024;\nconst ROBOTS_BYTES = 512 * 1024;\nconst LAST_DATE_MS = 8.64e15;\nconst WEB_PROTOCOLS: ReadonlySet<string> = new Set([\"http:\", \"https:\"]);\nconst HEADER_TEXT = /^[\\t -~]*$/;\nconst PERMANENT_REDIRECTS: ReadonlySet<number> = new Set([301, 308]);\nconst HEURISTIC_STATUSES: ReadonlySet<number> = new Set([\n 200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501,\n]);\n/** A caller cannot set these headers. The crawler sets them, or HTTP does. */\nconst RESERVED_HEADERS: ReadonlySet<string> = new Set([\n \"user-agent\",\n \"from\",\n \"accept-encoding\",\n \"if-none-match\",\n \"if-modified-since\",\n \"if-match\",\n \"if-unmodified-since\",\n \"if-range\",\n \"range\",\n \"host\",\n \"connection\",\n \"content-length\",\n \"transfer-encoding\",\n \"te\",\n \"upgrade\",\n \"keep-alive\",\n]);\n/** A redirect to a different origin removes these headers, as the Fetch standard does. */\nconst CREDENTIAL_HEADERS: ReadonlySet<string> = new Set([\n \"authorization\",\n \"cookie\",\n \"proxy-authorization\",\n]);\n/** The crawler sends the same value of these headers in each request, so a `Vary` on them changes nothing. */\nconst CONSTANT_HEADERS: ReadonlySet<string> = new Set([\n \"accept-encoding\",\n \"user-agent\",\n \"from\",\n // ── signed requests: the identity is the same in each request ──\n \"signature\",\n \"signature-input\",\n \"signature-agent\",\n]);\n// ── signed requests (Web Bot Auth) ──\n/** With a signer, the crawler sets these headers, and a caller cannot. */\nconst SIGNATURE_HEADERS: ReadonlySet<string> = new Set([\n \"signature\",\n \"signature-input\",\n \"signature-agent\",\n]);\n// ── end of signed requests ──\n// ── archived fallback ──\n/** These response headers hold usage signals. With `originalHeaderPrefix`, the archived forms of them count too. */\nconst SIGNAL_HEADERS = [\n \"content-usage\",\n \"content-signal\",\n \"x-robots-tag\",\n \"tdm-reservation\",\n \"tdm-policy\",\n] as const;\n// ── end of archived fallback ──\n/**\n * These content codings change nothing in the body. A server that stored an object with\n * `aws-chunked` can send the name with the decoded body.\n */\nconst IDENTITY_CODINGS: ReadonlySet<string> = new Set([\"identity\", \"aws-chunked\"]);\n\n/**\n * This function tells if an answer is a web page that the request did not ask for. The answer is\n * HTML, and the `Accept` of the request admits no HTML (`application/json`). A login or an error\n * page in place of the data of an API is such an answer. A media range with `q=0` admits nothing.\n */\nconst unaskedPage = (accept: string | undefined, contentType: string | undefined): boolean => {\n const type = (contentType ?? \"\").split(\";\")[0]!.trim().toLowerCase();\n if (\n accept === undefined ||\n accept.trim() === \"\" ||\n (type !== \"text/html\" && type !== \"application/xhtml+xml\")\n )\n return false;\n return !accept.split(\",\").some((range) => {\n const [media = \"\", ...params] = range.split(\";\").map((part) => part.trim().toLowerCase());\n const q = params.find((param) => param.startsWith(\"q=\"));\n if (q !== undefined && Number(q.slice(2)) === 0) return false;\n return (\n media === \"*/*\" ||\n media === type ||\n (media.endsWith(\"/*\") && type.startsWith(media.slice(0, -1)))\n );\n });\n};\n\n/** This function reads the TDMRep `<meta>` tags at the start of an HTML body. */\nconst pageSignals = (body: Uint8Array | undefined, contentType: string | undefined): string[] =>\n body === undefined || !/html/i.test(contentType ?? \"\")\n ? []\n : metaSignals(new TextDecoder(\"latin1\").decode(body.subarray(0, 65_536)));\n\n/** A count of slots: `max` holders at one time at most, and the others wait in sequence. */\nclass Slots {\n private used = 0;\n private readonly waiting: Array<() => void> = [];\n\n constructor(private readonly max: number) {}\n\n /** This method gives a function that frees the slot. It rejects when the signal aborts before a slot is free. */\n async acquire(signal: AbortSignal): Promise<() => void> {\n if (this.used < this.max) {\n this.used += 1;\n return this.releaser();\n }\n await new Promise<void>((resolve, reject) => {\n const stop = (): void => {\n const at = this.waiting.indexOf(go);\n if (at !== -1) this.waiting.splice(at, 1);\n reject(signal.reason instanceof Error ? signal.reason : new Error(\"cancelled\"));\n };\n const go = (): void => {\n signal.removeEventListener(\"abort\", stop);\n resolve();\n };\n if (signal.aborted) {\n stop();\n return;\n }\n this.waiting.push(go);\n signal.addEventListener(\"abort\", stop, { once: true });\n });\n // the slot of the holder that freed it passes to this waiter, so the count stays\n return this.releaser();\n }\n\n private releaser(): () => void {\n let freed = false;\n return () => {\n if (freed) return;\n freed = true;\n const next = this.waiting.shift();\n if (next !== undefined) next();\n else this.used -= 1;\n };\n }\n}\n\n/** This function tells if an error of a renderer says that it has no browser: no `playwright-core`, or no browser on the disk. */\nconst missingBrowser = (message: string): boolean =>\n message.startsWith(NO_PLAYWRIGHT) ||\n /Executable doesn't exist|Cannot find (?:module|package) 'playwright-core'/.test(message);\n\n/** This text tells why the crawler did not fetch a page: a rule of the site, or no robots.txt to read. */\nconst robotsRefusal = (robots: Robots, by: string): string =>\n robots.unreachable === undefined\n ? `robots.txt disallows it for ${by}`\n : `robots.txt could not be read (${robots.unreachable}), so nothing on the site is fetched`;\n\nconst messageOf = (error: unknown): string =>\n error instanceof Error ? (error.message.split(/\\r?\\n/)[0] ?? \"\") : String(error);\n\nconst isoOf = (ms: number): string | undefined =>\n Number.isFinite(ms) && Math.abs(ms) < LAST_DATE_MS ? new Date(ms).toISOString() : undefined;\n\nconst stopNote = (reason: string, until: number): string => {\n const end = isoOf(until);\n return end === undefined ? `host stopped: ${reason}` : `host stopped until ${end}: ${reason}`;\n};\n\n/** A request did not start before its deadline. The crawler gives a skip with `DEADLINE_NOTE` for it. */\nclass DeadlineReached extends Error {\n constructor(\n /** This value is the earliest start of the request on its host, in milliseconds since the epoch. */\n readonly readyAt: number,\n ) {\n super(DEADLINE_NOTE);\n }\n}\n\nconst deadlineNote = (error: DeadlineReached, deadline: number | undefined): string =>\n `${DEADLINE_NOTE}: the host is free at ${isoOf(error.readyAt) ?? \"a later time\"}, and the request must start before ${isoOf(deadline ?? Number.NaN) ?? \"its deadline\"}`;\n\nconst secretSet = (secretParams: readonly string[]): Set<string> =>\n new Set([...DEFAULT_SECRET_PARAMS, ...secretParams].map((name) => name.toLowerCase()));\n\nconst nameOfParam = (raw: string): string => {\n try {\n return decodeURIComponent(raw.replace(/\\+/g, \" \")).toLowerCase();\n } catch {\n return raw.toLowerCase();\n }\n};\n\n/**\n * This function gives the URL of a capture. It removes a user name and a password, and it writes\n * `REDACTED` in place of the value of each secret parameter. The other parameters stay as they\n * were written.\n */\nexport const redact = (url: URL, secretParams: readonly string[] = []): string => {\n const copy = new URL(url.href);\n copy.username = \"\";\n copy.password = \"\";\n if (copy.search.length > 1) {\n const secrets = secretSet(secretParams);\n const parts = copy.search\n .slice(1)\n .split(\"&\")\n .map((part) => {\n const equals = part.indexOf(\"=\");\n if (equals === -1) return part;\n const name = part.slice(0, equals);\n return secrets.has(nameOfParam(name)) ? `${name}=REDACTED` : part;\n });\n copy.search = `?${parts.join(\"&\")}`;\n }\n return copy.href;\n};\n\n/** This function tells if a URL holds a secret: a user name, a password or a secret parameter. */\nconst holdsSecrets = (url: URL, secretParams: readonly string[]): boolean => {\n if (url.username !== \"\" || url.password !== \"\") return true;\n const secrets = secretSet(secretParams);\n return url.search\n .slice(1)\n .split(\"&\")\n .some(\n (part) => part.includes(\"=\") && secrets.has(nameOfParam(part.slice(0, part.indexOf(\"=\")))),\n );\n};\n\n/** The pace, the stops and the robots.txt of a host do not change with a trailing dot or the case of the name. */\nconst hostKey = (url: URL): string =>\n `${url.hostname.toLowerCase().replace(/\\.$/, \"\")}${url.port === \"\" ? \"\" : `:${url.port}`}`;\n\nconst originKey = (url: URL): string => `${url.protocol}//${hostKey(url)}`;\n\nconst headerOf =\n (headers: Readonly<Record<string, string | string[] | undefined>>) =>\n (name: string): string | undefined => {\n const value = headers[name.toLowerCase()];\n return Array.isArray(value) ? value.join(\", \") : value;\n };\n\nclass Lru<K, V> {\n private readonly map = new Map<K, V>();\n constructor(private readonly max: number) {}\n\n get(key: K): V | undefined {\n const value = this.map.get(key);\n if (value !== undefined) {\n this.map.delete(key);\n this.map.set(key, value);\n }\n return value;\n }\n\n peek(key: K): V | undefined {\n return this.map.get(key);\n }\n\n set(key: K, value: V): void {\n this.map.delete(key);\n this.map.set(key, value);\n if (this.map.size > this.max) this.map.delete(this.map.keys().next().value as K);\n }\n\n delete(key: K): void {\n this.map.delete(key);\n }\n\n keys(): K[] {\n return [...this.map.keys()];\n }\n\n get size(): number {\n return this.map.size;\n }\n}\n\n/** A body failed in transport (a reset, a timeout) or in decompression. */\nclass BodyError extends Error {\n constructor(\n readonly kind: \"transport\" | \"decode\",\n cause: unknown,\n ) {\n super(messageOf(cause), { cause });\n }\n}\n\n/** RFC 9110 says that `deflate` is the zlib format, but some servers send raw deflate. This stream reads both. */\nconst inflateAny = (): Transform => {\n let inner: Transform | undefined;\n const outer: Transform = new Transform({\n transform(chunk: Buffer, _encoding, done) {\n if (inner === undefined) {\n const zlib =\n chunk.length >= 2 &&\n (chunk[0]! & 0x0f) === 8 &&\n ((chunk[0]! << 8) | chunk[1]!) % 31 === 0;\n inner = zlib ? createInflate() : createInflateRaw();\n inner.on(\"data\", (data: Buffer) => outer.push(data));\n inner.on(\"error\", (error) => outer.destroy(error));\n }\n inner.write(chunk, () => done());\n },\n flush(done) {\n if (inner === undefined) {\n done();\n return;\n }\n inner.once(\"end\", () => done());\n inner.end();\n },\n destroy(error, callback) {\n inner?.destroy();\n callback(error);\n },\n });\n return outer;\n};\n\nconst decoderFor = (coding: string): Transform => {\n if (coding === \"gzip\" || coding === \"x-gzip\") return createGunzip();\n if (coding === \"br\") return createBrotliDecompress();\n if (coding === \"deflate\") return inflateAny();\n throw new BodyError(\"decode\", new Error(`unsupported content-encoding \"${coding}\"`));\n};\n\n/** This function tells if a decoder stopped because the compressed body ended early: the server cut it. */\nconst endedEarly = (error: unknown): boolean =>\n typeof error === \"object\" &&\n error !== null &&\n \"code\" in error &&\n (error.code === \"Z_BUF_ERROR\" || /unexpected end of file/i.test(messageOf(error)));\n\n/**\n * This function reads a body as a stream and decodes each content coding in sequence. It counts\n * the decoded bytes and keeps at most `maxBytes`. Thus a decompression bomb stops at the\n * maximum, and a long page keeps its first part. A body that fails gives a `BodyError`. A\n * compressed body that ends early gives the part that it decoded, marked `cut`.\n */\nconst readBody = async (\n body: Readable,\n encoding: string | undefined,\n maxBytes: number,\n): Promise<{ readonly bytes: Uint8Array; readonly truncated: boolean; readonly cut: boolean }> => {\n const codings = (encoding ?? \"\")\n .split(\",\")\n .map((coding) => coding.trim().toLowerCase())\n .filter((coding) => coding.length > 0 && !IDENTITY_CODINGS.has(coding))\n .reverse();\n // undici gives an error when a body that did not end is destroyed; the loop below reports real errors\n body.on(\"error\", () => undefined);\n let decoders: Transform[];\n try {\n decoders = codings.map(decoderFor);\n } catch (error) {\n body.destroy();\n throw error;\n }\n // a compressed stream that gives almost no output stops at this length of input\n const rawCap = maxBytes * 4 + 65_536;\n let rawBytes = 0;\n let stream: Readable = body;\n if (decoders.length > 0) {\n body.on(\"data\", (chunk: Buffer) => {\n rawBytes += chunk.length;\n });\n for (const decoder of decoders) {\n const source = stream;\n source.on(\"error\", (error) => decoder.destroy(error));\n stream = source.pipe(decoder);\n }\n }\n const chunks: Buffer[] = [];\n let size = 0;\n let truncated = false;\n let cut = false;\n try {\n for await (const chunk of stream as AsyncIterable<Buffer>) {\n const room = maxBytes - size;\n if (chunk.length > room) {\n // the chunk that crosses the maximum gives its first part\n if (room > 0) chunks.push(chunk.subarray(0, room));\n size += Math.max(0, room);\n truncated = true;\n break;\n }\n chunks.push(chunk);\n size += chunk.length;\n if (rawBytes > rawCap) {\n truncated = true;\n break;\n }\n }\n } catch (error) {\n const transport = body.errored !== null || decoders.length === 0;\n // a server that closes a compressed body early: the part before the end is still the page\n if (!transport && size > 0 && endedEarly(error)) {\n cut = true;\n truncated = true;\n } else {\n throw new BodyError(transport ? \"transport\" : \"decode\", error);\n }\n } finally {\n body.destroy();\n for (const decoder of decoders) decoder.destroy();\n }\n // one copy into a buffer of its own, so no reader sees the shared pool of Node.js\n const bytes = new Uint8Array(size);\n let at = 0;\n for (const chunk of chunks) {\n bytes.set(chunk, at);\n at += chunk.length;\n }\n return { bytes, truncated, cut };\n};\n\nconst directives = (value: string | undefined): Map<string, string> => {\n const out = new Map<string, string>();\n for (const part of (value ?? \"\").split(\",\")) {\n const equals = part.indexOf(\"=\");\n const name = (equals === -1 ? part : part.slice(0, equals)).trim().toLowerCase();\n if (name.length === 0) continue;\n const raw = equals === -1 ? \"\" : part.slice(equals + 1).trim();\n out.set(\n name,\n raw.length >= 2 && raw.startsWith('\"') && raw.endsWith('\"') ? raw.slice(1, -1) : raw,\n );\n }\n return out;\n};\n\nconst TEN_YEARS_MS = 10 * 365 * 24 * 60 * 60 * 1000;\n\n/**\n * This function gives the end of the freshness of a response, as RFC 9111 §4.2 calculates it.\n * The sources in sequence are `max-age`, `Expires`, and 10 percent of the age of\n * `Last-Modified`, with a maximum. A response with `no-store` or `no-cache` is stale at once. A\n * response with no freshness gives null.\n */\nconst freshUntil = (\n header: (name: string) => string | undefined,\n status: number,\n fetchedAt: number,\n lastModified: string | null | undefined,\n capMs: number,\n): string | null => {\n const control = directives(header(\"cache-control\"));\n if (control.has(\"no-store\") || control.has(\"no-cache\")) return isoOf(fetchedAt) ?? null;\n const ageText = header(\"age\")?.trim() ?? \"\";\n const age = /^\\d+$/.test(ageText) ? Number(ageText) * 1000 : 0;\n const date = parseHttpDate(header(\"date\")) ?? fetchedAt;\n let lifetime: number | undefined;\n const maxAge = control.get(\"max-age\");\n const expires = header(\"expires\");\n if (maxAge !== undefined && /^\\d+$/.test(maxAge)) lifetime = Number(maxAge) * 1000;\n else if (expires !== undefined) {\n // an Expires that is not a date means that the response is already stale\n const at = parseHttpDate(expires);\n lifetime = at === undefined ? 0 : at - date;\n } else if (HEURISTIC_STATUSES.has(status)) {\n const modified = parseHttpDate(lastModified ?? undefined);\n if (modified !== undefined && modified < date)\n lifetime = Math.min(capMs, (date - modified) / 10);\n else if (PERMANENT_REDIRECTS.has(status)) lifetime = capMs;\n }\n if (lifetime === undefined) return null;\n return isoOf(fetchedAt + Math.max(0, Math.min(lifetime, TEN_YEARS_MS) - age)) ?? null;\n};\n\n/** This function gives the request headers that a response varies on, with their values. A `Vary: *` gives `*`. */\nconst varyKeyOf = (\n vary: string | undefined,\n sent: Readonly<Record<string, string>>,\n): string | null => {\n if (vary === undefined) return null;\n const names = vary\n .split(\",\")\n .map((name) => name.trim().toLowerCase())\n .filter((name) => name.length > 0 && !CONSTANT_HEADERS.has(name));\n if (names.includes(\"*\")) return \"*\";\n if (names.length === 0) return null;\n return [...new Set(names)]\n .sort()\n .map((name) => `${name}=${sent[name] ?? \"\"}`)\n .join(\"\\n\");\n};\n\nconst varyMatches = (\n stored: string | null | undefined,\n sent: Readonly<Record<string, string>>,\n): boolean => {\n if (stored === undefined || stored === null) return true;\n if (stored === \"*\") return false;\n return stored.split(\"\\n\").every((line) => {\n const equals = line.indexOf(\"=\");\n return (sent[line.slice(0, equals)] ?? \"\") === line.slice(equals + 1);\n });\n};\n\nconst withoutCredentials = (\n headers: Readonly<Record<string, string>> | undefined,\n): Readonly<Record<string, string>> | undefined =>\n headers === undefined\n ? undefined\n : Object.fromEntries(\n Object.entries(headers).filter(([name]) => !CREDENTIAL_HEADERS.has(name.toLowerCase())),\n );\n\nconst invalidNumber = (\n name: string,\n value: number | \"auto\" | undefined,\n auto: boolean,\n): string | undefined => {\n if (value === undefined || (auto && value === \"auto\")) return undefined;\n return typeof value === \"number\" && value >= 0\n ? undefined\n : `invalid option: ${name} is ${String(value)}`;\n};\n\nconst closedResult = (url: string): Fetched => ({\n url,\n status: 0,\n outcome: \"skipped\",\n fromCache: false,\n capture: undefined,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note: \"crawler closed\",\n});\n\n/** One step of a fetch: the answer for one URL, and the next URL when the answer is a redirect to follow. */\ninterface Step {\n readonly fetched: Fetched;\n /** This URL is the target of the redirect, with its secrets. It stays in memory. */\n readonly next?: string | undefined;\n readonly retryAfterMs?: number | undefined;\n}\n\n/** A robots verdict is a parsed robots.txt and the capture that holds it. */\ninterface RobotsVerdict {\n readonly robots: Robots;\n readonly capture: string | undefined;\n /** When the crawler could not read robots.txt and used a stored copy, this text gives the reason. */\n readonly stale?: string | undefined;\n}\n\n/** The scope and the labels of the requests that wait for one fetch of a robots.txt. The record of the fetch reads them when it is written. */\ninterface Waiting {\n scope: string | undefined;\n labels: Record<string, string> | undefined;\n}\n\ninterface RobotsEntry {\n readonly at: number;\n readonly waiting: Waiting;\n ttlMs: number;\n verdict: Promise<RobotsVerdict>;\n value: RobotsVerdict | undefined;\n /** This value counts the unreachable verdicts in sequence, for the backoff. */\n misses: number;\n /** This verdict is the last good robots.txt, with its time. */\n good: { readonly verdict: RobotsVerdict; readonly at: number } | undefined;\n}\n\ninterface Settle {\n readonly outcome: FetchOutcome;\n readonly status: number;\n readonly retryAfterMs: number | undefined;\n /** The pace of the host: the kind of the request, its duration, the crawl delay and the documented interval. */\n readonly rest: RestInput;\n /** A 401 or a 403 on robots.txt means \"no robots.txt\" (RFC 9309), so it does not stop the host. */\n readonly robotsFile: boolean;\n /** This result replaces the result that the outcome gives. */\n readonly result?: HostResult | undefined;\n}\n\ntype RenderDone =\n | {\n readonly kind: \"rendered\";\n readonly rendered: Rendered;\n readonly classified: Classified;\n readonly elapsedMs: number;\n }\n | { readonly kind: \"failed\"; readonly note: string; readonly elapsedMs: number }\n | {\n readonly kind: \"refused\";\n readonly reason: RefusedReason;\n readonly note: string;\n /** This URL is the request that the renderer refused. */\n readonly url?: string | undefined;\n readonly elapsedMs: number;\n }\n | { readonly kind: \"cancelled\"; readonly elapsedMs: number }\n | { readonly kind: \"closed\" };\n\n/** These are the parts of a request that the admission reads. */\ninterface AdmitSpec {\n readonly transport: \"http\" | \"browser\";\n readonly url: string;\n readonly mode: \"api\" | \"page\";\n readonly base: Record<string, unknown>;\n readonly secretParams: readonly string[];\n readonly invalid: string | undefined;\n readonly onBehalfOf: readonly string[] | undefined;\n readonly scope: string | undefined;\n readonly labels: Readonly<Record<string, string>> | undefined;\n /** This function answers from the cache, when the stored copy is young enough. */\n readonly cached: (url: string) => Promise<Step | undefined>;\n}\n\n/** A request that the admission let through: the facts that the transport needs. */\ninterface Admitted {\n readonly target: URL;\n readonly url: string;\n readonly host: string;\n readonly policy: PolitenessPolicy;\n readonly tokens: readonly string[];\n readonly verdict: RobotsVerdict | undefined;\n readonly check: Record<string, unknown> | undefined;\n readonly robotsLines: readonly string[];\n readonly crawlDelaySeconds: number;\n readonly skip: (note: string, extra?: Record<string, unknown>) => Promise<Fetched>;\n}\n\nconst robotsCheck = (\n verdict: RobotsVerdict,\n tokens: readonly string[],\n decision: RobotsDecision,\n): Record<string, unknown> => ({\n capture: verdict.capture ?? null,\n tokens,\n rule:\n decision.rule === undefined\n ? null\n : {\n line: decision.rule.line ?? null,\n pattern: decision.rule.pattern,\n allow: decision.rule.allow,\n },\n group_line: decision.groups?.[0]?.line ?? null,\n ...(verdict.robots.unreachable !== undefined ? { unreachable: verdict.robots.unreachable } : {}),\n ...(verdict.stale !== undefined ? { stale: verdict.stale } : {}),\n});\n\nconst raceAbort = <T>(work: Promise<T>, signal: AbortSignal): Promise<T> => {\n if (signal.aborted) {\n work.catch(() => undefined);\n return Promise.reject(signal.reason);\n }\n return new Promise<T>((resolve, reject) => {\n const cancel = (): void => reject(signal.reason);\n signal.addEventListener(\"abort\", cancel, { once: true });\n work.then(\n (value) => {\n signal.removeEventListener(\"abort\", cancel);\n resolve(value);\n },\n (error: unknown) => {\n signal.removeEventListener(\"abort\", cancel);\n reject(error);\n },\n );\n });\n};\n\n/** The result of reading robots.txt: the rules, the capture that holds them, and the wait that the server asked for. */\nexport interface RobotsRead {\n readonly robots: Robots;\n readonly capture: string | undefined;\n readonly retryAfterMs: number | undefined;\n}\n\n/** @beta */\nexport class Crawler {\n private readonly clock: Clock;\n private readonly random: () => number;\n private readonly scheduler: HostScheduler;\n private readonly dispatcher: Dispatcher;\n private readonly ownsDispatcher: boolean;\n private readonly egress: EgressOptions;\n private readonly robots = new Lru<string, RobotsEntry>(4096);\n /** The fields of a cache entry that the index does not keep, and the raw target of a redirect. */\n private readonly meta = new Lru<\n string,\n { readonly entry: CacheEntry; readonly location?: string | undefined }\n >(10_000);\n private readonly inFlight = new Map<string, Promise<Step>>();\n private readonly running = new Set<Promise<unknown>>();\n private readonly closing = new AbortController();\n private closed = false;\n private closingDone: Promise<void> | undefined;\n /** This note tells why the renderer has no browser. After the first page that finds it, the crawler renders nothing. */\n private renderOff: string | undefined;\n /** The renders that run at one time (`maxRenders`). */\n private readonly renders: Slots;\n /** These tokens apply because the `User-Agent` links the bot page (`obeyDefaultToken`). */\n private readonly pageTokens: readonly string[];\n /** The deadlines of the requests that have one, by the signal that the scheduler gets for each of them. */\n private readonly deadlines = new WeakMap<AbortSignal, number>();\n // ── signed requests (Web Bot Auth) ──\n /** This signer signs each request. It holds the key, and the crawler writes only its key id and its agent. */\n private readonly signer: RequestSigner | undefined;\n // ── end of signed requests ──\n\n constructor(private readonly options: CrawlerOptions) {\n // ── signed requests: a bad key throws here, as each bad option does ──\n this.signer = signerOf(options.signer);\n if (!isProductToken(options.token)) {\n throw new Error(\n `the token \"${options.token}\" is not a product token: use only letters, \"_\" and \"-\" (RFC 9309)`,\n );\n }\n for (const [name, value] of [\n [\"userAgent\", options.userAgent],\n [\"from\", options.from],\n ] as const) {\n if (value !== undefined && !HEADER_TEXT.test(value)) {\n throw new Error(\n `${name} \"${value}\" has a character that an HTTP header cannot hold: use printable ASCII`,\n );\n }\n }\n checkPolicy(options.politeness);\n this.clock = options.clock ?? realClock;\n this.random = options.random ?? Math.random;\n this.egress = options.egress ?? {};\n // the scheduler waits on this clock: a wait that ends at the deadline of its request or later\n // does not start, so a request with a deadline does not wait for nothing\n const clock = this.clock;\n const deadlines = this.deadlines;\n this.scheduler = new HostScheduler(\n {\n now: () => clock.now(),\n sleep: (ms, signal) => {\n const deadline = signal === undefined ? undefined : deadlines.get(signal);\n const readyAt = clock.now() + ms;\n return deadline !== undefined && readyAt >= deadline\n ? Promise.reject(new DeadlineReached(readyAt))\n : clock.sleep(ms, signal);\n },\n },\n options.maxActive ?? 16,\n { store: options.hostState },\n );\n // the bot page tells site owners to write rules for PoliteCrawler, so an application that links it obeys them\n this.pageTokens =\n options.obeyDefaultToken !== false &&\n options.token.toLowerCase() !== DEFAULT_TOKEN.toLowerCase() &&\n options.userAgent.includes(BOT_PAGE)\n ? [DEFAULT_TOKEN]\n : [];\n const maxRenders = options.maxRenders ?? 2;\n if (!Number.isInteger(maxRenders) || maxRenders < 1) {\n throw new RangeError(`maxRenders must be an integer of 1 or more, not ${String(maxRenders)}`);\n }\n this.renders = new Slots(maxRenders);\n this.ownsDispatcher = options.dispatcher === undefined;\n this.dispatcher =\n options.dispatcher ??\n new Agent({\n // past the per-host delay, or every polite request reconnects\n keepAliveTimeout: 60_000,\n keepAliveMaxTimeout: 120_000,\n allowH2: true,\n // the egress policy checks each address after DNS, at the time of the connection\n connect: { timeout: 10_000, lookup: egressLookup(this.egress) },\n headersTimeout: 30_000,\n bodyTimeout: 60_000,\n });\n }\n\n private iso(): string {\n return new Date(this.clock.now()).toISOString();\n }\n\n private policyOf(url: URL): PolitenessPolicy {\n return policyFor(this.options.politeness, url.hostname);\n }\n\n /**\n * This method gives the signal of a request for the scheduler. A request with a deadline gets a\n * signal of its own, and the clock of the scheduler reads the deadline of that signal.\n */\n private paced(signal: AbortSignal, deadline: number | undefined): AbortSignal {\n if (deadline === undefined) return signal;\n const own = AbortSignal.any([signal]);\n this.deadlines.set(own, deadline);\n return own;\n }\n\n /** This method throws `DeadlineReached` when the deadline of a request came before its slot. */\n private beforeDeadline(deadline: number | undefined): void {\n const now = this.clock.now();\n if (deadline !== undefined && now >= deadline) throw new DeadlineReached(now);\n }\n\n private emit(event: CrawlEvent): void {\n try {\n this.options.onEvent?.(event);\n } catch {\n // a listener that fails does not change the crawl\n }\n }\n\n private track<T>(work: Promise<T>): Promise<T> {\n this.running.add(work);\n void work.then(\n () => this.running.delete(work),\n () => this.running.delete(work),\n );\n return work;\n }\n\n /** This method gives the validator memo of a host. The memo is a hint, so an error of its store gives no memo. */\n private memoOf(host: string): ValidatorMemo | undefined {\n try {\n return this.scheduler.store.validators?.(host);\n } catch {\n return undefined;\n }\n }\n\n private stopHost(host: string, until: number, reason: string): void {\n this.scheduler.store.stop(host, until, reason);\n this.emit({ type: \"stop\", host, until, reason });\n }\n\n private async record(\n kind: \"capture\" | \"revisit\" | \"skipped\",\n payload: Record<string, unknown>,\n url: string,\n labels: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<string | undefined> {\n // a closed crawler writes nothing, so its log stays closed\n if (this.closed) return undefined;\n const at = this.iso();\n const scope = labels.scope ?? this.options.scope;\n const record = makeRecord({\n source: \"fetch\",\n kind,\n payload: { schema: CAPTURE_SCHEMA, ...payload, fetched_at: at },\n received_at: at,\n occurred_at: at,\n external_id: url,\n ...(scope !== undefined ? { scope } : {}),\n ...(labels.labels !== undefined ? { labels: labels.labels } : {}),\n });\n await this.options.log.append(record);\n return record.id;\n }\n\n private async skip(\n base: Record<string, unknown>,\n url: string,\n note: string,\n extra: Record<string, unknown> = {},\n labels: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<Fetched> {\n this.emit({ type: \"skip\", url, note });\n return {\n url,\n status: 0,\n outcome: \"skipped\",\n fromCache: false,\n capture: await this.record(\"skipped\", { ...base, url, note, ...extra }, url, labels),\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note,\n };\n }\n\n /** This method reads the cache entry of a URL, with the fields that the crawler keeps in memory. */\n private lookup(url: string, variant: Variant = \"plain\"): CacheEntry | undefined {\n const key = cacheKeyOf(url);\n const row = this.options.index.latest(key, variant);\n if (row === undefined) return undefined;\n const meta = this.meta.get(`${variant} ${key}`);\n if (meta === undefined || meta.entry.capture !== row.capture) return row;\n const kept = Object.fromEntries(Object.entries(row).filter(([, value]) => value !== undefined));\n return { ...meta.entry, ...kept } as CacheEntry;\n }\n\n private store(entry: CacheEntry, location?: string): void {\n const variant = entry.variant ?? \"plain\";\n const keyed = { ...entry, url: cacheKeyOf(entry.url), variant };\n this.options.index.setLatest(keyed);\n this.meta.set(`${variant} ${keyed.url}`, {\n entry: keyed,\n ...(location !== undefined ? { location } : {}),\n });\n }\n\n /** This method gives the target of a stored redirect, when the crawler can follow it. */\n private followable(entry: CacheEntry, secretParams: readonly string[]): string | undefined {\n const meta = this.meta.peek(`${entry.variant ?? \"plain\"} ${cacheKeyOf(entry.url)}`);\n if (meta?.location !== undefined && meta.entry.capture === entry.capture) return meta.location;\n if (entry.location === undefined || entry.location === null) return undefined;\n try {\n const target = new URL(entry.location);\n // a stored target with secrets holds REDACTED in their place, and is not the real target\n return holdsSecrets(target, secretParams) ? undefined : target.href;\n } catch {\n return undefined;\n }\n }\n\n /** This method answers from the cache, when the stored entry is young or fresh enough. */\n private async fromCache(\n url: string,\n variant: Variant,\n maxAge: number | \"auto\" | undefined,\n sent: Readonly<Record<string, string>>,\n secretParams: readonly string[],\n ): Promise<Step | undefined> {\n if (maxAge === undefined) return undefined;\n const entry = this.lookup(url, variant);\n if (entry === undefined || !varyMatches(entry.vary_key, sent)) return undefined;\n const now = this.clock.now();\n const fetchedAt = Date.parse(entry.fetched_at);\n if (!Number.isFinite(fetchedAt)) return undefined;\n const age = now - fetchedAt;\n const until =\n entry.fresh_until === undefined || entry.fresh_until === null\n ? Number.NaN\n : Date.parse(entry.fresh_until);\n const fresh = Number.isFinite(until) && until > now;\n const redirect = entry.status >= 300 && entry.status < 400;\n // an explicit age wins; `auto` uses the freshness of the server; a temporary redirect needs both\n const young =\n maxAge === \"auto\"\n ? fresh\n : redirect && !PERMANENT_REDIRECTS.has(entry.status)\n ? fresh && age < maxAge\n : age < maxAge;\n if (!young) return undefined;\n if (redirect) {\n const next = this.followable(entry, secretParams);\n if (next === undefined) return undefined;\n return {\n fetched: {\n url,\n status: entry.status,\n outcome: \"redirect\",\n fromCache: true,\n capture: entry.capture,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note: `location ${entry.location ?? redact(new URL(next), secretParams)}`,\n fetchedAt: entry.fetched_at,\n },\n next,\n };\n }\n if (entry.blob === null || entry.status < 200 || entry.status >= 300) return undefined;\n const body = await this.options.blobs.get(entry.blob);\n if (body === undefined) return undefined;\n return {\n fetched: {\n url,\n status: entry.status,\n outcome: \"ok\",\n fromCache: true,\n capture: entry.capture,\n blob: entry.blob,\n body,\n contentType: entry.content_type ?? undefined,\n signals: { ai_input: entry.ai_input ?? \"unset\", raw: [] },\n note: entry.truncated === true ? \"a truncated copy\" : undefined,\n ...(entry.truncated === true ? { truncated: true } : {}),\n fetchedAt: entry.fetched_at,\n },\n };\n }\n\n // ── signed requests (Web Bot Auth) ──\n /**\n * This method gives the signature headers of one request, and the facts of the capture: the key id\n * and the agent. With no signer, it gives nothing. A signer that throws gives the note of a skip,\n * because the crawler sends no unsigned request when it has a signer.\n */\n private signFor(target: URL):\n | {\n readonly headers: Readonly<Record<string, string>>;\n readonly facts: Record<string, unknown>;\n }\n | { readonly error: string }\n | undefined {\n if (this.signer === undefined) return undefined;\n let given: Readonly<Record<string, string>>;\n try {\n given = this.signer.sign(target, this.clock.now());\n } catch (error) {\n return { error: `the signer did not sign the request: ${messageOf(error)}` };\n }\n // a signer adds headers: it cannot replace the User-Agent, From or another header of the crawler\n const headers: Record<string, string> = {};\n for (const [name, value] of Object.entries(given)) {\n const lower = name.toLowerCase();\n if (RESERVED_HEADERS.has(lower)) continue;\n if (typeof value !== \"string\" || !HEADER_TEXT.test(value))\n return { error: `the signer gave a ${lower} header that HTTP cannot send` };\n headers[lower] = value;\n }\n return {\n headers,\n facts: { signed: { key_id: this.signer.keyId, agent: this.signer.signatureAgent } },\n };\n }\n // ── end of signed requests ──\n\n private requestHeaders(request_: FetchRequest): Record<string, string> {\n const headers: Record<string, string> = {};\n // the caller's headers go first, so the crawler's own headers replace them\n for (const [name, value] of Object.entries(request_.headers ?? {})) {\n const lower = name.toLowerCase();\n if (RESERVED_HEADERS.has(lower)) continue;\n // with a signer, the signature headers are the crawler's own\n if (this.signer !== undefined && SIGNATURE_HEADERS.has(lower)) continue;\n headers[lower] = value;\n }\n headers[\"accept\"] = request_.accept ?? headers[\"accept\"] ?? \"*/*\";\n headers[\"user-agent\"] = this.options.userAgent;\n headers[\"accept-encoding\"] = \"gzip, deflate, br\";\n if (this.options.from !== undefined) headers[\"from\"] = this.options.from;\n return headers;\n }\n\n /**\n * This method records the result of a request for the host, and it gives the rest of the host.\n * The rest comes from `restAfter`. A 429 or 503 waits, an error counts toward a stop, and a\n * refusal stops the host for a time.\n */\n private settle(host: string, policy: PolitenessPolicy, step: Settle): number {\n const store = this.scheduler.store;\n const now = this.clock.now();\n const rest = restAfter(policy, step.rest);\n const delayMs = rest.ms;\n const waited = (ms: number, reason: string): number => {\n if (ms > delayMs) this.emit({ type: \"wait\", host, ms, reason });\n return ms;\n };\n if (step.result !== undefined) {\n store.record(host, step.result);\n return delayMs;\n }\n const outcome = step.outcome;\n if (outcome === \"rate-limited\" || outcome === \"unavailable\") {\n const { waits } = store.record(host, \"wait\");\n const asked = step.retryAfterMs;\n if (asked !== undefined && asked > policy.maxRetryAfterMs) {\n // a wait longer than the budget stops the host, so no request sleeps for days\n this.stopHost(\n host,\n now + Math.min(asked, MAX_STOP_MS),\n `asked to wait ${String(Math.round(asked / 1000))} s`,\n );\n return delayMs;\n }\n return waited(\n Math.max(\n delayMs,\n asked ?? equalJitterMs(waits, policy.backoffBaseMs, policy.backoffCapMs, this.random),\n ),\n asked === undefined ? \"backoff\" : \"retry-after\",\n );\n }\n if (outcome === \"error\") {\n if (isClientError(step.status)) {\n store.record(host, \"neutral\");\n return delayMs;\n }\n const { failures } = store.record(host, \"failure\");\n if (failures >= policy.maxFailures)\n this.stopHost(host, now + policy.failureStopMs, `${String(failures)} failures in a row`);\n return waited(\n Math.max(\n delayMs,\n equalJitterMs(failures, policy.backoffBaseMs, policy.backoffCapMs, this.random),\n ),\n \"backoff\",\n );\n }\n if (outcome === \"challenge\") this.stopHost(host, now + policy.challengeStopMs, outcome);\n else if (outcome === \"payment-required\")\n this.stopHost(host, now + policy.paymentStopMs, outcome);\n else if (outcome === \"blocked\" && !step.robotsFile)\n this.stopHost(host, now + policy.blockedStopMs, outcome);\n store.record(host, \"success\");\n return delayMs;\n }\n\n /**\n * This method stops the host of a URL as a challenge does: for the `challengeStopMs` of the policy.\n * The runtime uses it when a page that answered 200 is a bot check (`meta.blocked`).\n */\n reportBlocked(url: string, what: string): void {\n let target: URL;\n try {\n target = new URL(url);\n } catch {\n return;\n }\n const policy = this.policyOf(target);\n this.stopHost(\n hostKey(target),\n this.clock.now() + policy.challengeStopMs,\n `challenge (${what})`,\n );\n }\n\n /**\n * This method applies the rules that a fetch and a render share. The sequence is the URL, the\n * options, the egress policy, the product tokens and the rules for an agent. Then come the\n * cache, the stops of the host and robots.txt. It gives the answer when a rule decides, or the\n * facts that the transport needs.\n */\n private async admit(\n spec: AdmitSpec,\n extra?: (\n admitted: Omit<Admitted, \"verdict\" | \"check\" | \"robotsLines\" | \"crawlDelaySeconds\">,\n ) => Promise<Fetched | undefined>,\n ): Promise<{ readonly answer: Step } | { readonly admitted: Admitted }> {\n const marks = { scope: spec.scope, labels: spec.labels };\n let target: URL;\n try {\n target = new URL(spec.url);\n } catch {\n return {\n answer: { fetched: await this.skip(spec.base, spec.url, \"not a valid URL\", {}, marks) },\n };\n }\n if (!WEB_PROTOCOLS.has(target.protocol)) {\n return {\n answer: {\n fetched: await this.skip(\n spec.base,\n redact(target, spec.secretParams),\n \"not an http(s) URL\",\n {},\n marks,\n ),\n },\n };\n }\n // a fragment never goes to the server, so it is not part of the cache key\n target.hash = \"\";\n const url = redact(target, spec.secretParams);\n const skip = (note: string, extra_?: Record<string, unknown>): Promise<Fetched> =>\n this.skip(spec.base, url, note, extra_, marks);\n const answer = async (note: string, extra_?: Record<string, unknown>) => ({\n answer: { fetched: await skip(note, extra_) },\n });\n if (spec.invalid !== undefined) return answer(spec.invalid);\n const refused = egressRefusal(this.egress, target.hostname);\n if (refused !== undefined) return answer(refused);\n const tokens = [this.options.token, ...(spec.onBehalfOf ?? []), ...this.pageTokens];\n const badToken = tokens.find((token) => !isProductToken(token));\n if (spec.mode === \"page\" && badToken !== undefined)\n return answer(`not a valid product token: \"${badToken}\"`);\n const host = hostKey(target);\n const policy = this.policyOf(target);\n const path = `${target.pathname}${target.search}`;\n\n // the rules for an agent apply before a stored copy answers it\n let verdict: RobotsVerdict | undefined;\n if (spec.mode === \"page\") {\n verdict =\n tokens.length > 1 ? await this.robotsFor(target, policy, spec) : this.knownRobots(target);\n if (verdict !== undefined) {\n const decision = decide(verdict.robots, tokens, path);\n if (!decision.allowed) {\n return answer(robotsRefusal(verdict.robots, decision.by ?? this.options.token), {\n robots_check: robotsCheck(verdict, tokens, decision),\n });\n }\n }\n }\n const cached = await spec.cached(url);\n if (cached !== undefined) return { answer: cached };\n\n const stop = this.scheduler.stopOf(host);\n if (stop !== undefined) return answer(stopNote(stop.reason, stop.until));\n const early = await extra?.({ target, url, host, policy, tokens, skip });\n if (early !== undefined) return { answer: { fetched: early } };\n\n let robotsLines: readonly string[] = [];\n let crawlDelaySeconds = 0;\n let check: Record<string, unknown> | undefined;\n if (spec.mode === \"page\") {\n verdict ??= await this.robotsFor(target, policy, spec);\n const decision = decide(verdict.robots, tokens, path);\n check = robotsCheck(verdict, tokens, decision);\n if (!decision.allowed) {\n return answer(robotsRefusal(verdict.robots, decision.by ?? this.options.token), {\n robots_check: check,\n });\n }\n robotsLines = signalsFor(verdict.robots, tokens, path);\n crawlDelaySeconds = decision.crawlDelaySeconds ?? 0;\n }\n if (this.closed) return { answer: { fetched: closedResult(url) } };\n return {\n admitted: {\n target,\n url,\n host,\n policy,\n tokens,\n verdict,\n check,\n robotsLines,\n crawlDelaySeconds,\n skip,\n },\n };\n }\n\n /** This method fetches a URL. It follows the redirects up to the maximum, and each step is a separate decision. */\n async fetch(request_: FetchRequest): Promise<Fetched> {\n if (this.closed) return closedResult(request_.url);\n return this.track(this.follow(request_));\n }\n\n private async follow(request_: FetchRequest): Promise<Fetched> {\n const signal =\n request_.signal === undefined\n ? this.closing.signal\n : AbortSignal.any([request_.signal, this.closing.signal]);\n let url = request_.url;\n let headers = request_.headers;\n let origin: string | undefined;\n let maxRedirects = DEFAULT_POLICY.maxRedirects;\n try {\n const first = new URL(url);\n origin = first.origin;\n maxRedirects = this.policyOf(first).maxRedirects;\n } catch {\n // fetchOnce records the URL that does not parse\n }\n try {\n for (let hop = 0; ; hop += 1) {\n const step = await this.hop({ ...request_, url, headers }, signal);\n if (step.next === undefined) return step.fetched;\n if (hop >= maxRedirects) {\n return {\n ...step.fetched,\n outcome: \"error\",\n note: `more than ${String(maxRedirects)} redirects`,\n };\n }\n const next = new URL(step.next);\n // a credential for one origin does not go to a different origin\n if (next.origin !== origin) headers = withoutCredentials(headers);\n origin = next.origin;\n url = step.next;\n }\n } catch (error) {\n if (this.closed) return closedResult(request_.url);\n return {\n ...closedResult(request_.url),\n outcome: \"error\",\n note: `crawler error: ${messageOf(error)}`,\n };\n }\n }\n\n /** This method joins a request to an equal request in progress, so one URL gets one request at a time. */\n private hop(request_: FetchRequest, signal: AbortSignal): Promise<Step> {\n // a request with a signal of its own runs alone, so its cancellation stops only that request\n if (request_.signal !== undefined) return this.fetchOnce(request_, signal);\n const key = JSON.stringify([\n cacheKeyOf(request_.url.split(\"#\")[0] ?? request_.url),\n request_.mode,\n request_.recipe,\n request_.accept ?? null,\n request_.headers ?? null,\n request_.secretParams ?? null,\n request_.minIntervalMs ?? null,\n request_.maxAgeMs ?? null,\n request_.maxBytes ?? null,\n request_.onBehalfOf ?? null,\n request_.job ?? null,\n request_.scope ?? null,\n request_.labels ?? null,\n request_.deadline ?? null,\n // ── archived fallback ──\n request_.range ?? null,\n request_.signalLines ?? null,\n request_.originalHeaderPrefix ?? null,\n ]);\n const running = this.inFlight.get(key);\n if (running !== undefined) return running;\n const work = this.fetchOnce(request_, signal).finally(() => this.inFlight.delete(key));\n this.inFlight.set(key, work);\n return work;\n }\n\n private knownRobots(target: URL): RobotsVerdict | undefined {\n const entry = this.robots.peek(originKey(target));\n return entry !== undefined && this.clock.now() - entry.at < entry.ttlMs\n ? entry.value\n : undefined;\n }\n\n /**\n * This method gives the robots.txt of the origin of a URL. A good copy stays for the TTL of the\n * policy. An unreachable verdict stays for some minutes, with a backoff and the `Retry-After`\n * of the server. During that time the last good copy applies, when there is one (RFC 9309 §2.4).\n */\n private robotsFor(\n target: URL,\n policy: PolitenessPolicy,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<RobotsVerdict> {\n const key = originKey(target);\n const now = this.clock.now();\n const cached = this.robots.get(key);\n if (cached !== undefined && now - cached.at < cached.ttlMs) {\n // a request that joins a fetch in progress adds its scope and its labels to the record\n if (cached.value === undefined) this.join(cached.waiting, marks);\n return cached.verdict;\n }\n const waiting: Waiting = { scope: undefined, labels: undefined };\n this.join(waiting, marks);\n const entry: RobotsEntry = {\n at: now,\n waiting,\n ttlMs: policy.robotsTtlMs,\n verdict: Promise.resolve({ robots: unreachableRobots(\"not read yet\"), capture: undefined }),\n value: undefined,\n misses: cached?.misses ?? 0,\n good: cached?.good,\n };\n entry.verdict = this.fetchRobots(target, policy, waiting)\n .catch((error: unknown) => ({\n robots: unreachableRobots(`robots.txt failed: ${messageOf(error)}`),\n capture: undefined,\n retryAfterMs: undefined,\n }))\n .then(async (read): Promise<RobotsVerdict> => {\n let verdict: RobotsVerdict = { robots: read.robots, capture: read.capture };\n if (read.robots.unreachable === undefined) {\n entry.misses = 0;\n entry.good = { verdict, at: this.clock.now() };\n } else {\n entry.misses += 1;\n const backoff = equalJitterMs(\n entry.misses,\n policy.robotsRetryMs,\n policy.robotsRetryCapMs,\n this.random,\n );\n entry.ttlMs = Math.min(\n policy.robotsRetryCapMs,\n Math.max(read.retryAfterMs ?? 0, backoff),\n );\n const good =\n entry.good !== undefined && this.clock.now() - entry.good.at <= policy.robotsStaleMs\n ? entry.good.verdict\n : await this.storedRobots(target, policy);\n if (good !== undefined) verdict = { ...good, stale: read.robots.unreachable };\n }\n entry.value = verdict;\n return verdict;\n });\n this.robots.set(key, entry);\n return entry.verdict;\n }\n\n /**\n * This method gives the robots.txt rules that apply to a URL, as the crawler applies them. They\n * come from its memory, or from a fetch of `/robots.txt` with its redirects. A robots.txt that\n * the crawler cannot read gives rules that permit nothing, and `unreachable` gives the reason.\n */\n async readRobots(\n url: string,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<{\n readonly robots: Robots;\n readonly capture: string | undefined;\n readonly stale?: string | undefined;\n }> {\n const target = new URL(url);\n if (!WEB_PROTOCOLS.has(target.protocol))\n return { robots: unreachableRobots(\"not an http(s) URL\"), capture: undefined };\n return this.robotsFor(target, this.policyOf(target), marks);\n }\n\n /**\n * This method tells if robots.txt lets the crawler fetch a URL as a page. The rules of the token\n * of the crawler and of the agents of `onBehalfOf` apply, as for a fetch. It reads robots.txt as\n * a fetch does, so a robots.txt that the crawler cannot read permits nothing. When the signal\n * aborts, the promise rejects, and a fetch of robots.txt in progress continues.\n */\n async robotsAllow(\n url: string,\n options: {\n readonly onBehalfOf?: readonly string[] | undefined;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n readonly signal?: AbortSignal | undefined;\n } = {},\n ): Promise<{\n readonly allowed: boolean;\n readonly note: string | undefined;\n /** These are the lines of usage signals (`Content-Signal`) of robots.txt that apply to the URL, when there are any. The archived fallback applies them to a copy. */\n readonly signals?: readonly string[] | undefined;\n }> {\n let target: URL;\n try {\n target = new URL(url);\n } catch {\n return { allowed: false, note: \"not a valid URL\" };\n }\n if (!WEB_PROTOCOLS.has(target.protocol)) return { allowed: false, note: \"not an http(s) URL\" };\n if (this.closed) return { allowed: false, note: \"crawler closed\" };\n const tokens = [this.options.token, ...(options.onBehalfOf ?? []), ...this.pageTokens];\n const badToken = tokens.find((token) => !isProductToken(token));\n if (badToken !== undefined)\n return { allowed: false, note: `not a valid product token: \"${badToken}\"` };\n const work = this.robotsFor(target, this.policyOf(target), options);\n const verdict = await (options.signal === undefined ? work : raceAbort(work, options.signal));\n const path = `${target.pathname}${target.search}`;\n const decision = decide(verdict.robots, tokens, path);\n // ── archived fallback: the content signals of the site apply to an archived copy ──\n // (only when robots.txt has such lines, so the result of a plain robots.txt stays the same)\n const lines = signalsFor(verdict.robots, tokens, path);\n const signals = lines.length > 0 ? { signals: lines } : {};\n return decision.allowed\n ? { allowed: true, note: undefined, ...signals }\n : {\n allowed: false,\n note: robotsRefusal(verdict.robots, decision.by ?? this.options.token),\n ...signals,\n };\n }\n\n /** This method reads the stored copy of a robots.txt, when it is younger than the limit of the policy. */\n private async storedRobots(\n target: URL,\n policy: PolitenessPolicy,\n ): Promise<RobotsVerdict | undefined> {\n try {\n const row = this.lookup(redact(new URL(\"/robots.txt\", target.origin)));\n if (row === undefined || row.blob === null || row.status < 200 || row.status >= 300)\n return undefined;\n if (this.clock.now() - Date.parse(row.fetched_at) > policy.robotsStaleMs) return undefined;\n const body = await this.options.blobs.get(row.blob);\n if (body === undefined) return undefined;\n const robots = robotsFromResponse(200, row.content_type, body);\n return robots.unreachable === undefined ? { robots, capture: row.capture } : undefined;\n } catch {\n return undefined;\n }\n }\n\n /** This method adds the scope and the labels of one request to the requests that wait for a robots.txt. The most private scope wins. */\n private join(\n waiting: Waiting,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n },\n ): void {\n const scope = marks.scope ?? this.options.scope;\n const rank = this.options.scopeRank;\n if (waiting.scope === undefined) waiting.scope = scope;\n else if (scope !== undefined && rank !== undefined && rank(scope) > rank(waiting.scope))\n waiting.scope = scope;\n if (marks.labels !== undefined) waiting.labels = { ...marks.labels, ...waiting.labels };\n }\n\n private async fetchRobots(\n target: URL,\n policy: PolitenessPolicy,\n waiting: Waiting,\n ): Promise<RobotsRead> {\n let url = new URL(\"/robots.txt\", target.origin);\n const unreachable = (\n reason: string,\n capture: string | undefined,\n retryAfterMs?: number,\n ): RobotsRead => ({\n robots: unreachableRobots(reason),\n capture,\n retryAfterMs,\n });\n for (let hop = 0; hop <= policy.maxRedirects; hop += 1) {\n const step = await this.fetchOnce(\n {\n url: url.href,\n mode: \"api\",\n recipe: \"robots.txt\",\n accept: \"text/plain\",\n maxBytes: ROBOTS_BYTES,\n minIntervalMs: policy.apiIntervalMs,\n // the record reads the requests that wait at the time that it is written, not at the start\n get scope() {\n return waiting.scope;\n },\n get labels() {\n return waiting.labels;\n },\n },\n this.closing.signal,\n );\n const result = step.fetched;\n if (result.outcome === \"redirect\") {\n if (step.next === undefined)\n return unreachable(result.note ?? \"a redirect with no Location header\", result.capture);\n url = new URL(step.next);\n continue;\n }\n if (result.outcome === \"skipped\" || result.status === 0)\n return unreachable(result.note ?? \"no response\", result.capture);\n // a body that the crawler cannot read is not an empty robots.txt: the site stays closed\n if (result.outcome === \"error\" && result.status >= 200 && result.status < 300)\n return unreachable(result.note ?? \"the body could not be read\", result.capture);\n // a body that the server cut can have lost a rule\n if (result.note?.startsWith(\"the compressed body ended early\") === true)\n return unreachable(result.note, result.capture);\n if (result.outcome === \"not-modified\") {\n return result.body === undefined\n ? unreachable(\"status 304, and no stored copy\", result.capture)\n : {\n robots: robotsFromResponse(200, result.contentType ?? null, result.body),\n capture: result.capture,\n retryAfterMs: undefined,\n };\n }\n const robots =\n result.status >= 200 && result.status < 300\n ? robotsFromResponse(\n result.status,\n result.contentType ?? null,\n result.body ?? new Uint8Array(),\n )\n : robotsFromStatus(result.status, undefined);\n return { robots, capture: result.capture, retryAfterMs: step.retryAfterMs };\n }\n return unreachable(`more than ${String(policy.maxRedirects)} redirects`, undefined);\n }\n\n private async fetchOnce(request_: FetchRequest, signal: AbortSignal): Promise<Step> {\n const base = {\n recipe: request_.recipe,\n mode: request_.mode,\n ...(request_.job ? { job: request_.job } : {}),\n };\n const secretParams = request_.secretParams ?? [];\n const sent = this.requestHeaders(request_);\n // read when a record is written: more requests can wait for the fetch of a robots.txt by then\n const marks = {\n get scope() {\n return request_.scope;\n },\n get labels() {\n return request_.labels;\n },\n };\n // ── archived fallback: a range is a part of a resource, never its stored copy ──\n const range = request_.range;\n const badRange =\n range === undefined ||\n (Number.isSafeInteger(range.offset) &&\n range.offset >= 0 &&\n Number.isSafeInteger(range.length) &&\n range.length >= 1)\n ? undefined\n : `invalid option: range is ${JSON.stringify(range)}`;\n // ── end of archived fallback ──\n const admission = await this.admit({\n transport: \"http\",\n url: request_.url,\n mode: request_.mode,\n base,\n secretParams,\n invalid:\n invalidNumber(\"minIntervalMs\", request_.minIntervalMs, false) ??\n invalidNumber(\"maxBytes\", request_.maxBytes, false) ??\n invalidNumber(\"maxAgeMs\", request_.maxAgeMs, true) ??\n badRange,\n onBehalfOf: request_.onBehalfOf,\n scope: request_.scope,\n labels: request_.labels,\n cached: (url) =>\n range !== undefined\n ? Promise.resolve(undefined)\n : this.fromCache(url, \"plain\", request_.maxAgeMs, sent, secretParams),\n });\n if (\"answer\" in admission) return admission.answer;\n const { target, url, host, policy, check, robotsLines, crawlDelaySeconds, skip } =\n admission.admitted;\n const maxBytes = Math.min(request_.maxBytes ?? policy.maxBytes, range?.length ?? Infinity);\n const rest = (elapsedMs: number): RestInput => ({\n kind: request_.mode,\n elapsedMs,\n crawlDelaySeconds,\n minIntervalMs: request_.minIntervalMs,\n });\n const robotsFile = request_.recipe === \"robots.txt\";\n\n try {\n return await this.scheduler.run(\n host,\n async (): Promise<Paced<Step>> => {\n // an equal request that waited in the queue can find a fresh copy now\n const again =\n range === undefined\n ? await this.fromCache(url, \"plain\", request_.maxAgeMs, sent, secretParams)\n : undefined;\n if (again !== undefined) return { value: again, delayMs: 0 };\n // no request starts at or after its deadline\n this.beforeDeadline(request_.deadline);\n // a range sends no validators: a 304 for a part has no stored body\n const latest = range === undefined ? this.lookup(url) : undefined;\n // validators go only with a stored body of the same variant, so a 304 always has a body\n const usable =\n latest !== undefined && latest.truncated !== true && varyMatches(latest.vary_key, sent)\n ? latest\n : undefined;\n let stored: Uint8Array | undefined;\n if (usable?.blob && (usable.etag || usable.last_modified))\n stored = await this.options.blobs.get(usable.blob);\n const headers: Record<string, string> = { ...sent };\n // the memo of the host can leave out an ETag that never matches; robots.txt is often a\n // static file with validators of its own, so it keeps the two and teaches the memo nothing\n const send =\n stored === undefined || usable === undefined\n ? { etag: false, lastModified: false }\n : validatorsToSend(\n robotsFile ? undefined : this.memoOf(host),\n { etag: Boolean(usable.etag), lastModified: Boolean(usable.last_modified) },\n this.clock.now(),\n );\n if (send.etag && usable?.etag) headers[\"if-none-match\"] = usable.etag;\n if (send.lastModified && usable?.last_modified)\n headers[\"if-modified-since\"] = usable.last_modified;\n const validators = [\"if-none-match\", \"if-modified-since\"].filter(\n (name) => headers[name] !== undefined,\n );\n // ── archived fallback: the part of a WARC file ──\n const rangeText =\n range === undefined\n ? undefined\n : `bytes=${String(range.offset)}-${String(range.offset + range.length - 1)}`;\n if (rangeText !== undefined) headers[\"range\"] = rangeText;\n // ── signed requests: the signature is made last, for this URL at this time ──\n const signed = this.signFor(target);\n if (signed !== undefined && \"error\" in signed)\n return { value: { fetched: await skip(signed.error) }, delayMs: 0 };\n if (signed !== undefined) Object.assign(headers, signed.headers);\n const signedFacts = signed?.facts ?? {};\n // ── end of signed requests ──\n const started = this.clock.now();\n const timeout = AbortSignal.timeout(policy.requestTimeoutMs);\n const abort = AbortSignal.any([timeout, signal]);\n const cancelled = (): boolean => request_.signal?.aborted === true;\n const failedNote = (prefix: string, error: unknown): string =>\n timeout.aborted\n ? `${prefix}: timed out after ${String(Math.round(policy.requestTimeoutMs / 1000))} s`\n : `${prefix}: ${messageOf(error)}`;\n\n this.emit({ type: \"request\", url, host, transport: \"http\" });\n let response: Dispatcher.ResponseData;\n try {\n response = await request(target, {\n method: \"GET\",\n headers,\n dispatcher: this.dispatcher,\n signal: abort,\n });\n } catch (error) {\n const egress = egressError(error);\n if (egress !== undefined)\n return { value: { fetched: await skip(egress.message) }, delayMs: 0 };\n if (this.closed) return { value: { fetched: closedResult(url) }, delayMs: 0 };\n const elapsedMs = this.clock.now() - started;\n const note = cancelled() ? \"cancelled by the caller\" : failedNote(\"no response\", error);\n const delayMs = this.settle(host, policy, {\n outcome: \"error\",\n status: 0,\n retryAfterMs: undefined,\n rest: rest(elapsedMs),\n robotsFile,\n ...(cancelled() ? { result: \"neutral\" as const } : {}),\n });\n this.emit({ type: \"response\", url, host, status: 0, outcome: \"error\", elapsedMs });\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n status: 0,\n outcome: \"error\",\n note,\n elapsed_ms: elapsedMs,\n ...(check ? { robots_check: check } : {}),\n ...(rangeText !== undefined ? { range: rangeText } : {}),\n ...signedFacts,\n },\n url,\n marks,\n );\n return {\n value: {\n fetched: {\n url,\n status: 0,\n outcome: \"error\",\n fromCache: false,\n capture,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note,\n fetchedAt: new Date(started).toISOString(),\n ...marks,\n },\n },\n delayMs,\n };\n }\n\n const header = headerOf(\n response.headers as Record<string, string | string[] | undefined>,\n );\n const ttfbMs = this.clock.now() - started;\n const classified = classify(response.statusCode, header, this.clock.now());\n let outcome: Outcome = classified.outcome;\n let note = classified.note;\n let body: Uint8Array | undefined;\n let blob: string | undefined;\n let truncated = false;\n let cut = false;\n let next: string | undefined;\n let result: HostResult | undefined;\n if (outcome === \"ok\") {\n try {\n const read = await readBody(response.body, header(\"content-encoding\"), maxBytes);\n body = read.bytes;\n truncated = read.truncated;\n cut = read.cut;\n if (read.cut)\n note = `the compressed body ended early, so the crawler keeps the ${String(read.bytes.length)} bytes before the end`;\n else if (truncated) note = `truncated at ${String(maxBytes)} bytes`;\n } catch (error) {\n if (this.closed) return { value: { fetched: closedResult(url) }, delayMs: 0 };\n outcome = \"error\";\n if (cancelled()) {\n note = \"cancelled by the caller\";\n result = \"neutral\";\n } else {\n const decode = error instanceof BodyError && error.kind === \"decode\";\n note = failedNote(decode ? \"undecodable body\" : \"body failed\", error);\n }\n }\n if (body !== undefined) {\n try {\n blob = (await this.options.blobs.put(body)).hash;\n } catch (error) {\n // a full disk is not a fault of the host\n outcome = \"error\";\n note = `blob store: ${messageOf(error)}`;\n body = undefined;\n result = \"neutral\";\n }\n }\n } else {\n try {\n await readBody(response.body, undefined, ERROR_BODY_BYTES);\n } catch {\n // the body of an error answer is not kept\n }\n }\n const elapsedMs = this.clock.now() - started;\n if (outcome === \"not-modified\") {\n if (stored !== undefined && usable?.blob) {\n body = stored;\n blob = usable.blob;\n } else note = \"not modified, but the crawler has no stored copy\";\n }\n let location: string | undefined;\n if (outcome === \"redirect\") {\n const raw = header(\"location\");\n if (raw === undefined) note = \"a redirect with no Location header\";\n else {\n try {\n const resolved = new URL(raw, target);\n next = resolved.href;\n location = redact(resolved, secretParams);\n note = `location ${location}`;\n } catch {\n outcome = \"error\";\n note = \"bad redirect target\";\n }\n }\n }\n\n const revisit = outcome === \"not-modified\" && usable !== undefined;\n // ── archived fallback: the signals of the site and of the archived headers apply to a copy ──\n const prefix = request_.originalHeaderPrefix?.toLowerCase();\n const archivedLines =\n prefix === undefined\n ? []\n : SIGNAL_HEADERS.flatMap((name) => {\n const value = header(`${prefix}${name}`);\n return value === undefined ? [] : [`${prefix}${name}: ${value}`];\n });\n let signals = readSignals(\n [...robotsLines, ...(request_.signalLines ?? []), ...archivedLines],\n header,\n pageSignals(body, header(\"content-type\")),\n );\n // ── end of archived fallback ──\n if (revisit) {\n // a 304 updates the stored answer and does not replace it (RFC 9111 §4.3.4)\n const merged: AiInput =\n signals.ai_input === \"no\" || usable.ai_input === \"no\"\n ? \"no\"\n : signals.ai_input !== \"unset\"\n ? signals.ai_input\n : usable.ai_input;\n signals = { ...signals, ai_input: merged };\n }\n const contentType =\n header(\"content-type\") ?? (revisit ? (usable.content_type ?? undefined) : undefined);\n // a web page in place of the data of an API (a login or an error page) is not the stored copy\n const unasked =\n request_.mode === \"api\" &&\n outcome === \"ok\" &&\n unaskedPage(request_.accept, contentType);\n if (unasked) {\n const said = `the server sent ${contentType ?? \"\"}, and the request accepts ${request_.accept ?? \"\"}, so the crawler keeps no stored copy of it`;\n note = note === undefined ? said : `${note}; ${said}`;\n }\n const etag = header(\"etag\") ?? (revisit ? usable.etag : null) ?? null;\n const lastModified =\n header(\"last-modified\") ?? (revisit ? usable.last_modified : null) ?? null;\n const fetchedAt = this.clock.now();\n const freshness = freshUntil(\n header,\n response.statusCode,\n fetchedAt,\n lastModified,\n policy.heuristicFreshnessCapMs,\n );\n const varyKey =\n header(\"vary\") !== undefined\n ? varyKeyOf(header(\"vary\"), sent)\n : revisit\n ? (usable.vary_key ?? null)\n : null;\n const cacheHeaders = Object.fromEntries(\n [\"cache-control\", \"expires\", \"age\", \"date\", \"vary\"].flatMap((name) => {\n const value = header(name);\n return value === undefined ? [] : [[name.replace(\"-\", \"_\"), value]];\n }),\n );\n this.emit({\n type: \"response\",\n url,\n host,\n status: response.statusCode,\n outcome,\n elapsedMs,\n });\n const capture = await this.record(\n revisit ? \"revisit\" : \"capture\",\n {\n ...base,\n url,\n status: response.statusCode,\n outcome,\n elapsed_ms: elapsedMs,\n ttfb_ms: ttfbMs,\n content_type: contentType ?? null,\n bytes: body?.length ?? 0,\n blob: blob ?? null,\n etag,\n last_modified: lastModified,\n ...(validators.length > 0 ? { validators } : {}),\n ...(revisit ? { revisit_of: usable.capture } : {}),\n robots: request_.mode === \"page\" ? \"allowed\" : \"not-checked\",\n ...(check !== undefined ? { robots_check: check } : {}),\n signals,\n ...(truncated ? { truncated: true } : {}),\n ...(cut ? { cut: true } : {}),\n ...(unasked ? { unasked: true } : {}),\n ...cacheHeaders,\n ...(freshness !== null ? { fresh_until: freshness } : {}),\n ...(varyKey !== null ? { vary_key: varyKey } : {}),\n ...(location !== undefined ? { location } : {}),\n ...(note !== undefined ? { note } : {}),\n ...(rangeText !== undefined ? { range: rangeText } : {}),\n ...signedFacts,\n },\n url,\n marks,\n );\n const fetchedIso = new Date(fetchedAt).toISOString();\n // a body that the server cut is not a copy to keep: the next request asks for the whole body\n // (a range is a part of the resource, so it is not the stored copy of the URL either)\n if (\n capture !== undefined &&\n (outcome === \"ok\" || outcome === \"not-modified\") &&\n blob !== undefined &&\n !cut &&\n !unasked &&\n range === undefined\n ) {\n this.store({\n url,\n capture,\n blob,\n // a truncated body keeps no validators, so a later request gets the whole body again\n etag: truncated ? null : etag,\n last_modified: truncated ? null : lastModified,\n content_type: contentType ?? null,\n fetched_at: fetchedIso,\n // an index that a replay built can hold 304 as the status of the stored body\n status: revisit ? (usable.status === 304 ? 200 : usable.status) : response.statusCode,\n ai_input: signals.ai_input,\n truncated: truncated ? true : undefined,\n fresh_until: freshness,\n vary_key: varyKey,\n });\n }\n if (\n capture !== undefined &&\n outcome === \"redirect\" &&\n next !== undefined &&\n (PERMANENT_REDIRECTS.has(response.statusCode) || freshness !== null)\n ) {\n this.store(\n {\n url,\n capture,\n blob: null,\n etag: null,\n last_modified: null,\n content_type: null,\n fetched_at: fetchedIso,\n status: response.statusCode,\n ai_input: \"unset\",\n location: location ?? null,\n fresh_until: freshness,\n vary_key: varyKey,\n },\n next,\n );\n }\n // the memo of the host learns which validator gets a 304 (a 200 with the stored body is a miss)\n const answer: ValidatorAnswer | undefined =\n validators.length === 0 || capture === undefined || robotsFile\n ? undefined\n : outcome === \"not-modified\"\n ? \"not-modified\"\n : outcome === \"ok\" && blob !== undefined && !cut\n ? blob === usable?.blob\n ? \"same\"\n : \"changed\"\n : undefined;\n if (answer !== undefined) {\n try {\n this.scheduler.store.noteValidators?.(\n host,\n {\n etag: headers[\"if-none-match\"] !== undefined,\n lastModified: headers[\"if-modified-since\"] !== undefined,\n answer,\n },\n fetchedAt,\n );\n } catch {\n // the memo is a hint: an error of its store does not fail the fetch\n }\n }\n const delayMs = this.settle(host, policy, {\n outcome,\n status: response.statusCode,\n retryAfterMs: classified.retryAfterMs,\n rest: rest(elapsedMs),\n robotsFile,\n ...(result !== undefined ? { result } : {}),\n });\n return {\n value: {\n fetched: {\n url,\n status: response.statusCode,\n outcome,\n fromCache: false,\n capture,\n blob,\n body,\n contentType,\n signals,\n note,\n ...(truncated ? { truncated: true } : {}),\n fetchedAt: fetchedIso,\n ...marks,\n },\n next,\n retryAfterMs: classified.retryAfterMs,\n },\n delayMs,\n };\n },\n this.paced(signal, request_.deadline),\n );\n } catch (error) {\n if (error instanceof HostStopped)\n return { fetched: await skip(stopNote(error.reason, error.until)) };\n if (error instanceof DeadlineReached)\n return { fetched: await skip(deadlineNote(error, request_.deadline)) };\n if (this.closed) return { fetched: closedResult(url) };\n if (request_.signal?.aborted === true)\n return { fetched: await skip(\"cancelled by the caller\") };\n throw error;\n }\n }\n\n /**\n * This method gives the function that decides each request of a rendered page. It refuses an\n * address that the egress policy refuses, and a URL of the page's host that robots.txt\n * disallows. For a different host, it uses a robots.txt that the crawler holds, when there is one.\n */\n private allowRequestFor(\n page: URL,\n robots: Robots,\n tokens: readonly string[],\n signal: AbortSignal,\n tally: { allowed: number; refused: number; readonly hosts: Set<string> },\n ): (url: string) => Promise<boolean> {\n const pageOrigin = originKey(page);\n const checked = new Map<string, Promise<string | undefined>>();\n const refuse = (): boolean => {\n tally.refused += 1;\n return false;\n };\n return async (raw: string): Promise<boolean> => {\n if (signal.aborted) return false;\n let target: URL;\n try {\n target = new URL(raw);\n } catch {\n return refuse();\n }\n if (\n target.protocol === \"data:\" ||\n target.protocol === \"blob:\" ||\n target.protocol === \"about:\"\n )\n return true;\n const web = WEB_PROTOCOLS.has(target.protocol);\n if (!web && target.protocol !== \"ws:\" && target.protocol !== \"wss:\") return refuse();\n let check = checked.get(target.hostname);\n if (check === undefined) {\n check = checkEgress(this.egress, target);\n checked.set(target.hostname, check);\n }\n if ((await check) !== undefined) return refuse();\n if (web) {\n const rules = originKey(target) === pageOrigin ? robots : this.knownRobots(target)?.robots;\n if (\n rules !== undefined &&\n !decide(rules, tokens, `${target.pathname}${target.search}`).allowed\n )\n return refuse();\n }\n tally.allowed += 1;\n if (tally.hosts.size < 50) tally.hosts.add(hostKey(target));\n return true;\n };\n }\n\n /**\n * This method tells why the browser did not go on to a URL that the page moved to: the egress\n * policy or robots.txt. For a different site, it uses a robots.txt that the crawler holds. It\n * gives undefined for the page itself, or for a URL that it cannot read.\n */\n private async movedRefusal(\n raw: string,\n page: URL,\n robots: Robots,\n tokens: readonly string[],\n ): Promise<{ readonly note: string; readonly url: string } | undefined> {\n let target: URL;\n try {\n target = new URL(raw);\n } catch {\n return undefined;\n }\n target.hash = \"\";\n if (target.href === page.href) return undefined;\n const url = redact(target);\n const moved = `the page moved to ${url}`;\n if (!WEB_PROTOCOLS.has(target.protocol))\n return { note: `${moved}, which is not an http(s) URL`, url };\n const egress = await checkEgress(this.egress, target);\n if (egress !== undefined) return { note: `${moved}: ${egress}`, url };\n const rules = originKey(target) === originKey(page) ? robots : this.knownRobots(target)?.robots;\n if (rules !== undefined) {\n const decision = decide(rules, tokens, `${target.pathname}${target.search}`);\n if (!decision.allowed)\n return {\n note: `${moved}: ${robotsRefusal(rules, decision.by ?? this.options.token)}`,\n url,\n };\n }\n return undefined;\n }\n\n /**\n * This method renders a page in the headless browser and captures what the page shows. The rules\n * of a fetch apply:\n *\n * - the egress policy and robots.txt, for the page, its final URL and each request of the page\n * - one request at a time for each host\n * - the backoff after a 429 or a 503\n * - a capture for each decision\n *\n * A render costs the site more than a fetch, so the host rests longer after one. A render that\n * the browser refuses (`RenderRefused`) does not count against the host.\n */\n async render(request_: RenderRequest): Promise<Fetched> {\n if (this.closed) return closedResult(request_.url);\n return this.track(\n this.renderOnce(request_).catch((error: unknown) =>\n this.closed\n ? closedResult(request_.url)\n : {\n ...closedResult(request_.url),\n outcome: \"error\" as const,\n note: `crawler error: ${messageOf(error)}`,\n },\n ),\n );\n }\n\n private async renderOnce(request_: RenderRequest): Promise<Fetched> {\n const base = {\n recipe: request_.recipe,\n mode: \"page\" as const,\n rendered: true,\n ...(request_.job ? { job: request_.job } : {}),\n };\n const marks = { scope: request_.scope, labels: request_.labels };\n const renderer = this.options.renderer;\n const admission = await this.admit(\n {\n transport: \"browser\",\n url: request_.url,\n mode: \"page\",\n base,\n secretParams: [],\n invalid: invalidNumber(\"maxAgeMs\", request_.maxAgeMs, true),\n onBehalfOf: request_.onBehalfOf,\n scope: request_.scope,\n labels: request_.labels,\n cached: async (url) => {\n const cached = await this.fromCache(url, \"rendered\", request_.maxAgeMs, {}, []);\n if (cached === undefined || cached.fetched.body === undefined) return undefined;\n return { fetched: { ...cached.fetched, url, contentType: RENDERED_TYPE } };\n },\n },\n async ({ target, skip }) => {\n if (renderer === undefined) return skip(\"no browser configured for rendering\");\n if (this.renderOff !== undefined) return skip(this.renderOff);\n // the browser does its own DNS, so the check here resolves the name too\n const resolved = await checkEgress(this.egress, target);\n return resolved === undefined ? undefined : skip(resolved);\n },\n );\n if (\"answer\" in admission) return admission.answer.fetched;\n const { target, url, host, policy, tokens, check, crawlDelaySeconds, skip } =\n admission.admitted;\n const verdict = admission.admitted.verdict!;\n const signal =\n request_.signal === undefined\n ? this.closing.signal\n : AbortSignal.any([request_.signal, this.closing.signal]);\n const tally = { allowed: 0, refused: 0, hosts: new Set<string>() };\n const allowRequest = this.allowRequestFor(target, verdict.robots, tokens, signal, tally);\n // ── signed requests: the browser signs each request of the page, or does not send it ──\n const sign =\n this.signer === undefined\n ? undefined\n : (raw: string): Readonly<Record<string, string>> | undefined => {\n let to: URL;\n try {\n to = new URL(raw);\n } catch {\n return undefined;\n }\n const signed = this.signFor(to);\n return signed === undefined || \"error\" in signed ? undefined : signed.headers;\n };\n const signedFacts =\n this.signer === undefined\n ? {}\n : { signed: { key_id: this.signer.keyId, agent: this.signer.signatureAgent } };\n // ── end of signed requests ──\n const rest = (elapsedMs: number): RestInput => ({\n kind: \"render\",\n elapsedMs,\n crawlDelaySeconds,\n });\n\n // the wait for a slot of the browser comes before the lease of the host, so the host does not wait for it\n let release: () => void;\n try {\n release = await this.renders.acquire(signal);\n } catch {\n if (this.closed) return closedResult(url);\n return skip(\"cancelled by the caller\");\n }\n let done: RenderDone;\n try {\n done = await this.scheduler.run(\n host,\n async (): Promise<Paced<RenderDone>> => {\n this.beforeDeadline(request_.deadline);\n const started = this.clock.now();\n this.emit({ type: \"request\", url, host, transport: \"browser\" });\n try {\n const rendered = await raceAbort(\n renderer!.render(target.href, {\n userAgent: this.options.userAgent,\n timeoutMs: policy.renderTimeoutMs,\n ...(this.options.from !== undefined\n ? { headers: { from: this.options.from } }\n : {}),\n allowRequest,\n maxRedirects: policy.maxRedirects,\n signal,\n ...(sign === undefined ? {} : { sign }),\n }),\n signal,\n );\n const header = (name: string): string | undefined =>\n rendered.headers[name.toLowerCase()];\n const classified: Classified =\n rendered.status === null\n ? { outcome: \"ok\", retryAfterMs: undefined, note: undefined }\n : classify(rendered.status, header, this.clock.now());\n const elapsedMs = this.clock.now() - started;\n const delayMs = this.settle(host, policy, {\n outcome: classified.outcome,\n status: rendered.status ?? 200,\n retryAfterMs: classified.retryAfterMs,\n rest: rest(elapsedMs),\n robotsFile: false,\n });\n return { value: { kind: \"rendered\", rendered, classified, elapsedMs }, delayMs };\n } catch (error) {\n const elapsedMs = this.clock.now() - started;\n if (this.closed) return { value: { kind: \"closed\" }, delayMs: 0 };\n if (request_.signal?.aborted === true) {\n return {\n value: { kind: \"cancelled\", elapsedMs },\n delayMs: restAfter(policy, rest(0)).ms,\n };\n }\n // a renderer of an application that has no browser is no fault of the host either\n const refused =\n error instanceof RenderRefused\n ? error\n : missingBrowser(messageOf(error))\n ? new RenderRefused(\"browser\", `rendering needs a browser: ${messageOf(error)}`)\n : undefined;\n if (refused !== undefined) {\n if (refused.reason === \"browser\") this.renderOff = refused.message;\n // a refused navigation, a file, a scheme or no browser is no fault of the host\n const delayMs = this.settle(host, policy, {\n outcome: \"skipped\",\n status: 0,\n retryAfterMs: undefined,\n rest: rest(0),\n robotsFile: false,\n result: \"neutral\",\n });\n return {\n value: {\n kind: \"refused\",\n reason: refused.reason,\n note: refused.message,\n url: refused.url,\n elapsedMs,\n },\n delayMs,\n };\n }\n const delayMs = this.settle(host, policy, {\n outcome: \"error\",\n status: 0,\n retryAfterMs: undefined,\n rest: rest(elapsedMs),\n robotsFile: false,\n });\n return {\n value: { kind: \"failed\", note: `render failed: ${messageOf(error)}`, elapsedMs },\n delayMs,\n };\n }\n },\n this.paced(signal, request_.deadline),\n );\n } catch (error) {\n if (error instanceof HostStopped) return skip(stopNote(error.reason, error.until));\n if (error instanceof DeadlineReached) return skip(deadlineNote(error, request_.deadline));\n if (this.closed) return closedResult(url);\n if (request_.signal?.aborted === true) return skip(\"cancelled by the caller\");\n throw error;\n } finally {\n release();\n }\n\n if (done.kind === \"closed\" || this.closed) return closedResult(url);\n const subrequests = { allowed: tally.allowed, refused: tally.refused, hosts: [...tally.hosts] };\n if (done.kind === \"refused\") {\n this.emit({\n type: \"response\",\n url,\n host,\n status: 0,\n outcome: done.reason === \"download\" ? \"not-html\" : \"skipped\",\n elapsedMs: done.elapsedMs,\n });\n if (done.reason !== \"download\") {\n const moved =\n done.url === undefined\n ? undefined\n : await this.movedRefusal(done.url, target, verdict.robots, tokens);\n return skip(moved?.note ?? done.note, {\n robots_check: check,\n subrequests,\n ...(moved === undefined ? {} : { final_url: moved.url }),\n });\n }\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n status: 0,\n outcome: \"not-html\",\n note: done.note,\n elapsed_ms: done.elapsedMs,\n robots_check: check,\n subrequests,\n ...signedFacts,\n },\n url,\n marks,\n );\n return { ...closedResult(url), outcome: \"not-html\", capture, note: done.note, ...marks };\n }\n if (done.kind === \"failed\" || done.kind === \"cancelled\") {\n const note = done.kind === \"failed\" ? done.note : \"cancelled by the caller\";\n this.emit({\n type: \"response\",\n url,\n host,\n status: 0,\n outcome: \"error\",\n elapsedMs: done.elapsedMs,\n });\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n status: 0,\n outcome: \"error\",\n note,\n elapsed_ms: done.elapsedMs,\n robots_check: check,\n subrequests,\n ...signedFacts,\n },\n url,\n marks,\n );\n return {\n url,\n status: 0,\n outcome: \"error\",\n fromCache: false,\n capture,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note,\n ...marks,\n };\n }\n\n const { rendered, classified, elapsedMs } = done;\n const header = (name: string): string | undefined => rendered.headers[name.toLowerCase()];\n // the page can move to a different URL: the same rules apply to it\n let finalTarget = target;\n let finalVerdict = verdict;\n try {\n finalTarget = new URL(rendered.finalUrl);\n finalTarget.hash = \"\";\n } catch {\n finalTarget = target;\n }\n const finalUrl = redact(finalTarget);\n const finalPath = `${finalTarget.pathname}${finalTarget.search}`;\n if (finalTarget.href !== target.href) {\n const moved = `the page moved to ${finalUrl}`;\n if (!WEB_PROTOCOLS.has(finalTarget.protocol))\n return skip(`${moved}, which is not an http(s) URL`, { final_url: finalUrl });\n const refusal = await checkEgress(this.egress, finalTarget);\n if (refusal !== undefined) return skip(`${moved}: ${refusal}`, { final_url: finalUrl });\n if (originKey(finalTarget) !== originKey(target)) {\n finalVerdict = await this.robotsFor(finalTarget, this.policyOf(finalTarget), request_);\n }\n const finalDecision = decide(finalVerdict.robots, tokens, finalPath);\n if (!finalDecision.allowed) {\n return skip(\n `${moved}: ${robotsRefusal(finalVerdict.robots, finalDecision.by ?? this.options.token)}`,\n {\n final_url: finalUrl,\n robots_check: robotsCheck(finalVerdict, tokens, finalDecision),\n },\n );\n }\n }\n let outcome: Outcome = classified.outcome;\n let note = classified.note;\n let body: Uint8Array | undefined;\n let blob: string | undefined;\n if (outcome === \"ok\") {\n body = new TextEncoder().encode(rendered.html);\n try {\n blob = (await this.options.blobs.put(body)).hash;\n } catch (error) {\n outcome = \"error\";\n note = `blob store: ${messageOf(error)}`;\n body = undefined;\n }\n }\n const signals = readSignals(\n signalsFor(finalVerdict.robots, tokens, finalPath),\n header,\n metaSignals(rendered.html),\n );\n const status = rendered.status ?? 200;\n const fetchedAt = this.clock.now();\n const freshness = freshUntil(\n header,\n status,\n fetchedAt,\n header(\"last-modified\"),\n policy.heuristicFreshnessCapMs,\n );\n this.emit({ type: \"response\", url, host, status, outcome, elapsedMs });\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n final_url: finalUrl,\n status,\n outcome,\n elapsed_ms: elapsedMs,\n content_type: RENDERED_TYPE,\n bytes: body?.length ?? 0,\n blob: blob ?? null,\n robots: \"allowed\",\n robots_check: check,\n signals,\n subrequests,\n ...(freshness !== null ? { fresh_until: freshness } : {}),\n ...(note !== undefined ? { note } : {}),\n ...signedFacts,\n },\n url,\n marks,\n );\n const fetchedIso = new Date(fetchedAt).toISOString();\n if (capture !== undefined && blob !== undefined) {\n this.store({\n url,\n variant: \"rendered\",\n capture,\n blob,\n etag: null,\n last_modified: null,\n content_type: RENDERED_TYPE,\n fetched_at: fetchedIso,\n status,\n ai_input: signals.ai_input,\n fresh_until: freshness,\n });\n }\n return {\n url,\n status,\n outcome,\n fromCache: false,\n capture,\n blob,\n body,\n contentType: RENDERED_TYPE,\n signals,\n note,\n fetchedAt: fetchedIso,\n ...marks,\n };\n }\n\n /**\n * This method removes the stops and the failure counts of one host, or of each host. It also\n * forgets the robots.txt verdicts that came from an unreachable robots.txt. The pace of each host\n * stays. A process that runs for a long time uses it, for example after the network comes back.\n */\n resetHosts(host?: string): void {\n const name = host?.toLowerCase().replace(/\\.$/, \"\");\n this.scheduler.reset(name);\n for (const key of this.robots.keys()) {\n const entry = this.robots.peek(key);\n const unreachable =\n entry?.value?.robots.unreachable !== undefined || entry?.value?.stale !== undefined;\n if (unreachable && (name === undefined || key.endsWith(`//${name}`))) this.robots.delete(key);\n }\n }\n\n // ── watch: the stop of a host ─────────────────────────────────────────────\n\n /**\n * This method gives the stop of the host of a URL, or undefined when the host is not stopped. A\n * watch uses it, so a page of a stopped host is due again at the end of the stop.\n */\n hostStop(url: string): ReturnType<HostStateStore[\"stopped\"]> {\n try {\n return this.scheduler.store.stopped(hostKey(new URL(url)), this.clock.now());\n } catch {\n return undefined;\n }\n }\n\n // ── end of watch ──────────────────────────────────────────────────────────\n\n /**\n * This method closes the crawler. Work in the queue ends as `skipped` with the note\n * `crawler closed`, and the crawler writes no capture after the close. Then it closes the\n * browser, the log and its own dispatcher. A second close waits for the first.\n */\n close(): Promise<void> {\n if (this.closingDone === undefined) {\n this.closed = true;\n this.closing.abort(new Error(\"crawler closed\"));\n this.closingDone = (async () => {\n await Promise.allSettled(this.running);\n await this.options.renderer?.close();\n await this.options.log.close?.();\n if (this.ownsDispatcher) await this.dispatcher.close();\n })();\n }\n return this.closingDone;\n }\n}\n","/**\n * This module makes two forms of a text equal when a reader thinks that they are equal. It has two\n * modes:\n *\n * - `quote`: for the verification of a quote. It applies NFKC (a ligature `fi` is `fi`, a\n * fullwidth `A` is `A`, an ellipsis `…` is `...`) to each character and its combining marks,\n * thus NFC and NFD are equal. It removes soft hyphens and zero-width characters. It makes\n * typographic quotes, dashes and the minus sign ASCII, makes each run of spaces one space, and\n * makes the letters lowercase. Accents stay, so \"résumé\" is not \"resume\".\n * - `search`: for the search index. It also removes the accents and the other nonspacing marks,\n * and it folds the case fully (`ß` is `ss`). It puts a space around each Chinese, Japanese and\n * Thai character, because these scripts write no space between words.\n *\n * Each character of the folded text knows the start and the end of its source in the original\n * text. Thus a match in the folded text gives exact positions in the original text.\n */\n\nexport type FoldMode = \"quote\" | \"search\";\n\n/** A folded text, with the position in the original text of each UTF-16 unit. */\nexport interface Folded {\n readonly text: string;\n /** This value is the start, in the original text, of the source of each unit. */\n readonly start: Int32Array;\n /** This value is the end, in the original text, of the source of each unit. */\n readonly end: Int32Array;\n}\n\nconst MARK = /\\p{M}/u;\nconst NONSPACING = /[\\p{Mn}\\p{Me}]/gu;\nconst IGNORABLE = /^\\p{Default_Ignorable_Code_Point}$/u;\nconst SPACE = /^\\s$/u;\nconst SEGMENTED =\n /[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Thai}\\p{Script=Lao}\\p{Script=Khmer}\\p{Script=Myanmar}]/u;\n/** These characters become ASCII: typographic quotes, dashes and the minus sign. */\nconst TYPOGRAPHIC: ReadonlyMap<number, number> = new Map([\n [0x2018, 0x27],\n [0x2019, 0x27],\n [0x201a, 0x27],\n [0x201b, 0x27],\n [0x2032, 0x27],\n [0x201c, 0x22],\n [0x201d, 0x22],\n [0x201e, 0x22],\n [0x201f, 0x22],\n [0x2033, 0x22],\n [0x2010, 0x2d],\n [0x2011, 0x2d],\n [0x2012, 0x2d],\n [0x2013, 0x2d],\n [0x2014, 0x2d],\n [0x2015, 0x2d],\n [0x2212, 0x2d],\n [0xfe58, 0x2d],\n [0xfe63, 0x2d],\n]);\n\nconst isAsciiSpace = (code: number): boolean => code === 0x20 || (code >= 0x09 && code <= 0x0d);\n\n/**\n * This function folds a text, and it keeps the position in the original text of each unit of the\n * result. Its time is linear. An ASCII character with no combining mark after it takes a short path.\n */\nexport const foldText = (text: string, mode: FoldMode): Folded => {\n let codes = new Uint16Array(Math.max(16, text.length + 16));\n let starts = new Int32Array(codes.length);\n let ends = new Int32Array(codes.length);\n let size = 0;\n let space = true;\n const grow = (need: number): void => {\n if (size + need <= codes.length) return;\n const length = Math.max(codes.length * 2, size + need);\n const nextCodes = new Uint16Array(length);\n nextCodes.set(codes.subarray(0, size));\n const nextStarts = new Int32Array(length);\n nextStarts.set(starts.subarray(0, size));\n const nextEnds = new Int32Array(length);\n nextEnds.set(ends.subarray(0, size));\n codes = nextCodes;\n starts = nextStarts;\n ends = nextEnds;\n };\n const push = (code: number, from: number, to: number): void => {\n grow(1);\n codes[size] = code;\n starts[size] = from;\n ends[size] = to;\n size += 1;\n };\n const pushSpace = (from: number, to: number): void => {\n if (space) return;\n push(0x20, from, to);\n space = true;\n };\n const length = text.length;\n let index = 0;\n while (index < length) {\n const code = text.charCodeAt(index);\n const next = index + 1 < length ? text.charCodeAt(index + 1) : 0;\n // the short path: an ASCII character that no combining mark follows\n if (code < 0x80 && next < 0x300) {\n if (isAsciiSpace(code)) pushSpace(index, index + 1);\n else {\n push(code >= 0x41 && code <= 0x5a ? code + 32 : code, index, index + 1);\n space = false;\n }\n index += 1;\n continue;\n }\n // a cluster: one code point and the combining marks after it\n const point = text.codePointAt(index)!;\n let end = index + (point > 0xffff ? 2 : 1);\n while (end < length) {\n const mark = text.codePointAt(end)!;\n if (!MARK.test(String.fromCodePoint(mark))) break;\n end += mark > 0xffff ? 2 : 1;\n }\n const cluster = text.slice(index, end);\n const first = String.fromCodePoint(point);\n if (IGNORABLE.test(first) && end === index + first.length) {\n index = end;\n continue;\n }\n if (SPACE.test(first)) {\n pushSpace(index, end);\n index = end;\n continue;\n }\n let folded =\n mode === \"quote\"\n ? cluster.normalize(\"NFKC\").toLowerCase()\n : cluster\n .normalize(\"NFKD\")\n .replace(NONSPACING, \"\")\n .normalize(\"NFC\")\n .toUpperCase()\n .toLowerCase();\n if (mode === \"search\" && SEGMENTED.test(first)) folded = ` ${folded} `;\n for (let at = 0; at < folded.length; at += 1) {\n let unit = folded.charCodeAt(at);\n unit = TYPOGRAPHIC.get(unit) ?? unit;\n if (unit === 0x20 || (unit < 0x80 && isAsciiSpace(unit)) || unit === 0xa0) {\n pushSpace(index, end);\n continue;\n }\n push(unit, index, end);\n space = false;\n }\n index = end;\n }\n if (size > 0 && codes[size - 1] === 0x20) size -= 1;\n let result = \"\";\n for (let at = 0; at < size; at += 8192) {\n result += String.fromCharCode(...codes.subarray(at, Math.min(size, at + 8192)));\n }\n return { text: result, start: starts.subarray(0, size), end: ends.subarray(0, size) };\n};\n\n/** This function gives the folded text alone, for a text with no need of positions. */\nexport const fold = (text: string, mode: FoldMode): string => foldText(text, mode).text;\n\n/** @beta */\nexport interface Verified {\n readonly found: boolean;\n /** These values are the character positions in the text of the document, when the quote is there. */\n readonly start: number | null;\n readonly end: number | null;\n /** This text is the exact quote as stored, with some text before and after it. */\n readonly excerpt: string | null;\n /** This value is the number of places where the quote is, with no overlap. */\n readonly count: number;\n}\n\nconst NOT_FOUND: Verified = { found: false, start: null, end: null, excerpt: null, count: 0 };\n\n/** This function moves a position off the second half of a surrogate pair, so a slice does not cut a character. */\nconst atBoundary = (text: string, position: number, forward: boolean): number => {\n const code = text.charCodeAt(position);\n if (position > 0 && position < text.length && code >= 0xdc00 && code <= 0xdfff) {\n return forward ? position + 1 : position - 1;\n }\n return position;\n};\n\n/**\n * This function finds a quote in a text, in the `quote` mode of `foldText`. It ignores case, runs\n * of spaces, typographic punctuation, soft hyphens, zero-width characters and the differences of\n * NFKC, NFC and NFD. The positions are in the original text.\n *\n * @beta\n */\nexport const locate = (text: string, quote: string): Verified => {\n const needle = fold(quote, \"quote\");\n if (needle.length === 0) return NOT_FOUND;\n const haystack = foldText(text, \"quote\");\n const hit = haystack.text.indexOf(needle);\n if (hit === -1) return NOT_FOUND;\n let count = 0;\n for (let at = hit; at !== -1; at = haystack.text.indexOf(needle, at + needle.length)) count += 1;\n const start = haystack.start[hit]!;\n const end = haystack.end[hit + needle.length - 1]!;\n const from = atBoundary(text, Math.max(0, start - 80), false);\n const to = atBoundary(text, Math.min(text.length, end + 80), true);\n return {\n found: true,\n start,\n end,\n excerpt: `${from > 0 ? \"…\" : \"\"}${text.slice(from, to)}${to < text.length ? \"…\" : \"\"}`,\n count,\n };\n};\n\n/** A token character of the search index: a letter, a digit, a private-use character or a spacing mark. */\nconst TOKEN_SPLIT = /[^\\p{L}\\p{N}\\p{Co}\\p{Mc}]+/u;\n\n/** This function gives the tokens of a text, as the search index reads them. */\nexport const tokensOf = (text: string): string[] =>\n fold(text, \"search\")\n .split(TOKEN_SPLIT)\n .filter((token) => token.length > 0);\n\n/** A query for the search index, in the two forms that `DocIndex.find` uses. */\nexport interface FtsQuery {\n /** Each part must match: the parts joined with AND. */\n readonly all: string;\n /** One part is enough: the parts joined with OR. */\n readonly any: string;\n /** These are the parts of `any`, each as FTS5 syntax. */\n readonly anyParts: readonly string[];\n /** These are the first tokens of the parts. A snippet marks them. */\n readonly terms: readonly string[];\n /** This value is the number of parts. */\n readonly parts: number;\n}\n\nconst MAX_PARTS = 16;\n\n/** This function removes the stars at the end of a word. It uses no regular expression, so its time is linear. */\nconst withoutStars = (word: string): string => {\n let end = word.length;\n while (end > 0 && word.charCodeAt(end - 1) === 0x2a) end -= 1;\n return word.slice(0, end);\n};\n\n/**\n * These English words are in almost every document. In the form that needs one part, they match\n * each document and cost the most time. Thus that form leaves them out when other words remain.\n */\nconst STOP_WORDS: ReadonlySet<string> = new Set([\n \"a\",\n \"an\",\n \"and\",\n \"are\",\n \"as\",\n \"at\",\n \"be\",\n \"by\",\n \"for\",\n \"from\",\n \"how\",\n \"in\",\n \"is\",\n \"it\",\n \"of\",\n \"on\",\n \"or\",\n \"that\",\n \"the\",\n \"this\",\n \"to\",\n \"was\",\n \"what\",\n \"when\",\n \"where\",\n \"which\",\n \"who\",\n \"with\",\n]);\n\n/**\n * This function reads a search query. The syntax:\n *\n * - A word is a term. The index folds case, accents and Unicode forms, so `cafe` finds `Café`. It\n * reduces English words to their stems, so `baking` finds `bake`.\n * - Text in double quotes is a phrase: its words must be next to each other.\n * - A `*` at the end of a word asks for a prefix: `bake*` finds `bakery`.\n * - A word of Chinese, Japanese or Thai is a phrase of its characters, so it matches in the text.\n * - Other punctuation is not syntax. The words `AND`, `OR`, `NOT` and `NEAR` are words.\n *\n * The function gives no query for a text with no word. It reads 16 parts or fewer. Each part is\n * quoted, so the text of a user cannot become FTS5 syntax. The form that needs one part (`any`)\n * leaves out the common English words (`STOP_WORDS`) when other words remain.\n */\nexport const parseQuery = (q: string): FtsQuery | undefined => {\n const parts: Array<{ readonly text: string; readonly term: string; readonly stop: boolean }> = [];\n for (const match of q.matchAll(/\"([^\"]*)\"(\\*?)|([^\\s\"]+)/g)) {\n if (parts.length >= MAX_PARTS) break;\n const phrase = match[1] !== undefined;\n const raw = phrase ? match[1]! : match[3]!;\n const prefix = phrase ? match[2] === \"*\" : raw.endsWith(\"*\");\n const tokens = tokensOf(phrase ? raw : withoutStars(raw));\n if (tokens.length === 0) continue;\n parts.push({\n text: `\"${tokens.join(\" \")}\"${prefix ? \" *\" : \"\"}`,\n term: tokens[0]!,\n stop: !phrase && !prefix && tokens.length === 1 && STOP_WORDS.has(tokens[0]!),\n });\n }\n if (parts.length === 0) return undefined;\n const words = parts.filter((part) => !part.stop);\n const any = words.length > 0 ? words : parts;\n return {\n all: parts.map((part) => part.text).join(\" AND \"),\n any: any.map((part) => part.text).join(\" OR \"),\n anyParts: any.map((part) => part.text),\n terms: any.map((part) => part.term),\n parts: parts.length,\n };\n};\n\n/** This function gives the FTS5 query that matches one part or more of a query. It is the form of version 0.1. */\nexport const ftsQuery = (q: string): string | undefined => parseQuery(q)?.any;\n\n/**\n * This function gives a short part of a text around the first match of a term, with the match in\n * square brackets. With no match, it gives the start of the text.\n */\nexport const snippetOf = (text: string, terms: readonly string[], width = 160): string => {\n if (text.length === 0) return \"\";\n let start = -1;\n let end = -1;\n const lower = text.toLowerCase();\n if (lower.length === text.length) {\n for (const term of terms) {\n const at = lower.indexOf(term);\n if (at !== -1 && (start === -1 || at < start)) {\n start = at;\n end = at + term.length;\n }\n }\n }\n if (start === -1) {\n const folded = foldText(text, \"search\");\n for (const term of terms) {\n const at = folded.text.indexOf(term);\n if (at !== -1) {\n const from = folded.start[at]!;\n if (start === -1 || from < start) {\n start = from;\n end = folded.end[at + term.length - 1]!;\n }\n }\n }\n }\n if (start === -1) {\n const cut = atBoundary(text, Math.min(text.length, width), false);\n return `${text.slice(0, cut)}${cut < text.length ? \"…\" : \"\"}`;\n }\n const from = atBoundary(text, Math.max(0, start - Math.floor(width / 3)), false);\n const to = atBoundary(text, Math.min(text.length, end + Math.ceil((width * 2) / 3)), true);\n const clean = (part: string): string => part.replace(/\\s+/g, \" \");\n return `${from > 0 ? \"…\" : \"\"}${clean(text.slice(from, start))}[${clean(text.slice(start, end))}]${clean(text.slice(end, to))}${to < text.length ? \"…\" : \"\"}`;\n};\n","import { constants, inflateSync } from \"node:zlib\";\nimport { getDocumentProxy, getMeta } from \"unpdf\";\n\n/**\n * The crawler reads the text of a PDF. Restaurant menus are often PDFs: 29 of 231 surveyed\n * restaurant sites link one. A menu that the crawler reads one time stays searchable. A paper in a\n * PDF reads the same way. A scanned PDF has no text, so the result is empty, not a guess.\n *\n * A small file can cost much work: a stream of 150 KB can inflate to 100 MB. Thus the reader:\n *\n * - Inflates the compressed streams first, with a limit, and refuses a file that inflates past it.\n * - Reads one page at a time, and lets other work run between the pages.\n * - Stops at a page limit, a text limit and a time limit, and says so (`truncated`). The time limit\n * stops a page in the middle too, and the text limit cuts the text of a page.\n *\n * @beta\n */\nexport interface PdfText {\n /** This is the title in the file, or null when the file names no real title (\"Microsoft Word - menu.docx\"). */\n readonly title: string | null;\n readonly text: string;\n readonly pages: number;\n /** This value is true when a limit stopped the reader before the last page. */\n readonly truncated: boolean;\n /** This note tells why the reader stopped or refused the file, or it is null. */\n readonly note: string | null;\n}\n\n/**\n * These are the limits of one read.\n *\n * @beta\n */\nexport interface PdfOptions {\n /** The reader reads no more pages than this. The default is 500. */\n readonly maxPages?: number | undefined;\n /** The reader stops when the text has this many characters, in the middle of a page too. The default is 200,000. */\n readonly maxChars?: number | undefined;\n /** The reader stops after this time in milliseconds, in the middle of a page too. The default is 20,000. */\n readonly timeBudgetMs?: number | undefined;\n /** The reader refuses a file whose streams inflate to more bytes than this. The default is 8 MB. */\n readonly maxInflatedBytes?: number | undefined;\n}\n\nconst MAX_PAGES = 500;\nconst MAX_CHARS = 200_000;\nconst TIME_BUDGET_MS = 20_000;\n/**\n * The text streams of 35 stored menus and papers inflate to 0.6 MB or less. The parser of pdf.js\n * reads a page with no pause. Thus a page of 8 MB of text operators stops the process for\n * approximately one second.\n */\nconst MAX_INFLATED_BYTES = 8 * 1024 * 1024;\n\n/**\n * This function finds a PDF from its first bytes or its content type. Some servers put bytes before `%PDF-`.\n *\n * @beta\n */\nexport const isPdf = (body: Uint8Array, contentType: string | null): boolean => {\n if (/\\bpdf\\b/i.test(contentType ?? \"\")) return true;\n const end = Math.min(body.length - 4, 1024);\n for (let index = 0; index < end; index += 1) {\n if (\n body[index] === 0x25 &&\n body[index + 1] === 0x50 &&\n body[index + 2] === 0x44 &&\n body[index + 3] === 0x46 &&\n body[index + 4] === 0x2d\n ) {\n return true;\n }\n }\n return false;\n};\n\n/** This function gives one line of text for each line of the PDF. It removes the empty lines. */\nconst tidy = (page: string): string =>\n page\n .split(/\\r?\\n/)\n .map((line) => line.replace(/\\s+/g, \" \").trim())\n .filter((line) => line.length > 0)\n .join(\"\\n\");\n\n/** A word processor writes the name of its file as the title (\"Microsoft Word - menu-final-v3.docx\"). */\nconst JUNK_TITLE =\n /^(?:microsoft (?:word|powerpoint|excel) - |untitled\\b|document\\d*$|presentation\\d*$)|\\.(?:docx?|pdf|pptx?|xlsx?|indd|ai|psd|rtf|odt|pages|txt)$/i;\n\nconst latin1 = (bytes: Uint8Array, start: number, end: number): string => {\n let out = \"\";\n for (let index = Math.max(0, start); index < end && index < bytes.length; index += 1) {\n out += String.fromCharCode(bytes[index]!);\n }\n return out;\n};\n\nconst indexOf = (bytes: Uint8Array, needle: string, from: number): number => {\n const first = needle.charCodeAt(0);\n for (\n let index = bytes.indexOf(first, from);\n index !== -1;\n index = bytes.indexOf(first, index + 1)\n ) {\n let match = true;\n for (let at = 1; at < needle.length; at += 1) {\n if (bytes[index + at] !== needle.charCodeAt(at)) {\n match = false;\n break;\n }\n }\n if (match) return index;\n }\n return -1;\n};\n\n/**\n * This function adds up the bytes that the compressed streams of a file inflate to, and stops at\n * the limit. It tries each stream: a stream in another format fails at its first bytes. An image\n * stream does not count, because the text reader does not decode it. The function gives true when\n * the file stays inside the limit.\n */\nconst inflatesWithin = (bytes: Uint8Array, limit: number): boolean => {\n let total = 0;\n let from = 0;\n for (;;) {\n const keyword = indexOf(bytes, \"stream\", from);\n if (keyword === -1) return true;\n from = keyword + 6;\n // \"endstream\" holds \"stream\" too\n if (latin1(bytes, keyword - 3, keyword) === \"end\") continue;\n const next = bytes[keyword + 6];\n if (next !== 0x0a && next !== 0x0d) continue;\n const dictionary = latin1(bytes, keyword - 1024, keyword);\n const head = dictionary.slice(Math.max(0, dictionary.lastIndexOf(\"<<\")));\n if (/\\/Subtype\\s*\\/Image\\b/.test(head)) continue;\n let start = keyword + 7;\n if (next === 0x0d && bytes[start] === 0x0a) start += 1;\n const declared = /\\/Length\\s+(\\d+)(?!\\s+\\d+\\s+R)/.exec(head);\n const stop = indexOf(bytes, \"endstream\", start);\n const end =\n declared !== null && start + Number(declared[1]) <= bytes.length\n ? start + Number(declared[1])\n : stop === -1\n ? bytes.length\n : stop;\n try {\n total += inflateSync(bytes.subarray(start, end), {\n maxOutputLength: Math.max(1, limit - total + 1),\n finishFlush: constants.Z_SYNC_FLUSH,\n }).length;\n } catch (error) {\n // a stream past the limit, or damaged data that pdf.js reads as far as it can\n if (error instanceof RangeError) return false;\n }\n if (total > limit) return false;\n from = Math.max(from, end);\n }\n};\n\n/** A page of pdf.js gives its text as items. A line ends where an item says so. */\ninterface TextItem {\n readonly str?: string;\n readonly hasEOL?: boolean;\n}\n\n/** This function lets the other work of the process run, for example another request. */\nconst pause = (): Promise<void> => new Promise((resolve) => setImmediate(resolve));\n\n/** This function gives the value of a promise, or undefined when the time ends first. */\nconst within = async <T>(promise: Promise<T>, ms: number): Promise<T | undefined> => {\n let timer: ReturnType<typeof setTimeout> | undefined;\n const late = new Promise<undefined>((resolve) => {\n timer = setTimeout(() => resolve(undefined), Math.max(0, ms));\n });\n try {\n return await Promise.race([promise, late]);\n } finally {\n clearTimeout(timer);\n }\n};\n\n/** This function keeps the first characters of a text, and does not cut a surrogate pair. */\nconst cutText = (text: string, length: number): string => {\n let end = Math.max(0, length);\n const code = text.charCodeAt(end);\n if (end > 0 && end < text.length && code >= 0xdc00 && code <= 0xdfff) end -= 1;\n return text.slice(0, end).trimEnd();\n};\n\n/** @beta */\nexport const readPdf = async (bytes: Uint8Array, options: PdfOptions = {}): Promise<PdfText> => {\n const maxPages = options.maxPages ?? MAX_PAGES;\n const maxChars = options.maxChars ?? MAX_CHARS;\n const budgetMs = options.timeBudgetMs ?? TIME_BUDGET_MS;\n const maxInflated = options.maxInflatedBytes ?? MAX_INFLATED_BYTES;\n if (!inflatesWithin(bytes, maxInflated)) {\n return {\n title: null,\n text: \"\",\n pages: 0,\n truncated: true,\n note: `the streams of the file inflate past ${String(Math.round(maxInflated / 1024 / 1024))} MB`,\n };\n }\n // pdf.js may take over the buffer it is handed, and this one is the caller's\n const pdf = await getDocumentProxy(new Uint8Array(bytes));\n try {\n const started = Date.now();\n const texts: string[] = [];\n let chars = 0;\n let note: string | null = null;\n const last = Math.min(pdf.numPages, maxPages);\n if (pdf.numPages > maxPages) note = `the reader stops after ${String(maxPages)} pages`;\n const late = (number: number): string =>\n `the reader stops after ${String(Math.round(budgetMs / 1000))} seconds, at page ${String(number - 1)}`;\n for (let number = 1; number <= last; number += 1) {\n if (number > 1) {\n await pause();\n if (Date.now() - started > budgetMs) {\n note = late(number);\n break;\n }\n }\n const page = await pdf.getPage(number);\n // pdf.js reads a content stream with no pause, so this limit stops only a page that waits (for\n // a font, for example). The limit of the inflated bytes keeps one page near one second\n const content = await within(page.getTextContent(), budgetMs - (Date.now() - started));\n if (content === undefined) {\n note = `the reader stops after ${String(Math.round(budgetMs / 1000))} seconds, in page ${String(number)}`;\n break;\n }\n const text = tidy(\n (content.items as readonly TextItem[])\n .filter((item) => item.str !== undefined)\n .map((item) => `${item.str ?? \"\"}${item.hasEOL === true ? \"\\n\" : \"\"}`)\n .join(\"\"),\n );\n page.cleanup();\n if (text.length > 0) {\n // the text and the blank lines between the pages so far, and the room that is left\n if (text.length > maxChars - chars) {\n texts.push(cutText(text, maxChars - chars));\n note = `the reader stops at ${String(maxChars)} characters, at page ${String(number)}`;\n break;\n }\n texts.push(text);\n chars += text.length + 2;\n }\n if (chars >= maxChars && number < pdf.numPages) {\n note = `the reader stops at ${String(maxChars)} characters, at page ${String(number)}`;\n break;\n }\n }\n const info = (await getMeta(pdf).catch(() => undefined))?.info;\n const named: unknown = info?.[\"Title\"];\n const title = typeof named === \"string\" ? named.replace(/\\s+/g, \" \").trim() : \"\";\n return {\n title: title.length > 0 && !JUNK_TITLE.test(title) ? title : null,\n text: texts.join(\"\\n\\n\"),\n pages: pdf.numPages,\n truncated: note !== null,\n note,\n };\n } finally {\n // the loading task owns the document and its worker\n await pdf.loadingTask.destroy();\n }\n};\n","import { brotliDecompressSync, gunzipSync, inflateRawSync, inflateSync } from \"node:zlib\";\nimport type { Fetched, FetchRequest } from \"./crawler.ts\";\nimport { redact } from \"./crawler.ts\";\nimport type { Doc, ExtractedDoc } from \"./docs.ts\";\nimport type { CrawlRuntime, Extractor, Indexed, PageOptions, PageResult } from \"./runtime.ts\";\nimport { HELD_BACK, fetchAndIndex, isHeldBack, pageExtractor } from \"./runtime.ts\";\nimport { readSignals } from \"./signals.ts\";\nimport { canonicalUrl } from \"./urls.ts\";\n\n/**\n * The archived fallback. When a site refuses a page, or is down, the crawler can give a copy from a\n * public web archive: the Wayback Machine, then Common Crawl. The crawler does not show the copy as a fresh fetch:\n *\n * - It is a document of its own (`archive:<url>`), of the kind `archived`. Its metadata and each\n * result that shows it have `archived`: the archive, the date of the copy and its URL there.\n * - The result keeps the outcome of the live fetch (`blocked`, for example), and its note names the\n * archive and the date.\n *\n * The fallback is opt-in (`fallback: \"archive\"`). The archives are third-party APIs with their own\n * politeness, so each request to them is a fetch of the same crawler. Their robots.txt, the pace of\n * each host, the egress policy and the identity of the crawler apply, and the capture log records\n * each request.\n *\n * By default, the crawler gives no archived copy of a page that the current robots.txt of its site\n * disallows for this agent. Only a stop, an error or a dead host gets the fallback.\n * `archiveDespiteRobots` widens it. The usage signals of the copy apply as for a live page.\n */\n\n/** The public web archives of the fallback. */\nexport type WebArchive = \"wayback\" | \"commoncrawl\";\n\n/** An archived copy: the archive, the time when the archive captured the page, and where the copy is. */\nexport interface Archived {\n readonly source: WebArchive;\n /** This time (ISO 8601) is when the archive captured the page. It is not the time of the fetch from the archive. */\n readonly timestamp: string;\n /** This URL is the copy in the archive: a Wayback Machine page, or the Common Crawl WARC file. */\n readonly url: string;\n /** This URL is the page that the archive captured. */\n readonly original: string;\n /** This id is the `WARC-Record-ID` of a Common Crawl copy in its WARC file. */\n readonly record?: string | undefined;\n}\n\n/** The recipe of the request for an archived copy. Its extractor reads a Wayback Machine copy or a Common Crawl WARC record. */\nexport const ARCHIVE_COPY_RECIPE = \"archive-copy\";\n\n/** The recipe of the requests to the indexes of the archives. They give no document. */\nexport const ARCHIVE_INDEX_RECIPE = \"archive-index\";\n\n/** The kind of the document of an archived copy. */\nexport const ARCHIVED_KIND = \"archived\";\n\n/**\n * This function gives the document id of the archived copy of a page.\n *\n * @beta\n */\nexport const archivedDocId = (url: string): string => `archive:${canonicalUrl(url)}`;\n\n/** The addresses of the archives. */\nconst WAYBACK_AVAILABLE = \"https://archive.org/wayback/available\";\nconst WAYBACK = \"https://web.archive.org/web/\";\nconst CC_INDEX = \"https://index.commoncrawl.org/\";\nconst CC_DATA = \"https://data.commoncrawl.org/\";\n\n/** The Common Crawl indexes that the fallback asks, the newest first. */\nconst CC_INDEXES = 3;\n/** The largest compressed WARC record that the fallback asks for. */\nconst MAX_RECORD_BYTES = 5 * 1024 * 1024;\n/** The largest decompressed WARC record or body that the fallback reads. */\nconst MAX_INFLATED_BYTES = 20 * 1024 * 1024;\n\n/** The Wayback Machine path of a raw copy: the 14 digits of the time, `id_`, and the URL of the page. */\nconst WAYBACK_RAW = /^\\/web\\/(\\d{14})id_\\/(https?:\\/\\/.+)$/s;\nconst HTML_ACCEPT = \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.5\";\nconst JSON_ACCEPT = \"application/json\";\n\n/** These are the reasons of a refusal that can get an archived copy. */\nexport type FallbackReason =\n | \"robots\"\n | \"unreachable\"\n | \"stopped\"\n | \"blocked\"\n | \"challenge\"\n | \"payment-required\"\n | \"unavailable\"\n | \"error\";\n\n// The notes of the crawler that tell why it did not send a request. The tests make sure that they agree.\nconst ROBOTS_DISALLOW = \"robots.txt disallows it\";\nconst ROBOTS_UNREACHABLE = \"robots.txt could not be read\";\nconst HOST_STOPPED = \"host stopped\";\n\n/**\n * This function tells if the result of a live fetch can get an archived copy, and why. These\n * refusals can: robots.txt, a stop of the host, a 401, a 402, a 403 and a challenge. A 502, 503 or\n * 504, a 5xx error and a dead host can too. A 404, a 429, a cancel and a refusal of the egress\n * policy cannot.\n */\nexport const fallbackReason = (indexed: Indexed): FallbackReason | undefined => {\n const { fetched } = indexed;\n if (indexed.blocked !== undefined) return \"challenge\";\n const note = fetched.note ?? \"\";\n switch (fetched.outcome) {\n case \"skipped\":\n if (note.startsWith(ROBOTS_DISALLOW)) return \"robots\";\n if (note.startsWith(ROBOTS_UNREACHABLE)) return \"unreachable\";\n if (note.startsWith(HOST_STOPPED)) return \"stopped\";\n return undefined;\n case \"blocked\":\n case \"challenge\":\n case \"payment-required\":\n case \"unavailable\":\n return fetched.outcome;\n case \"error\":\n if (fetched.status === 0 && note.startsWith(\"no response\")) return \"unreachable\";\n return fetched.status >= 500 ? \"error\" : undefined;\n default:\n return undefined;\n }\n};\n\n/** This function gives the `archived` of the metadata of a document, when the document is an archived copy. */\nexport const archivedOf = (meta: Readonly<Record<string, unknown>>): Archived | undefined => {\n const value = meta[\"archived\"];\n if (typeof value !== \"object\" || value === null) return undefined;\n const { source, timestamp, url, original, record } = value as Record<string, unknown>;\n if (\n (source !== \"wayback\" && source !== \"commoncrawl\") ||\n typeof timestamp !== \"string\" ||\n typeof url !== \"string\" ||\n typeof original !== \"string\"\n )\n return undefined;\n return { source, timestamp, url, original, ...(typeof record === \"string\" ? { record } : {}) };\n};\n\nconst ARCHIVE_NAMES: Readonly<Record<WebArchive, string>> = {\n wayback: \"the Wayback Machine\",\n commoncrawl: \"Common Crawl\",\n};\n\n/** This function gives the note of an archived copy: the archive and the date, and that it is not a fresh fetch. */\nexport const archivedNote = (archived: Archived): string =>\n `an archived copy from ${ARCHIVE_NAMES[archived.source]}, captured ${archived.timestamp}, not a fresh fetch of the page`;\n\n/** This function gives an ISO 8601 time with no milliseconds, or undefined for a time that is not valid. */\nconst isoSeconds = (ms: number): string | undefined =>\n Number.isFinite(ms) ? new Date(ms).toISOString().replace(/\\.\\d{3}Z$/, \"Z\") : undefined;\n\n/** This function reads the 14 digits of a Wayback Machine time (`20240131235959`) as an ISO 8601 time. */\nconst waybackTime = (digits: string): string | undefined => {\n const match = /^(\\d{4})(\\d{2})(\\d{2})(\\d{2})(\\d{2})(\\d{2})$/.exec(digits);\n if (match === null) return undefined;\n const [, year, month, day, hour, minute, second] = match;\n const ms = Date.UTC(\n Number(year),\n Number(month) - 1,\n Number(day),\n Number(hour),\n Number(minute),\n Number(second),\n );\n const iso = isoSeconds(ms);\n // a date such as 20240231 rolls over; it is not the date that the archive wrote\n return iso !== undefined && iso.replace(/\\D/g, \"\").startsWith(digits) ? iso : undefined;\n};\n\nconst httpUrlOf = (text: string): string | undefined => {\n try {\n const url = new URL(text);\n return url.protocol === \"http:\" || url.protocol === \"https:\" ? url.href : undefined;\n } catch {\n return undefined;\n }\n};\n\n// ── the reader of a WARC record (Common Crawl) ──\n\ninterface WarcResponse {\n readonly target: string;\n readonly date: string;\n readonly record: string | undefined;\n readonly status: number;\n readonly header: (name: string) => string | undefined;\n readonly body: Uint8Array;\n}\n\nconst CRLF2 = Buffer.from(\"\\r\\n\\r\\n\");\n\n/** This function reads the header lines of a block as a map. A name can be in any case, and two lines of one name join with \", \". */\nconst headerMap = (text: string): Map<string, string> => {\n const out = new Map<string, string>();\n for (const line of text.split(/\\r?\\n/)) {\n const colon = line.indexOf(\":\");\n if (colon <= 0) continue;\n const name = line.slice(0, colon).trim().toLowerCase();\n const value = line.slice(colon + 1).trim();\n const known = out.get(name);\n out.set(name, known === undefined ? value : `${known}, ${value}`);\n }\n return out;\n};\n\n/** This function joins the chunks of a body with `Transfer-Encoding: chunked`. A body that is not in that form gives undefined. */\nconst dechunk = (body: Buffer): Buffer | undefined => {\n const parts: Buffer[] = [];\n let at = 0;\n let total = 0;\n for (;;) {\n const end = body.indexOf(\"\\r\\n\", at);\n if (end === -1) return undefined;\n const size = Number.parseInt(\n body.subarray(at, end).toString(\"latin1\").split(\";\")[0]!.trim(),\n 16,\n );\n if (!Number.isFinite(size) || size < 0) return undefined;\n if (size === 0) return Buffer.concat(parts);\n const start = end + 2;\n if (start + size > body.length) return undefined;\n total += size;\n if (total > MAX_INFLATED_BYTES) return undefined;\n parts.push(body.subarray(start, start + size));\n at = start + size + 2;\n }\n};\n\n/** This function decodes the content coding of an archived body, with a limit on the output. */\nconst decodeCoding = (body: Buffer, coding: string | undefined): Buffer | undefined => {\n const options = { maxOutputLength: MAX_INFLATED_BYTES };\n let out = body;\n for (const name of (coding ?? \"\")\n .split(\",\")\n .map((part) => part.trim().toLowerCase())\n .filter((part) => part.length > 0 && part !== \"identity\")\n .reverse()) {\n if (name === \"gzip\" || name === \"x-gzip\") out = gunzipSync(out, options);\n else if (name === \"br\") out = brotliDecompressSync(out, options);\n else if (name === \"deflate\") {\n try {\n out = inflateSync(out, options);\n } catch {\n out = inflateRawSync(out, options);\n }\n } else return undefined;\n }\n return out;\n};\n\n/**\n * This function reads one WARC record of the type `response`, as Common Crawl gives it. The\n * record is a gzip member with the WARC headers, the HTTP headers and the body. It gives undefined for a different\n * record, or for bytes that do not decode. Common Crawl keeps the decoded body and renames\n * `Content-Encoding` to `X-Crawler-Content-Encoding`. A record with the original codings is decoded too.\n */\nexport const readWarcResponse = (bytes: Uint8Array): WarcResponse | undefined => {\n if (bytes.length < 2 || bytes[0] !== 0x1f || bytes[1] !== 0x8b) return undefined;\n let record: Buffer;\n try {\n record = gunzipSync(bytes, { maxOutputLength: MAX_INFLATED_BYTES });\n } catch {\n return undefined;\n }\n if (!record.subarray(0, 8).toString(\"latin1\").startsWith(\"WARC/1.\")) return undefined;\n const warcEnd = record.indexOf(CRLF2);\n if (warcEnd === -1) return undefined;\n const warc = headerMap(record.subarray(0, warcEnd).toString(\"utf8\"));\n if (warc.get(\"warc-type\")?.toLowerCase() !== \"response\") return undefined;\n const target = httpUrlOf(warc.get(\"warc-target-uri\")?.replace(/^<|>$/g, \"\") ?? \"\");\n const date = isoSeconds(Date.parse(warc.get(\"warc-date\") ?? \"\"));\n if (target === undefined || date === undefined) return undefined;\n const length = Number(warc.get(\"content-length\") ?? Number.NaN);\n const start = warcEnd + 4;\n const block = record.subarray(\n start,\n Number.isInteger(length) && length >= 0\n ? Math.min(record.length, start + length)\n : record.length,\n );\n const httpEnd = block.indexOf(CRLF2);\n if (httpEnd === -1) return undefined;\n const head = block.subarray(0, httpEnd).toString(\"latin1\");\n const statusLine = /^HTTP\\/\\d(?:\\.\\d)? (\\d{3})/.exec(head);\n if (statusLine === null) return undefined;\n const headers = headerMap(head.slice(head.indexOf(\"\\n\") + 1));\n let body: Buffer | undefined = block.subarray(httpEnd + 4);\n try {\n if (/\\bchunked\\b/i.test(headers.get(\"transfer-encoding\") ?? \"\")) body = dechunk(body);\n if (body !== undefined) body = decodeCoding(body, headers.get(\"content-encoding\"));\n } catch {\n body = undefined;\n }\n if (body === undefined) return undefined;\n return {\n target,\n date,\n record: warc.get(\"warc-record-id\")?.replace(/^<|>$/g, \"\"),\n status: Number(statusLine[1]),\n header: (name) => headers.get(name.toLowerCase()),\n body: new Uint8Array(body),\n };\n};\n\n// ── the extractor of the recipe `archive-copy` ──\n\n/** This function changes the documents of an archived body into the documents of the copy. A bot check is not the content, so it gives no document. */\nconst asArchived = (\n docs: ReadonlyArray<ExtractedDoc>,\n archived: Archived,\n aiInput: \"no\" | undefined,\n): ExtractedDoc[] =>\n docs\n .filter((doc) => typeof doc.meta[\"blocked\"] !== \"string\" || doc.meta[\"blocked\"] === \"\")\n .map((doc) => ({\n ...doc,\n doc: archivedDocId(archived.original),\n kind: ARCHIVED_KIND,\n url: archived.url,\n meta: { ...doc.meta, archived_kind: doc.kind, archived },\n ...(aiInput === \"no\" ? { ai_input: \"no\" as const } : {}),\n }));\n\n/**\n * This extractor reads the captures of the recipe `archive-copy`. It has no side effects, so\n * `reindex` replays them. It reads two forms:\n *\n * - A Wayback Machine copy in the raw form (`/web/<time>id_/<url>`). The body is the body of the page.\n * - A Common Crawl WARC record (a gzip member). It reads the HTTP headers and the body from the record.\n *\n * It reads the body with the page reader, and gives a document of the kind `archived` with\n * `archived` in its metadata. The usage signals of the HTTP headers in a WARC record apply too.\n *\n * @beta\n */\nexport const archiveExtractor: Extractor = async (body, capture) => {\n let url: URL;\n try {\n url = new URL(capture.url);\n } catch {\n return [];\n }\n const raw = WAYBACK_RAW.exec(`${url.pathname}${url.search}`);\n if (raw !== null) {\n const timestamp = waybackTime(raw[1]!);\n const original = httpUrlOf(raw[2]!);\n if (timestamp === undefined || original === undefined) return [];\n const archived: Archived = {\n source: \"wayback\",\n timestamp,\n url: `${url.origin}/web/${raw[1]!}/${original}`,\n original,\n };\n return asArchived(\n await pageExtractor(body, { url: original, contentType: capture.contentType }),\n archived,\n undefined,\n );\n }\n const response = readWarcResponse(body);\n if (response === undefined || response.status < 200 || response.status >= 300) return [];\n const archived: Archived = {\n source: \"commoncrawl\",\n timestamp: response.date,\n url: capture.url,\n original: response.target,\n ...(response.record === undefined ? {} : { record: response.record }),\n };\n const signals = readSignals([], response.header);\n return asArchived(\n await pageExtractor(response.body, {\n url: response.target,\n contentType: response.header(\"content-type\") ?? null,\n }),\n archived,\n signals.ai_input === \"no\" ? \"no\" : undefined,\n );\n};\n\n// ── the fallback ──\n\n/** The result of `archivedCopy`. */\nexport interface ArchiveAnswer {\n /** This document is the archived copy, when an archive had one. */\n readonly doc?: Doc | undefined;\n readonly archived?: Archived | undefined;\n /** This value is true when the store answered, with no request. */\n readonly fromCache: boolean;\n /** These notes tell what each archive answered, and why no copy came. */\n readonly notes: readonly string[];\n}\n\n/** The parts of each request to an archive: the identity of the request, its budget and its labels. */\nconst common = (\n options: PageOptions,\n): Pick<\n FetchRequest,\n \"maxAgeMs\" | \"onBehalfOf\" | \"job\" | \"signal\" | \"scope\" | \"labels\" | \"deadline\"\n> => ({\n maxAgeMs: options.maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n job: options.job,\n signal: options.signal,\n scope: options.scope,\n labels: options.labels,\n deadline: options.deadline,\n});\n\nconst got = (fetched: Fetched): boolean =>\n (fetched.outcome === \"ok\" || fetched.outcome === \"not-modified\") && fetched.body !== undefined;\n\nconst said = (fetched: Fetched): string =>\n `${fetched.outcome}${fetched.status > 0 ? ` ${String(fetched.status)}` : \"\"}${fetched.note === undefined ? \"\" : `: ${fetched.note}`}`;\n\nconst jsonOf = (fetched: Fetched): unknown => {\n try {\n return JSON.parse(new TextDecoder().decode(fetched.body)) as unknown;\n } catch {\n return undefined;\n }\n};\n\n/** This function gives the archived document of a fetch of a copy, or a note that tells why there is none. */\nconst copyOf = (\n indexed: Indexed,\n name: string,\n): { readonly doc: Doc; readonly archived: Archived } | { readonly note: string } => {\n const doc = indexed.docs.find((one) => one.kind === ARCHIVED_KIND);\n const archived = doc === undefined ? undefined : archivedOf(doc.meta);\n if (doc !== undefined && archived !== undefined) return { doc, archived };\n if (!got(indexed.fetched))\n return { note: `${name}: the copy did not come (${said(indexed.fetched)})` };\n return {\n note: `${name}: the copy gave no text${indexed.error === undefined ? \" (a bot check or an empty page)\" : ` (${indexed.error})`}`,\n };\n};\n\n/** This function asks the Wayback Machine for its newest copy of a page, and fetches the raw copy. */\nconst fromWayback = async (\n runtime: CrawlRuntime,\n url: string,\n signalLines: readonly string[],\n options: PageOptions,\n): Promise<\n | { readonly doc: Doc; readonly archived: Archived; readonly fromCache: boolean }\n | { readonly note: string }\n> => {\n const name = ARCHIVE_NAMES.wayback;\n const lookup = await runtime.crawler.fetch({\n url: `${WAYBACK_AVAILABLE}?url=${encodeURIComponent(url)}`,\n mode: \"page\",\n recipe: ARCHIVE_INDEX_RECIPE,\n accept: JSON_ACCEPT,\n ...common(options),\n });\n if (!got(lookup)) return { note: `${name}: the index did not answer (${said(lookup)})` };\n const closest = (\n jsonOf(lookup) as { archived_snapshots?: { closest?: Record<string, unknown> } } | undefined\n )?.archived_snapshots?.closest;\n if (closest === undefined || closest[\"available\"] === false) return { note: `${name}: no copy` };\n const status = closest[\"status\"];\n if (status !== undefined && status !== \"200\")\n return {\n note: `${name}: the newest copy has the status ${JSON.stringify(status) ?? \"unknown\"}, not 200`,\n };\n const digits = typeof closest[\"timestamp\"] === \"string\" ? closest[\"timestamp\"] : \"\";\n const shown = typeof closest[\"url\"] === \"string\" ? closest[\"url\"] : \"\";\n const original = httpUrlOf(/\\/web\\/\\d{14}[a-z_]*\\/(.+)$/s.exec(shown)?.[1] ?? \"\");\n if (waybackTime(digits) === undefined || original === undefined)\n return { note: `${name}: the index gave no usable time or URL` };\n const copy = await fetchAndIndex(runtime, {\n url: `${WAYBACK}${digits}id_/${original}`,\n mode: \"page\",\n recipe: ARCHIVE_COPY_RECIPE,\n accept: HTML_ACCEPT,\n originalHeaderPrefix: \"x-archive-orig-\",\n signalLines,\n ...common(options),\n });\n const found = copyOf(copy, name);\n return \"note\" in found\n ? found\n : { ...found, fromCache: lookup.fromCache && copy.fetched.fromCache };\n};\n\ninterface CdxEntry {\n readonly timestamp: string;\n readonly filename: string;\n readonly offset: number;\n readonly length: number;\n}\n\n/** This function reads the JSON lines of a Common Crawl index answer, and gives the newest copy with the status 200. */\nconst newestOf = (fetched: Fetched): CdxEntry | undefined => {\n const text = new TextDecoder().decode(fetched.body);\n let best: CdxEntry | undefined;\n for (const line of text.split(\"\\n\")) {\n if (line.trim() === \"\") continue;\n let row: Record<string, unknown>;\n try {\n row = JSON.parse(line) as Record<string, unknown>;\n } catch {\n continue;\n }\n const { timestamp, filename, status } = row;\n const offset = Number(row[\"offset\"]);\n const length = Number(row[\"length\"]);\n // the file name comes from a third party: only a path of the crawl data, on the host of the data\n if (\n status !== \"200\" ||\n typeof timestamp !== \"string\" ||\n !/^\\d{14}$/.test(timestamp) ||\n typeof filename !== \"string\" ||\n !/^crawl-data\\/[\\w\\-./]+\\.warc\\.gz$/.test(filename) ||\n filename.includes(\"..\") ||\n !Number.isSafeInteger(offset) ||\n offset < 0 ||\n !Number.isSafeInteger(length) ||\n length < 1 ||\n length > MAX_RECORD_BYTES\n )\n continue;\n if (best === undefined || timestamp > best.timestamp)\n best = { timestamp, filename, offset, length };\n }\n return best;\n};\n\n/** This function asks the newest Common Crawl indexes for a page, and fetches its WARC record with a range request. */\nconst fromCommonCrawl = async (\n runtime: CrawlRuntime,\n url: string,\n signalLines: readonly string[],\n options: PageOptions,\n): Promise<\n | { readonly doc: Doc; readonly archived: Archived; readonly fromCache: boolean }\n | { readonly note: string }\n> => {\n const name = ARCHIVE_NAMES.commoncrawl;\n const info = await runtime.crawler.fetch({\n url: `${CC_INDEX}collinfo.json`,\n mode: \"page\",\n recipe: ARCHIVE_INDEX_RECIPE,\n accept: JSON_ACCEPT,\n ...common(options),\n });\n if (!got(info)) return { note: `${name}: the list of indexes did not come (${said(info)})` };\n const listed = jsonOf(info);\n const apis = (Array.isArray(listed) ? listed : [])\n .map((entry: unknown) =>\n typeof entry === \"object\" && entry !== null\n ? (entry as Record<string, unknown>)[\"cdx-api\"]\n : undefined,\n )\n .filter((api): api is string => typeof api === \"string\" && api.startsWith(CC_INDEX))\n .slice(0, CC_INDEXES);\n if (apis.length === 0) return { note: `${name}: the list of indexes is empty` };\n let entry: CdxEntry | undefined;\n let fromCache = info.fromCache;\n const misses: string[] = [];\n for (const api of apis) {\n const query = await runtime.crawler.fetch({\n url: `${api}?url=${encodeURIComponent(url)}&output=json`,\n mode: \"page\",\n recipe: ARCHIVE_INDEX_RECIPE,\n accept: JSON_ACCEPT,\n ...common(options),\n });\n fromCache &&= query.fromCache;\n if (got(query)) entry = newestOf(query);\n else if (query.outcome !== \"not-found\") misses.push(said(query));\n if (entry !== undefined) break;\n if (query.outcome === \"skipped\") break;\n }\n if (entry === undefined)\n return {\n note: `${name}: no copy in the ${String(apis.length)} newest indexes${misses.length > 0 ? ` (${misses.join(\"; \")})` : \"\"}`,\n };\n const copy = await fetchAndIndex(runtime, {\n url: `${CC_DATA}${entry.filename}`,\n mode: \"page\",\n recipe: ARCHIVE_COPY_RECIPE,\n accept: \"*/*\",\n range: { offset: entry.offset, length: entry.length },\n signalLines,\n ...common(options),\n });\n if (got(copy.fetched) && copy.fetched.status !== 206)\n return {\n note: `${name}: the server sent the status ${String(copy.fetched.status)}, not the part of the file`,\n };\n const found = copyOf(copy, name);\n return \"note\" in found ? found : { ...found, fromCache: false };\n};\n\n/**\n * This function gives an archived copy of a page after a refusal. It applies the rule for\n * robots.txt first: with no `archiveDespiteRobots`, a page that the current robots.txt of its site\n * disallows gets no copy. A stored copy younger than `maxAgeMs` answers with no request. Then it asks\n * the Wayback Machine, then Common Crawl, each one with polite fetches of the crawler.\n */\nexport const archivedCopy = async (\n runtime: CrawlRuntime,\n url: string,\n reason: FallbackReason,\n options: PageOptions,\n): Promise<ArchiveAnswer> => {\n let target: URL;\n try {\n target = new URL(url);\n } catch {\n return { fromCache: false, notes: [] };\n }\n target.hash = \"\";\n // an archive is a third party: a URL with a password or a token does not go to it\n if (redact(target) !== target.href)\n return {\n fromCache: false,\n notes: [\"the URL holds a secret, so the crawler does not send it to a web archive\"],\n };\n const despite = options.archiveDespiteRobots === true;\n const refused =\n \"the current robots.txt of the site disallows the page for this agent, so the crawler gives no archived copy (archiveDespiteRobots permits one)\";\n if (reason === \"robots\" && !despite) return { fromCache: false, notes: [refused] };\n // the rules now: a stop does not tell them, and the content signals of the site apply to the copy\n let rules: Awaited<ReturnType<CrawlRuntime[\"crawler\"][\"robotsAllow\"]>>;\n try {\n rules = await runtime.crawler.robotsAllow(target.href, {\n onBehalfOf: options.onBehalfOf,\n scope: options.scope,\n labels: options.labels,\n signal: options.signal,\n });\n } catch {\n // a cancel during the check: the result of the live fetch stays, with no copy\n return { fromCache: false, notes: [\"the fallback stopped before it read robots.txt\"] };\n }\n if (!rules.allowed && (rules.note ?? \"\").startsWith(ROBOTS_DISALLOW) && !despite)\n return { fromCache: false, notes: [refused] };\n const signalLines = rules.signals ?? [];\n // the store answers first, as for a live page\n const known = runtime.index.get(archivedDocId(target.href));\n const archivedKnown = known === undefined ? undefined : archivedOf(known.meta);\n if (\n known !== undefined &&\n archivedKnown !== undefined &&\n typeof options.maxAgeMs === \"number\" &&\n Date.parse(runtime.now()) - Date.parse(known.fetched_at) < options.maxAgeMs\n )\n return { doc: known, archived: archivedKnown, fromCache: true, notes: [] };\n const notes: string[] = [];\n for (const ask of [fromWayback, fromCommonCrawl]) {\n if (options.signal?.aborted === true) break;\n const answer = await ask(runtime, target.href, signalLines, options);\n if (\"note\" in answer) notes.push(answer.note);\n else return { doc: answer.doc, archived: answer.archived, fromCache: answer.fromCache, notes };\n }\n return { fromCache: false, notes: [`no archived copy: ${notes.join(\"; \")}`] };\n};\n\n/**\n * This function gives the result of a page after a refusal, with an archived copy when an archive\n * had one. The result keeps the outcome and the status of the live fetch. It gives undefined when\n * the refusal gets no copy, and `notes` tell why.\n */\nexport const archivedPage = async (\n runtime: CrawlRuntime,\n url: string,\n live: Indexed,\n options: PageOptions,\n): Promise<{ readonly page?: PageResult | undefined; readonly notes: readonly string[] }> => {\n const reason = fallbackReason(live);\n if (reason === undefined) return { notes: [] };\n const answer = await archivedCopy(runtime, url, reason, options);\n if (answer.doc === undefined || answer.archived === undefined) return { notes: answer.notes };\n const doc = runtime.index.get(answer.doc.doc) ?? answer.doc;\n const archived = archivedOf(doc.meta) ?? answer.archived;\n const held = isHeldBack(doc);\n const types = doc.meta[\"jsonld_types\"];\n const links = doc.meta[\"links\"];\n const note = [\n live.fetched.note,\n live.error,\n ...answer.notes,\n archivedNote(archived),\n ...(held ? [HELD_BACK] : []),\n ].filter((part): part is string => part !== undefined && part.length > 0);\n return {\n page: {\n url,\n doc: doc.doc,\n outcome: live.fetched.outcome,\n status: live.fetched.status,\n title: held ? null : doc.title,\n chars: doc.text.length,\n rendered: false,\n structured: Array.isArray(types) ? (types as string[]) : [],\n links: Array.isArray(links) ? links.length : 0,\n captures: [...doc.captures],\n capture: doc.capture,\n capture_count: doc.capture_count,\n fetched_at: doc.fetched_at,\n from_cache: answer.fromCache,\n held_back: held,\n archived,\n note: note.join(\"; \"),\n },\n notes: [],\n };\n};\n","import type { BlobStore } from \"./blobs.ts\";\nimport type { Crawler, FetchRequest, Fetched, RenderRequest } from \"./crawler.ts\";\nimport type {\n Doc,\n DocInput,\n ExtractedDoc,\n Found,\n Latest,\n RankMode,\n ReadingIndex,\n Variant,\n VersionAt,\n} from \"./docs.ts\";\nimport { locate } from \"./docs.ts\";\nimport type { PageExtract } from \"./html.ts\";\nimport { extractPage, typesOf } from \"./html.ts\";\nimport type { CaptureLog, CaptureReader } from \"./log.ts\";\nimport { isPdf, readPdf } from \"./pdf.ts\";\nimport type { CaptureRecord } from \"./record.ts\";\nimport { makeRecord } from \"./record.ts\";\nimport type { AiInput } from \"./signals.ts\";\nimport { decodeBody } from \"./text.ts\";\nimport { canonicalUrl } from \"./urls.ts\";\n// ── archived fallback ──\nimport type { Archived } from \"./fallback.ts\";\nimport { archiveExtractor, archivedNote, archivedOf, archivedPage } from \"./fallback.ts\";\n// ── end of archived fallback ──\n\n/**\n * An extractor changes one captured body into the documents that it holds. It has no side effects,\n * so `reindex` can replay each capture through it and build the index again with no request. It can\n * answer later, because the crawler reads a PDF asynchronously.\n *\n * An extractor that throws does not stop a fetch, a render or a rebuild. The runtime records the\n * error and gives no document for that capture.\n */\nexport type Extractor = (\n body: Uint8Array,\n capture: { readonly url: string; readonly contentType: string | null },\n) => ReadonlyArray<ExtractedDoc> | Promise<ReadonlyArray<ExtractedDoc>>;\n\n/**\n * This function gives the document id of a web page. Each page has one id.\n *\n * This export is internal: the root entry point does not have it since 0.2.0.\n */\nexport const pageDocId = (url: string): string => `url:${canonicalUrl(url)}`;\n\n/** The JSON-LD that a document keeps, in characters of JSON. */\nconst MAX_JSONLD_CHARS = 30_000;\n/** These properties make a JSON-LD record large, and they say little about the place: they go first when a record is too large. */\nconst BULKY = [\n \"review\",\n \"reviews\",\n \"comment\",\n \"hasMenu\",\n \"hasMenuSection\",\n \"hasMenuItem\",\n \"menu\",\n \"photo\",\n \"image\",\n \"subjectOf\",\n \"makesOffer\",\n \"hasOfferCatalog\",\n];\n/** These properties say where and when a place is open. A record with one of them is kept first. */\nconst PLACE_KEYS = [\"openingHours\", \"openingHoursSpecification\", \"address\", \"geo\", \"telephone\"];\n\n/**\n * This function keeps the JSON-LD records that fit the limit. A record with hours, an address or a\n * position goes first. A large record loses its reviews and its menu before it is left out. A\n * record that does not fit does not stop the records after it.\n */\nconst keepJsonLd = (\n records: readonly Record<string, unknown>[],\n): { readonly kept: Record<string, unknown>[]; readonly dropped: number } => {\n const place = (record: Record<string, unknown>): boolean =>\n PLACE_KEYS.some((key) => record[key] !== undefined);\n const ordered = [...records.filter(place), ...records.filter((record) => !place(record))];\n const kept: Record<string, unknown>[] = [];\n let dropped = 0;\n let room = MAX_JSONLD_CHARS;\n for (const record of ordered) {\n let size = JSON.stringify(record).length;\n let candidate = record;\n if (size > room) {\n candidate = Object.fromEntries(\n Object.entries(record).filter(([key]) => !BULKY.includes(key)),\n );\n size = JSON.stringify(candidate).length;\n }\n if (size > room) {\n dropped += 1;\n continue;\n }\n kept.push(candidate);\n room -= size;\n }\n return { kept, dropped };\n};\n\n/**\n * This extractor is the built-in recipe for a web page, plain or rendered. It keeps the visible\n * text, the JSON-LD (whole objects, up to a limit) and the links. It decodes the body in the\n * encoding that the page declares, with `decodeBody`. It does not throw: a page that it cannot read\n * gives a document that names the error. When the reader cut the text, the document keeps the whole\n * text in `full_text`, so `verify` finds a quote past the cut.\n *\n * This export is internal: the root entry point does not have it since 0.2.0.\n */\nexport const pageExtractor: Extractor = (body, capture) => {\n if (isPdf(body, capture.contentType)) return pdfDocs(body, capture.url);\n if (capture.contentType !== null && !/html|xml/i.test(capture.contentType)) return [];\n const doc = pageDocId(capture.url);\n let page: PageExtract;\n try {\n page = extractPage(decodeBody(body, capture.contentType), capture.url);\n } catch (error) {\n // one page that breaks the reader must not break the fetch, or each later reindex\n return [\n {\n doc,\n kind: \"page\",\n title: capture.url,\n text: capture.url,\n url: capture.url,\n meta: {\n extract_error: error instanceof Error ? error.message.slice(0, 300) : String(error),\n },\n },\n ];\n }\n const jsonld = keepJsonLd(page.jsonld);\n return [\n {\n doc,\n kind: \"page\",\n title: page.title,\n text: page.text.length > 0 ? page.text : page.title,\n url: capture.url,\n meta: {\n description: page.description,\n lang: page.lang,\n jsonld_types: [...new Set(page.jsonld.flatMap(typesOf))],\n jsonld: jsonld.kept,\n ...(jsonld.dropped > 0 ? { jsonld_dropped: jsonld.dropped } : {}),\n links: page.links.slice(0, 100),\n visible_chars: page.visibleChars,\n embedded: page.embedded,\n needs_render: page.needsRender,\n render_reason: page.renderReason,\n canonical: page.canonical,\n ...(page.feeds !== undefined && page.feeds.length > 0 ? { feeds: page.feeds } : {}),\n redirect: page.redirect,\n blocked: page.blocked,\n ...(page.notes.length > 0 ? { notes: page.notes } : {}),\n },\n ...(page.fullText !== undefined ? { full_text: page.fullText } : {}),\n },\n ];\n};\n\nconst MAX_PDF_TEXT_CHARS = 200_000;\n\n/**\n * This function changes a PDF into one document of its text. A PDF with no text, a scan for\n * example, gives a document of its title only, and the document says that it needs OCR. A damaged\n * file gives no document.\n */\nconst pdfDocs = async (body: Uint8Array, url: string): Promise<ReadonlyArray<ExtractedDoc>> => {\n let read;\n try {\n read = await readPdf(body, { maxChars: MAX_PDF_TEXT_CHARS });\n } catch {\n return [];\n }\n const file = new URL(url).pathname.split(\"/\").pop() ?? \"\";\n let named = file;\n try {\n named = decodeURIComponent(file);\n } catch {\n // a malformed escape: the name as written\n }\n const title = read.title ?? (named || url);\n const notes = {\n ...(read.truncated ? { truncated: true } : {}),\n ...(read.note !== null ? { note: read.note } : {}),\n };\n if (read.text.length === 0) {\n return [\n {\n doc: pageDocId(url),\n kind: \"pdf\",\n title,\n text: title,\n url,\n meta: {\n format: \"pdf\",\n pages: read.pages,\n visible_chars: 0,\n // a file that a budget stopped is not a scan\n ocr_needed: read.note === null,\n ...notes,\n },\n },\n ];\n }\n return [\n {\n doc: pageDocId(url),\n kind: \"pdf\",\n title,\n text: read.text.slice(0, MAX_PDF_TEXT_CHARS),\n url,\n meta: { format: \"pdf\", pages: read.pages, visible_chars: read.text.length, ...notes },\n },\n ];\n};\n\n/** @beta */\nexport class Extractors {\n // the archived fallback reads its copies with the built-in recipe `archive-copy`\n private readonly byRecipe = new Map<string, Extractor>([\n [\"page\", pageExtractor],\n [\"archive-copy\", archiveExtractor],\n ]);\n\n // 0.1.1 let an application register `archive-copy`. Thus an application can replace that built-in once.\n private readonly replaceable = new Set([\"archive-copy\"]);\n\n register(recipe: string, extractor: Extractor): void {\n if (this.byRecipe.has(recipe) && !this.replaceable.delete(recipe)) {\n throw new Error(`recipe ${recipe} already has an extractor`);\n }\n this.byRecipe.set(recipe, extractor);\n }\n\n get(recipe: string): Extractor | undefined {\n return this.byRecipe.get(recipe);\n }\n\n /** This method gives the names of the recipes that have an extractor. */\n recipes(): string[] {\n return [...this.byRecipe.keys()];\n }\n}\n\n/** @beta */\nexport interface CrawlRuntime {\n readonly crawler: Crawler;\n /** This index keeps the documents and the cache entries. `DocIndex` is the SQLite index. */\n readonly index: ReadingIndex;\n readonly blobs: BlobStore;\n /** This reader is the read side of the capture log. The function `reindex` replays it. */\n readonly captures: CaptureReader;\n /** This log is the write side of the capture log. */\n readonly log: CaptureLog;\n readonly extractors: Extractors;\n readonly now: () => string;\n}\n\n/**\n * This is the payload of a capture record, as the crawler writes it.\n *\n * @beta\n */\nexport interface CapturePayload {\n readonly url: string;\n readonly recipe: string;\n readonly mode: \"api\" | \"page\";\n readonly status?: number;\n readonly outcome?: string;\n readonly blob?: string | null;\n readonly content_type?: string | null;\n readonly etag?: string | null;\n readonly last_modified?: string | null;\n readonly fetched_at: string;\n readonly signals?: { readonly ai_input?: AiInput };\n /** This value is true when the headless browser made the capture, not a plain request. */\n readonly rendered?: boolean;\n /** This value is true when the body is only the first part of the response. */\n readonly truncated?: boolean;\n /** This value is true when the server cut a compressed body. The capture is not a copy to keep. */\n readonly cut?: boolean;\n /** This value is true when an API answered with a web page that the request did not accept. The capture is not a copy to keep. */\n readonly unasked?: boolean;\n /** This time (ISO 8601) is the end of the freshness that the server gave. Captures before version 0.2 do not have it. */\n readonly fresh_until?: string | null;\n /** This text holds the request headers that the response varies on. Captures before version 0.2 do not have it. */\n readonly vary_key?: string | null;\n /** This URL is the target of a redirect, with its secrets as `REDACTED`. */\n readonly location?: string | null;\n readonly note?: string;\n /** This text is the byte range of a range request (`bytes=0-99`). The capture is a part of the resource, not its stored copy. */\n readonly range?: string;\n}\n\n/** The structural fields of the metadata of a page: they hold no text of the page. A held-back page shows only these. */\nexport const STRUCTURAL_META: readonly string[] = [\n \"jsonld_types\",\n \"jsonld_dropped\",\n \"visible_chars\",\n \"lang\",\n \"embedded\",\n \"needs_render\",\n \"render_reason\",\n \"canonical\",\n \"redirect\",\n \"blocked\",\n \"format\",\n \"pages\",\n \"ocr_needed\",\n \"truncated\",\n // the label of an archived copy holds no text of the page\n \"archived\",\n \"archived_kind\",\n];\n\n/**\n * The note of a document that is held back.\n *\n * @beta\n */\nexport const HELD_BACK =\n \"This page asked not to be used as AI input, so its text is held back. Open the URL to read it, or read again with userProvided when a person gave this URL directly (the crawler logs that).\";\n\n/**\n * This function tells if a document is held back: the page asked not to be AI input, and no person\n * gave its URL directly. The only way to read a held-back document is `userProvided`.\n *\n * @beta\n */\nexport const isHeldBack = (doc: { readonly ai_input: AiInput }, userProvided = false): boolean =>\n doc.ai_input === \"no\" && !userProvided;\n\n/**\n * This function gives the metadata that a held-back document shows: only the structural fields, which hold no text of the page.\n *\n * @beta\n */\nexport const heldBackMeta = (meta: Readonly<Record<string, unknown>>): Record<string, unknown> =>\n Object.fromEntries(Object.entries(meta).filter(([key]) => STRUCTURAL_META.includes(key)));\n\nconst messageOf = (error: unknown): string =>\n error instanceof Error ? (error.message.split(/\\r?\\n/)[0] ?? \"\") : String(error);\n\n/** This function tells which bot check or firewall sent its page in place of the content, or gives undefined. */\nconst blockedBy = (doc: ExtractedDoc): string | undefined => {\n const blocked = doc.meta[\"blocked\"];\n return typeof blocked === \"string\" && blocked.length > 0 ? blocked : undefined;\n};\n\n/**\n * The result of `fetchAndIndex` and `renderAndIndex`.\n *\n * @beta\n */\nexport interface Indexed {\n readonly fetched: Fetched;\n readonly docs: readonly Doc[];\n /** This text tells why the capture gave no document: the error of the extractor, or a bot check. */\n readonly error?: string | undefined;\n /** This value names the bot check or the firewall that sent its page in place of the content. */\n readonly blocked?: string | undefined;\n}\n\n/** This function reads one capture with its extractor, and adds each document to the index as a version of the capture. */\nconst indexCapture = async (\n runtime: CrawlRuntime,\n fetched: Fetched,\n recipe: string,\n rendered: boolean,\n): Promise<Omit<Indexed, \"fetched\">> => {\n const capture = fetched.capture;\n if (fetched.body === undefined || capture === undefined) return { docs: [] };\n // a stored capture that the index already read gives its documents with no new reading\n if (fetched.fromCache) {\n const known = runtime.index.docsOfCapture(capture);\n if (known.length > 0) return { docs: known };\n }\n const extractor = runtime.extractors.get(recipe);\n if (extractor === undefined) return { docs: [] };\n let extracted: ReadonlyArray<ExtractedDoc>;\n try {\n extracted = await extractor(fetched.body, {\n url: fetched.url,\n contentType: rendered ? \"text/html; charset=utf-8\" : (fetched.contentType ?? null),\n });\n } catch (error) {\n return { docs: [], error: `extract failed (${recipe}): ${messageOf(error)}` };\n }\n const blocked = extracted.map(blockedBy).find((name) => name !== undefined);\n if (blocked !== undefined) {\n // a bot check is not the content: the index keeps the earlier text, and the host stops\n runtime.crawler.reportBlocked(fetched.url, blocked);\n return {\n docs: [],\n blocked,\n error: `the page is a bot check (${blocked}), not the content, so the crawler keeps nothing of it and stops the host`,\n };\n }\n const now = runtime.now();\n const docs = extracted.map((doc) =>\n runtime.index.upsert(\n {\n ...doc,\n captures: [capture],\n capture,\n fetched_at: fetched.fetchedAt ?? now,\n blob: fetched.blob ?? null,\n recipe,\n rendered,\n // an extractor can find a \"no\" in the body (the headers of an archived response), never a \"yes\"\n ai_input: doc.ai_input === \"no\" ? \"no\" : fetched.signals.ai_input,\n ...(fetched.scope !== undefined ? { scope: fetched.scope } : {}),\n ...(fetched.labels !== undefined ? { labels: fetched.labels } : {}),\n },\n now,\n ),\n );\n return { docs };\n};\n\n/**\n * This function fetches one URL and adds its documents to the index. Each recipe uses it. A fetch\n * from the cache gives the documents that the index has for that capture. Thus a repeated\n * question sends no request and reads nothing again. An extractor that throws gives no document and an\n * `error`, and the fetch still succeeds.\n *\n * @beta\n */\nexport const fetchAndIndex = async (\n runtime: CrawlRuntime,\n request: FetchRequest,\n): Promise<Indexed> => {\n const fetched = await runtime.crawler.fetch(request);\n if (fetched.outcome !== \"ok\" && fetched.outcome !== \"not-modified\") return { fetched, docs: [] };\n return { fetched, ...(await indexCapture(runtime, fetched, request.recipe, false)) };\n};\n\n/**\n * This function renders a page in the headless browser and adds what the page shows to the index.\n *\n * This export is internal: the root entry point does not have it since 0.2.0.\n */\nexport const renderAndIndex = async (\n runtime: CrawlRuntime,\n request: RenderRequest,\n): Promise<Indexed> => {\n const fetched = await runtime.crawler.render(request);\n if (fetched.outcome !== \"ok\") return { fetched, docs: [] };\n return { fetched, ...(await indexCapture(runtime, fetched, request.recipe, true)) };\n};\n\nexport interface PageResult {\n readonly url: string;\n readonly doc: string | null;\n readonly outcome: string;\n readonly status: number;\n /** This value is the title of the page. A held-back page shows no title. */\n readonly title: string | null;\n readonly chars: number;\n readonly rendered: boolean;\n /** These are the schema.org types that the JSON-LD of the page declares. */\n readonly structured: readonly string[];\n readonly links: number;\n /** These ids are the captures behind the current text, the newest first, 10 or fewer. */\n readonly captures: readonly string[];\n /** This id is the capture of the text that the document shows. A citation names it. */\n readonly capture: string | null;\n /** This value is the number of captures of the document. */\n readonly capture_count: number;\n /** This time (ISO 8601) is the fetch time of the text that the document shows. */\n readonly fetched_at: string | null;\n /** This value is true when the stored copy answered, and the crawler sent no request. */\n readonly from_cache: boolean;\n /** This value is true when the page asked not to be AI input. Its title and text are then held back. */\n readonly held_back: boolean;\n readonly note: string | null;\n /**\n * This value tells that the document is an archived copy, not a fresh fetch: the archive, the date\n * of the copy and its URL there. The outcome and the status stay those of the live fetch.\n */\n readonly archived?: Archived | undefined;\n}\n\n/**\n * These options control one page.\n *\n * @beta\n */\nexport interface PageOptions {\n readonly render: \"auto\" | \"never\" | \"always\";\n readonly maxAgeMs?: number | \"auto\" | undefined;\n readonly onBehalfOf?: readonly string[] | undefined;\n readonly job?: string | undefined;\n /** When this signal aborts, the crawler stops the fetch and the render. */\n readonly signal?: AbortSignal | undefined;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n /** This time (milliseconds since the epoch) is the end of a budget. No request of the page starts at or after it (`FetchRequest.deadline`). */\n readonly deadline?: number | undefined;\n // ── archived fallback ──\n /** The value `archive` gives a copy from a public web archive when the site refuses the page, or is down. The default is `none`. */\n readonly fallback?: \"archive\" | \"none\" | undefined;\n /** The value true also gives an archived copy of a page that the current robots.txt of its site disallows. The default is false. */\n readonly archiveDespiteRobots?: boolean | undefined;\n // ── end of archived fallback ──\n}\n\nconst HTML_TYPE = /html|xml/i;\n\n/**\n * This function reads one web page. It first sends a plain request. It opens the browser only when\n * the page shows almost nothing without JavaScript, or when the caller asks for it. It renders\n * only an HTML page: not a PDF, a different file or a bot check. A page that sends the reader on\n * with a `<meta http-equiv=\"refresh\">` gets one more plain fetch, of its target.\n *\n * @beta\n */\nexport const fetchPage = async (\n runtime: CrawlRuntime,\n url: string,\n options: PageOptions,\n): Promise<PageResult> => {\n const plainRequest = (target: string): FetchRequest => ({\n url: target,\n mode: \"page\",\n recipe: \"page\",\n accept: \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.5\",\n maxAgeMs: options.maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n job: options.job,\n signal: options.signal,\n scope: options.scope,\n labels: options.labels,\n deadline: options.deadline,\n });\n let plain = await fetchAndIndex(runtime, plainRequest(url));\n const notes: string[] = [];\n if (plain.error !== undefined) notes.push(plain.error);\n const target = plain.docs[0]?.meta[\"redirect\"];\n if (typeof target === \"string\" && target.length > 0 && plain.blocked === undefined) {\n const followed = await fetchAndIndex(runtime, plainRequest(target));\n notes.push(\n `the page sends the reader on to ${followed.fetched.url}, so the crawler fetched that page`,\n );\n if (followed.error !== undefined) notes.push(followed.error);\n plain = followed;\n }\n let docs = plain.docs;\n let rendered = false;\n let fromCache = plain.fetched.fromCache;\n const reachable = plain.fetched.outcome === \"ok\" || plain.fetched.outcome === \"not-modified\";\n const first = docs[0];\n const html =\n first?.kind === \"page\" &&\n (plain.fetched.contentType === undefined || HTML_TYPE.test(plain.fetched.contentType));\n const wanted =\n options.render === \"always\" ||\n (options.render === \"auto\" && first?.meta[\"needs_render\"] === true);\n if (reachable && wanted) {\n if (plain.blocked !== undefined) notes.push(\"the crawler does not render a bot check\");\n else if (first === undefined || !html) {\n notes.push(\"the crawler renders only an HTML page, not a PDF or a different file\");\n } else {\n // a render of an earlier body is not the page now: the render is fresh when the plain body changed\n const current = runtime.index.get(first.doc);\n const stale = current !== undefined && !current.rendered && current.capture_count > 1;\n const shown = await renderAndIndex(runtime, {\n url: plain.fetched.url,\n recipe: \"page\",\n maxAgeMs: stale ? 0 : options.maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n job: options.job,\n signal: options.signal,\n scope: options.scope,\n labels: options.labels,\n deadline: options.deadline,\n });\n if (shown.docs.length > 0) {\n docs = shown.docs;\n rendered = true;\n fromCache = fromCache && shown.fetched.fromCache;\n } else {\n notes.push(shown.error ?? shown.fetched.note ?? \"the render gave no document\");\n }\n }\n }\n // ── archived fallback: a refusal can get a copy from a web archive, clearly marked ──\n if (options.fallback === \"archive\" && docs.length === 0) {\n const fallback = await archivedPage(runtime, url, plain, options);\n if (fallback.page !== undefined) return fallback.page;\n notes.push(...fallback.notes);\n }\n // ── end of archived fallback ──\n const doc = docs[0] === undefined ? undefined : (runtime.index.get(docs[0].doc) ?? docs[0]);\n const held = doc !== undefined && isHeldBack(doc);\n const types = doc?.meta[\"jsonld_types\"];\n const links = doc?.meta[\"links\"];\n if (held) notes.push(HELD_BACK);\n const note = [plain.fetched.note, ...notes].filter(\n (part): part is string => part !== undefined && part.length > 0,\n );\n return {\n url,\n doc: doc?.doc ?? null,\n outcome: plain.fetched.outcome,\n status: plain.fetched.status,\n title: doc === undefined || held ? null : doc.title,\n chars: doc?.text.length ?? 0,\n rendered: doc?.rendered ?? rendered,\n structured: Array.isArray(types) ? (types as string[]) : [],\n links: Array.isArray(links) ? links.length : 0,\n captures: doc === undefined ? [] : [...doc.captures],\n capture: doc?.capture ?? null,\n capture_count: doc?.capture_count ?? 0,\n fetched_at: doc?.fetched_at ?? plain.fetched.fetchedAt ?? null,\n from_cache: fromCache,\n held_back: held,\n note: note.length > 0 ? note.join(\"; \") : null,\n };\n};\n\n// ── reading, verifying and finding ──────────────────────────────────────────\n\n/** This result is a part of one stored document. */\nexport interface ReadResult {\n readonly doc: string;\n readonly kind: string;\n /** This value is the title of the document. A held-back document shows no title. */\n readonly title: string;\n readonly url: string | null;\n /** This value is the metadata of the document. A held-back document shows only the structural fields. */\n readonly meta: Readonly<Record<string, unknown>>;\n readonly from: number;\n readonly to: number;\n readonly total: number;\n readonly text: string;\n readonly captures: readonly string[];\n /** This id is the capture of the text that the result reads. */\n readonly capture: string;\n /** This time (ISO 8601) is the fetch time of the text that the result reads. */\n readonly fetched_at: string;\n readonly ai_input: AiInput;\n /** This value is true when the page asked not to be AI input. The text is then empty. */\n readonly held_back: boolean;\n readonly note: string | null;\n /** This time (ISO 8601) is the `at` of the read. The result has it only when the read gave `at`. */\n readonly at?: string | undefined;\n /**\n * This value is true when `at` is before the first capture of the document. Then no version\n * answers: the text, the title, `capture` and `fetched_at` are empty, and the note tells why.\n */\n readonly before_first?: boolean | undefined;\n /** This value tells that the text is an archived copy, with the archive and the date of the copy. */\n readonly archived?: Archived | undefined;\n}\n\n/** This result tells if a quote is in a stored document, and where. */\nexport interface VerifyResult {\n readonly doc: string;\n /** This value is true when the quote is in the text. A held-back document gives false. */\n readonly found: boolean;\n /** These values are the positions of the quote in the text of `capture`. */\n readonly start: number | null;\n readonly end: number | null;\n readonly excerpt: string | null;\n readonly url: string | null;\n readonly captures: readonly string[];\n /** This id is the capture whose text holds the quote, or the capture that the result examined. */\n readonly capture: string | null;\n /** This hash is the blob of that capture. */\n readonly blob: string | null;\n /** This time (ISO 8601) is the fetch time of that capture. */\n readonly fetched_at: string | null;\n /** This value is true when only an earlier version of the document holds the quote. */\n readonly stale: boolean;\n /** This value is true when the quote is in the whole text of a page that the reader cut. The positions are then in that whole text. */\n readonly in_full_text: boolean;\n /** This value is false when the examined text is not complete (a PDF that the reader stopped). A quote that is not found can then be in the file. */\n readonly complete: boolean;\n /** This value is the number of places where the quote is. */\n readonly count: number;\n readonly held_back: boolean;\n readonly note: string | null;\n /** This time (ISO 8601) is the `at` of the verification. The result has it only when the verification gave `at`. */\n readonly at?: string | undefined;\n /** This value is true when `at` is before the first capture of the document. Then no version holds the quote, and the note tells why. */\n readonly before_first?: boolean | undefined;\n /**\n * This value tells that the examined text is an archived copy, with the archive and the date of\n * the copy. A citation of it names the archive and that date, not a date of a fresh fetch.\n */\n readonly archived?: Archived | undefined;\n}\n\n/** These options control a read. */\nexport interface ReadOptions {\n /** This value is the first character to read. The default is 0. */\n readonly from?: number | undefined;\n /** This value is the number of characters to read. The default is 4000. */\n readonly length?: number | undefined;\n /** This value is true when a person gave this URL directly. It is the only way to read a held-back document. The crawler logs it. */\n readonly userProvided?: boolean | undefined;\n /** This id reads the version of one capture, not the current version. */\n readonly capture?: string | undefined;\n /**\n * This time reads the version that was current at that time: an ISO 8601 text or a `Date`. A\n * date with no time is the start of that day in UTC. Give `capture` or `at`, not the two.\n */\n readonly at?: string | Date | undefined;\n}\n\n/** These options control a verification. */\nexport interface VerifyOptions {\n /** This id examines the version of one capture only. A citation of that capture thus stays true after a new fetch. */\n readonly capture?: string | undefined;\n /** This value is true when a person gave this URL directly. It is the only way to verify a quote in a held-back document. */\n readonly userProvided?: boolean | undefined;\n /** This time examines only the version that was current at that time (`ReadOptions.at`). Give `capture` or `at`, not the two. */\n readonly at?: string | Date | undefined;\n}\n\nconst knownDoc = (runtime: CrawlRuntime, id: string): Doc => {\n const doc = runtime.index.get(id);\n if (doc === undefined)\n throw new Error(`no stored document ${id}: find one with find(), or fetch its page`);\n return doc;\n};\n\n// ── time travel ─────────────────────────────────────────────────────────────\n\n/**\n * This function gives the ISO 8601 form of a time. A text that is not a time throws a `RangeError`.\n * A date with no time is the start of that day in UTC.\n */\nexport const isoTime = (name: string, value: string | Date): string => {\n const ms =\n value instanceof Date ? value.getTime() : typeof value === \"string\" ? Date.parse(value) : NaN;\n if (!Number.isFinite(ms)) {\n throw new RangeError(\n `${name} must be a time, such as 2026-10-01T12:00:00Z, not ${JSON.stringify(String(value))}`,\n );\n }\n return new Date(ms).toISOString();\n};\n\n/**\n * This function gives the version of a document that was current at a time (ISO 8601). It uses\n * `ReadingIndex.versionAt` when the index has it. Else it takes the newest version at or before\n * the time.\n *\n * @beta\n */\nexport const versionAtTime = (runtime: CrawlRuntime, id: string, at: string): VersionAt => {\n const index = runtime.index;\n if (index.versionAt !== undefined) return index.versionAt(id, at);\n const all = index.versions(id, { limit: 1_000_000 });\n const pick = all.find((version) => version.fetched_at <= at);\n return {\n version: pick === undefined ? undefined : index.versionText(id, pick.capture),\n first: all.at(-1)?.fetched_at,\n };\n};\n\n/** This function gives the note of a time before the first version of a document. */\nconst beforeFirstNote = (id: string, at: string, first: string | undefined): string =>\n `the document ${id} has no version at ${at} or before${first === undefined ? \"\" : `: its first capture is from ${first}`}`;\n\n/** This function gives the time of `at`, or undefined. It refuses `capture` and `at` together. */\nconst atOf = (options: {\n readonly capture?: string | undefined;\n readonly at?: string | Date | undefined;\n}): string | undefined => {\n if (options.at === undefined) return undefined;\n if (options.capture !== undefined)\n throw new RangeError(\"give capture or at, not the two: each one chooses a version\");\n return isoTime(\"at\", options.at);\n};\n\n// ── end of time travel ──────────────────────────────────────────────────────\n\nconst wholeNumber = (name: string, value: number | undefined, fallback: number): number => {\n if (value === undefined) return fallback;\n if (!Number.isFinite(value) || value < 0) {\n throw new RangeError(`${name} must be a number of 0 or more, not ${String(value)}`);\n }\n return Math.floor(value);\n};\n\n/** This function moves a position off the second half of a surrogate pair, so a slice does not cut a character. */\nconst onBoundary = (text: string, position: number): number => {\n const code = text.charCodeAt(position);\n return position > 0 && position < text.length && code >= 0xdc00 && code <= 0xdfff\n ? position - 1\n : position;\n};\n\n/**\n * This function writes an `override` record to the capture log. A person gave the URL of a\n * held-back document directly, and the hold stops for one action (`read` or `verify`). The record\n * takes the scope and the labels of `marks`, or else those of the document, as its captures do.\n *\n * @beta\n */\nexport const recordOverride = async (\n runtime: CrawlRuntime,\n doc: {\n readonly doc: string;\n readonly url: string | null;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n },\n action: string,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n): Promise<void> => {\n const at = runtime.now();\n const scope = marks.scope ?? doc.scope;\n const labels = marks.labels ?? doc.labels;\n await runtime.log.append(\n makeRecord({\n source: \"fetch\",\n kind: \"override\",\n payload: { doc: doc.doc, url: doc.url, action, reason: \"a person provided this URL\", at },\n received_at: at,\n external_id: doc.url ?? doc.doc,\n ...(scope !== undefined ? { scope } : {}),\n ...(labels !== undefined ? { labels } : {}),\n }),\n );\n};\n\n/**\n * This function reads a range of characters of one stored document. It applies the rule for a\n * held-back document: no title, no text, and only the structural metadata, unless `userProvided`.\n * The positions do not cut a character in two.\n *\n * @beta\n */\nexport const readDocument = async (\n runtime: CrawlRuntime,\n id: string,\n options: ReadOptions = {},\n): Promise<ReadResult> => {\n const from = wholeNumber(\"from\", options.from, 0);\n const length = wholeNumber(\"length\", options.length, 4000);\n const at = atOf(options);\n const doc = knownDoc(runtime, id);\n let version =\n options.capture === undefined ? undefined : runtime.index.versionText(id, options.capture);\n if (options.capture !== undefined && version === undefined) {\n throw new Error(`the document ${id} has no version from the capture ${options.capture}`);\n }\n const timed = at === undefined ? {} : { at, before_first: false };\n if (at !== undefined) {\n const found = versionAtTime(runtime, id, at);\n if (found.version === undefined) {\n // no version answers, so there is no text to hold back\n return {\n doc: doc.doc,\n kind: doc.kind,\n title: \"\",\n url: doc.url,\n meta: {},\n from: 0,\n to: 0,\n total: 0,\n text: \"\",\n captures: doc.captures,\n capture: \"\",\n fetched_at: \"\",\n ai_input: doc.ai_input,\n held_back: false,\n note: beforeFirstNote(id, at, found.first),\n at,\n before_first: true,\n };\n }\n version = found.version;\n }\n const aiInput: AiInput =\n doc.ai_input === \"no\" || version?.ai_input === \"no\"\n ? \"no\"\n : (version?.ai_input ?? doc.ai_input);\n const userProvided = options.userProvided ?? false;\n const held = isHeldBack({ ai_input: aiInput }, userProvided);\n if (aiInput === \"no\" && userProvided) await recordOverride(runtime, doc, \"read\");\n const text = version?.text ?? doc.text;\n const start = onBoundary(text, Math.min(from, text.length));\n const end = held ? start : onBoundary(text, Math.min(text.length, start + length));\n const meta = version?.meta ?? doc.meta;\n // ── archived fallback: an archived copy says so in each read ──\n const archived = archivedOf(meta);\n const notes = [...(held ? [HELD_BACK] : []), ...(archived ? [archivedNote(archived)] : [])];\n return {\n doc: doc.doc,\n kind: version?.kind ?? doc.kind,\n title: held ? \"\" : (version?.title ?? doc.title),\n url: version?.url ?? doc.url,\n meta: held ? heldBackMeta(meta) : meta,\n from: start,\n to: end,\n total: text.length,\n text: held ? \"\" : text.slice(start, end),\n captures: doc.captures,\n capture: version?.capture ?? doc.capture,\n fetched_at: version?.fetched_at ?? doc.fetched_at,\n ai_input: aiInput,\n held_back: held,\n note: notes.length > 0 ? notes.join(\" \") : null,\n ...timed,\n ...(archived !== undefined ? { archived } : {}),\n };\n};\n\n// ── archived fallback: the labels of an archived copy ──\n\n/** This function gives the `archived` field of a result, for the metadata of an archived copy. */\nconst archivedField = (\n meta: Readonly<Record<string, unknown>>,\n): { readonly archived?: Archived } => {\n const archived = archivedOf(meta);\n return archived === undefined ? {} : { archived };\n};\n\n/** This function gives the `archived` field and the note of a result: the note of an archived copy goes after the other note. */\nconst withArchived = (\n meta: Readonly<Record<string, unknown>>,\n note: string | null,\n): { readonly note: string | null; readonly archived?: Archived } => {\n const archived = archivedOf(meta);\n if (archived === undefined) return { note };\n const said = archivedNote(archived);\n return { note: note === null ? said : `${note}; ${said}`, archived };\n};\n\n// ── end of archived fallback ──\n\n/**\n * This function examines the versions of a document for a quote, and writes nothing. It also tells\n * if `userProvided` opened a held-back version, so that `verifyQuote` logs the override.\n */\nconst examineQuote = (\n runtime: CrawlRuntime,\n id: string,\n quote: string,\n options: VerifyOptions,\n): { readonly result: VerifyResult; readonly override: boolean } => {\n const at = atOf(options);\n const doc = knownDoc(runtime, id);\n let pinned =\n options.capture === undefined ? undefined : runtime.index.versionText(id, options.capture);\n if (options.capture !== undefined && pinned === undefined) {\n throw new Error(`the document ${id} has no version from the capture ${options.capture}`);\n }\n const userProvided = options.userProvided ?? false;\n const base = { doc: doc.doc, url: doc.url, captures: doc.captures };\n const timed = at === undefined ? {} : { at, before_first: false };\n const nothing = {\n found: false,\n start: null,\n end: null,\n excerpt: null,\n capture: null,\n blob: null,\n fetched_at: null,\n stale: false,\n in_full_text: false,\n count: 0,\n } as const;\n if (at !== undefined) {\n // the version that was current at the time is the only version that the check examines\n const found = versionAtTime(runtime, id, at);\n if (found.version === undefined) {\n const result: VerifyResult = {\n ...base,\n ...nothing,\n complete: true,\n held_back: false,\n note: beforeFirstNote(id, at, found.first),\n at,\n before_first: true,\n };\n return { result, override: false };\n }\n pinned = found.version;\n }\n const aiInput: AiInput = doc.ai_input === \"no\" || pinned?.ai_input === \"no\" ? \"no\" : doc.ai_input;\n if (isHeldBack({ ai_input: aiInput }, userProvided)) {\n const result: VerifyResult = {\n ...base,\n ...nothing,\n complete: false,\n held_back: true,\n note: HELD_BACK,\n ...timed,\n ...archivedField(pinned?.meta ?? doc.meta),\n };\n return { result, override: false };\n }\n const current = pinned ?? runtime.index.versionText(id);\n const candidates =\n pinned !== undefined\n ? [pinned]\n : [...(current === undefined ? [] : [current]), ...runtime.index.earlierTexts(id)];\n // a version that asked not to be AI input stays held back after the page drops the signal\n const held = candidates.filter((version) => version.ai_input === \"no\").length;\n const override = userProvided && (aiInput === \"no\" || held > 0);\n for (const [index, version] of candidates.entries()) {\n if (isHeldBack(version, userProvided)) continue;\n let hit = locate(version.text, quote);\n let inFull = false;\n if (!hit.found && version.full_text !== undefined) {\n hit = locate(version.full_text, quote);\n inFull = hit.found;\n }\n if (!hit.found) continue;\n const stale = pinned === undefined && index > 0;\n const result: VerifyResult = {\n ...base,\n found: true,\n start: hit.start,\n end: hit.end,\n excerpt: hit.excerpt,\n capture: version.capture,\n blob: version.blob,\n fetched_at: version.fetched_at,\n stale,\n in_full_text: inFull,\n complete: true,\n count: hit.count,\n held_back: false,\n ...withArchived(\n version.meta,\n stale ? \"only an earlier capture of the document holds the quote\" : null,\n ),\n ...timed,\n };\n return { result, override };\n }\n const incomplete =\n current !== undefined && current.meta[\"truncated\"] === true && current.full_text === undefined;\n // the note does not depend on the quote, so it tells nothing about the held-back text\n const notes = [\n ...(incomplete\n ? [\n \"the reader stopped before the end of the file, so the quote can be in the part that it did not read\",\n ]\n : []),\n ...(held > 0 && !userProvided\n ? [`the check did not examine the versions that asked not to be AI input (${String(held)})`]\n : []),\n ];\n const result: VerifyResult = {\n ...base,\n found: false,\n start: null,\n end: null,\n excerpt: null,\n capture: current?.capture ?? null,\n blob: current?.blob ?? null,\n fetched_at: current?.fetched_at ?? null,\n stale: false,\n in_full_text: false,\n complete: !incomplete,\n count: 0,\n held_back: false,\n ...withArchived(current?.meta ?? doc.meta, notes.length > 0 ? notes.join(\"; \") : null),\n ...timed,\n };\n return { result, override };\n};\n\n/**\n * This function finds a quote in a stored document, with `locate`, and writes nothing. It is the\n * synchronous part of `verifyQuote`. A held-back document gives `found: false` and no positions,\n * unless `userProvided`, and this function does not log the override. With no capture, it does not\n * examine a version that asked not to be AI input, also when a later capture dropped the signal.\n */\nexport const checkQuote = (\n runtime: CrawlRuntime,\n id: string,\n quote: string,\n options: VerifyOptions = {},\n): VerifyResult => examineQuote(runtime, id, quote, options).result;\n\n/**\n * This function finds a quote in a stored document, with `locate`. It tells which capture holds the\n * quote, with its blob, its fetch time and the positions:\n *\n * - With `capture`, it examines the version of that capture only.\n * - With no capture, it examines the current text first. When the quote is not there, it examines\n * the earlier texts, the newest first. A match there has `stale: true`.\n * - When the reader cut a text, it also examines the whole text.\n *\n * A held-back document gives `found: false` and no positions, so a quote cannot test its text,\n * unless `userProvided`. The same rule applies to each version. A version that asked not to be AI\n * input stays held back after a later capture drops the signal. The crawler logs an override when\n * `userProvided` opens a held-back version.\n *\n * @beta\n */\nexport const verifyQuote = async (\n runtime: CrawlRuntime,\n id: string,\n quote: string,\n options: VerifyOptions = {},\n): Promise<VerifyResult> => {\n const { result, override } = examineQuote(runtime, id, quote, options);\n if (override) await recordOverride(runtime, knownDoc(runtime, id), \"verify\");\n return result;\n};\n\n/**\n * This function searches the stored documents. The syntax of the query is the syntax of\n * `parseQuery`. A held-back document is not in the results. The option `rank` chooses the\n * ranking (`RankMode`), and the default is `exact`.\n *\n * @beta\n */\nexport const findDocuments = async (\n runtime: CrawlRuntime,\n q: string,\n options: {\n readonly limit?: number | undefined;\n readonly kind?: string | null | undefined;\n readonly rank?: RankMode | undefined;\n } = {},\n): Promise<Found[]> => {\n await Promise.resolve();\n const found = runtime.index.find(q, {\n limit: options.limit ?? 10,\n kind: options.kind ?? null,\n ...(options.rank !== undefined ? { rank: options.rank } : {}),\n });\n // ── archived fallback: a hit of an archived copy says so ──\n return found.map((hit) => {\n if (hit.kind !== \"archived\") return hit;\n const meta = runtime.index.get(hit.doc)?.meta;\n return meta === undefined ? hit : { ...hit, ...archivedField(meta) };\n });\n};\n\n// ── the rebuild ─────────────────────────────────────────────────────────────\n\n/** The counts of a rebuild. */\nexport interface ReindexResult {\n /** This value is the number of captures that the rebuild read again. */\n readonly captures: number;\n readonly docs: number;\n readonly urls: number;\n /** This value is the number of captures whose extractor threw. */\n readonly failed: number;\n /** This value is the number of captures whose blob is missing or damaged. */\n readonly missing: number;\n /** This value is the number of damaged lines of the log that the rebuild skipped. */\n readonly damaged: number;\n /** This value is the number of documents of recipes with no extractor, kept from the old index. */\n readonly carried: number;\n /** These recipes have captures in the log and no extractor in this runtime. */\n readonly unknown_recipes: readonly string[];\n /** These are the first errors of the extractors. */\n readonly errors: ReadonlyArray<{ readonly capture: string; readonly error: string }>;\n}\n\nconst RENDERED_MARK = \"#rendered\";\nconst MAX_ERRORS = 20;\n/**\n * The crawler reads the captures of these recipes for itself. They are robots.txt, the sitemaps\n * and feeds of `discover`, and the indexes of the web archives. They give no document.\n */\nconst CRAWLER_RECIPES: ReadonlySet<string> = new Set([\n \"robots.txt\",\n \"sitemap\",\n \"feed\",\n \"archive-index\",\n]);\n\n/** This function gives the URL of a capture and its variant. A URL of version 0.1 can end in `#rendered`. */\nconst captureTarget = (\n payload: CapturePayload,\n): { readonly url: string; readonly variant: Variant } => {\n if (payload.url.endsWith(RENDERED_MARK)) {\n return { url: payload.url.slice(0, -RENDERED_MARK.length), variant: \"rendered\" };\n }\n return { url: payload.url, variant: payload.rendered === true ? \"rendered\" : \"plain\" };\n};\n\n/** A replay writes into a new generation of the index (a rebuild), or into the index itself (the tail of the log). */\ninterface ReplayTarget {\n upsert(input: DocInput, now: string): unknown;\n setLatest(entry: Latest): void;\n latest(url: string, variant?: Variant): Latest | undefined;\n}\n\n/** These are the counts of a replay. */\ninterface ReplayTally {\n captures: number;\n failed: number;\n missing: number;\n readonly unknown: Set<string>;\n readonly errors: Array<{ readonly capture: string; readonly error: string }>;\n}\n\nconst newTally = (): ReplayTally => ({\n captures: 0,\n failed: 0,\n missing: 0,\n unknown: new Set(),\n errors: [],\n});\n\n/** This function replays the cache entry of one capture, as the crawler made it at the time of the fetch. */\nconst replayLatest = (\n build: ReplayTarget,\n envelope: CaptureRecord,\n payload: CapturePayload,\n): void => {\n const { url, variant } = captureTarget(payload);\n const fetchedAt = payload.fetched_at ?? envelope.received_at;\n if (payload.outcome === \"redirect\") {\n const location =\n payload.location ??\n (typeof payload.note === \"string\" && payload.note.startsWith(\"location \")\n ? payload.note.slice(9)\n : null);\n const permanent = payload.status === 301 || payload.status === 308;\n if (location !== null && (permanent || typeof payload.fresh_until === \"string\")) {\n build.setLatest({\n url,\n variant,\n capture: envelope.id,\n blob: null,\n etag: null,\n last_modified: null,\n content_type: null,\n fetched_at: fetchedAt,\n status: payload.status ?? 301,\n ai_input: \"unset\",\n location,\n fresh_until: payload.fresh_until ?? null,\n vary_key: payload.vary_key ?? null,\n });\n }\n return;\n }\n // a body that the server cut, a web page in place of the data of an API, or a part of a resource\n // from a range request was never the stored copy\n if (payload.cut === true || payload.unasked === true || payload.range !== undefined) return;\n const previous = build.latest(url, variant);\n const revisit = envelope.kind === \"revisit\" || payload.outcome === \"not-modified\";\n const truncated = payload.truncated === true;\n // a 304 updates the stored answer: its status stays, and its validators fill the gaps\n const etag = truncated ? null : (payload.etag ?? (revisit ? (previous?.etag ?? null) : null));\n const lastModified = truncated\n ? null\n : (payload.last_modified ?? (revisit ? (previous?.last_modified ?? null) : null));\n const kept = revisit ? (previous?.content_type ?? undefined) : undefined;\n const contentType =\n payload.content_type ?? kept ?? (variant === \"rendered\" ? \"text/html; charset=utf-8\" : null);\n const status = revisit\n ? previous === undefined || previous.status < 200 || previous.status >= 300\n ? 200\n : previous.status\n : (payload.status ?? 200);\n build.setLatest({\n url,\n variant,\n capture: envelope.id,\n blob: payload.blob ?? null,\n etag,\n last_modified: lastModified,\n content_type: contentType,\n fetched_at: fetchedAt,\n status,\n ai_input: payload.signals?.ai_input ?? \"unset\",\n ...(truncated ? { truncated: true } : {}),\n fresh_until: payload.fresh_until ?? null,\n vary_key: payload.vary_key ?? (revisit ? (previous?.vary_key ?? null) : null),\n });\n};\n\n/** This function replays one record of the log: its cache entry, and the documents that its extractor gives. */\nconst replayRecord = async (\n runtime: CrawlRuntime,\n target: ReplayTarget,\n envelope: CaptureRecord,\n tally: ReplayTally,\n): Promise<void> => {\n if (envelope.kind !== \"capture\" && envelope.kind !== \"revisit\") return;\n const payload = envelope.payload as CapturePayload | null;\n if (payload === null || typeof payload !== \"object\" || typeof payload.url !== \"string\") return;\n if (payload.outcome === \"redirect\") {\n replayLatest(target, envelope, payload);\n return;\n }\n if (typeof payload.blob !== \"string\") return;\n if (payload.outcome !== \"ok\" && payload.outcome !== \"not-modified\") return;\n tally.captures += 1;\n replayLatest(target, envelope, payload);\n const extractor = runtime.extractors.get(payload.recipe);\n if (extractor === undefined) {\n // the crawler reads robots.txt, the sitemaps and the feeds for itself; they give no document\n if (!CRAWLER_RECIPES.has(payload.recipe)) tally.unknown.add(payload.recipe);\n return;\n }\n const body = await runtime.blobs.get(payload.blob);\n if (body === undefined) {\n tally.missing += 1;\n return;\n }\n const { url, variant } = captureTarget(payload);\n const latest = target.latest(url, variant);\n let extracted: ReadonlyArray<ExtractedDoc>;\n try {\n extracted = await extractor(body, {\n url,\n contentType:\n variant === \"rendered\"\n ? \"text/html; charset=utf-8\"\n : (payload.content_type ?? latest?.content_type ?? null),\n });\n } catch (error) {\n tally.failed += 1;\n if (tally.errors.length < MAX_ERRORS)\n tally.errors.push({ capture: envelope.id, error: messageOf(error) });\n return;\n }\n // a bot check is not the content, so it is not a version of the document\n if (extracted.some((doc) => blockedBy(doc) !== undefined)) return;\n for (const doc of extracted) {\n target.upsert(\n {\n ...doc,\n captures: [envelope.id],\n capture: envelope.id,\n fetched_at: payload.fetched_at ?? envelope.received_at,\n blob: payload.blob,\n recipe: payload.recipe,\n rendered: variant === \"rendered\",\n ai_input: doc.ai_input === \"no\" ? \"no\" : (payload.signals?.ai_input ?? \"unset\"),\n ...(envelope.scope !== undefined ? { scope: envelope.scope } : {}),\n ...(envelope.labels !== undefined ? { labels: envelope.labels } : {}),\n },\n envelope.received_at,\n );\n }\n};\n\n/**\n * This function builds the reading index again from the capture log and the blobs. The index is a\n * projection, and this function is the proof that its deletion loses nothing. These are its rules:\n *\n * - It builds a new generation of the tables and commits it in batches. The old index answers\n * until the end, and a failed rebuild leaves it as it was.\n * - It reads the log as a stream, so its memory does not grow with the log.\n * - It counts and skips a damaged line, a missing blob and an extractor that throws.\n * - The documents of a recipe that has no extractor in this runtime stay, from the old index.\n * - A capture that a different writer adds during the rebuild goes into the new index.\n *\n * @beta\n */\nexport const reindex = async (\n runtime: CrawlRuntime,\n options: { readonly batch?: number | undefined } = {},\n): Promise<ReindexResult> => {\n const since = runtime.now();\n const build = runtime.index.beginRebuild(\n options.batch === undefined ? {} : { batch: options.batch },\n );\n const tally = newTally();\n let damaged = 0;\n try {\n const scan = runtime.captures.scan(\n { source: \"fetch\" },\n {\n onDamaged: () => {\n damaged += 1;\n },\n },\n );\n for await (const envelope of scan) await replayRecord(runtime, build, envelope, tally);\n const carried = build.carry(new Set(runtime.extractors.recipes()));\n const counts = build.commit(since);\n return {\n captures: tally.captures,\n ...counts,\n failed: tally.failed,\n missing: tally.missing,\n damaged,\n carried,\n unknown_recipes: [...tally.unknown].sort(),\n errors: tally.errors,\n };\n } catch (error) {\n build.abort();\n throw error;\n }\n};\n\n/**\n * The result of `catchUpIndex`.\n *\n * @beta\n */\nexport interface CatchUpResult extends ReindexResult {\n /**\n * The value `rebuild` tells that the index was stale, and the function built it again from the\n * whole log. The value `tail` tells that it read only the captures that the index missed.\n */\n readonly mode: \"rebuild\" | \"tail\";\n}\n\n/** The tail starts this long before the newest cache entry. Two writers and their clocks can differ. */\nconst TAIL_SLACK_MS = 10 * 60_000;\n\n/** This function tells if the index missed a record of the log: no cache entry as new as it, or no version of its documents. */\nconst missed = (runtime: CrawlRuntime, envelope: CaptureRecord): boolean => {\n if (envelope.kind !== \"capture\" && envelope.kind !== \"revisit\") return false;\n const payload = envelope.payload as CapturePayload | null;\n if (payload === null || typeof payload !== \"object\" || typeof payload.url !== \"string\")\n return false;\n const fetchedAt = payload.fetched_at ?? envelope.received_at;\n const { url, variant } = captureTarget(payload);\n const entry = runtime.index.latest(url, variant);\n if (payload.outcome === \"redirect\") return entry === undefined || entry.fetched_at < fetchedAt;\n if (\n typeof payload.blob !== \"string\" ||\n (payload.outcome !== \"ok\" && payload.outcome !== \"not-modified\")\n )\n return false;\n // a range request leaves no cache entry: only its documents tell if the index has it\n if (payload.range !== undefined)\n return (\n runtime.extractors.get(payload.recipe) !== undefined && !runtime.index.hasVersion(envelope.id)\n );\n if (entry === undefined || entry.fetched_at < fetchedAt) return true;\n return (\n runtime.extractors.get(payload.recipe) !== undefined && !runtime.index.hasVersion(envelope.id)\n );\n};\n\n/**\n * This function brings the reading index up to the capture log, with no request. `openCrawler`\n * uses it at open. An application that uses `createRuntime` uses it the same way:\n *\n * - A stale index (a new schema, or a deleted file) is built again with `reindex`.\n * - An index that is behind the log gets the captures that it missed. A process can stop between\n * the log and the index, and SQLite can lose its last transactions after a power cut. The\n * function reads the log from a short time before the newest cache entry.\n *\n * It gives the counts, or undefined when the index missed nothing.\n *\n * @beta\n */\nexport const catchUpIndex = async (runtime: CrawlRuntime): Promise<CatchUpResult | undefined> => {\n if (runtime.index.stale) return { mode: \"rebuild\", ...(await reindex(runtime)) };\n const newest = runtime.index.newestFetch();\n const since =\n newest === undefined ? undefined : new Date(Date.parse(newest) - TAIL_SLACK_MS).toISOString();\n const tally = newTally();\n let damaged = 0;\n const scan = runtime.captures.scan(\n { source: \"fetch\", ...(since === undefined ? {} : { since }) },\n {\n onDamaged: () => {\n damaged += 1;\n },\n },\n );\n for await (const envelope of scan) {\n if (missed(runtime, envelope)) await replayRecord(runtime, runtime.index, envelope, tally);\n }\n if (tally.captures === 0) return undefined;\n return {\n mode: \"tail\",\n captures: tally.captures,\n ...runtime.index.counts(),\n failed: tally.failed,\n missing: tally.missing,\n damaged,\n carried: 0,\n unknown_recipes: [...tally.unknown].sort(),\n errors: tally.errors,\n };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAmCA,IAAa,gBAAb,cAAmC,MAAM;CAG5B;CACA;CAHX,OAAwB;CACxB,YACE,SACA,MACA;EACA,MAAM,WAAW,SAAS,IAAI,CAAC;EAHtB,KAAA,UAAA;EACA,KAAA,OAAA;EAGT,KAAK,OAAO;CACd;AACF;;AAGA,MAAa,cAAc,SAAiB,SAC1C,kBAAkB,QAAQ,MAAM,YAAY,KAAK,IAAI,IAAI,OAAO,IAAI,GAAG,KAAK;;;;;;AAS9E,MAAa,aAAa,SAAoC;CAC5D,MAAM,QAAQ,KAAK,MAAM,GAAG;CAC5B,IAAI,MAAM,SAAS,KAAK,MAAM,MAAM,SAAS,OAAO,IAAI,MAAM,IAAI;CAClE,IAAI,MAAM,WAAW,KAAK,MAAM,SAAS,GAAG,OAAO,KAAA;CACnD,MAAM,UAAoB,CAAC;CAC3B,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI;EACJ,IAAI,sBAAsB,KAAK,IAAI,GACjC,QAAQ,KAAK,WAAW,IAAI,IAAI,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE;OAC9D,IAAI,YAAY,KAAK,IAAI,GAAG,QAAQ,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC;OACpE,IAAI,oBAAoB,KAAK,IAAI,GAAG,QAAQ,OAAO,IAAI;OACvD,OAAO,KAAA;EACZ,QAAQ,KAAK,KAAK;CACpB;CACA,MAAM,OAAO,QAAQ,IAAI;CACzB,IAAI,QAAQ,MAAM,UAAU,QAAQ,GAAG,GAAG,OAAO,KAAA;CACjD,IAAI,QAAQ,QAAQ,IAAI,QAAQ,SAAS,OAAO,KAAA;CAChD,IAAI,OAAO;CACX,MAAM,OAAiB,CAAC;CACxB,KAAK,IAAI,QAAQ,QAAQ,QAAQ,QAAQ,GAAG,SAAS,GAAG;EACtD,KAAK,QAAQ,OAAO,GAAG;EACvB,OAAO,KAAK,MAAM,OAAO,GAAG;CAC9B;CACA,OAAO,CAAC,GAAG,SAAS,GAAG,IAAI;AAC7B;;AAGA,MAAa,aAAa,SAAoC;CAC5D,IAAI,QAAQ,KAAK,WAAW,GAAG,KAAK,KAAK,SAAS,GAAG,IAAI,KAAK,MAAM,GAAG,EAAE,IAAI;CAC7E,MAAM,OAAO,MAAM,QAAQ,GAAG;CAC9B,IAAI,SAAS,IAAI,QAAQ,MAAM,MAAM,GAAG,IAAI;CAC5C,IAAI,KAAK,KAAK,MAAM,GAAG,OAAO,KAAA;CAC9B,MAAM,YAAY,MAAM,YAAY,GAAG;CACvC,MAAM,OAAO,MAAM,MAAM,YAAY,CAAC;CACtC,IAAI,KAAK,SAAS,GAAG,GAAG;EACtB,MAAM,KAAK,UAAU,IAAI;EACzB,IAAI,OAAO,KAAA,GAAW,OAAO,KAAA;EAC7B,MAAM,OAAQ,GAAG,MAAO,IAAK,GAAG;EAChC,MAAM,MAAO,GAAG,MAAO,IAAK,GAAG;EAC/B,QAAQ,GAAG,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,KAAK,SAAS,EAAE,EAAE,GAAG,IAAI,SAAS,EAAE;CACjF;CACA,MAAM,CAAC,OAAO,IAAI,QAAQ,MAAM,MAAM,IAAI;CAC1C,MAAM,YAAY,SAChB,KAAK,WAAW,IAAI,CAAC,IAAI,KAAK,MAAM,GAAG,CAAC,CAAC,KAAK,UAAU,OAAO,SAAS,OAAO,EAAE,CAAC;CACpF,MAAM,QAAQ,SAAS,IAAI;CAC3B,MAAM,OAAO,SAAS,KAAA,IAAY,CAAC,IAAI,SAAS,IAAI;CACpD,MAAM,SAAS;EAAC,GAAG;EAAO,GAAG,MAAc,IAAI,MAAM,SAAS,KAAK,MAAM,CAAC,CAAC,KAAK,CAAC;EAAG,GAAG;CAAI;CAC3F,IAAI,OAAO,WAAW,GAAG,OAAO,KAAA;CAChC,OAAO,OAAO,SAAS,UAAU,CAAC,SAAS,GAAG,QAAQ,GAAI,CAAC;AAC7D;AAEA,MAAM,MAAM,SAAwB,UAAU,IAAI;AAClD,MAAM,MAAM,SAAwB,UAAU,IAAI;;AAGlD,MAAM,YAAY,SAAgB,QAAe,SAA0B;CACzE,IAAI,QAAQ,WAAW,OAAO,QAAQ,OAAO;CAC7C,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;EACtD,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,OAAO,QAAQ,CAAC,CAAC;EACtD,IAAI,SAAS,GAAG,OAAO;EACvB,MAAM,OAAQ,OAAS,IAAI,OAAS;EACpC,KAAK,QAAQ,SAAU,WAAW,OAAO,SAAU,OAAO,OAAO;CACnE;CACA,OAAO;AACT;AAGA,MAAM,YAA6D;CACjE;EAAC,GAAG,iBAAiB;EAAG;EAAI;CAAU;CACtC;EAAC,GAAG,eAAe;EAAG;EAAI;CAAU;CACpC;EAAC,GAAG,iBAAiB;EAAG;EAAI;CAAU;CACtC;EAAC,GAAG,SAAS;EAAG;EAAG;CAAa;CAChC;EAAC,GAAG,UAAU;EAAG;EAAG;CAAS;CAC7B;EAAC,GAAG,YAAY;EAAG;EAAI;CAAmB;CAC1C;EAAC,GAAG,WAAW;EAAG;EAAG;CAAU;CAC/B;EAAC,GAAG,aAAa;EAAG;EAAI;CAAY;CACpC;EAAC,GAAG,YAAY;EAAG;EAAI;CAAS;CAChC;EAAC,GAAG,WAAW;EAAG;EAAI;CAAU;CAChC;EAAC,GAAG,WAAW;EAAG;EAAI;CAAe;CACrC;EAAC,GAAG,aAAa;EAAG;EAAI;CAAS;CACjC;EAAC,GAAG,YAAY;EAAG;EAAI;CAAW;CAClC;EAAC,GAAG,cAAc;EAAG;EAAI;CAAe;CACxC;EAAC,GAAG,aAAa;EAAG;EAAI;CAAe;CACvC;EAAC,GAAG,WAAW;EAAG;EAAG;CAAW;CAChC;EAAC,GAAG,WAAW;EAAG;EAAG;CAAU;AACjC;AAEA,MAAM,YAA6D;CACjE;EAAC,GAAG,eAAe;EAAG;EAAK;CAAU;CACrC;EAAC,GAAG,IAAI;EAAG;EAAK;CAAa;CAC7B;EAAC,GAAG,KAAK;EAAG;EAAK;CAAU;CAC3B;EAAC,GAAG,aAAa;EAAG;EAAI;CAAU;CAClC;EAAC,GAAG,OAAO;EAAG;EAAI;CAAU;CAC5B;EAAC,GAAG,QAAQ;EAAG;EAAI;CAAU;CAC7B;EAAC,GAAG,YAAY;EAAG;EAAI;CAAe;CACtC;EAAC,GAAG,QAAQ;EAAG;EAAG;CAAc;CAChC;EAAC,GAAG,QAAQ;EAAG;EAAI;CAAY;CAC/B;EAAC,GAAG,QAAQ;EAAG;EAAI;CAAS;CAC5B;EAAC,GAAG,QAAQ;EAAG;EAAG;CAAW;AAC/B;AAEA,MAAM,YAAY,YAChB,UAAU,MAAM,CAAC,QAAQ,UAAU,SAAS,SAAS,QAAQ,IAAI,CAAC,CAAC,GAAG;AAExE,MAAM,YAAY,YAAuC;CACvD,MAAM,UAAU,MAAc,OAC5B,QAAQ,MAAM,MAAM,EAAE,CAAC,CAAC,OAAO,SAAS,SAAS,CAAC;CACpD,MAAM,YAAY,SAAqC,SAAS,QAAQ,MAAM,MAAM,OAAO,CAAC,CAAC;CAE7F,IAAI,OAAO,GAAG,EAAE,KAAK,QAAQ,QAAQ,OAAQ,QAAQ,QAAQ,KAAM,OAAO,SAAS,EAAE;CACrF,IAAI,SAAS,SAAS,GAAG,WAAW,GAAG,EAAE,GAAG,OAAO,SAAS,EAAE;CAC9D,IAAI,QAAQ,OAAO,MAAQ,QAAQ,OAAO,GAAM,OAAO,SAAS,CAAC;CACjE,MAAM,SAAS,UAAU,MAAM,CAAC,QAAQ,UAAU,SAAS,SAAS,QAAQ,IAAI,CAAC,CAAC,GAAG;CACrF,IAAI,WAAW,KAAA,GAAW,OAAO;CAEjC,IAAI,OAAO,GAAG,EAAE,GAAG,OAAO,SAAS,EAAE,KAAK;CAC1C,IAAI,QAAQ,OAAO,GAAG,OAAO;AAE/B;;;;;;AAOA,MAAa,eAAe,YAAwC;CAClE,MAAM,OAAO,UAAU,OAAO;CAC9B,IAAI,SAAS,KAAA,GAAW,OAAO,SAAS,IAAI;CAC5C,MAAM,MAAM,UAAU,OAAO;CAC7B,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,SAAS,GAAG;AACrD;AAEA,MAAM,UAAU,SAAyB,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;;AAGpF,MAAM,aAAuD;CAC3D,CAAC,oBAAoB,UAAU;CAC/B,CAAC,gCAAgC,UAAU;CAC3C,CAAC,oBAAoB,UAAU;AACjC;AAEA,MAAM,aAAa,SAAoC,UAAU,IAAI,KAAK,UAAU,IAAI;AAExF,MAAM,WAAW,OAAe,YAA4B;CAC1D,MAAM,QAAQ,MAAM,QAAQ,GAAG;CAC/B,MAAM,OAAO,UAAU,UAAU,KAAK,QAAQ,MAAM,MAAM,GAAG,KAAK,CAAC;CACnE,IAAI,SAAS,KAAA,KAAa,KAAK,WAAW,QAAQ,QAAQ,OAAO;CACjE,MAAM,OAAO,UAAU,KAAK,KAAK,SAAS,IAAI,OAAO,MAAM,MAAM,QAAQ,CAAC,CAAC;CAC3E,OACE,OAAO,UAAU,IAAI,KAAK,QAAQ,KAAK,QAAQ,KAAK,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI;AAElG;;AAGA,MAAM,aAAa,QAAuB,MAAc,YAAyC;CAC/F,IAAI,OAAO,iBAAiB,MAAM,OAAO;CACzC,MAAM,OAAO,OAAO,IAAI;CACxB,MAAM,QAAQ,YAAY,KAAA,IAAY,UAAU,IAAI,IAAI,UAAU,OAAO;CACzE,KAAK,MAAM,OAAO,OAAO,cAAc,CAAC,GAAG;EACzC,MAAM,QAAQ,OAAO,GAAG;EACxB,IAAI,UAAU,MAAM,OAAO;EAC3B,IAAI,MAAM,WAAW,IAAI,KAAK,KAAK,SAAS,MAAM,MAAM,CAAC,CAAC,GAAG,OAAO;EACpE,IAAI,UAAU,KAAA,KAAa,QAAQ,OAAO,KAAK,GAAG,OAAO;CAC3D;CACA,OAAO;AACT;;AAGA,MAAM,SAAS,SAAyB;CACtC,MAAM,QAAQ,UAAU,IAAI;CAC5B,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,MAAM,WAAW,GAAG,OAAO,MAAM,KAAK,GAAG;CAC7C,MAAM,SAAmB,CAAC;CAC1B,KAAK,IAAI,QAAQ,GAAG,QAAQ,IAAI,SAAS,GACvC,OAAO,MAAO,MAAM,UAAW,IAAK,MAAM,QAAQ,GAAA,CAAK,SAAS,EAAE,CAAC;CACrE,OAAO,OAAO,KAAK,GAAG,CAAC,CAAC,QAAQ,oBAAoB,IAAI;AAC1D;;;;;;AAOA,MAAa,iBAAiB,QAAuB,SAAqC;CACxF,IAAI,OAAO,iBAAiB,MAAM,OAAO,KAAA;CACzC,MAAM,OAAO,OAAO,IAAI;CAExB,IADgB,UAAU,IAChB,MAAM,KAAA,GAAW;EACzB,MAAM,OAAO,YAAY,IAAI;EAC7B,OAAO,SAAS,KAAA,KAAa,UAAU,QAAQ,MAAM,KAAA,CAAS,IAC1D,KAAA,IACA,WAAW,MAAM,IAAI,GAAG,IAAI;CAClC;CACA,KAAK,MAAM,CAAC,SAAS,SAAS,YAC5B,IAAI,QAAQ,KAAK,IAAI,KAAK,CAAC,UAAU,QAAQ,MAAM,KAAA,CAAS,GAAG,OAAO,WAAW,MAAM,IAAI;AAG/F;AAaA,MAAM,cAA6B,MAAM,SAAS,aAChDA,OAAa,MAAM,SAAS,QAAQ;;AAGtC,MAAM,UACJ,QACA,MACA,cACsF;CACtF,MAAM,SAA0B,CAAC;CACjC,IAAI;CACJ,KAAK,MAAM,SAAS,WAAW;EAC7B,MAAM,OAAO,YAAY,MAAM,OAAO;EACtC,IAAI,SAAS,KAAA,KAAa,UAAU,QAAQ,MAAM,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK;OAC9E,YAAY,IAAI,cAAc,MAAM,MAAM,OAAO,GAAG,IAAI;CAC/D;CACA,OAAO;EACL;EACA,SAAS,OAAO,WAAW,IAAK,WAAW,IAAI,cAAc,MAAM,YAAY,IAAK,KAAA;CACtF;AACF;;;;;;;;AASA,MAAa,gBACV,QAAuB,UAAyB,gBAChD,MAAc,SAAwB,aAAa;CAClD,QAAQ,MAAM;EAAE,GAAG;EAAS,KAAK;CAAK,IAAI,OAAO,cAAc;EAC7D,IAAI,UAAU,MAAM;GAClB,SAAS,OAAO,CAAC,CAAC;GAClB;EACF;EACA,MAAM,EAAE,QAAQ,YACd,OAAO,iBAAiB,OACpB;GAAE,QAAQ;GAAW,SAAS,KAAA;EAAU,IACxC,OAAO,QAAQ,MAAM,SAAS;EACpC,IAAI,YAAY,KAAA,GAAW;GACzB,SAAS,SAAS,CAAC,CAAC;GACpB;EACF;EACA,IAAI,QAAQ,QAAQ,MAAM,SAAS,MAAM,MAAM;OAC1C,SAAS,MAAM,OAAO,EAAE,CAAE,SAAS,OAAO,EAAE,CAAE,MAAM;CAC3D,CAAC;AACH;;;;;;;;;;AAWF,MAAa,cAAc,OACzB,QACA,KACA,UAAyB,eACO;CAChC,IAAI,OAAO,iBAAiB,MAAM,OAAO,KAAA;CACzC,MAAM,SAAS,cAAc,QAAQ,IAAI,QAAQ;CACjD,IAAI,WAAW,KAAA,GAAW,OAAO;CACjC,MAAM,OAAO,OAAO,IAAI,QAAQ;CAChC,IAAI,UAAU,IAAI,MAAM,KAAA,KAAa,UAAU,QAAQ,MAAM,KAAA,CAAS,GAAG,OAAO,KAAA;CAChF,MAAM,YAAY,MAAM,IAAI,SAAkC,WAC5D,QAAQ,MAAM,EAAE,KAAK,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS,KAAK,CAAC,CACvE;CACA,IAAI,qBAAqB,OAAO,OAAO,KAAA;CACvC,KAAK,MAAM,SAAS,WAAW;EAC7B,MAAM,OAAO,YAAY,MAAM,OAAO;EACtC,IAAI,SAAS,KAAA,KAAa,CAAC,UAAU,QAAQ,MAAM,MAAM,OAAO,GAC9D,OAAO,WAAW,MAAM,MAAM,OAAO,GAAG,IAAI;CAChD;AAEF;;AAGA,MAAa,eAAe,UAA8C;CACxE,KACE,IAAI,UAAU,OAAO,QAAQ,GAC7B,YAAY,KAAA,KAAa,YAAY,QAAQ,QAAQ,GACrD,SAAS,GACT;EACA,IAAI,mBAAmB,eAAe,OAAO;EAC7C,IAAI,mBAAmB,gBAAgB;GACrC,MAAM,QAAQ,QAAQ,OAAO,MAAM,SAAkB,gBAAgB,aAAa;GAElF,IAAI,UAAU,KAAA,GAAW,OAAO;EAClC;EACA,UAAW,QAAgC;CAC7C;AAEF;;;;;;;ACtTA,MAAa,mBAAmB,UAA2B;CACzD,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO,KAAK,UAAU,KAAK,KAAK;CAElC,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,IAAI,MAAM,KAAK,SAAS,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE;CAMlE,OAAO,IAJS,OAAO,QAAQ,KAAgC,CAAC,CAC7D,QAAQ,GAAG,UAAU,SAAS,KAAA,CAAS,CAAC,CACxC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,CACtC,KAAK,CAAC,KAAK,UAAU,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG,gBAAgB,IAAI,GACrD,CAAC,CAAC,KAAK,GAAG,EAAE;AAC/B;;AAGA,MAAa,UAAU,SACrB,WAAW,QAAQ,CAAC,CAAC,OAAO,MAAM,MAAM,CAAC,CAAC,OAAO,KAAK;;;;;;;;AASxD,MAAa,cAAc,UAAsC;CAC/D,MAAM,eAAe,OACnB,gBAAgB;EACd,QAAQ,MAAM;EACd,MAAM,MAAM;EACZ,aAAa,MAAM,eAAe;EAClC,SAAS,MAAM;CACjB,CAAC,CACH;CACA,OAAO;EACL,IAAI,GAAG,MAAM,OAAO,GAAG,aAAa,MAAM,GAAG,EAAE;EAC/C,QAAQ,MAAM;EACd,MAAM,MAAM;EACZ,aAAa,MAAM;EACnB,SAAS,MAAM;EACf;EACA,GAAI,MAAM,gBAAgB,KAAA,IAAY,EAAE,aAAa,MAAM,YAAY,IAAI,CAAC;EAC5E,GAAI,MAAM,gBAAgB,KAAA,IAAY,EAAE,aAAa,MAAM,YAAY,IAAI,CAAC;EAC5E,GAAI,MAAM,UAAU,KAAA,IAAY,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;EAC1D,GAAI,MAAM,WAAW,KAAA,KAAa,OAAO,KAAK,MAAM,MAAM,CAAC,CAAC,SAAS,IACjE,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,EAAE,IAC9B,CAAC;CACP;AACF;;;;;;;;AC4OA,MAAa,gBAAgB;;;;;;AAe7B,MAAa,iBAAiB;;AAG9B,MAAa,gBAAgB;;;;;;;AAQ7B,MAAa,wBAA2C;CACtD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,MAAM,mBAAmB;AACzB,MAAM,eAAe;AACrB,MAAM,eAAe;AACrB,MAAM,gCAAqC,IAAI,IAAI,CAAC,SAAS,QAAQ,CAAC;AACtE,MAAM,cAAc;AACpB,MAAM,sCAA2C,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC;AACnE,MAAM,qCAA0C,IAAI,IAAI;CACtD;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;AACzD,CAAC;;AAED,MAAM,mCAAwC,IAAI,IAAI;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;AAED,MAAM,qCAA0C,IAAI,IAAI;CACtD;CACA;CACA;AACF,CAAC;;AAED,MAAM,mCAAwC,IAAI,IAAI;CACpD;CACA;CACA;CAEA;CACA;CACA;AACF,CAAC;;AAGD,MAAM,oCAAyC,IAAI,IAAI;CACrD;CACA;CACA;AACF,CAAC;;AAID,MAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;AACF;;;;;AAMA,MAAM,mCAAwC,IAAI,IAAI,CAAC,YAAY,aAAa,CAAC;;;;;;AAOjF,MAAM,eAAe,QAA4B,gBAA6C;CAC5F,MAAM,QAAQ,eAAe,GAAA,CAAI,MAAM,GAAG,CAAC,CAAC,EAAE,CAAE,KAAK,CAAC,CAAC,YAAY;CACnE,IACE,WAAW,KAAA,KACX,OAAO,KAAK,MAAM,MACjB,SAAS,eAAe,SAAS,yBAElC,OAAO;CACT,OAAO,CAAC,OAAO,MAAM,GAAG,CAAC,CAAC,MAAM,UAAU;EACxC,MAAM,CAAC,QAAQ,IAAI,GAAG,UAAU,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC;EACxF,MAAM,IAAI,OAAO,MAAM,UAAU,MAAM,WAAW,IAAI,CAAC;EACvD,IAAI,MAAM,KAAA,KAAa,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,GAAG,OAAO;EACxD,OACE,UAAU,SACV,UAAU,QACT,MAAM,SAAS,IAAI,KAAK,KAAK,WAAW,MAAM,MAAM,GAAG,EAAE,CAAC;CAE/D,CAAC;AACH;;AAGA,MAAM,eAAe,MAA8B,gBACjD,SAAS,KAAA,KAAa,CAAC,QAAQ,KAAK,eAAe,EAAE,IACjD,CAAC,IACD,YAAY,IAAI,YAAY,QAAQ,CAAC,CAAC,OAAO,KAAK,SAAS,GAAG,KAAM,CAAC,CAAC;;AAG5E,IAAM,QAAN,MAAY;CAImB;CAH7B,OAAe;CACf,UAA8C,CAAC;CAE/C,YAAY,KAA8B;EAAb,KAAA,MAAA;CAAc;;CAG3C,MAAM,QAAQ,QAA0C;EACtD,IAAI,KAAK,OAAO,KAAK,KAAK;GACxB,KAAK,QAAQ;GACb,OAAO,KAAK,SAAS;EACvB;EACA,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,aAAmB;IACvB,MAAM,KAAK,KAAK,QAAQ,QAAQ,EAAE;IAClC,IAAI,OAAO,IAAI,KAAK,QAAQ,OAAO,IAAI,CAAC;IACxC,OAAO,OAAO,kBAAkB,QAAQ,OAAO,yBAAS,IAAI,MAAM,WAAW,CAAC;GAChF;GACA,MAAM,WAAiB;IACrB,OAAO,oBAAoB,SAAS,IAAI;IACxC,QAAQ;GACV;GACA,IAAI,OAAO,SAAS;IAClB,KAAK;IACL;GACF;GACA,KAAK,QAAQ,KAAK,EAAE;GACpB,OAAO,iBAAiB,SAAS,MAAM,EAAE,MAAM,KAAK,CAAC;EACvD,CAAC;EAED,OAAO,KAAK,SAAS;CACvB;CAEA,WAA+B;EAC7B,IAAI,QAAQ;EACZ,aAAa;GACX,IAAI,OAAO;GACX,QAAQ;GACR,MAAM,OAAO,KAAK,QAAQ,MAAM;GAChC,IAAI,SAAS,KAAA,GAAW,KAAK;QACxB,KAAK,QAAQ;EACpB;CACF;AACF;;AAGA,MAAM,kBAAkB,YACtB,QAAQ,WAAA,iQAAwB,KAChC,4EAA4E,KAAK,OAAO;;AAG1F,MAAM,iBAAiB,QAAgB,OACrC,OAAO,gBAAgB,KAAA,IACnB,+BAA+B,OAC/B,iCAAiC,OAAO,YAAY;AAE1D,MAAMC,eAAa,UACjB,iBAAiB,QAAS,MAAM,QAAQ,MAAM,OAAO,CAAC,CAAC,MAAM,KAAM,OAAO,KAAK;AAEjF,MAAM,SAAS,OACb,OAAO,SAAS,EAAE,KAAK,KAAK,IAAI,EAAE,IAAI,eAAe,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY,IAAI,KAAA;AAEpF,MAAM,YAAY,QAAgB,UAA0B;CAC1D,MAAM,MAAM,MAAM,KAAK;CACvB,OAAO,QAAQ,KAAA,IAAY,iBAAiB,WAAW,sBAAsB,IAAI,IAAI;AACvF;;AAGA,IAAM,kBAAN,cAA8B,MAAM;CAGvB;CAFX,YAEE,SACA;EACA,MAAM,aAAa;EAFV,KAAA,UAAA;CAGX;AACF;AAEA,MAAM,gBAAgB,OAAwB,aAC5C,GAAG,cAAc,wBAAwB,MAAM,MAAM,OAAO,KAAK,eAAe,sCAAsC,MAAM,YAAY,GAAU,KAAK;AAEzJ,MAAM,aAAa,iBACjB,IAAI,IAAI,CAAC,GAAG,uBAAuB,GAAG,YAAY,CAAC,CAAC,KAAK,SAAS,KAAK,YAAY,CAAC,CAAC;AAEvF,MAAM,eAAe,QAAwB;CAC3C,IAAI;EACF,OAAO,mBAAmB,IAAI,QAAQ,OAAO,GAAG,CAAC,CAAC,CAAC,YAAY;CACjE,QAAQ;EACN,OAAO,IAAI,YAAY;CACzB;AACF;;;;;;AAOA,MAAa,UAAU,KAAU,eAAkC,CAAC,MAAc;CAChF,MAAM,OAAO,IAAI,IAAI,IAAI,IAAI;CAC7B,KAAK,WAAW;CAChB,KAAK,WAAW;CAChB,IAAI,KAAK,OAAO,SAAS,GAAG;EAC1B,MAAM,UAAU,UAAU,YAAY;EAUtC,KAAK,SAAS,IATA,KAAK,OAChB,MAAM,CAAC,CAAC,CACR,MAAM,GAAG,CAAC,CACV,KAAK,SAAS;GACb,MAAM,SAAS,KAAK,QAAQ,GAAG;GAC/B,IAAI,WAAW,IAAI,OAAO;GAC1B,MAAM,OAAO,KAAK,MAAM,GAAG,MAAM;GACjC,OAAO,QAAQ,IAAI,YAAY,IAAI,CAAC,IAAI,GAAG,KAAK,aAAa;EAC/D,CACoB,CAAC,CAAC,KAAK,GAAG;CAClC;CACA,OAAO,KAAK;AACd;;AAGA,MAAM,gBAAgB,KAAU,iBAA6C;CAC3E,IAAI,IAAI,aAAa,MAAM,IAAI,aAAa,IAAI,OAAO;CACvD,MAAM,UAAU,UAAU,YAAY;CACtC,OAAO,IAAI,OACR,MAAM,CAAC,CAAC,CACR,MAAM,GAAG,CAAC,CACV,MACE,SAAS,KAAK,SAAS,GAAG,KAAK,QAAQ,IAAI,YAAY,KAAK,MAAM,GAAG,KAAK,QAAQ,GAAG,CAAC,CAAC,CAAC,CAC3F;AACJ;;AAGA,MAAM,WAAW,QACf,GAAG,IAAI,SAAS,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE,IAAI,IAAI,SAAS,KAAK,KAAK,IAAI,IAAI;AAEpF,MAAM,aAAa,QAAqB,GAAG,IAAI,SAAS,IAAI,QAAQ,GAAG;AAEvE,MAAM,YACH,aACA,SAAqC;CACpC,MAAM,QAAQ,QAAQ,KAAK,YAAY;CACvC,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,KAAK,IAAI,IAAI;AACnD;AAEF,IAAM,MAAN,MAAgB;CAEe;CAD7B,sBAAuB,IAAI,IAAU;CACrC,YAAY,KAA8B;EAAb,KAAA,MAAA;CAAc;CAE3C,IAAI,KAAuB;EACzB,MAAM,QAAQ,KAAK,IAAI,IAAI,GAAG;EAC9B,IAAI,UAAU,KAAA,GAAW;GACvB,KAAK,IAAI,OAAO,GAAG;GACnB,KAAK,IAAI,IAAI,KAAK,KAAK;EACzB;EACA,OAAO;CACT;CAEA,KAAK,KAAuB;EAC1B,OAAO,KAAK,IAAI,IAAI,GAAG;CACzB;CAEA,IAAI,KAAQ,OAAgB;EAC1B,KAAK,IAAI,OAAO,GAAG;EACnB,KAAK,IAAI,IAAI,KAAK,KAAK;EACvB,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,KAAK,IAAI,OAAO,KAAK,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAU;CACjF;CAEA,OAAO,KAAc;EACnB,KAAK,IAAI,OAAO,GAAG;CACrB;CAEA,OAAY;EACV,OAAO,CAAC,GAAG,KAAK,IAAI,KAAK,CAAC;CAC5B;CAEA,IAAI,OAAe;EACjB,OAAO,KAAK,IAAI;CAClB;AACF;;AAGA,IAAM,YAAN,cAAwB,MAAM;CAEjB;CADX,YACE,MACA,OACA;EACA,MAAMA,YAAU,KAAK,GAAG,EAAE,MAAM,CAAC;EAHxB,KAAA,OAAA;CAIX;AACF;;AAGA,MAAM,mBAA8B;CAClC,IAAI;CACJ,MAAM,QAAmB,IAAI,UAAU;EACrC,UAAU,OAAe,WAAW,MAAM;GACxC,IAAI,UAAU,KAAA,GAAW;IAKvB,QAHE,MAAM,UAAU,MACf,MAAM,KAAM,QAAU,MACrB,MAAM,MAAO,IAAK,MAAM,MAAO,OAAO,IAC3B,cAAc,IAAI,iBAAiB;IAClD,MAAM,GAAG,SAAS,SAAiB,MAAM,KAAK,IAAI,CAAC;IACnD,MAAM,GAAG,UAAU,UAAU,MAAM,QAAQ,KAAK,CAAC;GACnD;GACA,MAAM,MAAM,aAAa,KAAK,CAAC;EACjC;EACA,MAAM,MAAM;GACV,IAAI,UAAU,KAAA,GAAW;IACvB,KAAK;IACL;GACF;GACA,MAAM,KAAK,aAAa,KAAK,CAAC;GAC9B,MAAM,IAAI;EACZ;EACA,QAAQ,OAAO,UAAU;GACvB,OAAO,QAAQ;GACf,SAAS,KAAK;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAEA,MAAM,cAAc,WAA8B;CAChD,IAAI,WAAW,UAAU,WAAW,UAAU,OAAO,aAAa;CAClE,IAAI,WAAW,MAAM,OAAO,uBAAuB;CACnD,IAAI,WAAW,WAAW,OAAO,WAAW;CAC5C,MAAM,IAAI,UAAU,0BAAU,IAAI,MAAM,iCAAiC,OAAO,EAAE,CAAC;AACrF;;AAGA,MAAM,cAAc,UAClB,OAAO,UAAU,YACjB,UAAU,QACV,UAAU,UACT,MAAM,SAAS,iBAAiB,0BAA0B,KAAKA,YAAU,KAAK,CAAC;;;;;;;AAQlF,MAAM,WAAW,OACf,MACA,UACA,aACgG;CAChG,MAAM,WAAW,YAAY,GAAA,CAC1B,MAAM,GAAG,CAAC,CACV,KAAK,WAAW,OAAO,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAC5C,QAAQ,WAAW,OAAO,SAAS,KAAK,CAAC,iBAAiB,IAAI,MAAM,CAAC,CAAC,CACtE,QAAQ;CAEX,KAAK,GAAG,eAAe,KAAA,CAAS;CAChC,IAAI;CACJ,IAAI;EACF,WAAW,QAAQ,IAAI,UAAU;CACnC,SAAS,OAAO;EACd,KAAK,QAAQ;EACb,MAAM;CACR;CAEA,MAAM,SAAS,WAAW,IAAI;CAC9B,IAAI,WAAW;CACf,IAAI,SAAmB;CACvB,IAAI,SAAS,SAAS,GAAG;EACvB,KAAK,GAAG,SAAS,UAAkB;GACjC,YAAY,MAAM;EACpB,CAAC;EACD,KAAK,MAAM,WAAW,UAAU;GAC9B,MAAM,SAAS;GACf,OAAO,GAAG,UAAU,UAAU,QAAQ,QAAQ,KAAK,CAAC;GACpD,SAAS,OAAO,KAAK,OAAO;EAC9B;CACF;CACA,MAAM,SAAmB,CAAC;CAC1B,IAAI,OAAO;CACX,IAAI,YAAY;CAChB,IAAI,MAAM;CACV,IAAI;EACF,WAAW,MAAM,SAAS,QAAiC;GACzD,MAAM,OAAO,WAAW;GACxB,IAAI,MAAM,SAAS,MAAM;IAEvB,IAAI,OAAO,GAAG,OAAO,KAAK,MAAM,SAAS,GAAG,IAAI,CAAC;IACjD,QAAQ,KAAK,IAAI,GAAG,IAAI;IACxB,YAAY;IACZ;GACF;GACA,OAAO,KAAK,KAAK;GACjB,QAAQ,MAAM;GACd,IAAI,WAAW,QAAQ;IACrB,YAAY;IACZ;GACF;EACF;CACF,SAAS,OAAO;EACd,MAAM,YAAY,KAAK,YAAY,QAAQ,SAAS,WAAW;EAE/D,IAAI,CAAC,aAAa,OAAO,KAAK,WAAW,KAAK,GAAG;GAC/C,MAAM;GACN,YAAY;EACd,OACE,MAAM,IAAI,UAAU,YAAY,cAAc,UAAU,KAAK;CAEjE,UAAU;EACR,KAAK,QAAQ;EACb,KAAK,MAAM,WAAW,UAAU,QAAQ,QAAQ;CAClD;CAEA,MAAM,QAAQ,IAAI,WAAW,IAAI;CACjC,IAAI,KAAK;CACT,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,IAAI,OAAO,EAAE;EACnB,MAAM,MAAM;CACd;CACA,OAAO;EAAE;EAAO;EAAW;CAAI;AACjC;AAEA,MAAM,cAAc,UAAmD;CACrE,MAAM,sBAAM,IAAI,IAAoB;CACpC,KAAK,MAAM,SAAS,SAAS,GAAA,CAAI,MAAM,GAAG,GAAG;EAC3C,MAAM,SAAS,KAAK,QAAQ,GAAG;EAC/B,MAAM,QAAQ,WAAW,KAAK,OAAO,KAAK,MAAM,GAAG,MAAM,EAAA,CAAG,KAAK,CAAC,CAAC,YAAY;EAC/E,IAAI,KAAK,WAAW,GAAG;EACvB,MAAM,MAAM,WAAW,KAAK,KAAK,KAAK,MAAM,SAAS,CAAC,CAAC,CAAC,KAAK;EAC7D,IAAI,IACF,MACA,IAAI,UAAU,KAAK,IAAI,WAAW,IAAG,KAAK,IAAI,SAAS,IAAG,IAAI,IAAI,MAAM,GAAG,EAAE,IAAI,GACnF;CACF;CACA,OAAO;AACT;AAEA,MAAM,eAAe;;;;;;;AAQrB,MAAM,cACJ,QACA,QACA,WACA,cACA,UACkB;CAClB,MAAM,UAAU,WAAW,OAAO,eAAe,CAAC;CAClD,IAAI,QAAQ,IAAI,UAAU,KAAK,QAAQ,IAAI,UAAU,GAAG,OAAO,MAAM,SAAS,KAAK;CACnF,MAAM,UAAU,OAAO,KAAK,CAAC,EAAE,KAAK,KAAK;CACzC,MAAM,MAAM,QAAQ,KAAK,OAAO,IAAI,OAAO,OAAO,IAAI,MAAO;CAC7D,MAAM,OAAO,cAAc,OAAO,MAAM,CAAC,KAAK;CAC9C,IAAI;CACJ,MAAM,SAAS,QAAQ,IAAI,SAAS;CACpC,MAAM,UAAU,OAAO,SAAS;CAChC,IAAI,WAAW,KAAA,KAAa,QAAQ,KAAK,MAAM,GAAG,WAAW,OAAO,MAAM,IAAI;MACzE,IAAI,YAAY,KAAA,GAAW;EAE9B,MAAM,KAAK,cAAc,OAAO;EAChC,WAAW,OAAO,KAAA,IAAY,IAAI,KAAK;CACzC,OAAO,IAAI,mBAAmB,IAAI,MAAM,GAAG;EACzC,MAAM,WAAW,cAAc,gBAAgB,KAAA,CAAS;EACxD,IAAI,aAAa,KAAA,KAAa,WAAW,MACvC,WAAW,KAAK,IAAI,QAAQ,OAAO,YAAY,EAAE;OAC9C,IAAI,oBAAoB,IAAI,MAAM,GAAG,WAAW;CACvD;CACA,IAAI,aAAa,KAAA,GAAW,OAAO;CACnC,OAAO,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,UAAU,YAAY,IAAI,GAAG,CAAC,KAAK;AACnF;;AAGA,MAAM,aACJ,MACA,SACkB;CAClB,IAAI,SAAS,KAAA,GAAW,OAAO;CAC/B,MAAM,QAAQ,KACX,MAAM,GAAG,CAAC,CACV,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CACxC,QAAQ,SAAS,KAAK,SAAS,KAAK,CAAC,iBAAiB,IAAI,IAAI,CAAC;CAClE,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO;CAChC,IAAI,MAAM,WAAW,GAAG,OAAO;CAC/B,OAAO,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,CAAC,CACvB,KAAK,CAAC,CACN,KAAK,SAAS,GAAG,KAAK,GAAG,KAAK,SAAS,IAAI,CAAC,CAC5C,KAAK,IAAI;AACd;AAEA,MAAM,eACJ,QACA,SACY;CACZ,IAAI,WAAW,KAAA,KAAa,WAAW,MAAM,OAAO;CACpD,IAAI,WAAW,KAAK,OAAO;CAC3B,OAAO,OAAO,MAAM,IAAI,CAAC,CAAC,OAAO,SAAS;EACxC,MAAM,SAAS,KAAK,QAAQ,GAAG;EAC/B,QAAQ,KAAK,KAAK,MAAM,GAAG,MAAM,MAAM,QAAQ,KAAK,MAAM,SAAS,CAAC;CACtE,CAAC;AACH;AAEA,MAAM,sBACJ,YAEA,YAAY,KAAA,IACR,KAAA,IACA,OAAO,YACL,OAAO,QAAQ,OAAO,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,IAAI,KAAK,YAAY,CAAC,CAAC,CACxF;AAEN,MAAM,iBACJ,MACA,OACA,SACuB;CACvB,IAAI,UAAU,KAAA,KAAc,QAAQ,UAAU,QAAS,OAAO,KAAA;CAC9D,OAAO,OAAO,UAAU,YAAY,SAAS,IACzC,KAAA,IACA,mBAAmB,KAAK,MAAM,OAAO,KAAK;AAChD;AAEA,MAAM,gBAAgB,SAA0B;CAC9C;CACA,QAAQ;CACR,SAAS;CACT,WAAW;CACX,SAAS,KAAA;CACT,MAAM,KAAA;CACN,MAAM,KAAA;CACN,aAAa,KAAA;CACb,SAAS;CACT,MAAM;AACR;AAgGA,MAAM,eACJ,SACA,QACA,cAC6B;CAC7B,SAAS,QAAQ,WAAW;CAC5B;CACA,MACE,SAAS,SAAS,KAAA,IACd,OACA;EACE,MAAM,SAAS,KAAK,QAAQ;EAC5B,SAAS,SAAS,KAAK;EACvB,OAAO,SAAS,KAAK;CACvB;CACN,YAAY,SAAS,SAAS,EAAE,EAAE,QAAQ;CAC1C,GAAI,QAAQ,OAAO,gBAAgB,KAAA,IAAY,EAAE,aAAa,QAAQ,OAAO,YAAY,IAAI,CAAC;CAC9F,GAAI,QAAQ,UAAU,KAAA,IAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAChE;AAEA,MAAM,aAAgB,MAAkB,WAAoC;CAC1E,IAAI,OAAO,SAAS;EAClB,KAAK,YAAY,KAAA,CAAS;EAC1B,OAAO,QAAQ,OAAO,OAAO,MAAM;CACrC;CACA,OAAO,IAAI,SAAY,SAAS,WAAW;EACzC,MAAM,eAAqB,OAAO,OAAO,MAAM;EAC/C,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;EACvD,KAAK,MACF,UAAU;GACT,OAAO,oBAAoB,SAAS,MAAM;GAC1C,QAAQ,KAAK;EACf,IACC,UAAmB;GAClB,OAAO,oBAAoB,SAAS,MAAM;GAC1C,OAAO,KAAK;EACd,CACF;CACF,CAAC;AACH;;AAUA,IAAa,UAAb,MAAqB;CA+BU;CA9B7B;CACA;CACA;CACA;CACA;CACA;CACA,SAA0B,IAAI,IAAyB,IAAI;;CAE3D,OAAwB,IAAI,IAG1B,GAAM;CACR,2BAA4B,IAAI,IAA2B;CAC3D,0BAA2B,IAAI,IAAsB;CACrD,UAA2B,IAAI,gBAAgB;CAC/C,SAAiB;CACjB;;CAEA;;CAEA;;CAEA;;CAEA,4BAA6B,IAAI,QAA6B;;CAG9D;CAGA,YAAY,SAA0C;EAAzB,KAAA,UAAA;EAE3B,KAAK,SAAS,SAAS,QAAQ,MAAM;EACrC,IAAI,CAAC,eAAe,QAAQ,KAAK,GAC/B,MAAM,IAAI,MACR,cAAc,QAAQ,MAAM,mEAC9B;EAEF,KAAK,MAAM,CAAC,MAAM,UAAU,CAC1B,CAAC,aAAa,QAAQ,SAAS,GAC/B,CAAC,QAAQ,QAAQ,IAAI,CACvB,GACE,IAAI,UAAU,KAAA,KAAa,CAAC,YAAY,KAAK,KAAK,GAChD,MAAM,IAAI,MACR,GAAG,KAAK,IAAI,MAAM,uEACpB;EAGJ,YAAY,QAAQ,UAAU;EAC9B,KAAK,QAAQ,QAAQ,SAAS;EAC9B,KAAK,SAAS,QAAQ,UAAU,KAAK;EACrC,KAAK,SAAS,QAAQ,UAAU,CAAC;EAGjC,MAAM,QAAQ,KAAK;EACnB,MAAM,YAAY,KAAK;EACvB,KAAK,YAAY,IAAI,cACnB;GACE,WAAW,MAAM,IAAI;GACrB,QAAQ,IAAI,WAAW;IACrB,MAAM,WAAW,WAAW,KAAA,IAAY,KAAA,IAAY,UAAU,IAAI,MAAM;IACxE,MAAM,UAAU,MAAM,IAAI,IAAI;IAC9B,OAAO,aAAa,KAAA,KAAa,WAAW,WACxC,QAAQ,OAAO,IAAI,gBAAgB,OAAO,CAAC,IAC3C,MAAM,MAAM,IAAI,MAAM;GAC5B;EACF,GACA,QAAQ,aAAa,IACrB,EAAE,OAAO,QAAQ,UAAU,CAC7B;EAEA,KAAK,aACH,QAAQ,qBAAqB,SAC7B,QAAQ,MAAM,YAAY,MAAA,gBAAoB,YAAY,KAC1D,QAAQ,UAAU,SAAA,8CAAiB,IAC/B,CAAC,aAAa,IACd,CAAC;EACP,MAAM,aAAa,QAAQ,cAAc;EACzC,IAAI,CAAC,OAAO,UAAU,UAAU,KAAK,aAAa,GAChD,MAAM,IAAI,WAAW,mDAAmD,OAAO,UAAU,GAAG;EAE9F,KAAK,UAAU,IAAI,MAAM,UAAU;EACnC,KAAK,iBAAiB,QAAQ,eAAe,KAAA;EAC7C,KAAK,aACH,QAAQ,cACR,IAAI,MAAM;GAER,kBAAkB;GAClB,qBAAqB;GACrB,SAAS;GAET,SAAS;IAAE,SAAS;IAAQ,QAAQ,aAAa,KAAK,MAAM;GAAE;GAC9D,gBAAgB;GAChB,aAAa;EACf,CAAC;CACL;CAEA,MAAsB;EACpB,OAAO,IAAI,KAAK,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,YAAY;CAChD;CAEA,SAAiB,KAA4B;EAC3C,OAAO,UAAU,KAAK,QAAQ,YAAY,IAAI,QAAQ;CACxD;;;;;CAMA,MAAc,QAAqB,UAA2C;EAC5E,IAAI,aAAa,KAAA,GAAW,OAAO;EACnC,MAAM,MAAM,YAAY,IAAI,CAAC,MAAM,CAAC;EACpC,KAAK,UAAU,IAAI,KAAK,QAAQ;EAChC,OAAO;CACT;;CAGA,eAAuB,UAAoC;EACzD,MAAM,MAAM,KAAK,MAAM,IAAI;EAC3B,IAAI,aAAa,KAAA,KAAa,OAAO,UAAU,MAAM,IAAI,gBAAgB,GAAG;CAC9E;CAEA,KAAa,OAAyB;EACpC,IAAI;GACF,KAAK,QAAQ,UAAU,KAAK;EAC9B,QAAQ,CAER;CACF;CAEA,MAAiB,MAA8B;EAC7C,KAAK,QAAQ,IAAI,IAAI;EACrB,KAAU,WACF,KAAK,QAAQ,OAAO,IAAI,SACxB,KAAK,QAAQ,OAAO,IAAI,CAChC;EACA,OAAO;CACT;;CAGA,OAAe,MAAyC;EACtD,IAAI;GACF,OAAO,KAAK,UAAU,MAAM,aAAa,IAAI;EAC/C,QAAQ;GACN;EACF;CACF;CAEA,SAAiB,MAAc,OAAe,QAAsB;EAClE,KAAK,UAAU,MAAM,KAAK,MAAM,OAAO,MAAM;EAC7C,KAAK,KAAK;GAAE,MAAM;GAAQ;GAAM;GAAO;EAAO,CAAC;CACjD;CAEA,MAAc,OACZ,MACA,SACA,KACA,SAGI,CAAC,GACwB;EAE7B,IAAI,KAAK,QAAQ,OAAO,KAAA;EACxB,MAAM,KAAK,KAAK,IAAI;EACpB,MAAM,QAAQ,OAAO,SAAS,KAAK,QAAQ;EAC3C,MAAM,SAAS,WAAW;GACxB,QAAQ;GACR;GACA,SAAS;IAAE,QAAA;IAAwB,GAAG;IAAS,YAAY;GAAG;GAC9D,aAAa;GACb,aAAa;GACb,aAAa;GACb,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;GACvC,GAAI,OAAO,WAAW,KAAA,IAAY,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;EACjE,CAAC;EACD,MAAM,KAAK,QAAQ,IAAI,OAAO,MAAM;EACpC,OAAO,OAAO;CAChB;CAEA,MAAc,KACZ,MACA,KACA,MACA,QAAiC,CAAC,GAClC,SAGI,CAAC,GACa;EAClB,KAAK,KAAK;GAAE,MAAM;GAAQ;GAAK;EAAK,CAAC;EACrC,OAAO;GACL;GACA,QAAQ;GACR,SAAS;GACT,WAAW;GACX,SAAS,MAAM,KAAK,OAAO,WAAW;IAAE,GAAG;IAAM;IAAK;IAAM,GAAG;GAAM,GAAG,KAAK,MAAM;GACnF,MAAM,KAAA;GACN,MAAM,KAAA;GACN,aAAa,KAAA;GACb,SAAS;GACT;EACF;CACF;;CAGA,OAAe,KAAa,UAAmB,SAAiC;EAC9E,MAAM,MAAM,WAAW,GAAG;EAC1B,MAAM,MAAM,KAAK,QAAQ,MAAM,OAAO,KAAK,OAAO;EAClD,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;EAC9B,MAAM,OAAO,KAAK,KAAK,IAAI,GAAG,QAAQ,GAAG,KAAK;EAC9C,IAAI,SAAS,KAAA,KAAa,KAAK,MAAM,YAAY,IAAI,SAAS,OAAO;EACrE,MAAM,OAAO,OAAO,YAAY,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,WAAW,UAAU,KAAA,CAAS,CAAC;EAC9F,OAAO;GAAE,GAAG,KAAK;GAAO,GAAG;EAAK;CAClC;CAEA,MAAc,OAAmB,UAAyB;EACxD,MAAM,UAAU,MAAM,WAAW;EACjC,MAAM,QAAQ;GAAE,GAAG;GAAO,KAAK,WAAW,MAAM,GAAG;GAAG;EAAQ;EAC9D,KAAK,QAAQ,MAAM,UAAU,KAAK;EAClC,KAAK,KAAK,IAAI,GAAG,QAAQ,GAAG,MAAM,OAAO;GACvC,OAAO;GACP,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;EAC/C,CAAC;CACH;;CAGA,WAAmB,OAAmB,cAAqD;EACzF,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,MAAM,WAAW,QAAQ,GAAG,WAAW,MAAM,GAAG,GAAG;EAClF,IAAI,MAAM,aAAa,KAAA,KAAa,KAAK,MAAM,YAAY,MAAM,SAAS,OAAO,KAAK;EACtF,IAAI,MAAM,aAAa,KAAA,KAAa,MAAM,aAAa,MAAM,OAAO,KAAA;EACpE,IAAI;GACF,MAAM,SAAS,IAAI,IAAI,MAAM,QAAQ;GAErC,OAAO,aAAa,QAAQ,YAAY,IAAI,KAAA,IAAY,OAAO;EACjE,QAAQ;GACN;EACF;CACF;;CAGA,MAAc,UACZ,KACA,SACA,QACA,MACA,cAC2B;EAC3B,IAAI,WAAW,KAAA,GAAW,OAAO,KAAA;EACjC,MAAM,QAAQ,KAAK,OAAO,KAAK,OAAO;EACtC,IAAI,UAAU,KAAA,KAAa,CAAC,YAAY,MAAM,UAAU,IAAI,GAAG,OAAO,KAAA;EACtE,MAAM,MAAM,KAAK,MAAM,IAAI;EAC3B,MAAM,YAAY,KAAK,MAAM,MAAM,UAAU;EAC7C,IAAI,CAAC,OAAO,SAAS,SAAS,GAAG,OAAO,KAAA;EACxC,MAAM,MAAM,MAAM;EAClB,MAAM,QACJ,MAAM,gBAAgB,KAAA,KAAa,MAAM,gBAAgB,OACrD,MACA,KAAK,MAAM,MAAM,WAAW;EAClC,MAAM,QAAQ,OAAO,SAAS,KAAK,KAAK,QAAQ;EAChD,MAAM,WAAW,MAAM,UAAU,OAAO,MAAM,SAAS;EAQvD,IAAI,EALF,WAAW,SACP,QACA,YAAY,CAAC,oBAAoB,IAAI,MAAM,MAAM,IAC/C,SAAS,MAAM,SACf,MAAM,SACF,OAAO,KAAA;EACnB,IAAI,UAAU;GACZ,MAAM,OAAO,KAAK,WAAW,OAAO,YAAY;GAChD,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;GAC/B,OAAO;IACL,SAAS;KACP;KACA,QAAQ,MAAM;KACd,SAAS;KACT,WAAW;KACX,SAAS,MAAM;KACf,MAAM,KAAA;KACN,MAAM,KAAA;KACN,aAAa,KAAA;KACb,SAAS;KACT,MAAM,YAAY,MAAM,YAAY,OAAO,IAAI,IAAI,IAAI,GAAG,YAAY;KACtE,WAAW,MAAM;IACnB;IACA;GACF;EACF;EACA,IAAI,MAAM,SAAS,QAAQ,MAAM,SAAS,OAAO,MAAM,UAAU,KAAK,OAAO,KAAA;EAC7E,MAAM,OAAO,MAAM,KAAK,QAAQ,MAAM,IAAI,MAAM,IAAI;EACpD,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;EAC/B,OAAO,EACL,SAAS;GACP;GACA,QAAQ,MAAM;GACd,SAAS;GACT,WAAW;GACX,SAAS,MAAM;GACf,MAAM,MAAM;GACZ;GACA,aAAa,MAAM,gBAAgB,KAAA;GACnC,SAAS;IAAE,UAAU,MAAM,YAAY;IAAS,KAAK,CAAC;GAAE;GACxD,MAAM,MAAM,cAAc,OAAO,qBAAqB,KAAA;GACtD,GAAI,MAAM,cAAc,OAAO,EAAE,WAAW,KAAK,IAAI,CAAC;GACtD,WAAW,MAAM;EACnB,EACF;CACF;;;;;;CAQA,QAAgB,QAMF;EACZ,IAAI,KAAK,WAAW,KAAA,GAAW,OAAO,KAAA;EACtC,IAAI;EACJ,IAAI;GACF,QAAQ,KAAK,OAAO,KAAK,QAAQ,KAAK,MAAM,IAAI,CAAC;EACnD,SAAS,OAAO;GACd,OAAO,EAAE,OAAO,wCAAwCA,YAAU,KAAK,IAAI;EAC7E;EAEA,MAAM,UAAkC,CAAC;EACzC,KAAK,MAAM,CAAC,MAAM,UAAU,OAAO,QAAQ,KAAK,GAAG;GACjD,MAAM,QAAQ,KAAK,YAAY;GAC/B,IAAI,iBAAiB,IAAI,KAAK,GAAG;GACjC,IAAI,OAAO,UAAU,YAAY,CAAC,YAAY,KAAK,KAAK,GACtD,OAAO,EAAE,OAAO,qBAAqB,MAAM,+BAA+B;GAC5E,QAAQ,SAAS;EACnB;EACA,OAAO;GACL;GACA,OAAO,EAAE,QAAQ;IAAE,QAAQ,KAAK,OAAO;IAAO,OAAO,KAAK,OAAO;GAAe,EAAE;EACpF;CACF;CAGA,eAAuB,UAAgD;EACrE,MAAM,UAAkC,CAAC;EAEzC,KAAK,MAAM,CAAC,MAAM,UAAU,OAAO,QAAQ,SAAS,WAAW,CAAC,CAAC,GAAG;GAClE,MAAM,QAAQ,KAAK,YAAY;GAC/B,IAAI,iBAAiB,IAAI,KAAK,GAAG;GAEjC,IAAI,KAAK,WAAW,KAAA,KAAa,kBAAkB,IAAI,KAAK,GAAG;GAC/D,QAAQ,SAAS;EACnB;EACA,QAAQ,YAAY,SAAS,UAAU,QAAQ,aAAa;EAC5D,QAAQ,gBAAgB,KAAK,QAAQ;EACrC,QAAQ,qBAAqB;EAC7B,IAAI,KAAK,QAAQ,SAAS,KAAA,GAAW,QAAQ,UAAU,KAAK,QAAQ;EACpE,OAAO;CACT;;;;;;CAOA,OAAe,MAAc,QAA0B,MAAsB;EAC3E,MAAM,QAAQ,KAAK,UAAU;EAC7B,MAAM,MAAM,KAAK,MAAM,IAAI;EAE3B,MAAM,UADO,UAAU,QAAQ,KAAK,IACjB,CAAC,CAAC;EACrB,MAAM,UAAU,IAAY,WAA2B;GACrD,IAAI,KAAK,SAAS,KAAK,KAAK;IAAE,MAAM;IAAQ;IAAM;IAAI;GAAO,CAAC;GAC9D,OAAO;EACT;EACA,IAAI,KAAK,WAAW,KAAA,GAAW;GAC7B,MAAM,OAAO,MAAM,KAAK,MAAM;GAC9B,OAAO;EACT;EACA,MAAM,UAAU,KAAK;EACrB,IAAI,YAAY,kBAAkB,YAAY,eAAe;GAC3D,MAAM,EAAE,UAAU,MAAM,OAAO,MAAM,MAAM;GAC3C,MAAM,QAAQ,KAAK;GACnB,IAAI,UAAU,KAAA,KAAa,QAAQ,OAAO,iBAAiB;IAEzD,KAAK,SACH,MACA,MAAM,KAAK,IAAI,OAAO,WAAW,GACjC,iBAAiB,OAAO,KAAK,MAAM,QAAQ,GAAI,CAAC,EAAE,GACpD;IACA,OAAO;GACT;GACA,OAAO,OACL,KAAK,IACH,SACA,SAAS,cAAc,OAAO,OAAO,eAAe,OAAO,cAAc,KAAK,MAAM,CACtF,GACA,UAAU,KAAA,IAAY,YAAY,aACpC;EACF;EACA,IAAI,YAAY,SAAS;GACvB,IAAI,cAAc,KAAK,MAAM,GAAG;IAC9B,MAAM,OAAO,MAAM,SAAS;IAC5B,OAAO;GACT;GACA,MAAM,EAAE,aAAa,MAAM,OAAO,MAAM,SAAS;GACjD,IAAI,YAAY,OAAO,aACrB,KAAK,SAAS,MAAM,MAAM,OAAO,eAAe,GAAG,OAAO,QAAQ,EAAE,mBAAmB;GACzF,OAAO,OACL,KAAK,IACH,SACA,cAAc,UAAU,OAAO,eAAe,OAAO,cAAc,KAAK,MAAM,CAChF,GACA,SACF;EACF;EACA,IAAI,YAAY,aAAa,KAAK,SAAS,MAAM,MAAM,OAAO,iBAAiB,OAAO;OACjF,IAAI,YAAY,oBACnB,KAAK,SAAS,MAAM,MAAM,OAAO,eAAe,OAAO;OACpD,IAAI,YAAY,aAAa,CAAC,KAAK,YACtC,KAAK,SAAS,MAAM,MAAM,OAAO,eAAe,OAAO;EACzD,MAAM,OAAO,MAAM,SAAS;EAC5B,OAAO;CACT;;;;;CAMA,cAAc,KAAa,MAAoB;EAC7C,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,GAAG;EACtB,QAAQ;GACN;EACF;EACA,MAAM,SAAS,KAAK,SAAS,MAAM;EACnC,KAAK,SACH,QAAQ,MAAM,GACd,KAAK,MAAM,IAAI,IAAI,OAAO,iBAC1B,cAAc,KAAK,EACrB;CACF;;;;;;;CAQA,MAAc,MACZ,MACA,OAGsE;EACtE,MAAM,QAAQ;GAAE,OAAO,KAAK;GAAO,QAAQ,KAAK;EAAO;EACvD,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,KAAK,GAAG;EAC3B,QAAQ;GACN,OAAO,EACL,QAAQ,EAAE,SAAS,MAAM,KAAK,KAAK,KAAK,MAAM,KAAK,KAAK,mBAAmB,CAAC,GAAG,KAAK,EAAE,EACxF;EACF;EACA,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GACpC,OAAO,EACL,QAAQ,EACN,SAAS,MAAM,KAAK,KAClB,KAAK,MACL,OAAO,QAAQ,KAAK,YAAY,GAChC,sBACA,CAAC,GACD,KACF,EACF,EACF;EAGF,OAAO,OAAO;EACd,MAAM,MAAM,OAAO,QAAQ,KAAK,YAAY;EAC5C,MAAM,QAAQ,MAAc,WAC1B,KAAK,KAAK,KAAK,MAAM,KAAK,MAAM,QAAQ,KAAK;EAC/C,MAAM,SAAS,OAAO,MAAc,YAAsC,EACxE,QAAQ,EAAE,SAAS,MAAM,KAAK,MAAM,MAAM,EAAE,EAC9C;EACA,IAAI,KAAK,YAAY,KAAA,GAAW,OAAO,OAAO,KAAK,OAAO;EAC1D,MAAM,UAAU,cAAc,KAAK,QAAQ,OAAO,QAAQ;EAC1D,IAAI,YAAY,KAAA,GAAW,OAAO,OAAO,OAAO;EAChD,MAAM,SAAS;GAAC,KAAK,QAAQ;GAAO,GAAI,KAAK,cAAc,CAAC;GAAI,GAAG,KAAK;EAAU;EAClF,MAAM,WAAW,OAAO,MAAM,UAAU,CAAC,eAAe,KAAK,CAAC;EAC9D,IAAI,KAAK,SAAS,UAAU,aAAa,KAAA,GACvC,OAAO,OAAO,+BAA+B,SAAS,EAAE;EAC1D,MAAM,OAAO,QAAQ,MAAM;EAC3B,MAAM,SAAS,KAAK,SAAS,MAAM;EACnC,MAAM,OAAO,GAAG,OAAO,WAAW,OAAO;EAGzC,IAAI;EACJ,IAAI,KAAK,SAAS,QAAQ;GACxB,UACE,OAAO,SAAS,IAAI,MAAM,KAAK,UAAU,QAAQ,QAAQ,IAAI,IAAI,KAAK,YAAY,MAAM;GAC1F,IAAI,YAAY,KAAA,GAAW;IACzB,MAAM,WAAW,OAAO,QAAQ,QAAQ,QAAQ,IAAI;IACpD,IAAI,CAAC,SAAS,SACZ,OAAO,OAAO,cAAc,QAAQ,QAAQ,SAAS,MAAM,KAAK,QAAQ,KAAK,GAAG,EAC9E,cAAc,YAAY,SAAS,QAAQ,QAAQ,EACrD,CAAC;GAEL;EACF;EACA,MAAM,SAAS,MAAM,KAAK,OAAO,GAAG;EACpC,IAAI,WAAW,KAAA,GAAW,OAAO,EAAE,QAAQ,OAAO;EAElD,MAAM,OAAO,KAAK,UAAU,OAAO,IAAI;EACvC,IAAI,SAAS,KAAA,GAAW,OAAO,OAAO,SAAS,KAAK,QAAQ,KAAK,KAAK,CAAC;EACvE,MAAM,QAAQ,MAAM,QAAQ;GAAE;GAAQ;GAAK;GAAM;GAAQ;GAAQ;EAAK,CAAC;EACvE,IAAI,UAAU,KAAA,GAAW,OAAO,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE;EAE7D,IAAI,cAAiC,CAAC;EACtC,IAAI,oBAAoB;EACxB,IAAI;EACJ,IAAI,KAAK,SAAS,QAAQ;GACxB,YAAY,MAAM,KAAK,UAAU,QAAQ,QAAQ,IAAI;GACrD,MAAM,WAAW,OAAO,QAAQ,QAAQ,QAAQ,IAAI;GACpD,QAAQ,YAAY,SAAS,QAAQ,QAAQ;GAC7C,IAAI,CAAC,SAAS,SACZ,OAAO,OAAO,cAAc,QAAQ,QAAQ,SAAS,MAAM,KAAK,QAAQ,KAAK,GAAG,EAC9E,cAAc,MAChB,CAAC;GAEH,cAAc,WAAW,QAAQ,QAAQ,QAAQ,IAAI;GACrD,oBAAoB,SAAS,qBAAqB;EACpD;EACA,IAAI,KAAK,QAAQ,OAAO,EAAE,QAAQ,EAAE,SAAS,aAAa,GAAG,EAAE,EAAE;EACjE,OAAO,EACL,UAAU;GACR;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;EACF,EACF;CACF;;CAGA,MAAM,MAAM,UAA0C;EACpD,IAAI,KAAK,QAAQ,OAAO,aAAa,SAAS,GAAG;EACjD,OAAO,KAAK,MAAM,KAAK,OAAO,QAAQ,CAAC;CACzC;CAEA,MAAc,OAAO,UAA0C;EAC7D,MAAM,SACJ,SAAS,WAAW,KAAA,IAChB,KAAK,QAAQ,SACb,YAAY,IAAI,CAAC,SAAS,QAAQ,KAAK,QAAQ,MAAM,CAAC;EAC5D,IAAI,MAAM,SAAS;EACnB,IAAI,UAAU,SAAS;EACvB,IAAI;EACJ,IAAI,eAAe,eAAe;EAClC,IAAI;GACF,MAAM,QAAQ,IAAI,IAAI,GAAG;GACzB,SAAS,MAAM;GACf,eAAe,KAAK,SAAS,KAAK,CAAC,CAAC;EACtC,QAAQ,CAER;EACA,IAAI;GACF,KAAK,IAAI,MAAM,IAAK,OAAO,GAAG;IAC5B,MAAM,OAAO,MAAM,KAAK,IAAI;KAAE,GAAG;KAAU;KAAK;IAAQ,GAAG,MAAM;IACjE,IAAI,KAAK,SAAS,KAAA,GAAW,OAAO,KAAK;IACzC,IAAI,OAAO,cACT,OAAO;KACL,GAAG,KAAK;KACR,SAAS;KACT,MAAM,aAAa,OAAO,YAAY,EAAE;IAC1C;IAEF,MAAM,OAAO,IAAI,IAAI,KAAK,IAAI;IAE9B,IAAI,KAAK,WAAW,QAAQ,UAAU,mBAAmB,OAAO;IAChE,SAAS,KAAK;IACd,MAAM,KAAK;GACb;EACF,SAAS,OAAO;GACd,IAAI,KAAK,QAAQ,OAAO,aAAa,SAAS,GAAG;GACjD,OAAO;IACL,GAAG,aAAa,SAAS,GAAG;IAC5B,SAAS;IACT,MAAM,kBAAkBA,YAAU,KAAK;GACzC;EACF;CACF;;CAGA,IAAY,UAAwB,QAAoC;EAEtE,IAAI,SAAS,WAAW,KAAA,GAAW,OAAO,KAAK,UAAU,UAAU,MAAM;EACzE,MAAM,MAAM,KAAK,UAAU;GACzB,WAAW,SAAS,IAAI,MAAM,GAAG,CAAC,CAAC,MAAM,SAAS,GAAG;GACrD,SAAS;GACT,SAAS;GACT,SAAS,UAAU;GACnB,SAAS,WAAW;GACpB,SAAS,gBAAgB;GACzB,SAAS,iBAAiB;GAC1B,SAAS,YAAY;GACrB,SAAS,YAAY;GACrB,SAAS,cAAc;GACvB,SAAS,OAAO;GAChB,SAAS,SAAS;GAClB,SAAS,UAAU;GACnB,SAAS,YAAY;GAErB,SAAS,SAAS;GAClB,SAAS,eAAe;GACxB,SAAS,wBAAwB;EACnC,CAAC;EACD,MAAM,UAAU,KAAK,SAAS,IAAI,GAAG;EACrC,IAAI,YAAY,KAAA,GAAW,OAAO;EAClC,MAAM,OAAO,KAAK,UAAU,UAAU,MAAM,CAAC,CAAC,cAAc,KAAK,SAAS,OAAO,GAAG,CAAC;EACrF,KAAK,SAAS,IAAI,KAAK,IAAI;EAC3B,OAAO;CACT;CAEA,YAAoB,QAAwC;EAC1D,MAAM,QAAQ,KAAK,OAAO,KAAK,UAAU,MAAM,CAAC;EAChD,OAAO,UAAU,KAAA,KAAa,KAAK,MAAM,IAAI,IAAI,MAAM,KAAK,MAAM,QAC9D,MAAM,QACN,KAAA;CACN;;;;;;CAOA,UACE,QACA,QACA,QAGI,CAAC,GACmB;EACxB,MAAM,MAAM,UAAU,MAAM;EAC5B,MAAM,MAAM,KAAK,MAAM,IAAI;EAC3B,MAAM,SAAS,KAAK,OAAO,IAAI,GAAG;EAClC,IAAI,WAAW,KAAA,KAAa,MAAM,OAAO,KAAK,OAAO,OAAO;GAE1D,IAAI,OAAO,UAAU,KAAA,GAAW,KAAK,KAAK,OAAO,SAAS,KAAK;GAC/D,OAAO,OAAO;EAChB;EACA,MAAM,UAAmB;GAAE,OAAO,KAAA;GAAW,QAAQ,KAAA;EAAU;EAC/D,KAAK,KAAK,SAAS,KAAK;EACxB,MAAM,QAAqB;GACzB,IAAI;GACJ;GACA,OAAO,OAAO;GACd,SAAS,QAAQ,QAAQ;IAAE,QAAQ,kBAAkB,cAAc;IAAG,SAAS,KAAA;GAAU,CAAC;GAC1F,OAAO,KAAA;GACP,QAAQ,QAAQ,UAAU;GAC1B,MAAM,QAAQ;EAChB;EACA,MAAM,UAAU,KAAK,YAAY,QAAQ,QAAQ,OAAO,CAAC,CACtD,OAAO,WAAoB;GAC1B,QAAQ,kBAAkB,sBAAsBA,YAAU,KAAK,GAAG;GAClE,SAAS,KAAA;GACT,cAAc,KAAA;EAChB,EAAE,CAAC,CACF,KAAK,OAAO,SAAiC;GAC5C,IAAI,UAAyB;IAAE,QAAQ,KAAK;IAAQ,SAAS,KAAK;GAAQ;GAC1E,IAAI,KAAK,OAAO,gBAAgB,KAAA,GAAW;IACzC,MAAM,SAAS;IACf,MAAM,OAAO;KAAE;KAAS,IAAI,KAAK,MAAM,IAAI;IAAE;GAC/C,OAAO;IACL,MAAM,UAAU;IAChB,MAAM,UAAU,cACd,MAAM,QACN,OAAO,eACP,OAAO,kBACP,KAAK,MACP;IACA,MAAM,QAAQ,KAAK,IACjB,OAAO,kBACP,KAAK,IAAI,KAAK,gBAAgB,GAAG,OAAO,CAC1C;IACA,MAAM,OACJ,MAAM,SAAS,KAAA,KAAa,KAAK,MAAM,IAAI,IAAI,MAAM,KAAK,MAAM,OAAO,gBACnE,MAAM,KAAK,UACX,MAAM,KAAK,aAAa,QAAQ,MAAM;IAC5C,IAAI,SAAS,KAAA,GAAW,UAAU;KAAE,GAAG;KAAM,OAAO,KAAK,OAAO;IAAY;GAC9E;GACA,MAAM,QAAQ;GACd,OAAO;EACT,CAAC;EACH,KAAK,OAAO,IAAI,KAAK,KAAK;EAC1B,OAAO,MAAM;CACf;;;;;;CAOA,MAAM,WACJ,KACA,QAGI,CAAC,GAKJ;EACD,MAAM,SAAS,IAAI,IAAI,GAAG;EAC1B,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GACpC,OAAO;GAAE,QAAQ,kBAAkB,oBAAoB;GAAG,SAAS,KAAA;EAAU;EAC/E,OAAO,KAAK,UAAU,QAAQ,KAAK,SAAS,MAAM,GAAG,KAAK;CAC5D;;;;;;;CAQA,MAAM,YACJ,KACA,UAKI,CAAC,GAMJ;EACD,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,GAAG;EACtB,QAAQ;GACN,OAAO;IAAE,SAAS;IAAO,MAAM;GAAkB;EACnD;EACA,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GAAG,OAAO;GAAE,SAAS;GAAO,MAAM;EAAqB;EAC7F,IAAI,KAAK,QAAQ,OAAO;GAAE,SAAS;GAAO,MAAM;EAAiB;EACjE,MAAM,SAAS;GAAC,KAAK,QAAQ;GAAO,GAAI,QAAQ,cAAc,CAAC;GAAI,GAAG,KAAK;EAAU;EACrF,MAAM,WAAW,OAAO,MAAM,UAAU,CAAC,eAAe,KAAK,CAAC;EAC9D,IAAI,aAAa,KAAA,GACf,OAAO;GAAE,SAAS;GAAO,MAAM,+BAA+B,SAAS;EAAG;EAC5E,MAAM,OAAO,KAAK,UAAU,QAAQ,KAAK,SAAS,MAAM,GAAG,OAAO;EAClE,MAAM,UAAU,OAAO,QAAQ,WAAW,KAAA,IAAY,OAAO,UAAU,MAAM,QAAQ,MAAM;EAC3F,MAAM,OAAO,GAAG,OAAO,WAAW,OAAO;EACzC,MAAM,WAAW,OAAO,QAAQ,QAAQ,QAAQ,IAAI;EAGpD,MAAM,QAAQ,WAAW,QAAQ,QAAQ,QAAQ,IAAI;EACrD,MAAM,UAAU,MAAM,SAAS,IAAI,EAAE,SAAS,MAAM,IAAI,CAAC;EACzD,OAAO,SAAS,UACZ;GAAE,SAAS;GAAM,MAAM,KAAA;GAAW,GAAG;EAAQ,IAC7C;GACE,SAAS;GACT,MAAM,cAAc,QAAQ,QAAQ,SAAS,MAAM,KAAK,QAAQ,KAAK;GACrE,GAAG;EACL;CACN;;CAGA,MAAc,aACZ,QACA,QACoC;EACpC,IAAI;GACF,MAAM,MAAM,KAAK,OAAO,OAAO,IAAI,IAAI,eAAe,OAAO,MAAM,CAAC,CAAC;GACrE,IAAI,QAAQ,KAAA,KAAa,IAAI,SAAS,QAAQ,IAAI,SAAS,OAAO,IAAI,UAAU,KAC9E,OAAO,KAAA;GACT,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,MAAM,IAAI,UAAU,IAAI,OAAO,eAAe,OAAO,KAAA;GACjF,MAAM,OAAO,MAAM,KAAK,QAAQ,MAAM,IAAI,IAAI,IAAI;GAClD,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;GAC/B,MAAM,SAAS,mBAAmB,KAAK,IAAI,cAAc,IAAI;GAC7D,OAAO,OAAO,gBAAgB,KAAA,IAAY;IAAE;IAAQ,SAAS,IAAI;GAAQ,IAAI,KAAA;EAC/E,QAAQ;GACN;EACF;CACF;;CAGA,KACE,SACA,OAIM;EACN,MAAM,QAAQ,MAAM,SAAS,KAAK,QAAQ;EAC1C,MAAM,OAAO,KAAK,QAAQ;EAC1B,IAAI,QAAQ,UAAU,KAAA,GAAW,QAAQ,QAAQ;OAC5C,IAAI,UAAU,KAAA,KAAa,SAAS,KAAA,KAAa,KAAK,KAAK,IAAI,KAAK,QAAQ,KAAK,GACpF,QAAQ,QAAQ;EAClB,IAAI,MAAM,WAAW,KAAA,GAAW,QAAQ,SAAS;GAAE,GAAG,MAAM;GAAQ,GAAG,QAAQ;EAAO;CACxF;CAEA,MAAc,YACZ,QACA,QACA,SACqB;EACrB,IAAI,MAAM,IAAI,IAAI,eAAe,OAAO,MAAM;EAC9C,MAAM,eACJ,QACA,SACA,kBACgB;GAChB,QAAQ,kBAAkB,MAAM;GAChC;GACA;EACF;EACA,KAAK,IAAI,MAAM,GAAG,OAAO,OAAO,cAAc,OAAO,GAAG;GACtD,MAAM,OAAO,MAAM,KAAK,UACtB;IACE,KAAK,IAAI;IACT,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,eAAe,OAAO;IAEtB,IAAI,QAAQ;KACV,OAAO,QAAQ;IACjB;IACA,IAAI,SAAS;KACX,OAAO,QAAQ;IACjB;GACF,GACA,KAAK,QAAQ,MACf;GACA,MAAM,SAAS,KAAK;GACpB,IAAI,OAAO,YAAY,YAAY;IACjC,IAAI,KAAK,SAAS,KAAA,GAChB,OAAO,YAAY,OAAO,QAAQ,sCAAsC,OAAO,OAAO;IACxF,MAAM,IAAI,IAAI,KAAK,IAAI;IACvB;GACF;GACA,IAAI,OAAO,YAAY,aAAa,OAAO,WAAW,GACpD,OAAO,YAAY,OAAO,QAAQ,eAAe,OAAO,OAAO;GAEjE,IAAI,OAAO,YAAY,WAAW,OAAO,UAAU,OAAO,OAAO,SAAS,KACxE,OAAO,YAAY,OAAO,QAAQ,8BAA8B,OAAO,OAAO;GAEhF,IAAI,OAAO,MAAM,WAAW,iCAAiC,MAAM,MACjE,OAAO,YAAY,OAAO,MAAM,OAAO,OAAO;GAChD,IAAI,OAAO,YAAY,gBACrB,OAAO,OAAO,SAAS,KAAA,IACnB,YAAY,kCAAkC,OAAO,OAAO,IAC5D;IACE,QAAQ,mBAAmB,KAAK,OAAO,eAAe,MAAM,OAAO,IAAI;IACvE,SAAS,OAAO;IAChB,cAAc,KAAA;GAChB;GAUN,OAAO;IAAE,QAPP,OAAO,UAAU,OAAO,OAAO,SAAS,MACpC,mBACE,OAAO,QACP,OAAO,eAAe,MACtB,OAAO,wBAAQ,IAAI,WAAW,CAChC,IACA,iBAAiB,OAAO,QAAQ,KAAA,CAAS;IAC9B,SAAS,OAAO;IAAS,cAAc,KAAK;GAAa;EAC5E;EACA,OAAO,YAAY,aAAa,OAAO,OAAO,YAAY,EAAE,aAAa,KAAA,CAAS;CACpF;CAEA,MAAc,UAAU,UAAwB,QAAoC;EAClF,MAAM,OAAO;GACX,QAAQ,SAAS;GACjB,MAAM,SAAS;GACf,GAAI,SAAS,MAAM,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC;EAC9C;EACA,MAAM,eAAe,SAAS,gBAAgB,CAAC;EAC/C,MAAM,OAAO,KAAK,eAAe,QAAQ;EAEzC,MAAM,QAAQ;GACZ,IAAI,QAAQ;IACV,OAAO,SAAS;GAClB;GACA,IAAI,SAAS;IACX,OAAO,SAAS;GAClB;EACF;EAEA,MAAM,QAAQ,SAAS;EACvB,MAAM,WACJ,UAAU,KAAA,KACT,OAAO,cAAc,MAAM,MAAM,KAChC,MAAM,UAAU,KAChB,OAAO,cAAc,MAAM,MAAM,KACjC,MAAM,UAAU,IACd,KAAA,IACA,4BAA4B,KAAK,UAAU,KAAK;EAEtD,MAAM,YAAY,MAAM,KAAK,MAAM;GACjC,WAAW;GACX,KAAK,SAAS;GACd,MAAM,SAAS;GACf;GACA;GACA,SACE,cAAc,iBAAiB,SAAS,eAAe,KAAK,KAC5D,cAAc,YAAY,SAAS,UAAU,KAAK,KAClD,cAAc,YAAY,SAAS,UAAU,IAAI,KACjD;GACF,YAAY,SAAS;GACrB,OAAO,SAAS;GAChB,QAAQ,SAAS;GACjB,SAAS,QACP,UAAU,KAAA,IACN,QAAQ,QAAQ,KAAA,CAAS,IACzB,KAAK,UAAU,KAAK,SAAS,SAAS,UAAU,MAAM,YAAY;EAC1E,CAAC;EACD,IAAI,YAAY,WAAW,OAAO,UAAU;EAC5C,MAAM,EAAE,QAAQ,KAAK,MAAM,QAAQ,OAAO,aAAa,mBAAmB,SACxE,UAAU;EACZ,MAAM,WAAW,KAAK,IAAI,SAAS,YAAY,OAAO,UAAU,OAAO,UAAU,QAAQ;EACzF,MAAM,QAAQ,eAAkC;GAC9C,MAAM,SAAS;GACf;GACA;GACA,eAAe,SAAS;EAC1B;EACA,MAAM,aAAa,SAAS,WAAW;EAEvC,IAAI;GACF,OAAO,MAAM,KAAK,UAAU,IAC1B,MACA,YAAkC;IAEhC,MAAM,QACJ,UAAU,KAAA,IACN,MAAM,KAAK,UAAU,KAAK,SAAS,SAAS,UAAU,MAAM,YAAY,IACxE,KAAA;IACN,IAAI,UAAU,KAAA,GAAW,OAAO;KAAE,OAAO;KAAO,SAAS;IAAE;IAE3D,KAAK,eAAe,SAAS,QAAQ;IAErC,MAAM,SAAS,UAAU,KAAA,IAAY,KAAK,OAAO,GAAG,IAAI,KAAA;IAExD,MAAM,SACJ,WAAW,KAAA,KAAa,OAAO,cAAc,QAAQ,YAAY,OAAO,UAAU,IAAI,IAClF,SACA,KAAA;IACN,IAAI;IACJ,IAAI,QAAQ,SAAS,OAAO,QAAQ,OAAO,gBACzC,SAAS,MAAM,KAAK,QAAQ,MAAM,IAAI,OAAO,IAAI;IACnD,MAAM,UAAkC,EAAE,GAAG,KAAK;IAGlD,MAAM,OACJ,WAAW,KAAA,KAAa,WAAW,KAAA,IAC/B;KAAE,MAAM;KAAO,cAAc;IAAM,IACnC,iBACE,aAAa,KAAA,IAAY,KAAK,OAAO,IAAI,GACzC;KAAE,MAAM,QAAQ,OAAO,IAAI;KAAG,cAAc,QAAQ,OAAO,aAAa;IAAE,GAC1E,KAAK,MAAM,IAAI,CACjB;IACN,IAAI,KAAK,QAAQ,QAAQ,MAAM,QAAQ,mBAAmB,OAAO;IACjE,IAAI,KAAK,gBAAgB,QAAQ,eAC/B,QAAQ,uBAAuB,OAAO;IACxC,MAAM,aAAa,CAAC,iBAAiB,mBAAmB,CAAC,CAAC,QACvD,SAAS,QAAQ,UAAU,KAAA,CAC9B;IAEA,MAAM,YACJ,UAAU,KAAA,IACN,KAAA,IACA,SAAS,OAAO,MAAM,MAAM,EAAE,GAAG,OAAO,MAAM,SAAS,MAAM,SAAS,CAAC;IAC7E,IAAI,cAAc,KAAA,GAAW,QAAQ,WAAW;IAEhD,MAAM,SAAS,KAAK,QAAQ,MAAM;IAClC,IAAI,WAAW,KAAA,KAAa,WAAW,QACrC,OAAO;KAAE,OAAO,EAAE,SAAS,MAAM,KAAK,OAAO,KAAK,EAAE;KAAG,SAAS;IAAE;IACpE,IAAI,WAAW,KAAA,GAAW,OAAO,OAAO,SAAS,OAAO,OAAO;IAC/D,MAAM,cAAc,QAAQ,SAAS,CAAC;IAEtC,MAAM,UAAU,KAAK,MAAM,IAAI;IAC/B,MAAM,UAAU,YAAY,QAAQ,OAAO,gBAAgB;IAC3D,MAAM,QAAQ,YAAY,IAAI,CAAC,SAAS,MAAM,CAAC;IAC/C,MAAM,kBAA2B,SAAS,QAAQ,YAAY;IAC9D,MAAM,cAAc,QAAgB,UAClC,QAAQ,UACJ,GAAG,OAAO,oBAAoB,OAAO,KAAK,MAAM,OAAO,mBAAmB,GAAI,CAAC,EAAE,MACjF,GAAG,OAAO,IAAIA,YAAU,KAAK;IAEnC,KAAK,KAAK;KAAE,MAAM;KAAW;KAAK;KAAM,WAAW;IAAO,CAAC;IAC3D,IAAI;IACJ,IAAI;KACF,WAAW,MAAM,QAAQ,QAAQ;MAC/B,QAAQ;MACR;MACA,YAAY,KAAK;MACjB,QAAQ;KACV,CAAC;IACH,SAAS,OAAO;KACd,MAAM,SAAS,YAAY,KAAK;KAChC,IAAI,WAAW,KAAA,GACb,OAAO;MAAE,OAAO,EAAE,SAAS,MAAM,KAAK,OAAO,OAAO,EAAE;MAAG,SAAS;KAAE;KACtE,IAAI,KAAK,QAAQ,OAAO;MAAE,OAAO,EAAE,SAAS,aAAa,GAAG,EAAE;MAAG,SAAS;KAAE;KAC5E,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;KACrC,MAAM,OAAO,UAAU,IAAI,4BAA4B,WAAW,eAAe,KAAK;KACtF,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;MACxC,SAAS;MACT,QAAQ;MACR,cAAc,KAAA;MACd,MAAM,KAAK,SAAS;MACpB;MACA,GAAI,UAAU,IAAI,EAAE,QAAQ,UAAmB,IAAI,CAAC;KACtD,CAAC;KACD,KAAK,KAAK;MAAE,MAAM;MAAY;MAAK;MAAM,QAAQ;MAAG,SAAS;MAAS;KAAU,CAAC;KACjF,MAAM,UAAU,MAAM,KAAK,OACzB,WACA;MACE,GAAG;MACH;MACA,QAAQ;MACR,SAAS;MACT;MACA,YAAY;MACZ,GAAI,QAAQ,EAAE,cAAc,MAAM,IAAI,CAAC;MACvC,GAAI,cAAc,KAAA,IAAY,EAAE,OAAO,UAAU,IAAI,CAAC;MACtD,GAAG;KACL,GACA,KACA,KACF;KACA,OAAO;MACL,OAAO,EACL,SAAS;OACP;OACA,QAAQ;OACR,SAAS;OACT,WAAW;OACX;OACA,MAAM,KAAA;OACN,MAAM,KAAA;OACN,aAAa,KAAA;OACb,SAAS;OACT;OACA,WAAW,IAAI,KAAK,OAAO,CAAC,CAAC,YAAY;OACzC,GAAG;MACL,EACF;MACA;KACF;IACF;IAEA,MAAM,SAAS,SACb,SAAS,OACX;IACA,MAAM,SAAS,KAAK,MAAM,IAAI,IAAI;IAClC,MAAM,aAAa,SAAS,SAAS,YAAY,QAAQ,KAAK,MAAM,IAAI,CAAC;IACzE,IAAI,UAAmB,WAAW;IAClC,IAAI,OAAO,WAAW;IACtB,IAAI;IACJ,IAAI;IACJ,IAAI,YAAY;IAChB,IAAI,MAAM;IACV,IAAI;IACJ,IAAI;IACJ,IAAI,YAAY,MAAM;KACpB,IAAI;MACF,MAAM,OAAO,MAAM,SAAS,SAAS,MAAM,OAAO,kBAAkB,GAAG,QAAQ;MAC/E,OAAO,KAAK;MACZ,YAAY,KAAK;MACjB,MAAM,KAAK;MACX,IAAI,KAAK,KACP,OAAO,6DAA6D,OAAO,KAAK,MAAM,MAAM,EAAE;WAC3F,IAAI,WAAW,OAAO,gBAAgB,OAAO,QAAQ,EAAE;KAC9D,SAAS,OAAO;MACd,IAAI,KAAK,QAAQ,OAAO;OAAE,OAAO,EAAE,SAAS,aAAa,GAAG,EAAE;OAAG,SAAS;MAAE;MAC5E,UAAU;MACV,IAAI,UAAU,GAAG;OACf,OAAO;OACP,SAAS;MACX,OAEE,OAAO,WADQ,iBAAiB,aAAa,MAAM,SAAS,WACjC,qBAAqB,eAAe,KAAK;KAExE;KACA,IAAI,SAAS,KAAA,GACX,IAAI;MACF,QAAQ,MAAM,KAAK,QAAQ,MAAM,IAAI,IAAI,EAAA,CAAG;KAC9C,SAAS,OAAO;MAEd,UAAU;MACV,OAAO,eAAeA,YAAU,KAAK;MACrC,OAAO,KAAA;MACP,SAAS;KACX;IAEJ,OACE,IAAI;KACF,MAAM,SAAS,SAAS,MAAM,KAAA,GAAW,gBAAgB;IAC3D,QAAQ,CAER;IAEF,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;IACrC,IAAI,YAAY,gBAAgB;KAC9B,IAAI,WAAW,KAAA,KAAa,QAAQ,MAAM;MACxC,OAAO;MACP,OAAO,OAAO;KAChB,OAAO,OAAO;IAChB;IACA,IAAI;IACJ,IAAI,YAAY,YAAY;KAC1B,MAAM,MAAM,OAAO,UAAU;KAC7B,IAAI,QAAQ,KAAA,GAAW,OAAO;UAE5B,IAAI;MACF,MAAM,WAAW,IAAI,IAAI,KAAK,MAAM;MACpC,OAAO,SAAS;MAChB,WAAW,OAAO,UAAU,YAAY;MACxC,OAAO,YAAY;KACrB,QAAQ;MACN,UAAU;MACV,OAAO;KACT;IAEJ;IAEA,MAAM,UAAU,YAAY,kBAAkB,WAAW,KAAA;IAEzD,MAAM,SAAS,SAAS,sBAAsB,YAAY;IAC1D,MAAM,gBACJ,WAAW,KAAA,IACP,CAAC,IACD,eAAe,SAAS,SAAS;KAC/B,MAAM,QAAQ,OAAO,GAAG,SAAS,MAAM;KACvC,OAAO,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,GAAG,SAAS,KAAK,IAAI,OAAO;IACjE,CAAC;IACP,IAAI,UAAU,YACZ;KAAC,GAAG;KAAa,GAAI,SAAS,eAAe,CAAC;KAAI,GAAG;IAAa,GAClE,QACA,YAAY,MAAM,OAAO,cAAc,CAAC,CAC1C;IAEA,IAAI,SAAS;KAEX,MAAM,SACJ,QAAQ,aAAa,QAAQ,OAAO,aAAa,OAC7C,OACA,QAAQ,aAAa,UACnB,QAAQ,WACR,OAAO;KACf,UAAU;MAAE,GAAG;MAAS,UAAU;KAAO;IAC3C;IACA,MAAM,cACJ,OAAO,cAAc,MAAM,UAAW,OAAO,gBAAgB,KAAA,IAAa,KAAA;IAE5E,MAAM,UACJ,SAAS,SAAS,SAClB,YAAY,QACZ,YAAY,SAAS,QAAQ,WAAW;IAC1C,IAAI,SAAS;KACX,MAAM,OAAO,mBAAmB,eAAe,GAAG,4BAA4B,SAAS,UAAU,GAAG;KACpG,OAAO,SAAS,KAAA,IAAY,OAAO,GAAG,KAAK,IAAI;IACjD;IACA,MAAM,OAAO,OAAO,MAAM,MAAM,UAAU,OAAO,OAAO,SAAS;IACjE,MAAM,eACJ,OAAO,eAAe,MAAM,UAAU,OAAO,gBAAgB,SAAS;IACxE,MAAM,YAAY,KAAK,MAAM,IAAI;IACjC,MAAM,YAAY,WAChB,QACA,SAAS,YACT,WACA,cACA,OAAO,uBACT;IACA,MAAM,UACJ,OAAO,MAAM,MAAM,KAAA,IACf,UAAU,OAAO,MAAM,GAAG,IAAI,IAC9B,UACG,OAAO,YAAY,OACpB;IACR,MAAM,eAAe,OAAO,YAC1B;KAAC;KAAiB;KAAW;KAAO;KAAQ;IAAM,CAAC,CAAC,SAAS,SAAS;KACpE,MAAM,QAAQ,OAAO,IAAI;KACzB,OAAO,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,CAAC,KAAK,QAAQ,KAAK,GAAG,GAAG,KAAK,CAAC;IACpE,CAAC,CACH;IACA,KAAK,KAAK;KACR,MAAM;KACN;KACA;KACA,QAAQ,SAAS;KACjB;KACA;IACF,CAAC;IACD,MAAM,UAAU,MAAM,KAAK,OACzB,UAAU,YAAY,WACtB;KACE,GAAG;KACH;KACA,QAAQ,SAAS;KACjB;KACA,YAAY;KACZ,SAAS;KACT,cAAc,eAAe;KAC7B,OAAO,MAAM,UAAU;KACvB,MAAM,QAAQ;KACd;KACA,eAAe;KACf,GAAI,WAAW,SAAS,IAAI,EAAE,WAAW,IAAI,CAAC;KAC9C,GAAI,UAAU,EAAE,YAAY,OAAO,QAAQ,IAAI,CAAC;KAChD,QAAQ,SAAS,SAAS,SAAS,YAAY;KAC/C,GAAI,UAAU,KAAA,IAAY,EAAE,cAAc,MAAM,IAAI,CAAC;KACrD;KACA,GAAI,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;KACvC,GAAI,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;KAC3B,GAAI,UAAU,EAAE,SAAS,KAAK,IAAI,CAAC;KACnC,GAAG;KACH,GAAI,cAAc,OAAO,EAAE,aAAa,UAAU,IAAI,CAAC;KACvD,GAAI,YAAY,OAAO,EAAE,UAAU,QAAQ,IAAI,CAAC;KAChD,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;KAC7C,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;KACrC,GAAI,cAAc,KAAA,IAAY,EAAE,OAAO,UAAU,IAAI,CAAC;KACtD,GAAG;IACL,GACA,KACA,KACF;IACA,MAAM,aAAa,IAAI,KAAK,SAAS,CAAC,CAAC,YAAY;IAGnD,IACE,YAAY,KAAA,MACX,YAAY,QAAQ,YAAY,mBACjC,SAAS,KAAA,KACT,CAAC,OACD,CAAC,WACD,UAAU,KAAA,GAEV,KAAK,MAAM;KACT;KACA;KACA;KAEA,MAAM,YAAY,OAAO;KACzB,eAAe,YAAY,OAAO;KAClC,cAAc,eAAe;KAC7B,YAAY;KAEZ,QAAQ,UAAW,OAAO,WAAW,MAAM,MAAM,OAAO,SAAU,SAAS;KAC3E,UAAU,QAAQ;KAClB,WAAW,YAAY,OAAO,KAAA;KAC9B,aAAa;KACb,UAAU;IACZ,CAAC;IAEH,IACE,YAAY,KAAA,KACZ,YAAY,cACZ,SAAS,KAAA,MACR,oBAAoB,IAAI,SAAS,UAAU,KAAK,cAAc,OAE/D,KAAK,MACH;KACE;KACA;KACA,MAAM;KACN,MAAM;KACN,eAAe;KACf,cAAc;KACd,YAAY;KACZ,QAAQ,SAAS;KACjB,UAAU;KACV,UAAU,YAAY;KACtB,aAAa;KACb,UAAU;IACZ,GACA,IACF;IAGF,MAAM,SACJ,WAAW,WAAW,KAAK,YAAY,KAAA,KAAa,aAChD,KAAA,IACA,YAAY,iBACV,iBACA,YAAY,QAAQ,SAAS,KAAA,KAAa,CAAC,MACzC,SAAS,QAAQ,OACf,SACA,YACF,KAAA;IACV,IAAI,WAAW,KAAA,GACb,IAAI;KACF,KAAK,UAAU,MAAM,iBACnB,MACA;MACE,MAAM,QAAQ,qBAAqB,KAAA;MACnC,cAAc,QAAQ,yBAAyB,KAAA;MAC/C;KACF,GACA,SACF;IACF,QAAQ,CAER;IAEF,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;KACxC;KACA,QAAQ,SAAS;KACjB,cAAc,WAAW;KACzB,MAAM,KAAK,SAAS;KACpB;KACA,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;IAC3C,CAAC;IACD,OAAO;KACL,OAAO;MACL,SAAS;OACP;OACA,QAAQ,SAAS;OACjB;OACA,WAAW;OACX;OACA;OACA;OACA;OACA;OACA;OACA,GAAI,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;OACvC,WAAW;OACX,GAAG;MACL;MACA;MACA,cAAc,WAAW;KAC3B;KACA;IACF;GACF,GACA,KAAK,MAAM,QAAQ,SAAS,QAAQ,CACtC;EACF,SAAS,OAAO;GACd,IAAI,iBAAiB,aACnB,OAAO,EAAE,SAAS,MAAM,KAAK,SAAS,MAAM,QAAQ,MAAM,KAAK,CAAC,EAAE;GACpE,IAAI,iBAAiB,iBACnB,OAAO,EAAE,SAAS,MAAM,KAAK,aAAa,OAAO,SAAS,QAAQ,CAAC,EAAE;GACvE,IAAI,KAAK,QAAQ,OAAO,EAAE,SAAS,aAAa,GAAG,EAAE;GACrD,IAAI,SAAS,QAAQ,YAAY,MAC/B,OAAO,EAAE,SAAS,MAAM,KAAK,yBAAyB,EAAE;GAC1D,MAAM;EACR;CACF;;;;;;CAOA,gBACE,MACA,QACA,QACA,QACA,OACmC;EACnC,MAAM,aAAa,UAAU,IAAI;EACjC,MAAM,0BAAU,IAAI,IAAyC;EAC7D,MAAM,eAAwB;GAC5B,MAAM,WAAW;GACjB,OAAO;EACT;EACA,OAAO,OAAO,QAAkC;GAC9C,IAAI,OAAO,SAAS,OAAO;GAC3B,IAAI;GACJ,IAAI;IACF,SAAS,IAAI,IAAI,GAAG;GACtB,QAAQ;IACN,OAAO,OAAO;GAChB;GACA,IACE,OAAO,aAAa,WACpB,OAAO,aAAa,WACpB,OAAO,aAAa,UAEpB,OAAO;GACT,MAAM,MAAM,cAAc,IAAI,OAAO,QAAQ;GAC7C,IAAI,CAAC,OAAO,OAAO,aAAa,SAAS,OAAO,aAAa,QAAQ,OAAO,OAAO;GACnF,IAAI,QAAQ,QAAQ,IAAI,OAAO,QAAQ;GACvC,IAAI,UAAU,KAAA,GAAW;IACvB,QAAQ,YAAY,KAAK,QAAQ,MAAM;IACvC,QAAQ,IAAI,OAAO,UAAU,KAAK;GACpC;GACA,IAAK,MAAM,UAAW,KAAA,GAAW,OAAO,OAAO;GAC/C,IAAI,KAAK;IACP,MAAM,QAAQ,UAAU,MAAM,MAAM,aAAa,SAAS,KAAK,YAAY,MAAM,CAAC,EAAE;IACpF,IACE,UAAU,KAAA,KACV,CAAC,OAAO,OAAO,QAAQ,GAAG,OAAO,WAAW,OAAO,QAAQ,CAAC,CAAC,SAE7D,OAAO,OAAO;GAClB;GACA,MAAM,WAAW;GACjB,IAAI,MAAM,MAAM,OAAO,IAAI,MAAM,MAAM,IAAI,QAAQ,MAAM,CAAC;GAC1D,OAAO;EACT;CACF;;;;;;CAOA,MAAc,aACZ,KACA,MACA,QACA,QACsE;EACtE,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,GAAG;EACtB,QAAQ;GACN;EACF;EACA,OAAO,OAAO;EACd,IAAI,OAAO,SAAS,KAAK,MAAM,OAAO,KAAA;EACtC,MAAM,MAAM,OAAO,MAAM;EACzB,MAAM,QAAQ,qBAAqB;EACnC,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GACpC,OAAO;GAAE,MAAM,GAAG,MAAM;GAAgC;EAAI;EAC9D,MAAM,SAAS,MAAM,YAAY,KAAK,QAAQ,MAAM;EACpD,IAAI,WAAW,KAAA,GAAW,OAAO;GAAE,MAAM,GAAG,MAAM,IAAI;GAAU;EAAI;EACpE,MAAM,QAAQ,UAAU,MAAM,MAAM,UAAU,IAAI,IAAI,SAAS,KAAK,YAAY,MAAM,CAAC,EAAE;EACzF,IAAI,UAAU,KAAA,GAAW;GACvB,MAAM,WAAW,OAAO,OAAO,QAAQ,GAAG,OAAO,WAAW,OAAO,QAAQ;GAC3E,IAAI,CAAC,SAAS,SACZ,OAAO;IACL,MAAM,GAAG,MAAM,IAAI,cAAc,OAAO,SAAS,MAAM,KAAK,QAAQ,KAAK;IACzE;GACF;EACJ;CAEF;;;;;;;;;;;;;CAcA,MAAM,OAAO,UAA2C;EACtD,IAAI,KAAK,QAAQ,OAAO,aAAa,SAAS,GAAG;EACjD,OAAO,KAAK,MACV,KAAK,WAAW,QAAQ,CAAC,CAAC,OAAO,UAC/B,KAAK,SACD,aAAa,SAAS,GAAG,IACzB;GACE,GAAG,aAAa,SAAS,GAAG;GAC5B,SAAS;GACT,MAAM,kBAAkBA,YAAU,KAAK;EACzC,CACN,CACF;CACF;CAEA,MAAc,WAAW,UAA2C;EAClE,MAAM,OAAO;GACX,QAAQ,SAAS;GACjB,MAAM;GACN,UAAU;GACV,GAAI,SAAS,MAAM,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC;EAC9C;EACA,MAAM,QAAQ;GAAE,OAAO,SAAS;GAAO,QAAQ,SAAS;EAAO;EAC/D,MAAM,WAAW,KAAK,QAAQ;EAC9B,MAAM,YAAY,MAAM,KAAK,MAC3B;GACE,WAAW;GACX,KAAK,SAAS;GACd,MAAM;GACN;GACA,cAAc,CAAC;GACf,SAAS,cAAc,YAAY,SAAS,UAAU,IAAI;GAC1D,YAAY,SAAS;GACrB,OAAO,SAAS;GAChB,QAAQ,SAAS;GACjB,QAAQ,OAAO,QAAQ;IACrB,MAAM,SAAS,MAAM,KAAK,UAAU,KAAK,YAAY,SAAS,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,WAAW,KAAA,KAAa,OAAO,QAAQ,SAAS,KAAA,GAAW,OAAO,KAAA;IACtE,OAAO,EAAE,SAAS;KAAE,GAAG,OAAO;KAAS;KAAK,aAAa;IAAc,EAAE;GAC3E;EACF,GACA,OAAO,EAAE,QAAQ,WAAW;GAC1B,IAAI,aAAa,KAAA,GAAW,OAAO,KAAK,qCAAqC;GAC7E,IAAI,KAAK,cAAc,KAAA,GAAW,OAAO,KAAK,KAAK,SAAS;GAE5D,MAAM,WAAW,MAAM,YAAY,KAAK,QAAQ,MAAM;GACtD,OAAO,aAAa,KAAA,IAAY,KAAA,IAAY,KAAK,QAAQ;EAC3D,CACF;EACA,IAAI,YAAY,WAAW,OAAO,UAAU,OAAO;EACnD,MAAM,EAAE,QAAQ,KAAK,MAAM,QAAQ,QAAQ,OAAO,mBAAmB,SACnE,UAAU;EACZ,MAAM,UAAU,UAAU,SAAS;EACnC,MAAM,SACJ,SAAS,WAAW,KAAA,IAChB,KAAK,QAAQ,SACb,YAAY,IAAI,CAAC,SAAS,QAAQ,KAAK,QAAQ,MAAM,CAAC;EAC5D,MAAM,QAAQ;GAAE,SAAS;GAAG,SAAS;GAAG,uBAAO,IAAI,IAAY;EAAE;EACjE,MAAM,eAAe,KAAK,gBAAgB,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,KAAK;EAEvF,MAAM,OACJ,KAAK,WAAW,KAAA,IACZ,KAAA,KACC,QAA8D;GAC7D,IAAI;GACJ,IAAI;IACF,KAAK,IAAI,IAAI,GAAG;GAClB,QAAQ;IACN;GACF;GACA,MAAM,SAAS,KAAK,QAAQ,EAAE;GAC9B,OAAO,WAAW,KAAA,KAAa,WAAW,SAAS,KAAA,IAAY,OAAO;EACxE;EACN,MAAM,cACJ,KAAK,WAAW,KAAA,IACZ,CAAC,IACD,EAAE,QAAQ;GAAE,QAAQ,KAAK,OAAO;GAAO,OAAO,KAAK,OAAO;EAAe,EAAE;EAEjF,MAAM,QAAQ,eAAkC;GAC9C,MAAM;GACN;GACA;EACF;EAGA,IAAI;EACJ,IAAI;GACF,UAAU,MAAM,KAAK,QAAQ,QAAQ,MAAM;EAC7C,QAAQ;GACN,IAAI,KAAK,QAAQ,OAAO,aAAa,GAAG;GACxC,OAAO,KAAK,yBAAyB;EACvC;EACA,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,KAAK,UAAU,IAC1B,MACA,YAAwC;IACtC,KAAK,eAAe,SAAS,QAAQ;IACrC,MAAM,UAAU,KAAK,MAAM,IAAI;IAC/B,KAAK,KAAK;KAAE,MAAM;KAAW;KAAK;KAAM,WAAW;IAAU,CAAC;IAC9D,IAAI;KACF,MAAM,WAAW,MAAM,UACrB,SAAU,OAAO,OAAO,MAAM;MAC5B,WAAW,KAAK,QAAQ;MACxB,WAAW,OAAO;MAClB,GAAI,KAAK,QAAQ,SAAS,KAAA,IACtB,EAAE,SAAS,EAAE,MAAM,KAAK,QAAQ,KAAK,EAAE,IACvC,CAAC;MACL;MACA,cAAc,OAAO;MACrB;MACA,GAAI,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,KAAK;KACvC,CAAC,GACD,MACF;KACA,MAAM,UAAU,SACd,SAAS,QAAQ,KAAK,YAAY;KACpC,MAAM,aACJ,SAAS,WAAW,OAChB;MAAE,SAAS;MAAM,cAAc,KAAA;MAAW,MAAM,KAAA;KAAU,IAC1D,SAAS,SAAS,QAAQ,QAAQ,KAAK,MAAM,IAAI,CAAC;KACxD,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;KACrC,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;MACxC,SAAS,WAAW;MACpB,QAAQ,SAAS,UAAU;MAC3B,cAAc,WAAW;MACzB,MAAM,KAAK,SAAS;MACpB,YAAY;KACd,CAAC;KACD,OAAO;MAAE,OAAO;OAAE,MAAM;OAAY;OAAU;OAAY;MAAU;MAAG;KAAQ;IACjF,SAAS,OAAO;KACd,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;KACrC,IAAI,KAAK,QAAQ,OAAO;MAAE,OAAO,EAAE,MAAM,SAAS;MAAG,SAAS;KAAE;KAChE,IAAI,SAAS,QAAQ,YAAY,MAC/B,OAAO;MACL,OAAO;OAAE,MAAM;OAAa;MAAU;MACtC,SAAS,UAAU,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;KACtC;KAGF,MAAM,UACJ,iBAAiB,gBACb,QACA,eAAeA,YAAU,KAAK,CAAC,IAC7B,IAAI,cAAc,WAAW,8BAA8BA,YAAU,KAAK,GAAG,IAC7E,KAAA;KACR,IAAI,YAAY,KAAA,GAAW;MACzB,IAAI,QAAQ,WAAW,WAAW,KAAK,YAAY,QAAQ;MAE3D,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;OACxC,SAAS;OACT,QAAQ;OACR,cAAc,KAAA;OACd,MAAM,KAAK,CAAC;OACZ,YAAY;OACZ,QAAQ;MACV,CAAC;MACD,OAAO;OACL,OAAO;QACL,MAAM;QACN,QAAQ,QAAQ;QAChB,MAAM,QAAQ;QACd,KAAK,QAAQ;QACb;OACF;OACA;MACF;KACF;KACA,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;MACxC,SAAS;MACT,QAAQ;MACR,cAAc,KAAA;MACd,MAAM,KAAK,SAAS;MACpB,YAAY;KACd,CAAC;KACD,OAAO;MACL,OAAO;OAAE,MAAM;OAAU,MAAM,kBAAkBA,YAAU,KAAK;OAAK;MAAU;MAC/E;KACF;IACF;GACF,GACA,KAAK,MAAM,QAAQ,SAAS,QAAQ,CACtC;EACF,SAAS,OAAO;GACd,IAAI,iBAAiB,aAAa,OAAO,KAAK,SAAS,MAAM,QAAQ,MAAM,KAAK,CAAC;GACjF,IAAI,iBAAiB,iBAAiB,OAAO,KAAK,aAAa,OAAO,SAAS,QAAQ,CAAC;GACxF,IAAI,KAAK,QAAQ,OAAO,aAAa,GAAG;GACxC,IAAI,SAAS,QAAQ,YAAY,MAAM,OAAO,KAAK,yBAAyB;GAC5E,MAAM;EACR,UAAU;GACR,QAAQ;EACV;EAEA,IAAI,KAAK,SAAS,YAAY,KAAK,QAAQ,OAAO,aAAa,GAAG;EAClE,MAAM,cAAc;GAAE,SAAS,MAAM;GAAS,SAAS,MAAM;GAAS,OAAO,CAAC,GAAG,MAAM,KAAK;EAAE;EAC9F,IAAI,KAAK,SAAS,WAAW;GAC3B,KAAK,KAAK;IACR,MAAM;IACN;IACA;IACA,QAAQ;IACR,SAAS,KAAK,WAAW,aAAa,aAAa;IACnD,WAAW,KAAK;GAClB,CAAC;GACD,IAAI,KAAK,WAAW,YAAY;IAC9B,MAAM,QACJ,KAAK,QAAQ,KAAA,IACT,KAAA,IACA,MAAM,KAAK,aAAa,KAAK,KAAK,QAAQ,QAAQ,QAAQ,MAAM;IACtE,OAAO,KAAK,OAAO,QAAQ,KAAK,MAAM;KACpC,cAAc;KACd;KACA,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,MAAM,IAAI;IACxD,CAAC;GACH;GACA,MAAM,UAAU,MAAM,KAAK,OACzB,WACA;IACE,GAAG;IACH;IACA,QAAQ;IACR,SAAS;IACT,MAAM,KAAK;IACX,YAAY,KAAK;IACjB,cAAc;IACd;IACA,GAAG;GACL,GACA,KACA,KACF;GACA,OAAO;IAAE,GAAG,aAAa,GAAG;IAAG,SAAS;IAAY;IAAS,MAAM,KAAK;IAAM,GAAG;GAAM;EACzF;EACA,IAAI,KAAK,SAAS,YAAY,KAAK,SAAS,aAAa;GACvD,MAAM,OAAO,KAAK,SAAS,WAAW,KAAK,OAAO;GAClD,KAAK,KAAK;IACR,MAAM;IACN;IACA;IACA,QAAQ;IACR,SAAS;IACT,WAAW,KAAK;GAClB,CAAC;GAiBD,OAAO;IACL;IACA,QAAQ;IACR,SAAS;IACT,WAAW;IACX,SAAA,MArBoB,KAAK,OACzB,WACA;KACE,GAAG;KACH;KACA,QAAQ;KACR,SAAS;KACT;KACA,YAAY,KAAK;KACjB,cAAc;KACd;KACA,GAAG;IACL,GACA,KACA,KACF;IAOE,MAAM,KAAA;IACN,MAAM,KAAA;IACN,aAAa,KAAA;IACb,SAAS;IACT;IACA,GAAG;GACL;EACF;EAEA,MAAM,EAAE,UAAU,YAAY,cAAc;EAC5C,MAAM,UAAU,SAAqC,SAAS,QAAQ,KAAK,YAAY;EAEvF,IAAI,cAAc;EAClB,IAAI,eAAe;EACnB,IAAI;GACF,cAAc,IAAI,IAAI,SAAS,QAAQ;GACvC,YAAY,OAAO;EACrB,QAAQ;GACN,cAAc;EAChB;EACA,MAAM,WAAW,OAAO,WAAW;EACnC,MAAM,YAAY,GAAG,YAAY,WAAW,YAAY;EACxD,IAAI,YAAY,SAAS,OAAO,MAAM;GACpC,MAAM,QAAQ,qBAAqB;GACnC,IAAI,CAAC,cAAc,IAAI,YAAY,QAAQ,GACzC,OAAO,KAAK,GAAG,MAAM,gCAAgC,EAAE,WAAW,SAAS,CAAC;GAC9E,MAAM,UAAU,MAAM,YAAY,KAAK,QAAQ,WAAW;GAC1D,IAAI,YAAY,KAAA,GAAW,OAAO,KAAK,GAAG,MAAM,IAAI,WAAW,EAAE,WAAW,SAAS,CAAC;GACtF,IAAI,UAAU,WAAW,MAAM,UAAU,MAAM,GAC7C,eAAe,MAAM,KAAK,UAAU,aAAa,KAAK,SAAS,WAAW,GAAG,QAAQ;GAEvF,MAAM,gBAAgB,OAAO,aAAa,QAAQ,QAAQ,SAAS;GACnE,IAAI,CAAC,cAAc,SACjB,OAAO,KACL,GAAG,MAAM,IAAI,cAAc,aAAa,QAAQ,cAAc,MAAM,KAAK,QAAQ,KAAK,KACtF;IACE,WAAW;IACX,cAAc,YAAY,cAAc,QAAQ,aAAa;GAC/D,CACF;EAEJ;EACA,IAAI,UAAmB,WAAW;EAClC,IAAI,OAAO,WAAW;EACtB,IAAI;EACJ,IAAI;EACJ,IAAI,YAAY,MAAM;GACpB,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,SAAS,IAAI;GAC7C,IAAI;IACF,QAAQ,MAAM,KAAK,QAAQ,MAAM,IAAI,IAAI,EAAA,CAAG;GAC9C,SAAS,OAAO;IACd,UAAU;IACV,OAAO,eAAeA,YAAU,KAAK;IACrC,OAAO,KAAA;GACT;EACF;EACA,MAAM,UAAU,YACd,WAAW,aAAa,QAAQ,QAAQ,SAAS,GACjD,QACA,YAAY,SAAS,IAAI,CAC3B;EACA,MAAM,SAAS,SAAS,UAAU;EAClC,MAAM,YAAY,KAAK,MAAM,IAAI;EACjC,MAAM,YAAY,WAChB,QACA,QACA,WACA,OAAO,eAAe,GACtB,OAAO,uBACT;EACA,KAAK,KAAK;GAAE,MAAM;GAAY;GAAK;GAAM;GAAQ;GAAS;EAAU,CAAC;EACrE,MAAM,UAAU,MAAM,KAAK,OACzB,WACA;GACE,GAAG;GACH;GACA,WAAW;GACX;GACA;GACA,YAAY;GACZ,cAAc;GACd,OAAO,MAAM,UAAU;GACvB,MAAM,QAAQ;GACd,QAAQ;GACR,cAAc;GACd;GACA;GACA,GAAI,cAAc,OAAO,EAAE,aAAa,UAAU,IAAI,CAAC;GACvD,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;GACrC,GAAG;EACL,GACA,KACA,KACF;EACA,MAAM,aAAa,IAAI,KAAK,SAAS,CAAC,CAAC,YAAY;EACnD,IAAI,YAAY,KAAA,KAAa,SAAS,KAAA,GACpC,KAAK,MAAM;GACT;GACA,SAAS;GACT;GACA;GACA,MAAM;GACN,eAAe;GACf,cAAc;GACd,YAAY;GACZ;GACA,UAAU,QAAQ;GAClB,aAAa;EACf,CAAC;EAEH,OAAO;GACL;GACA;GACA;GACA,WAAW;GACX;GACA;GACA;GACA,aAAa;GACb;GACA;GACA,WAAW;GACX,GAAG;EACL;CACF;;;;;;CAOA,WAAW,MAAqB;EAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;EAClD,KAAK,UAAU,MAAM,IAAI;EACzB,KAAK,MAAM,OAAO,KAAK,OAAO,KAAK,GAAG;GACpC,MAAM,QAAQ,KAAK,OAAO,KAAK,GAAG;GAGlC,KADE,OAAO,OAAO,OAAO,gBAAgB,KAAA,KAAa,OAAO,OAAO,UAAU,KAAA,OACxD,SAAS,KAAA,KAAa,IAAI,SAAS,KAAK,MAAM,IAAI,KAAK,OAAO,OAAO,GAAG;EAC9F;CACF;;;;;CAQA,SAAS,KAAoD;EAC3D,IAAI;GACF,OAAO,KAAK,UAAU,MAAM,QAAQ,QAAQ,IAAI,IAAI,GAAG,CAAC,GAAG,KAAK,MAAM,IAAI,CAAC;EAC7E,QAAQ;GACN;EACF;CACF;;;;;;CASA,QAAuB;EACrB,IAAI,KAAK,gBAAgB,KAAA,GAAW;GAClC,KAAK,SAAS;GACd,KAAK,QAAQ,sBAAM,IAAI,MAAM,gBAAgB,CAAC;GAC9C,KAAK,eAAe,YAAY;IAC9B,MAAM,QAAQ,WAAW,KAAK,OAAO;IACrC,MAAM,KAAK,QAAQ,UAAU,MAAM;IACnC,MAAM,KAAK,QAAQ,IAAI,QAAQ;IAC/B,IAAI,KAAK,gBAAgB,MAAM,KAAK,WAAW,MAAM;GACvD,EAAA,CAAG;EACL;EACA,OAAO,KAAK;CACd;AACF;;;ACr4FA,MAAM,OAAO;AACb,MAAM,aAAa;AACnB,MAAM,YAAY;AAClB,MAAM,QAAQ;AACd,MAAM,YACJ;;AAEF,MAAM,8BAA2C,IAAI,IAAI;CACvD,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,OAAQ,EAAI;CACb,CAAC,OAAQ,EAAI;AACf,CAAC;AAED,MAAM,gBAAgB,SAA0B,SAAS,MAAS,QAAQ,KAAQ,QAAQ;;;;;AAM1F,MAAa,YAAY,MAAc,SAA2B;CAChE,IAAI,QAAQ,IAAI,YAAY,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;CAC1D,IAAI,SAAS,IAAI,WAAW,MAAM,MAAM;CACxC,IAAI,OAAO,IAAI,WAAW,MAAM,MAAM;CACtC,IAAI,OAAO;CACX,IAAI,QAAQ;CACZ,MAAM,QAAQ,SAAuB;EACnC,IAAI,OAAO,QAAQ,MAAM,QAAQ;EACjC,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,GAAG,OAAO,IAAI;EACrD,MAAM,YAAY,IAAI,YAAY,MAAM;EACxC,UAAU,IAAI,MAAM,SAAS,GAAG,IAAI,CAAC;EACrC,MAAM,aAAa,IAAI,WAAW,MAAM;EACxC,WAAW,IAAI,OAAO,SAAS,GAAG,IAAI,CAAC;EACvC,MAAM,WAAW,IAAI,WAAW,MAAM;EACtC,SAAS,IAAI,KAAK,SAAS,GAAG,IAAI,CAAC;EACnC,QAAQ;EACR,SAAS;EACT,OAAO;CACT;CACA,MAAM,QAAQ,MAAc,MAAc,OAAqB;EAC7D,KAAK,CAAC;EACN,MAAM,QAAQ;EACd,OAAO,QAAQ;EACf,KAAK,QAAQ;EACb,QAAQ;CACV;CACA,MAAM,aAAa,MAAc,OAAqB;EACpD,IAAI,OAAO;EACX,KAAK,IAAM,MAAM,EAAE;EACnB,QAAQ;CACV;CACA,MAAM,SAAS,KAAK;CACpB,IAAI,QAAQ;CACZ,OAAO,QAAQ,QAAQ;EACrB,MAAM,OAAO,KAAK,WAAW,KAAK;EAClC,MAAM,OAAO,QAAQ,IAAI,SAAS,KAAK,WAAW,QAAQ,CAAC,IAAI;EAE/D,IAAI,OAAO,OAAQ,OAAO,KAAO;GAC/B,IAAI,aAAa,IAAI,GAAG,UAAU,OAAO,QAAQ,CAAC;QAC7C;IACH,KAAK,QAAQ,MAAQ,QAAQ,KAAO,OAAO,KAAK,MAAM,OAAO,QAAQ,CAAC;IACtE,QAAQ;GACV;GACA,SAAS;GACT;EACF;EAEA,MAAM,QAAQ,KAAK,YAAY,KAAK;EACpC,IAAI,MAAM,SAAS,QAAQ,QAAS,IAAI;EACxC,OAAO,MAAM,QAAQ;GACnB,MAAM,OAAO,KAAK,YAAY,GAAG;GACjC,IAAI,CAAC,KAAK,KAAK,OAAO,cAAc,IAAI,CAAC,GAAG;GAC5C,OAAO,OAAO,QAAS,IAAI;EAC7B;EACA,MAAM,UAAU,KAAK,MAAM,OAAO,GAAG;EACrC,MAAM,QAAQ,OAAO,cAAc,KAAK;EACxC,IAAI,UAAU,KAAK,KAAK,KAAK,QAAQ,QAAQ,MAAM,QAAQ;GACzD,QAAQ;GACR;EACF;EACA,IAAI,MAAM,KAAK,KAAK,GAAG;GACrB,UAAU,OAAO,GAAG;GACpB,QAAQ;GACR;EACF;EACA,IAAI,SACF,SAAS,UACL,QAAQ,UAAU,MAAM,CAAC,CAAC,YAAY,IACtC,QACG,UAAU,MAAM,CAAC,CACjB,QAAQ,YAAY,EAAE,CAAC,CACvB,UAAU,KAAK,CAAC,CAChB,YAAY,CAAC,CACb,YAAY;EACrB,IAAI,SAAS,YAAY,UAAU,KAAK,KAAK,GAAG,SAAS,IAAI,OAAO;EACpE,KAAK,IAAI,KAAK,GAAG,KAAK,OAAO,QAAQ,MAAM,GAAG;GAC5C,IAAI,OAAO,OAAO,WAAW,EAAE;GAC/B,OAAO,YAAY,IAAI,IAAI,KAAK;GAChC,IAAI,SAAS,MAAS,OAAO,OAAQ,aAAa,IAAI,KAAM,SAAS,KAAM;IACzE,UAAU,OAAO,GAAG;IACpB;GACF;GACA,KAAK,MAAM,OAAO,GAAG;GACrB,QAAQ;EACV;EACA,QAAQ;CACV;CACA,IAAI,OAAO,KAAK,MAAM,OAAO,OAAO,IAAM,QAAQ;CAClD,IAAI,SAAS;CACb,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,MAAM,MAChC,UAAU,OAAO,aAAa,GAAG,MAAM,SAAS,IAAI,KAAK,IAAI,MAAM,KAAK,IAAI,CAAC,CAAC;CAEhF,OAAO;EAAE,MAAM;EAAQ,OAAO,OAAO,SAAS,GAAG,IAAI;EAAG,KAAK,KAAK,SAAS,GAAG,IAAI;CAAE;AACtF;;AAGA,MAAa,QAAQ,MAAc,SAA2B,SAAS,MAAM,IAAI,CAAC,CAAC;AAcnF,MAAM,YAAsB;CAAE,OAAO;CAAO,OAAO;CAAM,KAAK;CAAM,SAAS;CAAM,OAAO;AAAE;;AAG5F,MAAM,cAAc,MAAc,UAAkB,YAA6B;CAC/E,MAAM,OAAO,KAAK,WAAW,QAAQ;CACrC,IAAI,WAAW,KAAK,WAAW,KAAK,UAAU,QAAQ,SAAU,QAAQ,OACtE,OAAO,UAAU,WAAW,IAAI,WAAW;CAE7C,OAAO;AACT;;;;;;;;AASA,MAAa,UAAU,MAAc,UAA4B;CAC/D,MAAM,SAAS,KAAK,OAAO,OAAO;CAClC,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,WAAW,SAAS,MAAM,OAAO;CACvC,MAAM,MAAM,SAAS,KAAK,QAAQ,MAAM;CACxC,IAAI,QAAQ,IAAI,OAAO;CACvB,IAAI,QAAQ;CACZ,KAAK,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,SAAS,KAAK,QAAQ,QAAQ,KAAK,OAAO,MAAM,GAAG,SAAS;CAC/F,MAAM,QAAQ,SAAS,MAAM;CAC7B,MAAM,MAAM,SAAS,IAAI,MAAM,OAAO,SAAS;CAC/C,MAAM,OAAO,WAAW,MAAM,KAAK,IAAI,GAAG,QAAQ,EAAE,GAAG,KAAK;CAC5D,MAAM,KAAK,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,MAAM,EAAE,GAAG,IAAI;CACjE,OAAO;EACL,OAAO;EACP;EACA;EACA,SAAS,GAAG,OAAO,IAAI,MAAM,KAAK,KAAK,MAAM,MAAM,EAAE,IAAI,KAAK,KAAK,SAAS,MAAM;EAClF;CACF;AACF;;AAGA,MAAM,cAAc;;AAGpB,MAAa,YAAY,SACvB,KAAK,MAAM,QAAQ,CAAC,CACjB,MAAM,WAAW,CAAC,CAClB,QAAQ,UAAU,MAAM,SAAS,CAAC;AAgBvC,MAAM,YAAY;;AAGlB,MAAM,gBAAgB,SAAyB;CAC7C,IAAI,MAAM,KAAK;CACf,OAAO,MAAM,KAAK,KAAK,WAAW,MAAM,CAAC,MAAM,IAAM,OAAO;CAC5D,OAAO,KAAK,MAAM,GAAG,GAAG;AAC1B;;;;;AAMA,MAAM,6BAAkC,IAAI,IAAI;CAC9C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;;;;;;;;;;AAgBD,MAAa,cAAc,MAAoC;CAC7D,MAAM,QAAyF,CAAC;CAChG,KAAK,MAAM,SAAS,EAAE,SAAS,2BAA2B,GAAG;EAC3D,IAAI,MAAM,UAAU,WAAW;EAC/B,MAAM,SAAS,MAAM,OAAO,KAAA;EAC5B,MAAM,MAAM,SAAS,MAAM,KAAM,MAAM;EACvC,MAAM,SAAS,SAAS,MAAM,OAAO,MAAM,IAAI,SAAS,GAAG;EAC3D,MAAM,SAAS,SAAS,SAAS,MAAM,aAAa,GAAG,CAAC;EACxD,IAAI,OAAO,WAAW,GAAG;EACzB,MAAM,KAAK;GACT,MAAM,IAAI,OAAO,KAAK,GAAG,EAAE,GAAG,SAAS,OAAO;GAC9C,MAAM,OAAO;GACb,MAAM,CAAC,UAAU,CAAC,UAAU,OAAO,WAAW,KAAK,WAAW,IAAI,OAAO,EAAG;EAC9E,CAAC;CACH;CACA,IAAI,MAAM,WAAW,GAAG,OAAO,KAAA;CAC/B,MAAM,QAAQ,MAAM,QAAQ,SAAS,CAAC,KAAK,IAAI;CAC/C,MAAM,MAAM,MAAM,SAAS,IAAI,QAAQ;CACvC,OAAO;EACL,KAAK,MAAM,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,OAAO;EAChD,KAAK,IAAI,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,MAAM;EAC7C,UAAU,IAAI,KAAK,SAAS,KAAK,IAAI;EACrC,OAAO,IAAI,KAAK,SAAS,KAAK,IAAI;EAClC,OAAO,MAAM;CACf;AACF;;;;;AASA,MAAa,aAAa,MAAc,OAA0B,QAAQ,QAAgB;CACxF,IAAI,KAAK,WAAW,GAAG,OAAO;CAC9B,IAAI,QAAQ;CACZ,IAAI,MAAM;CACV,MAAM,QAAQ,KAAK,YAAY;CAC/B,IAAI,MAAM,WAAW,KAAK,QACxB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,KAAK,MAAM,QAAQ,IAAI;EAC7B,IAAI,OAAO,OAAO,UAAU,MAAM,KAAK,QAAQ;GAC7C,QAAQ;GACR,MAAM,KAAK,KAAK;EAClB;CACF;CAEF,IAAI,UAAU,IAAI;EAChB,MAAM,SAAS,SAAS,MAAM,QAAQ;EACtC,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,KAAK,OAAO,KAAK,QAAQ,IAAI;GACnC,IAAI,OAAO,IAAI;IACb,MAAM,OAAO,OAAO,MAAM;IAC1B,IAAI,UAAU,MAAM,OAAO,OAAO;KAChC,QAAQ;KACR,MAAM,OAAO,IAAI,KAAK,KAAK,SAAS;IACtC;GACF;EACF;CACF;CACA,IAAI,UAAU,IAAI;EAChB,MAAM,MAAM,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,KAAK,GAAG,KAAK;EAChE,OAAO,GAAG,KAAK,MAAM,GAAG,GAAG,IAAI,MAAM,KAAK,SAAS,MAAM;CAC3D;CACA,MAAM,OAAO,WAAW,MAAM,KAAK,IAAI,GAAG,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC,GAAG,KAAK;CAC/E,MAAM,KAAK,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,MAAM,KAAK,KAAM,QAAQ,IAAK,CAAC,CAAC,GAAG,IAAI;CACzF,MAAM,SAAS,SAAyB,KAAK,QAAQ,QAAQ,GAAG;CAChE,OAAO,GAAG,OAAO,IAAI,MAAM,KAAK,MAAM,KAAK,MAAM,MAAM,KAAK,CAAC,EAAE,GAAG,MAAM,KAAK,MAAM,OAAO,GAAG,CAAC,EAAE,GAAG,MAAM,KAAK,MAAM,KAAK,EAAE,CAAC,IAAI,KAAK,KAAK,SAAS,MAAM;AAC3J;;;AC9TA,MAAM,YAAY;AAClB,MAAM,YAAY;AAClB,MAAM,iBAAiB;;;;;;AAMvB,MAAMC,uBAAqB;;;;;;AAO3B,MAAa,SAAS,MAAkB,gBAAwC;CAC9E,IAAI,WAAW,KAAK,eAAe,EAAE,GAAG,OAAO;CAC/C,MAAM,MAAM,KAAK,IAAI,KAAK,SAAS,GAAG,IAAI;CAC1C,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,SAAS,GACxC,IACE,KAAK,WAAW,MAChB,KAAK,QAAQ,OAAO,MACpB,KAAK,QAAQ,OAAO,MACpB,KAAK,QAAQ,OAAO,MACpB,KAAK,QAAQ,OAAO,IAEpB,OAAO;CAGX,OAAO;AACT;;AAGA,MAAM,QAAQ,SACZ,KACG,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAC/C,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,IAAI;;AAGd,MAAM,aACJ;AAEF,MAAM,UAAU,OAAmB,OAAe,QAAwB;CACxE,IAAI,MAAM;CACV,KAAK,IAAI,QAAQ,KAAK,IAAI,GAAG,KAAK,GAAG,QAAQ,OAAO,QAAQ,MAAM,QAAQ,SAAS,GACjF,OAAO,OAAO,aAAa,MAAM,MAAO;CAE1C,OAAO;AACT;AAEA,MAAM,WAAW,OAAmB,QAAgB,SAAyB;CAC3E,MAAM,QAAQ,OAAO,WAAW,CAAC;CACjC,KACE,IAAI,QAAQ,MAAM,QAAQ,OAAO,IAAI,GACrC,UAAU,IACV,QAAQ,MAAM,QAAQ,OAAO,QAAQ,CAAC,GACtC;EACA,IAAI,QAAQ;EACZ,KAAK,IAAI,KAAK,GAAG,KAAK,OAAO,QAAQ,MAAM,GACzC,IAAI,MAAM,QAAQ,QAAQ,OAAO,WAAW,EAAE,GAAG;GAC/C,QAAQ;GACR;EACF;EAEF,IAAI,OAAO,OAAO;CACpB;CACA,OAAO;AACT;;;;;;;AAQA,MAAM,kBAAkB,OAAmB,UAA2B;CACpE,IAAI,QAAQ;CACZ,IAAI,OAAO;CACX,SAAS;EACP,MAAM,UAAU,QAAQ,OAAO,UAAU,IAAI;EAC7C,IAAI,YAAY,IAAI,OAAO;EAC3B,OAAO,UAAU;EAEjB,IAAI,OAAO,OAAO,UAAU,GAAG,OAAO,MAAM,OAAO;EACnD,MAAM,OAAO,MAAM,UAAU;EAC7B,IAAI,SAAS,MAAQ,SAAS,IAAM;EACpC,MAAM,aAAa,OAAO,OAAO,UAAU,MAAM,OAAO;EACxD,MAAM,OAAO,WAAW,MAAM,KAAK,IAAI,GAAG,WAAW,YAAY,IAAI,CAAC,CAAC;EACvE,IAAI,wBAAwB,KAAK,IAAI,GAAG;EACxC,IAAI,QAAQ,UAAU;EACtB,IAAI,SAAS,MAAQ,MAAM,WAAW,IAAM,SAAS;EACrD,MAAM,WAAW,iCAAiC,KAAK,IAAI;EAC3D,MAAM,OAAO,QAAQ,OAAO,aAAa,KAAK;EAC9C,MAAM,MACJ,aAAa,QAAQ,QAAQ,OAAO,SAAS,EAAE,KAAK,MAAM,SACtD,QAAQ,OAAO,SAAS,EAAE,IAC1B,SAAS,KACP,MAAM,SACN;EACR,IAAI;GACF,SAAS,YAAY,MAAM,SAAS,OAAO,GAAG,GAAG;IAC/C,iBAAiB,KAAK,IAAI,GAAG,QAAQ,QAAQ,CAAC;IAC9C,aAAa,UAAU;GACzB,CAAC,CAAC,CAAC;EACL,SAAS,OAAO;GAEd,IAAI,iBAAiB,YAAY,OAAO;EAC1C;EACA,IAAI,QAAQ,OAAO,OAAO;EAC1B,OAAO,KAAK,IAAI,MAAM,GAAG;CAC3B;AACF;;AASA,MAAM,cAA6B,IAAI,SAAS,YAAY,aAAa,OAAO,CAAC;;AAGjF,MAAM,SAAS,OAAU,SAAqB,OAAuC;CACnF,IAAI;CACJ,MAAM,OAAO,IAAI,SAAoB,YAAY;EAC/C,QAAQ,iBAAiB,QAAQ,KAAA,CAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAC;CAC9D,CAAC;CACD,IAAI;EACF,OAAO,MAAM,QAAQ,KAAK,CAAC,SAAS,IAAI,CAAC;CAC3C,UAAU;EACR,aAAa,KAAK;CACpB;AACF;;AAGA,MAAM,WAAW,MAAc,WAA2B;CACxD,IAAI,MAAM,KAAK,IAAI,GAAG,MAAM;CAC5B,MAAM,OAAO,KAAK,WAAW,GAAG;CAChC,IAAI,MAAM,KAAK,MAAM,KAAK,UAAU,QAAQ,SAAU,QAAQ,OAAQ,OAAO;CAC7E,OAAO,KAAK,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ;AACpC;;AAGA,MAAa,UAAU,OAAO,OAAmB,UAAsB,CAAC,MAAwB;CAC9F,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,WAAW,QAAQ,gBAAgB;CACzC,MAAM,cAAc,QAAQ,oBAAoBA;CAChD,IAAI,CAAC,eAAe,OAAO,WAAW,GACpC,OAAO;EACL,OAAO;EACP,MAAM;EACN,OAAO;EACP,WAAW;EACX,MAAM,wCAAwC,OAAO,KAAK,MAAM,cAAc,OAAO,IAAI,CAAC,EAAE;CAC9F;CAGF,MAAM,MAAM,MAAM,iBAAiB,IAAI,WAAW,KAAK,CAAC;CACxD,IAAI;EACF,MAAM,UAAU,KAAK,IAAI;EACzB,MAAM,QAAkB,CAAC;EACzB,IAAI,QAAQ;EACZ,IAAI,OAAsB;EAC1B,MAAM,OAAO,KAAK,IAAI,IAAI,UAAU,QAAQ;EAC5C,IAAI,IAAI,WAAW,UAAU,OAAO,0BAA0B,OAAO,QAAQ,EAAE;EAC/E,MAAM,QAAQ,WACZ,0BAA0B,OAAO,KAAK,MAAM,WAAW,GAAI,CAAC,EAAE,oBAAoB,OAAO,SAAS,CAAC;EACrG,KAAK,IAAI,SAAS,GAAG,UAAU,MAAM,UAAU,GAAG;GAChD,IAAI,SAAS,GAAG;IACd,MAAM,MAAM;IACZ,IAAI,KAAK,IAAI,IAAI,UAAU,UAAU;KACnC,OAAO,KAAK,MAAM;KAClB;IACF;GACF;GACA,MAAM,OAAO,MAAM,IAAI,QAAQ,MAAM;GAGrC,MAAM,UAAU,MAAM,OAAO,KAAK,eAAe,GAAG,YAAY,KAAK,IAAI,IAAI,QAAQ;GACrF,IAAI,YAAY,KAAA,GAAW;IACzB,OAAO,0BAA0B,OAAO,KAAK,MAAM,WAAW,GAAI,CAAC,EAAE,oBAAoB,OAAO,MAAM;IACtG;GACF;GACA,MAAM,OAAO,KACV,QAAQ,MACN,QAAQ,SAAS,KAAK,QAAQ,KAAA,CAAS,CAAC,CACxC,KAAK,SAAS,GAAG,KAAK,OAAO,KAAK,KAAK,WAAW,OAAO,OAAO,IAAI,CAAC,CACrE,KAAK,EAAE,CACZ;GACA,KAAK,QAAQ;GACb,IAAI,KAAK,SAAS,GAAG;IAEnB,IAAI,KAAK,SAAS,WAAW,OAAO;KAClC,MAAM,KAAK,QAAQ,MAAM,WAAW,KAAK,CAAC;KAC1C,OAAO,uBAAuB,OAAO,QAAQ,EAAE,uBAAuB,OAAO,MAAM;KACnF;IACF;IACA,MAAM,KAAK,IAAI;IACf,SAAS,KAAK,SAAS;GACzB;GACA,IAAI,SAAS,YAAY,SAAS,IAAI,UAAU;IAC9C,OAAO,uBAAuB,OAAO,QAAQ,EAAE,uBAAuB,OAAO,MAAM;IACnF;GACF;EACF;EAEA,MAAM,UADQ,MAAM,QAAQ,GAAG,CAAC,CAAC,YAAY,KAAA,CAAS,EAAA,EAAI,KAAA,GAC5B;EAC9B,MAAM,QAAQ,OAAO,UAAU,WAAW,MAAM,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,IAAI;EAC9E,OAAO;GACL,OAAO,MAAM,SAAS,KAAK,CAAC,WAAW,KAAK,KAAK,IAAI,QAAQ;GAC7D,MAAM,MAAM,KAAK,MAAM;GACvB,OAAO,IAAI;GACX,WAAW,SAAS;GACpB;EACF;CACF,UAAU;EAER,MAAM,IAAI,YAAY,QAAQ;CAChC;AACF;;;;AC9NA,MAAa,sBAAsB;;AAGnC,MAAa,uBAAuB;;AAGpC,MAAa,gBAAgB;;;;;;AAO7B,MAAa,iBAAiB,QAAwB,WAAW,aAAa,GAAG;;AAGjF,MAAM,oBAAoB;AAC1B,MAAM,UAAU;AAChB,MAAM,WAAW;AACjB,MAAM,UAAU;;AAGhB,MAAM,aAAa;;AAEnB,MAAM,mBAAmB;;AAEzB,MAAM,qBAAqB;;AAG3B,MAAM,cAAc;AACpB,MAAM,cAAc;AACpB,MAAM,cAAc;AAcpB,MAAM,kBAAkB;AACxB,MAAM,qBAAqB;AAC3B,MAAM,eAAe;;;;;;;AAQrB,MAAa,kBAAkB,YAAiD;CAC9E,MAAM,EAAE,YAAY;CACpB,IAAI,QAAQ,YAAY,KAAA,GAAW,OAAO;CAC1C,MAAM,OAAO,QAAQ,QAAQ;CAC7B,QAAQ,QAAQ,SAAhB;EACE,KAAK;GACH,IAAI,KAAK,WAAW,eAAe,GAAG,OAAO;GAC7C,IAAI,KAAK,WAAW,kBAAkB,GAAG,OAAO;GAChD,IAAI,KAAK,WAAW,YAAY,GAAG,OAAO;GAC1C;EACF,KAAK;EACL,KAAK;EACL,KAAK;EACL,KAAK,eACH,OAAO,QAAQ;EACjB,KAAK;GACH,IAAI,QAAQ,WAAW,KAAK,KAAK,WAAW,aAAa,GAAG,OAAO;GACnE,OAAO,QAAQ,UAAU,MAAM,UAAU,KAAA;EAC3C,SACE;CACJ;AACF;;AAGA,MAAa,cAAc,SAAkE;CAC3F,MAAM,QAAQ,KAAK;CACnB,IAAI,OAAO,UAAU,YAAY,UAAU,MAAM,OAAO,KAAA;CACxD,MAAM,EAAE,QAAQ,WAAW,KAAK,UAAU,WAAW;CACrD,IACG,WAAW,aAAa,WAAW,iBACpC,OAAO,cAAc,YACrB,OAAO,QAAQ,YACf,OAAO,aAAa,UAEpB,OAAO,KAAA;CACT,OAAO;EAAE;EAAQ;EAAW;EAAK;EAAU,GAAI,OAAO,WAAW,WAAW,EAAE,OAAO,IAAI,CAAC;CAAG;AAC/F;AAEA,MAAM,gBAAsD;CAC1D,SAAS;CACT,aAAa;AACf;;AAGA,MAAa,gBAAgB,aAC3B,yBAAyB,cAAc,SAAS,QAAQ,aAAa,SAAS,UAAU;;AAG1F,MAAM,cAAc,OAClB,OAAO,SAAS,EAAE,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,aAAa,GAAG,IAAI,KAAA;;AAG/E,MAAM,eAAe,WAAuC;CAC1D,MAAM,QAAQ,+CAA+C,KAAK,MAAM;CACxE,IAAI,UAAU,MAAM,OAAO,KAAA;CAC3B,MAAM,GAAG,MAAM,OAAO,KAAK,MAAM,QAAQ,UAAU;CACnD,MAAM,KAAK,KAAK,IACd,OAAO,IAAI,GACX,OAAO,KAAK,IAAI,GAChB,OAAO,GAAG,GACV,OAAO,IAAI,GACX,OAAO,MAAM,GACb,OAAO,MAAM,CACf;CACA,MAAM,MAAM,WAAW,EAAE;CAEzB,OAAO,QAAQ,KAAA,KAAa,IAAI,QAAQ,OAAO,EAAE,CAAC,CAAC,WAAW,MAAM,IAAI,MAAM,KAAA;AAChF;AAEA,MAAM,aAAa,SAAqC;CACtD,IAAI;EACF,MAAM,MAAM,IAAI,IAAI,IAAI;EACxB,OAAO,IAAI,aAAa,WAAW,IAAI,aAAa,WAAW,IAAI,OAAO,KAAA;CAC5E,QAAQ;EACN;CACF;AACF;AAaA,MAAM,QAAQ,OAAO,KAAK,UAAU;;AAGpC,MAAM,aAAa,SAAsC;CACvD,MAAM,sBAAM,IAAI,IAAoB;CACpC,KAAK,MAAM,QAAQ,KAAK,MAAM,OAAO,GAAG;EACtC,MAAM,QAAQ,KAAK,QAAQ,GAAG;EAC9B,IAAI,SAAS,GAAG;EAChB,MAAM,OAAO,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,YAAY;EACrD,MAAM,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC,CAAC,KAAK;EACzC,MAAM,QAAQ,IAAI,IAAI,IAAI;EAC1B,IAAI,IAAI,MAAM,UAAU,KAAA,IAAY,QAAQ,GAAG,MAAM,IAAI,OAAO;CAClE;CACA,OAAO;AACT;;AAGA,MAAM,WAAW,SAAqC;CACpD,MAAM,QAAkB,CAAC;CACzB,IAAI,KAAK;CACT,IAAI,QAAQ;CACZ,SAAS;EACP,MAAM,MAAM,KAAK,QAAQ,QAAQ,EAAE;EACnC,IAAI,QAAQ,IAAI,OAAO,KAAA;EACvB,MAAM,OAAO,OAAO,SAClB,KAAK,SAAS,IAAI,GAAG,CAAC,CAAC,SAAS,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAE,KAAK,GAC9D,EACF;EACA,IAAI,CAAC,OAAO,SAAS,IAAI,KAAK,OAAO,GAAG,OAAO,KAAA;EAC/C,IAAI,SAAS,GAAG,OAAO,OAAO,OAAO,KAAK;EAC1C,MAAM,QAAQ,MAAM;EACpB,IAAI,QAAQ,OAAO,KAAK,QAAQ,OAAO,KAAA;EACvC,SAAS;EACT,IAAI,QAAQ,oBAAoB,OAAO,KAAA;EACvC,MAAM,KAAK,KAAK,SAAS,OAAO,QAAQ,IAAI,CAAC;EAC7C,KAAK,QAAQ,OAAO;CACtB;AACF;;AAGA,MAAM,gBAAgB,MAAc,WAAmD;CACrF,MAAM,UAAU,EAAE,iBAAiB,mBAAmB;CACtD,IAAI,MAAM;CACV,KAAK,MAAM,SAAS,UAAU,GAAA,CAC3B,MAAM,GAAG,CAAC,CACV,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CACxC,QAAQ,SAAS,KAAK,SAAS,KAAK,SAAS,UAAU,CAAC,CACxD,QAAQ,GACT,IAAI,SAAS,UAAU,SAAS,UAAU,MAAM,WAAW,KAAK,OAAO;MAClE,IAAI,SAAS,MAAM,MAAM,qBAAqB,KAAK,OAAO;MAC1D,IAAI,SAAS,WAChB,IAAI;EACF,MAAM,YAAY,KAAK,OAAO;CAChC,QAAQ;EACN,MAAM,eAAe,KAAK,OAAO;CACnC;MACK,OAAO,KAAA;CAEhB,OAAO;AACT;;;;;;;AAQA,MAAa,oBAAoB,UAAgD;CAC/E,IAAI,MAAM,SAAS,KAAK,MAAM,OAAO,MAAQ,MAAM,OAAO,KAAM,OAAO,KAAA;CACvE,IAAI;CACJ,IAAI;EACF,SAAS,WAAW,OAAO,EAAE,iBAAiB,mBAAmB,CAAC;CACpE,QAAQ;EACN;CACF;CACA,IAAI,CAAC,OAAO,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,QAAQ,CAAC,CAAC,WAAW,SAAS,GAAG,OAAO,KAAA;CAC5E,MAAM,UAAU,OAAO,QAAQ,KAAK;CACpC,IAAI,YAAY,IAAI,OAAO,KAAA;CAC3B,MAAM,OAAO,UAAU,OAAO,SAAS,GAAG,OAAO,CAAC,CAAC,SAAS,MAAM,CAAC;CACnE,IAAI,KAAK,IAAI,WAAW,CAAC,EAAE,YAAY,MAAM,YAAY,OAAO,KAAA;CAChE,MAAM,SAAS,UAAU,KAAK,IAAI,iBAAiB,CAAC,EAAE,QAAQ,UAAU,EAAE,KAAK,EAAE;CACjF,MAAM,OAAO,WAAW,KAAK,MAAM,KAAK,IAAI,WAAW,KAAK,EAAE,CAAC;CAC/D,IAAI,WAAW,KAAA,KAAa,SAAS,KAAA,GAAW,OAAO,KAAA;CACvD,MAAM,SAAS,OAAO,KAAK,IAAI,gBAAgB,KAAK,GAAU;CAC9D,MAAM,QAAQ,UAAU;CACxB,MAAM,QAAQ,OAAO,SACnB,OACA,OAAO,UAAU,MAAM,KAAK,UAAU,IAClC,KAAK,IAAI,OAAO,QAAQ,QAAQ,MAAM,IACtC,OAAO,MACb;CACA,MAAM,UAAU,MAAM,QAAQ,KAAK;CACnC,IAAI,YAAY,IAAI,OAAO,KAAA;CAC3B,MAAM,OAAO,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,SAAS,QAAQ;CACzD,MAAM,aAAa,6BAA6B,KAAK,IAAI;CACzD,IAAI,eAAe,MAAM,OAAO,KAAA;CAChC,MAAM,UAAU,UAAU,KAAK,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC;CAC5D,IAAI,OAA2B,MAAM,SAAS,UAAU,CAAC;CACzD,IAAI;EACF,IAAI,eAAe,KAAK,QAAQ,IAAI,mBAAmB,KAAK,EAAE,GAAG,OAAO,QAAQ,IAAI;EACpF,IAAI,SAAS,KAAA,GAAW,OAAO,aAAa,MAAM,QAAQ,IAAI,kBAAkB,CAAC;CACnF,QAAQ;EACN,OAAO,KAAA;CACT;CACA,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;CAC/B,OAAO;EACL;EACA;EACA,QAAQ,KAAK,IAAI,gBAAgB,CAAC,EAAE,QAAQ,UAAU,EAAE;EACxD,QAAQ,OAAO,WAAW,EAAE;EAC5B,SAAS,SAAS,QAAQ,IAAI,KAAK,YAAY,CAAC;EAChD,MAAM,IAAI,WAAW,IAAI;CAC3B;AACF;;AAKA,MAAM,cACJ,MACA,UACA,YAEA,KACG,QAAQ,QAAQ,OAAO,IAAI,KAAK,eAAe,YAAY,IAAI,KAAK,eAAe,EAAE,CAAC,CACtF,KAAK,SAAS;CACb,GAAG;CACH,KAAK,cAAc,SAAS,QAAQ;CACpC,MAAM;CACN,KAAK,SAAS;CACd,MAAM;EAAE,GAAG,IAAI;EAAM,eAAe,IAAI;EAAM;CAAS;CACvD,GAAI,YAAY,OAAO,EAAE,UAAU,KAAc,IAAI,CAAC;AACxD,EAAE;;;;;;;;;;;;;AAcN,MAAa,mBAA8B,OAAO,MAAM,YAAY;CAClE,IAAI;CACJ,IAAI;EACF,MAAM,IAAI,IAAI,QAAQ,GAAG;CAC3B,QAAQ;EACN,OAAO,CAAC;CACV;CACA,MAAM,MAAM,YAAY,KAAK,GAAG,IAAI,WAAW,IAAI,QAAQ;CAC3D,IAAI,QAAQ,MAAM;EAChB,MAAM,YAAY,YAAY,IAAI,EAAG;EACrC,MAAM,WAAW,UAAU,IAAI,EAAG;EAClC,IAAI,cAAc,KAAA,KAAa,aAAa,KAAA,GAAW,OAAO,CAAC;EAC/D,MAAM,WAAqB;GACzB,QAAQ;GACR;GACA,KAAK,GAAG,IAAI,OAAO,OAAO,IAAI,GAAI,GAAG;GACrC;EACF;EACA,OAAO,WACL,MAAM,cAAc,MAAM;GAAE,KAAK;GAAU,aAAa,QAAQ;EAAY,CAAC,GAC7E,UACA,KAAA,CACF;CACF;CACA,MAAM,WAAW,iBAAiB,IAAI;CACtC,IAAI,aAAa,KAAA,KAAa,SAAS,SAAS,OAAO,SAAS,UAAU,KAAK,OAAO,CAAC;CACvF,MAAM,WAAqB;EACzB,QAAQ;EACR,WAAW,SAAS;EACpB,KAAK,QAAQ;EACb,UAAU,SAAS;EACnB,GAAI,SAAS,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,SAAS,OAAO;CACrE;CACA,MAAM,UAAU,YAAY,CAAC,GAAG,SAAS,MAAM;CAC/C,OAAO,WACL,MAAM,cAAc,SAAS,MAAM;EACjC,KAAK,SAAS;EACd,aAAa,SAAS,OAAO,cAAc,KAAK;CAClD,CAAC,GACD,UACA,QAAQ,aAAa,OAAO,OAAO,KAAA,CACrC;AACF;;AAgBA,MAAM,UACJ,aAII;CACJ,UAAU,QAAQ;CAClB,YAAY,QAAQ;CACpB,KAAK,QAAQ;CACb,QAAQ,QAAQ;CAChB,OAAO,QAAQ;CACf,QAAQ,QAAQ;CAChB,UAAU,QAAQ;AACpB;AAEA,MAAM,OAAO,aACV,QAAQ,YAAY,QAAQ,QAAQ,YAAY,mBAAmB,QAAQ,SAAS,KAAA;AAEvF,MAAM,QAAQ,YACZ,GAAG,QAAQ,UAAU,QAAQ,SAAS,IAAI,IAAI,OAAO,QAAQ,MAAM,MAAM,KAAK,QAAQ,SAAS,KAAA,IAAY,KAAK,KAAK,QAAQ;AAE/H,MAAM,UAAU,YAA8B;CAC5C,IAAI;EACF,OAAO,KAAK,MAAM,IAAI,YAAY,CAAC,CAAC,OAAO,QAAQ,IAAI,CAAC;CAC1D,QAAQ;EACN;CACF;AACF;;AAGA,MAAM,UACJ,SACA,SACmF;CACnF,MAAM,MAAM,QAAQ,KAAK,MAAM,QAAQ,IAAI,SAAS,aAAa;CACjE,MAAM,WAAW,QAAQ,KAAA,IAAY,KAAA,IAAY,WAAW,IAAI,IAAI;CACpE,IAAI,QAAQ,KAAA,KAAa,aAAa,KAAA,GAAW,OAAO;EAAE;EAAK;CAAS;CACxE,IAAI,CAAC,IAAI,QAAQ,OAAO,GACtB,OAAO,EAAE,MAAM,GAAG,KAAK,2BAA2B,KAAK,QAAQ,OAAO,EAAE,GAAG;CAC7E,OAAO,EACL,MAAM,GAAG,KAAK,yBAAyB,QAAQ,UAAU,KAAA,IAAY,oCAAoC,KAAK,QAAQ,MAAM,KAC9H;AACF;;AAGA,MAAM,cAAc,OAClB,SACA,KACA,aACA,YAIG;CACH,MAAM,OAAO,cAAc;CAC3B,MAAM,SAAS,MAAM,QAAQ,QAAQ,MAAM;EACzC,KAAK,GAAG,kBAAkB,OAAO,mBAAmB,GAAG;EACvD,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,GAAG,OAAO,OAAO;CACnB,CAAC;CACD,IAAI,CAAC,IAAI,MAAM,GAAG,OAAO,EAAE,MAAM,GAAG,KAAK,8BAA8B,KAAK,MAAM,EAAE,GAAG;CACvF,MAAM,UACJ,OAAO,MAAM,CAAC,EACb,oBAAoB;CACvB,IAAI,YAAY,KAAA,KAAa,QAAQ,iBAAiB,OAAO,OAAO,EAAE,MAAM,GAAG,KAAK,WAAW;CAC/F,MAAM,SAAS,QAAQ;CACvB,IAAI,WAAW,KAAA,KAAa,WAAW,OACrC,OAAO,EACL,MAAM,GAAG,KAAK,mCAAmC,KAAK,UAAU,MAAM,KAAK,UAAU,WACvF;CACF,MAAM,SAAS,OAAO,QAAQ,iBAAiB,WAAW,QAAQ,eAAe;CACjF,MAAM,QAAQ,OAAO,QAAQ,WAAW,WAAW,QAAQ,SAAS;CACpE,MAAM,WAAW,UAAU,+BAA+B,KAAK,KAAK,CAAC,GAAG,MAAM,EAAE;CAChF,IAAI,YAAY,MAAM,MAAM,KAAA,KAAa,aAAa,KAAA,GACpD,OAAO,EAAE,MAAM,GAAG,KAAK,wCAAwC;CACjE,MAAM,OAAO,MAAM,cAAc,SAAS;EACxC,KAAK,GAAG,UAAU,OAAO,MAAM;EAC/B,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,sBAAsB;EACtB;EACA,GAAG,OAAO,OAAO;CACnB,CAAC;CACD,MAAM,QAAQ,OAAO,MAAM,IAAI;CAC/B,OAAO,UAAU,QACb,QACA;EAAE,GAAG;EAAO,WAAW,OAAO,aAAa,KAAK,QAAQ;CAAU;AACxE;;AAUA,MAAM,YAAY,YAA2C;CAC3D,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,QAAQ,IAAI;CAClD,IAAI;CACJ,KAAK,MAAM,QAAQ,KAAK,MAAM,IAAI,GAAG;EACnC,IAAI,KAAK,KAAK,MAAM,IAAI;EACxB,IAAI;EACJ,IAAI;GACF,MAAM,KAAK,MAAM,IAAI;EACvB,QAAQ;GACN;EACF;EACA,MAAM,EAAE,WAAW,UAAU,WAAW;EACxC,MAAM,SAAS,OAAO,IAAI,SAAS;EACnC,MAAM,SAAS,OAAO,IAAI,SAAS;EAEnC,IACE,WAAW,SACX,OAAO,cAAc,YACrB,CAAC,WAAW,KAAK,SAAS,KAC1B,OAAO,aAAa,YACpB,CAAC,oCAAoC,KAAK,QAAQ,KAClD,SAAS,SAAS,IAAI,KACtB,CAAC,OAAO,cAAc,MAAM,KAC5B,SAAS,KACT,CAAC,OAAO,cAAc,MAAM,KAC5B,SAAS,KACT,SAAS,kBAET;EACF,IAAI,SAAS,KAAA,KAAa,YAAY,KAAK,WACzC,OAAO;GAAE;GAAW;GAAU;GAAQ;EAAO;CACjD;CACA,OAAO;AACT;;AAGA,MAAM,kBAAkB,OACtB,SACA,KACA,aACA,YAIG;CACH,MAAM,OAAO,cAAc;CAC3B,MAAM,OAAO,MAAM,QAAQ,QAAQ,MAAM;EACvC,KAAK,GAAG,SAAS;EACjB,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,GAAG,OAAO,OAAO;CACnB,CAAC;CACD,IAAI,CAAC,IAAI,IAAI,GAAG,OAAO,EAAE,MAAM,GAAG,KAAK,sCAAsC,KAAK,IAAI,EAAE,GAAG;CAC3F,MAAM,SAAS,OAAO,IAAI;CAC1B,MAAM,QAAQ,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,EAAA,CAC7C,KAAK,UACJ,OAAO,UAAU,YAAY,UAAU,OAClC,MAAkC,aACnC,KAAA,CACN,CAAC,CACA,QAAQ,QAAuB,OAAO,QAAQ,YAAY,IAAI,WAAW,QAAQ,CAAC,CAAC,CACnF,MAAM,GAAG,UAAU;CACtB,IAAI,KAAK,WAAW,GAAG,OAAO,EAAE,MAAM,GAAG,KAAK,gCAAgC;CAC9E,IAAI;CACJ,IAAI,YAAY,KAAK;CACrB,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,MAAM,QAAQ,QAAQ,MAAM;GACxC,KAAK,GAAG,IAAI,OAAO,mBAAmB,GAAG,EAAE;GAC3C,MAAM;GACN,QAAQ;GACR,QAAQ;GACR,GAAG,OAAO,OAAO;EACnB,CAAC;EACD,cAAc,MAAM;EACpB,IAAI,IAAI,KAAK,GAAG,QAAQ,SAAS,KAAK;OACjC,IAAI,MAAM,YAAY,aAAa,OAAO,KAAK,KAAK,KAAK,CAAC;EAC/D,IAAI,UAAU,KAAA,GAAW;EACzB,IAAI,MAAM,YAAY,WAAW;CACnC;CACA,IAAI,UAAU,KAAA,GACZ,OAAO,EACL,MAAM,GAAG,KAAK,mBAAmB,OAAO,KAAK,MAAM,EAAE,iBAAiB,OAAO,SAAS,IAAI,KAAK,OAAO,KAAK,IAAI,EAAE,KAAK,KACxH;CACF,MAAM,OAAO,MAAM,cAAc,SAAS;EACxC,KAAK,GAAG,UAAU,MAAM;EACxB,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,OAAO;GAAE,QAAQ,MAAM;GAAQ,QAAQ,MAAM;EAAO;EACpD;EACA,GAAG,OAAO,OAAO;CACnB,CAAC;CACD,IAAI,IAAI,KAAK,OAAO,KAAK,KAAK,QAAQ,WAAW,KAC/C,OAAO,EACL,MAAM,GAAG,KAAK,+BAA+B,OAAO,KAAK,QAAQ,MAAM,EAAE,4BAC3E;CACF,MAAM,QAAQ,OAAO,MAAM,IAAI;CAC/B,OAAO,UAAU,QAAQ,QAAQ;EAAE,GAAG;EAAO,WAAW;CAAM;AAChE;;;;;;;AAQA,MAAa,eAAe,OAC1B,SACA,KACA,QACA,YAC2B;CAC3B,IAAI;CACJ,IAAI;EACF,SAAS,IAAI,IAAI,GAAG;CACtB,QAAQ;EACN,OAAO;GAAE,WAAW;GAAO,OAAO,CAAC;EAAE;CACvC;CACA,OAAO,OAAO;CAEd,IAAI,OAAO,MAAM,MAAM,OAAO,MAC5B,OAAO;EACL,WAAW;EACX,OAAO,CAAC,0EAA0E;CACpF;CACF,MAAM,UAAU,QAAQ,yBAAyB;CACjD,MAAM,UACJ;CACF,IAAI,WAAW,YAAY,CAAC,SAAS,OAAO;EAAE,WAAW;EAAO,OAAO,CAAC,OAAO;CAAE;CAEjF,IAAI;CACJ,IAAI;EACF,QAAQ,MAAM,QAAQ,QAAQ,YAAY,OAAO,MAAM;GACrD,YAAY,QAAQ;GACpB,OAAO,QAAQ;GACf,QAAQ,QAAQ;GAChB,QAAQ,QAAQ;EAClB,CAAC;CACH,QAAQ;EAEN,OAAO;GAAE,WAAW;GAAO,OAAO,CAAC,gDAAgD;EAAE;CACvF;CACA,IAAI,CAAC,MAAM,YAAY,MAAM,QAAQ,GAAA,CAAI,WAAW,eAAe,KAAK,CAAC,SACvE,OAAO;EAAE,WAAW;EAAO,OAAO,CAAC,OAAO;CAAE;CAC9C,MAAM,cAAc,MAAM,WAAW,CAAC;CAEtC,MAAM,QAAQ,QAAQ,MAAM,IAAI,cAAc,OAAO,IAAI,CAAC;CAC1D,MAAM,gBAAgB,UAAU,KAAA,IAAY,KAAA,IAAY,WAAW,MAAM,IAAI;CAC7E,IACE,UAAU,KAAA,KACV,kBAAkB,KAAA,KAClB,OAAO,QAAQ,aAAa,YAC5B,KAAK,MAAM,QAAQ,IAAI,CAAC,IAAI,KAAK,MAAM,MAAM,UAAU,IAAI,QAAQ,UAEnE,OAAO;EAAE,KAAK;EAAO,UAAU;EAAe,WAAW;EAAM,OAAO,CAAC;CAAE;CAC3E,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,OAAO,CAAC,aAAa,eAAe,GAAG;EAChD,IAAI,QAAQ,QAAQ,YAAY,MAAM;EACtC,MAAM,SAAS,MAAM,IAAI,SAAS,OAAO,MAAM,aAAa,OAAO;EACnE,IAAI,UAAU,QAAQ,MAAM,KAAK,OAAO,IAAI;OACvC,OAAO;GAAE,KAAK,OAAO;GAAK,UAAU,OAAO;GAAU,WAAW,OAAO;GAAW;EAAM;CAC/F;CACA,OAAO;EAAE,WAAW;EAAO,OAAO,CAAC,qBAAqB,MAAM,KAAK,IAAI,GAAG;CAAE;AAC9E;;;;;;AAOA,MAAa,eAAe,OAC1B,SACA,KACA,MACA,YAC2F;CAC3F,MAAM,SAAS,eAAe,IAAI;CAClC,IAAI,WAAW,KAAA,GAAW,OAAO,EAAE,OAAO,CAAC,EAAE;CAC7C,MAAM,SAAS,MAAM,aAAa,SAAS,KAAK,QAAQ,OAAO;CAC/D,IAAI,OAAO,QAAQ,KAAA,KAAa,OAAO,aAAa,KAAA,GAAW,OAAO,EAAE,OAAO,OAAO,MAAM;CAC5F,MAAM,MAAM,QAAQ,MAAM,IAAI,OAAO,IAAI,GAAG,KAAK,OAAO;CACxD,MAAM,WAAW,WAAW,IAAI,IAAI,KAAK,OAAO;CAChD,MAAM,OAAO,WAAW,GAAG;CAC3B,MAAM,QAAQ,IAAI,KAAK;CACvB,MAAM,QAAQ,IAAI,KAAK;CACvB,MAAM,OAAO;EACX,KAAK,QAAQ;EACb,KAAK;EACL,GAAG,OAAO;EACV,aAAa,QAAQ;EACrB,GAAI,OAAO,CAAC,SAAS,IAAI,CAAC;CAC5B,CAAC,CAAC,QAAQ,SAAyB,SAAS,KAAA,KAAa,KAAK,SAAS,CAAC;CACxE,OAAO;EACL,MAAM;GACJ;GACA,KAAK,IAAI;GACT,SAAS,KAAK,QAAQ;GACtB,QAAQ,KAAK,QAAQ;GACrB,OAAO,OAAO,OAAO,IAAI;GACzB,OAAO,IAAI,KAAK;GAChB,UAAU;GACV,YAAY,MAAM,QAAQ,KAAK,IAAK,QAAqB,CAAC;GAC1D,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,SAAS;GAC7C,UAAU,CAAC,GAAG,IAAI,QAAQ;GAC1B,SAAS,IAAI;GACb,eAAe,IAAI;GACnB,YAAY,IAAI;GAChB,YAAY,OAAO;GACnB,WAAW;GACX;GACA,MAAM,KAAK,KAAK,IAAI;EACtB;EACA,OAAO,CAAC;CACV;AACF;;;;;;;;ACvpBA,MAAa,aAAa,QAAwB,OAAO,aAAa,GAAG;;AAGzE,MAAM,mBAAmB;;AAEzB,MAAM,QAAQ;CACZ;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;AAEA,MAAM,aAAa;CAAC;CAAgB;CAA6B;CAAW;CAAO;AAAW;;;;;;AAO9F,MAAM,cACJ,YAC2E;CAC3E,MAAM,SAAS,WACb,WAAW,MAAM,QAAQ,OAAO,SAAS,KAAA,CAAS;CACpD,MAAM,UAAU,CAAC,GAAG,QAAQ,OAAO,KAAK,GAAG,GAAG,QAAQ,QAAQ,WAAW,CAAC,MAAM,MAAM,CAAC,CAAC;CACxF,MAAM,OAAkC,CAAC;CACzC,IAAI,UAAU;CACd,IAAI,OAAO;CACX,KAAK,MAAM,UAAU,SAAS;EAC5B,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,CAAC;EAClC,IAAI,YAAY;EAChB,IAAI,OAAO,MAAM;GACf,YAAY,OAAO,YACjB,OAAO,QAAQ,MAAM,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,SAAS,GAAG,CAAC,CAC/D;GACA,OAAO,KAAK,UAAU,SAAS,CAAC,CAAC;EACnC;EACA,IAAI,OAAO,MAAM;GACf,WAAW;GACX;EACF;EACA,KAAK,KAAK,SAAS;EACnB,QAAQ;CACV;CACA,OAAO;EAAE;EAAM;CAAQ;AACzB;;;;;;;;;;AAWA,MAAa,iBAA4B,MAAM,YAAY;CACzD,IAAI,MAAM,MAAM,QAAQ,WAAW,GAAG,OAAO,QAAQ,MAAM,QAAQ,GAAG;CACtE,IAAI,QAAQ,gBAAgB,QAAQ,CAAC,YAAY,KAAK,QAAQ,WAAW,GAAG,OAAO,CAAC;CACpF,MAAM,MAAM,UAAU,QAAQ,GAAG;CACjC,IAAI;CACJ,IAAI;EACF,OAAO,YAAY,WAAW,MAAM,QAAQ,WAAW,GAAG,QAAQ,GAAG;CACvE,SAAS,OAAO;EAEd,OAAO,CACL;GACE;GACA,MAAM;GACN,OAAO,QAAQ;GACf,MAAM,QAAQ;GACd,KAAK,QAAQ;GACb,MAAM,EACJ,eAAe,iBAAiB,QAAQ,MAAM,QAAQ,MAAM,GAAG,GAAG,IAAI,OAAO,KAAK,EACpF;EACF,CACF;CACF;CACA,MAAM,SAAS,WAAW,KAAK,MAAM;CACrC,OAAO,CACL;EACE;EACA,MAAM;EACN,OAAO,KAAK;EACZ,MAAM,KAAK,KAAK,SAAS,IAAI,KAAK,OAAO,KAAK;EAC9C,KAAK,QAAQ;EACb,MAAM;GACJ,aAAa,KAAK;GAClB,MAAM,KAAK;GACX,cAAc,CAAC,GAAG,IAAI,IAAI,KAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;GACvD,QAAQ,OAAO;GACf,GAAI,OAAO,UAAU,IAAI,EAAE,gBAAgB,OAAO,QAAQ,IAAI,CAAC;GAC/D,OAAO,KAAK,MAAM,MAAM,GAAG,GAAG;GAC9B,eAAe,KAAK;GACpB,UAAU,KAAK;GACf,cAAc,KAAK;GACnB,eAAe,KAAK;GACpB,WAAW,KAAK;GAChB,GAAI,KAAK,UAAU,KAAA,KAAa,KAAK,MAAM,SAAS,IAAI,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;GACjF,UAAU,KAAK;GACf,SAAS,KAAK;GACd,GAAI,KAAK,MAAM,SAAS,IAAI,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;EACvD;EACA,GAAI,KAAK,aAAa,KAAA,IAAY,EAAE,WAAW,KAAK,SAAS,IAAI,CAAC;CACpE,CACF;AACF;AAEA,MAAM,qBAAqB;;;;;;AAO3B,MAAM,UAAU,OAAO,MAAkB,QAAsD;CAC7F,IAAI;CACJ,IAAI;EACF,OAAO,MAAM,QAAQ,MAAM,EAAE,UAAU,mBAAmB,CAAC;CAC7D,QAAQ;EACN,OAAO,CAAC;CACV;CACA,MAAM,OAAO,IAAI,IAAI,GAAG,CAAC,CAAC,SAAS,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;CACvD,IAAI,QAAQ;CACZ,IAAI;EACF,QAAQ,mBAAmB,IAAI;CACjC,QAAQ,CAER;CACA,MAAM,QAAQ,KAAK,UAAU,SAAS;CACtC,MAAM,QAAQ;EACZ,GAAI,KAAK,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;EAC5C,GAAI,KAAK,SAAS,OAAO,EAAE,MAAM,KAAK,KAAK,IAAI,CAAC;CAClD;CACA,IAAI,KAAK,KAAK,WAAW,GACvB,OAAO,CACL;EACE,KAAK,UAAU,GAAG;EAClB,MAAM;EACN;EACA,MAAM;EACN;EACA,MAAM;GACJ,QAAQ;GACR,OAAO,KAAK;GACZ,eAAe;GAEf,YAAY,KAAK,SAAS;GAC1B,GAAG;EACL;CACF,CACF;CAEF,OAAO,CACL;EACE,KAAK,UAAU,GAAG;EAClB,MAAM;EACN;EACA,MAAM,KAAK,KAAK,MAAM,GAAG,kBAAkB;EAC3C;EACA,MAAM;GAAE,QAAQ;GAAO,OAAO,KAAK;GAAO,eAAe,KAAK,KAAK;GAAQ,GAAG;EAAM;CACtF,CACF;AACF;;AAGA,IAAa,aAAb,MAAwB;CAEtB,2BAA4B,IAAI,IAAuB,CACrD,CAAC,QAAQ,aAAa,GACtB,CAAC,gBAAgB,gBAAgB,CACnC,CAAC;CAGD,8BAA+B,IAAI,IAAI,CAAC,cAAc,CAAC;CAEvD,SAAS,QAAgB,WAA4B;EACnD,IAAI,KAAK,SAAS,IAAI,MAAM,KAAK,CAAC,KAAK,YAAY,OAAO,MAAM,GAC9D,MAAM,IAAI,MAAM,UAAU,OAAO,0BAA0B;EAE7D,KAAK,SAAS,IAAI,QAAQ,SAAS;CACrC;CAEA,IAAI,QAAuC;EACzC,OAAO,KAAK,SAAS,IAAI,MAAM;CACjC;;CAGA,UAAoB;EAClB,OAAO,CAAC,GAAG,KAAK,SAAS,KAAK,CAAC;CACjC;AACF;;AAqDA,MAAa,kBAAqC;CAChD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;AACF;;;;;;AAOA,MAAa,YACX;;;;;;;AAQF,MAAa,cAAc,KAAqC,eAAe,UAC7E,IAAI,aAAa,QAAQ,CAAC;;;;;;AAO5B,MAAa,gBAAgB,SAC3B,OAAO,YAAY,OAAO,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,gBAAgB,SAAS,GAAG,CAAC,CAAC;AAE1F,MAAM,aAAa,UACjB,iBAAiB,QAAS,MAAM,QAAQ,MAAM,OAAO,CAAC,CAAC,MAAM,KAAM,OAAO,KAAK;;AAGjF,MAAM,aAAa,QAA0C;CAC3D,MAAM,UAAU,IAAI,KAAK;CACzB,OAAO,OAAO,YAAY,YAAY,QAAQ,SAAS,IAAI,UAAU,KAAA;AACvE;;AAiBA,MAAM,eAAe,OACnB,SACA,SACA,QACA,aACsC;CACtC,MAAM,UAAU,QAAQ;CACxB,IAAI,QAAQ,SAAS,KAAA,KAAa,YAAY,KAAA,GAAW,OAAO,EAAE,MAAM,CAAC,EAAE;CAE3E,IAAI,QAAQ,WAAW;EACrB,MAAM,QAAQ,QAAQ,MAAM,cAAc,OAAO;EACjD,IAAI,MAAM,SAAS,GAAG,OAAO,EAAE,MAAM,MAAM;CAC7C;CACA,MAAM,YAAY,QAAQ,WAAW,IAAI,MAAM;CAC/C,IAAI,cAAc,KAAA,GAAW,OAAO,EAAE,MAAM,CAAC,EAAE;CAC/C,IAAI;CACJ,IAAI;EACF,YAAY,MAAM,UAAU,QAAQ,MAAM;GACxC,KAAK,QAAQ;GACb,aAAa,WAAW,6BAA8B,QAAQ,eAAe;EAC/E,CAAC;CACH,SAAS,OAAO;EACd,OAAO;GAAE,MAAM,CAAC;GAAG,OAAO,mBAAmB,OAAO,KAAK,UAAU,KAAK;EAAI;CAC9E;CACA,MAAM,UAAU,UAAU,IAAI,SAAS,CAAC,CAAC,MAAM,SAAS,SAAS,KAAA,CAAS;CAC1E,IAAI,YAAY,KAAA,GAAW;EAEzB,QAAQ,QAAQ,cAAc,QAAQ,KAAK,OAAO;EAClD,OAAO;GACL,MAAM,CAAC;GACP;GACA,OAAO,4BAA4B,QAAQ;EAC7C;CACF;CACA,MAAM,MAAM,QAAQ,IAAI;CAmBxB,OAAO,EAAE,MAlBI,UAAU,KAAK,QAC1B,QAAQ,MAAM,OACZ;EACE,GAAG;EACH,UAAU,CAAC,OAAO;EAClB;EACA,YAAY,QAAQ,aAAa;EACjC,MAAM,QAAQ,QAAQ;EACtB;EACA;EAEA,UAAU,IAAI,aAAa,OAAO,OAAO,QAAQ,QAAQ;EACzD,GAAI,QAAQ,UAAU,KAAA,IAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;EAC9D,GAAI,QAAQ,WAAW,KAAA,IAAY,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;CACnE,GACA,GACF,CAEU,EAAE;AAChB;;;;;;;;;AAUA,MAAa,gBAAgB,OAC3B,SACA,YACqB;CACrB,MAAM,UAAU,MAAM,QAAQ,QAAQ,MAAM,OAAO;CACnD,IAAI,QAAQ,YAAY,QAAQ,QAAQ,YAAY,gBAAgB,OAAO;EAAE;EAAS,MAAM,CAAC;CAAE;CAC/F,OAAO;EAAE;EAAS,GAAI,MAAM,aAAa,SAAS,SAAS,QAAQ,QAAQ,KAAK;CAAG;AACrF;;;;;;AAOA,MAAa,iBAAiB,OAC5B,SACA,YACqB;CACrB,MAAM,UAAU,MAAM,QAAQ,QAAQ,OAAO,OAAO;CACpD,IAAI,QAAQ,YAAY,MAAM,OAAO;EAAE;EAAS,MAAM,CAAC;CAAE;CACzD,OAAO;EAAE;EAAS,GAAI,MAAM,aAAa,SAAS,SAAS,QAAQ,QAAQ,IAAI;CAAG;AACpF;AA0DA,MAAM,YAAY;;;;;;;;;AAUlB,MAAa,YAAY,OACvB,SACA,KACA,YACwB;CACxB,MAAM,gBAAgB,YAAkC;EACtD,KAAK;EACL,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,UAAU,QAAQ;EAClB,YAAY,QAAQ;EACpB,KAAK,QAAQ;EACb,QAAQ,QAAQ;EAChB,OAAO,QAAQ;EACf,QAAQ,QAAQ;EAChB,UAAU,QAAQ;CACpB;CACA,IAAI,QAAQ,MAAM,cAAc,SAAS,aAAa,GAAG,CAAC;CAC1D,MAAM,QAAkB,CAAC;CACzB,IAAI,MAAM,UAAU,KAAA,GAAW,MAAM,KAAK,MAAM,KAAK;CACrD,MAAM,SAAS,MAAM,KAAK,EAAE,EAAE,KAAK;CACnC,IAAI,OAAO,WAAW,YAAY,OAAO,SAAS,KAAK,MAAM,YAAY,KAAA,GAAW;EAClF,MAAM,WAAW,MAAM,cAAc,SAAS,aAAa,MAAM,CAAC;EAClE,MAAM,KACJ,mCAAmC,SAAS,QAAQ,IAAI,mCAC1D;EACA,IAAI,SAAS,UAAU,KAAA,GAAW,MAAM,KAAK,SAAS,KAAK;EAC3D,QAAQ;CACV;CACA,IAAI,OAAO,MAAM;CACjB,IAAI,WAAW;CACf,IAAI,YAAY,MAAM,QAAQ;CAC9B,MAAM,YAAY,MAAM,QAAQ,YAAY,QAAQ,MAAM,QAAQ,YAAY;CAC9E,MAAM,QAAQ,KAAK;CACnB,MAAM,OACJ,OAAO,SAAS,WACf,MAAM,QAAQ,gBAAgB,KAAA,KAAa,UAAU,KAAK,MAAM,QAAQ,WAAW;CACtF,MAAM,SACJ,QAAQ,WAAW,YAClB,QAAQ,WAAW,UAAU,OAAO,KAAK,oBAAoB;CAChE,IAAI,aAAa,QAAQ;EACvB,IAAI,MAAM,YAAY,KAAA,GAAW,MAAM,KAAK,yCAAyC;OAChF,IAAI,UAAU,KAAA,KAAa,CAAC,MAC/B,MAAM,KAAK,sEAAsE;OAC5E;GAEL,MAAM,UAAU,QAAQ,MAAM,IAAI,MAAM,GAAG;GAC3C,MAAM,QAAQ,YAAY,KAAA,KAAa,CAAC,QAAQ,YAAY,QAAQ,gBAAgB;GACpF,MAAM,QAAQ,MAAM,eAAe,SAAS;IAC1C,KAAK,MAAM,QAAQ;IACnB,QAAQ;IACR,UAAU,QAAQ,IAAI,QAAQ;IAC9B,YAAY,QAAQ;IACpB,KAAK,QAAQ;IACb,QAAQ,QAAQ;IAChB,OAAO,QAAQ;IACf,QAAQ,QAAQ;IAChB,UAAU,QAAQ;GACpB,CAAC;GACD,IAAI,MAAM,KAAK,SAAS,GAAG;IACzB,OAAO,MAAM;IACb,WAAW;IACX,YAAY,aAAa,MAAM,QAAQ;GACzC,OACE,MAAM,KAAK,MAAM,SAAS,MAAM,QAAQ,QAAQ,6BAA6B;EAEjF;CACF;CAEA,IAAI,QAAQ,aAAa,aAAa,KAAK,WAAW,GAAG;EACvD,MAAM,WAAW,MAAM,aAAa,SAAS,KAAK,OAAO,OAAO;EAChE,IAAI,SAAS,SAAS,KAAA,GAAW,OAAO,SAAS;EACjD,MAAM,KAAK,GAAG,SAAS,KAAK;CAC9B;CAEA,MAAM,MAAM,KAAK,OAAO,KAAA,IAAY,KAAA,IAAa,QAAQ,MAAM,IAAI,KAAK,EAAE,CAAC,GAAG,KAAK,KAAK;CACxF,MAAM,OAAO,QAAQ,KAAA,KAAa,WAAW,GAAG;CAChD,MAAM,QAAQ,KAAK,KAAK;CACxB,MAAM,QAAQ,KAAK,KAAK;CACxB,IAAI,MAAM,MAAM,KAAK,SAAS;CAC9B,MAAM,OAAO,CAAC,MAAM,QAAQ,MAAM,GAAG,KAAK,CAAC,CAAC,QACzC,SAAyB,SAAS,KAAA,KAAa,KAAK,SAAS,CAChE;CACA,OAAO;EACL;EACA,KAAK,KAAK,OAAO;EACjB,SAAS,MAAM,QAAQ;EACvB,QAAQ,MAAM,QAAQ;EACtB,OAAO,QAAQ,KAAA,KAAa,OAAO,OAAO,IAAI;EAC9C,OAAO,KAAK,KAAK,UAAU;EAC3B,UAAU,KAAK,YAAY;EAC3B,YAAY,MAAM,QAAQ,KAAK,IAAK,QAAqB,CAAC;EAC1D,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,SAAS;EAC7C,UAAU,QAAQ,KAAA,IAAY,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ;EACnD,SAAS,KAAK,WAAW;EACzB,eAAe,KAAK,iBAAiB;EACrC,YAAY,KAAK,cAAc,MAAM,QAAQ,aAAa;EAC1D,YAAY;EACZ,WAAW;EACX,MAAM,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI;CAC5C;AACF;AAsGA,MAAM,YAAY,SAAuB,OAAoB;CAC3D,MAAM,MAAM,QAAQ,MAAM,IAAI,EAAE;CAChC,IAAI,QAAQ,KAAA,GACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,0CAA0C;CACrF,OAAO;AACT;;;;;AAQA,MAAa,WAAW,MAAc,UAAiC;CACrE,MAAM,KACJ,iBAAiB,OAAO,MAAM,QAAQ,IAAI,OAAO,UAAU,WAAW,KAAK,MAAM,KAAK,IAAI;CAC5F,IAAI,CAAC,OAAO,SAAS,EAAE,GACrB,MAAM,IAAI,WACR,GAAG,KAAK,qDAAqD,KAAK,UAAU,OAAO,KAAK,CAAC,GAC3F;CAEF,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY;AAClC;;;;;;;;AASA,MAAa,iBAAiB,SAAuB,IAAY,OAA0B;CACzF,MAAM,QAAQ,QAAQ;CACtB,IAAI,MAAM,cAAc,KAAA,GAAW,OAAO,MAAM,UAAU,IAAI,EAAE;CAChE,MAAM,MAAM,MAAM,SAAS,IAAI,EAAE,OAAO,IAAU,CAAC;CACnD,MAAM,OAAO,IAAI,MAAM,YAAY,QAAQ,cAAc,EAAE;CAC3D,OAAO;EACL,SAAS,SAAS,KAAA,IAAY,KAAA,IAAY,MAAM,YAAY,IAAI,KAAK,OAAO;EAC5E,OAAO,IAAI,GAAG,EAAE,CAAC,EAAE;CACrB;AACF;;AAGA,MAAM,mBAAmB,IAAY,IAAY,UAC/C,gBAAgB,GAAG,qBAAqB,GAAG,YAAY,UAAU,KAAA,IAAY,KAAK,+BAA+B;;AAGnH,MAAM,QAAQ,YAGY;CACxB,IAAI,QAAQ,OAAO,KAAA,GAAW,OAAO,KAAA;CACrC,IAAI,QAAQ,YAAY,KAAA,GACtB,MAAM,IAAI,WAAW,6DAA6D;CACpF,OAAO,QAAQ,MAAM,QAAQ,EAAE;AACjC;AAIA,MAAM,eAAe,MAAc,OAA2B,aAA6B;CACzF,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GACrC,MAAM,IAAI,WAAW,GAAG,KAAK,sCAAsC,OAAO,KAAK,GAAG;CAEpF,OAAO,KAAK,MAAM,KAAK;AACzB;;AAGA,MAAM,cAAc,MAAc,aAA6B;CAC7D,MAAM,OAAO,KAAK,WAAW,QAAQ;CACrC,OAAO,WAAW,KAAK,WAAW,KAAK,UAAU,QAAQ,SAAU,QAAQ,QACvE,WAAW,IACX;AACN;;;;;;;;AASA,MAAa,iBAAiB,OAC5B,SACA,KAMA,QACA,QAGI,CAAC,MACa;CAClB,MAAM,KAAK,QAAQ,IAAI;CACvB,MAAM,QAAQ,MAAM,SAAS,IAAI;CACjC,MAAM,SAAS,MAAM,UAAU,IAAI;CACnC,MAAM,QAAQ,IAAI,OAChB,WAAW;EACT,QAAQ;EACR,MAAM;EACN,SAAS;GAAE,KAAK,IAAI;GAAK,KAAK,IAAI;GAAK;GAAQ,QAAQ;GAA8B;EAAG;EACxF,aAAa;EACb,aAAa,IAAI,OAAO,IAAI;EAC5B,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;EACvC,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CAC3C,CAAC,CACH;AACF;;;;;;;;AASA,MAAa,eAAe,OAC1B,SACA,IACA,UAAuB,CAAC,MACA;CACxB,MAAM,OAAO,YAAY,QAAQ,QAAQ,MAAM,CAAC;CAChD,MAAM,SAAS,YAAY,UAAU,QAAQ,QAAQ,GAAI;CACzD,MAAM,KAAK,KAAK,OAAO;CACvB,MAAM,MAAM,SAAS,SAAS,EAAE;CAChC,IAAI,UACF,QAAQ,YAAY,KAAA,IAAY,KAAA,IAAY,QAAQ,MAAM,YAAY,IAAI,QAAQ,OAAO;CAC3F,IAAI,QAAQ,YAAY,KAAA,KAAa,YAAY,KAAA,GAC/C,MAAM,IAAI,MAAM,gBAAgB,GAAG,mCAAmC,QAAQ,SAAS;CAEzF,MAAM,QAAQ,OAAO,KAAA,IAAY,CAAC,IAAI;EAAE;EAAI,cAAc;CAAM;CAChE,IAAI,OAAO,KAAA,GAAW;EACpB,MAAM,QAAQ,cAAc,SAAS,IAAI,EAAE;EAC3C,IAAI,MAAM,YAAY,KAAA,GAEpB,OAAO;GACL,KAAK,IAAI;GACT,MAAM,IAAI;GACV,OAAO;GACP,KAAK,IAAI;GACT,MAAM,CAAC;GACP,MAAM;GACN,IAAI;GACJ,OAAO;GACP,MAAM;GACN,UAAU,IAAI;GACd,SAAS;GACT,YAAY;GACZ,UAAU,IAAI;GACd,WAAW;GACX,MAAM,gBAAgB,IAAI,IAAI,MAAM,KAAK;GACzC;GACA,cAAc;EAChB;EAEF,UAAU,MAAM;CAClB;CACA,MAAM,UACJ,IAAI,aAAa,QAAQ,SAAS,aAAa,OAC3C,OACC,SAAS,YAAY,IAAI;CAChC,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,OAAO,WAAW,EAAE,UAAU,QAAQ,GAAG,YAAY;CAC3D,IAAI,YAAY,QAAQ,cAAc,MAAM,eAAe,SAAS,KAAK,MAAM;CAC/E,MAAM,OAAO,SAAS,QAAQ,IAAI;CAClC,MAAM,QAAQ,WAAW,MAAM,KAAK,IAAI,MAAM,KAAK,MAAM,CAAC;CAC1D,MAAM,MAAM,OAAO,QAAQ,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,QAAQ,MAAM,CAAC;CACjF,MAAM,OAAO,SAAS,QAAQ,IAAI;CAElC,MAAM,WAAW,WAAW,IAAI;CAChC,MAAM,QAAQ,CAAC,GAAI,OAAO,CAAC,SAAS,IAAI,CAAC,GAAI,GAAI,WAAW,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,CAAE;CAC1F,OAAO;EACL,KAAK,IAAI;EACT,MAAM,SAAS,QAAQ,IAAI;EAC3B,OAAO,OAAO,KAAM,SAAS,SAAS,IAAI;EAC1C,KAAK,SAAS,OAAO,IAAI;EACzB,MAAM,OAAO,aAAa,IAAI,IAAI;EAClC,MAAM;EACN,IAAI;EACJ,OAAO,KAAK;EACZ,MAAM,OAAO,KAAK,KAAK,MAAM,OAAO,GAAG;EACvC,UAAU,IAAI;EACd,SAAS,SAAS,WAAW,IAAI;EACjC,YAAY,SAAS,cAAc,IAAI;EACvC,UAAU;EACV,WAAW;EACX,MAAM,MAAM,SAAS,IAAI,MAAM,KAAK,GAAG,IAAI;EAC3C,GAAG;EACH,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;CAC/C;AACF;;AAKA,MAAM,iBACJ,SACqC;CACrC,MAAM,WAAW,WAAW,IAAI;CAChC,OAAO,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS;AAClD;;AAGA,MAAM,gBACJ,MACA,SACmE;CACnE,MAAM,WAAW,WAAW,IAAI;CAChC,IAAI,aAAa,KAAA,GAAW,OAAO,EAAE,KAAK;CAC1C,MAAM,OAAO,aAAa,QAAQ;CAClC,OAAO;EAAE,MAAM,SAAS,OAAO,OAAO,GAAG,KAAK,IAAI;EAAQ;CAAS;AACrE;;;;;AAQA,MAAM,gBACJ,SACA,IACA,OACA,YACkE;CAClE,MAAM,KAAK,KAAK,OAAO;CACvB,MAAM,MAAM,SAAS,SAAS,EAAE;CAChC,IAAI,SACF,QAAQ,YAAY,KAAA,IAAY,KAAA,IAAY,QAAQ,MAAM,YAAY,IAAI,QAAQ,OAAO;CAC3F,IAAI,QAAQ,YAAY,KAAA,KAAa,WAAW,KAAA,GAC9C,MAAM,IAAI,MAAM,gBAAgB,GAAG,mCAAmC,QAAQ,SAAS;CAEzF,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,OAAO;EAAE,KAAK,IAAI;EAAK,KAAK,IAAI;EAAK,UAAU,IAAI;CAAS;CAClE,MAAM,QAAQ,OAAO,KAAA,IAAY,CAAC,IAAI;EAAE;EAAI,cAAc;CAAM;CAChE,MAAM,UAAU;EACd,OAAO;EACP,OAAO;EACP,KAAK;EACL,SAAS;EACT,SAAS;EACT,MAAM;EACN,YAAY;EACZ,OAAO;EACP,cAAc;EACd,OAAO;CACT;CACA,IAAI,OAAO,KAAA,GAAW;EAEpB,MAAM,QAAQ,cAAc,SAAS,IAAI,EAAE;EAC3C,IAAI,MAAM,YAAY,KAAA,GAUpB,OAAO;GAAE,QAAA;IARP,GAAG;IACH,GAAG;IACH,UAAU;IACV,WAAW;IACX,MAAM,gBAAgB,IAAI,IAAI,MAAM,KAAK;IACzC;IACA,cAAc;GAEF;GAAG,UAAU;EAAM;EAEnC,SAAS,MAAM;CACjB;CACA,MAAM,UAAmB,IAAI,aAAa,QAAQ,QAAQ,aAAa,OAAO,OAAO,IAAI;CACzF,IAAI,WAAW,EAAE,UAAU,QAAQ,GAAG,YAAY,GAUhD,OAAO;EAAE,QAAA;GARP,GAAG;GACH,GAAG;GACH,UAAU;GACV,WAAW;GACX,MAAM;GACN,GAAG;GACH,GAAG,cAAc,QAAQ,QAAQ,IAAI,IAAI;EAE7B;EAAG,UAAU;CAAM;CAEnC,MAAM,UAAU,UAAU,QAAQ,MAAM,YAAY,EAAE;CACtD,MAAM,aACJ,WAAW,KAAA,IACP,CAAC,MAAM,IACP,CAAC,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,CAAC,OAAO,GAAI,GAAG,QAAQ,MAAM,aAAa,EAAE,CAAC;CAErF,MAAM,OAAO,WAAW,QAAQ,YAAY,QAAQ,aAAa,IAAI,CAAC,CAAC;CACvE,MAAM,WAAW,iBAAiB,YAAY,QAAQ,OAAO;CAC7D,KAAK,MAAM,CAAC,OAAO,YAAY,WAAW,QAAQ,GAAG;EACnD,IAAI,WAAW,SAAS,YAAY,GAAG;EACvC,IAAI,MAAM,OAAO,QAAQ,MAAM,KAAK;EACpC,IAAI,SAAS;EACb,IAAI,CAAC,IAAI,SAAS,QAAQ,cAAc,KAAA,GAAW;GACjD,MAAM,OAAO,QAAQ,WAAW,KAAK;GACrC,SAAS,IAAI;EACf;EACA,IAAI,CAAC,IAAI,OAAO;EAChB,MAAM,QAAQ,WAAW,KAAA,KAAa,QAAQ;EAqB9C,OAAO;GAAE,QAAA;IAnBP,GAAG;IACH,OAAO;IACP,OAAO,IAAI;IACX,KAAK,IAAI;IACT,SAAS,IAAI;IACb,SAAS,QAAQ;IACjB,MAAM,QAAQ;IACd,YAAY,QAAQ;IACpB;IACA,cAAc;IACd,UAAU;IACV,OAAO,IAAI;IACX,WAAW;IACX,GAAG,aACD,QAAQ,MACR,QAAQ,4DAA4D,IACtE;IACA,GAAG;GAES;GAAG;EAAS;CAC5B;CACA,MAAM,aACJ,YAAY,KAAA,KAAa,QAAQ,KAAK,iBAAiB,QAAQ,QAAQ,cAAc,KAAA;CAEvF,MAAM,QAAQ,CACZ,GAAI,aACA,CACE,qGACF,IACA,CAAC,GACL,GAAI,OAAO,KAAK,CAAC,eACb,CAAC,yEAAyE,OAAO,IAAI,EAAE,EAAE,IACzF,CAAC,CACP;CAkBA,OAAO;EAAE,QAAA;GAhBP,GAAG;GACH,OAAO;GACP,OAAO;GACP,KAAK;GACL,SAAS;GACT,SAAS,SAAS,WAAW;GAC7B,MAAM,SAAS,QAAQ;GACvB,YAAY,SAAS,cAAc;GACnC,OAAO;GACP,cAAc;GACd,UAAU,CAAC;GACX,OAAO;GACP,WAAW;GACX,GAAG,aAAa,SAAS,QAAQ,IAAI,MAAM,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,IAAI;GACrF,GAAG;EAES;EAAG;CAAS;AAC5B;;;;;;;;;;;;;;;;;AA+BA,MAAa,cAAc,OACzB,SACA,IACA,OACA,UAAyB,CAAC,MACA;CAC1B,MAAM,EAAE,QAAQ,aAAa,aAAa,SAAS,IAAI,OAAO,OAAO;CACrE,IAAI,UAAU,MAAM,eAAe,SAAS,SAAS,SAAS,EAAE,GAAG,QAAQ;CAC3E,OAAO;AACT;;;;;;;;AASA,MAAa,gBAAgB,OAC3B,SACA,GACA,UAII,CAAC,MACgB;CACrB,MAAM,QAAQ,QAAQ;CAOtB,OANc,QAAQ,MAAM,KAAK,GAAG;EAClC,OAAO,QAAQ,SAAS;EACxB,MAAM,QAAQ,QAAQ;EACtB,GAAI,QAAQ,SAAS,KAAA,IAAY,EAAE,MAAM,QAAQ,KAAK,IAAI,CAAC;CAC7D,CAEW,CAAC,CAAC,KAAK,QAAQ;EACxB,IAAI,IAAI,SAAS,YAAY,OAAO;EACpC,MAAM,OAAO,QAAQ,MAAM,IAAI,IAAI,GAAG,CAAC,EAAE;EACzC,OAAO,SAAS,KAAA,IAAY,MAAM;GAAE,GAAG;GAAK,GAAG,cAAc,IAAI;EAAE;CACrE,CAAC;AACH;AAwBA,MAAM,gBAAgB;AACtB,MAAM,aAAa;;;;;AAKnB,MAAM,kCAAuC,IAAI,IAAI;CACnD;CACA;CACA;CACA;AACF,CAAC;;AAGD,MAAM,iBACJ,YACwD;CACxD,IAAI,QAAQ,IAAI,SAAS,aAAa,GACpC,OAAO;EAAE,KAAK,QAAQ,IAAI,MAAM,GAAG,EAAqB;EAAG,SAAS;CAAW;CAEjF,OAAO;EAAE,KAAK,QAAQ;EAAK,SAAS,QAAQ,aAAa,OAAO,aAAa;CAAQ;AACvF;AAkBA,MAAM,kBAA+B;CACnC,UAAU;CACV,QAAQ;CACR,SAAS;CACT,yBAAS,IAAI,IAAI;CACjB,QAAQ,CAAC;AACX;;AAGA,MAAM,gBACJ,OACA,UACA,YACS;CACT,MAAM,EAAE,KAAK,YAAY,cAAc,OAAO;CAC9C,MAAM,YAAY,QAAQ,cAAc,SAAS;CACjD,IAAI,QAAQ,YAAY,YAAY;EAClC,MAAM,WACJ,QAAQ,aACP,OAAO,QAAQ,SAAS,YAAY,QAAQ,KAAK,WAAW,WAAW,IACpE,QAAQ,KAAK,MAAM,CAAC,IACpB;EACN,MAAM,YAAY,QAAQ,WAAW,OAAO,QAAQ,WAAW;EAC/D,IAAI,aAAa,SAAS,aAAa,OAAO,QAAQ,gBAAgB,WACpE,MAAM,UAAU;GACd;GACA;GACA,SAAS,SAAS;GAClB,MAAM;GACN,MAAM;GACN,eAAe;GACf,cAAc;GACd,YAAY;GACZ,QAAQ,QAAQ,UAAU;GAC1B,UAAU;GACV;GACA,aAAa,QAAQ,eAAe;GACpC,UAAU,QAAQ,YAAY;EAChC,CAAC;EAEH;CACF;CAGA,IAAI,QAAQ,QAAQ,QAAQ,QAAQ,YAAY,QAAQ,QAAQ,UAAU,KAAA,GAAW;CACrF,MAAM,WAAW,MAAM,OAAO,KAAK,OAAO;CAC1C,MAAM,UAAU,SAAS,SAAS,aAAa,QAAQ,YAAY;CACnE,MAAM,YAAY,QAAQ,cAAc;CAExC,MAAM,OAAO,YAAY,OAAQ,QAAQ,SAAS,UAAW,UAAU,QAAQ,OAAQ;CACvF,MAAM,eAAe,YACjB,OACC,QAAQ,kBAAkB,UAAW,UAAU,iBAAiB,OAAQ;CAC7E,MAAM,OAAO,UAAW,UAAU,gBAAgB,KAAA,IAAa,KAAA;CAC/D,MAAM,cACJ,QAAQ,gBAAgB,SAAS,YAAY,aAAa,6BAA6B;CACzF,MAAM,SAAS,UACX,aAAa,KAAA,KAAa,SAAS,SAAS,OAAO,SAAS,UAAU,MACpE,MACA,SAAS,SACV,QAAQ,UAAU;CACvB,MAAM,UAAU;EACd;EACA;EACA,SAAS,SAAS;EAClB,MAAM,QAAQ,QAAQ;EACtB;EACA,eAAe;EACf,cAAc;EACd,YAAY;EACZ;EACA,UAAU,QAAQ,SAAS,YAAY;EACvC,GAAI,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;EACvC,aAAa,QAAQ,eAAe;EACpC,UAAU,QAAQ,aAAa,UAAW,UAAU,YAAY,OAAQ;CAC1E,CAAC;AACH;;AAGA,MAAM,eAAe,OACnB,SACA,QACA,UACA,UACkB;CAClB,IAAI,SAAS,SAAS,aAAa,SAAS,SAAS,WAAW;CAChE,MAAM,UAAU,SAAS;CACzB,IAAI,YAAY,QAAQ,OAAO,YAAY,YAAY,OAAO,QAAQ,QAAQ,UAAU;CACxF,IAAI,QAAQ,YAAY,YAAY;EAClC,aAAa,QAAQ,UAAU,OAAO;EACtC;CACF;CACA,IAAI,OAAO,QAAQ,SAAS,UAAU;CACtC,IAAI,QAAQ,YAAY,QAAQ,QAAQ,YAAY,gBAAgB;CACpE,MAAM,YAAY;CAClB,aAAa,QAAQ,UAAU,OAAO;CACtC,MAAM,YAAY,QAAQ,WAAW,IAAI,QAAQ,MAAM;CACvD,IAAI,cAAc,KAAA,GAAW;EAE3B,IAAI,CAAC,gBAAgB,IAAI,QAAQ,MAAM,GAAG,MAAM,QAAQ,IAAI,QAAQ,MAAM;EAC1E;CACF;CACA,MAAM,OAAO,MAAM,QAAQ,MAAM,IAAI,QAAQ,IAAI;CACjD,IAAI,SAAS,KAAA,GAAW;EACtB,MAAM,WAAW;EACjB;CACF;CACA,MAAM,EAAE,KAAK,YAAY,cAAc,OAAO;CAC9C,MAAM,SAAS,OAAO,OAAO,KAAK,OAAO;CACzC,IAAI;CACJ,IAAI;EACF,YAAY,MAAM,UAAU,MAAM;GAChC;GACA,aACE,YAAY,aACR,6BACC,QAAQ,gBAAgB,QAAQ,gBAAgB;EACzD,CAAC;CACH,SAAS,OAAO;EACd,MAAM,UAAU;EAChB,IAAI,MAAM,OAAO,SAAS,YACxB,MAAM,OAAO,KAAK;GAAE,SAAS,SAAS;GAAI,OAAO,UAAU,KAAK;EAAE,CAAC;EACrE;CACF;CAEA,IAAI,UAAU,MAAM,QAAQ,UAAU,GAAG,MAAM,KAAA,CAAS,GAAG;CAC3D,KAAK,MAAM,OAAO,WAChB,OAAO,OACL;EACE,GAAG;EACH,UAAU,CAAC,SAAS,EAAE;EACtB,SAAS,SAAS;EAClB,YAAY,QAAQ,cAAc,SAAS;EAC3C,MAAM,QAAQ;EACd,QAAQ,QAAQ;EAChB,UAAU,YAAY;EACtB,UAAU,IAAI,aAAa,OAAO,OAAQ,QAAQ,SAAS,YAAY;EACvE,GAAI,SAAS,UAAU,KAAA,IAAY,EAAE,OAAO,SAAS,MAAM,IAAI,CAAC;EAChE,GAAI,SAAS,WAAW,KAAA,IAAY,EAAE,QAAQ,SAAS,OAAO,IAAI,CAAC;CACrE,GACA,SAAS,WACX;AAEJ;;;;;;;;;;;;;;AAeA,MAAa,UAAU,OACrB,SACA,UAAmD,CAAC,MACzB;CAC3B,MAAM,QAAQ,QAAQ,IAAI;CAC1B,MAAM,QAAQ,QAAQ,MAAM,aAC1B,QAAQ,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,QAAQ,MAAM,CAC5D;CACA,MAAM,QAAQ,SAAS;CACvB,IAAI,UAAU;CACd,IAAI;EACF,MAAM,OAAO,QAAQ,SAAS,KAC5B,EAAE,QAAQ,QAAQ,GAClB,EACE,iBAAiB;GACf,WAAW;EACb,EACF,CACF;EACA,WAAW,MAAM,YAAY,MAAM,MAAM,aAAa,SAAS,OAAO,UAAU,KAAK;EACrF,MAAM,UAAU,MAAM,MAAM,IAAI,IAAI,QAAQ,WAAW,QAAQ,CAAC,CAAC;EACjE,MAAM,SAAS,MAAM,OAAO,KAAK;EACjC,OAAO;GACL,UAAU,MAAM;GAChB,GAAG;GACH,QAAQ,MAAM;GACd,SAAS,MAAM;GACf;GACA;GACA,iBAAiB,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,KAAK;GACzC,QAAQ,MAAM;EAChB;CACF,SAAS,OAAO;EACd,MAAM,MAAM;EACZ,MAAM;CACR;AACF;;AAgBA,MAAM,gBAAgB;;AAGtB,MAAM,UAAU,SAAuB,aAAqC;CAC1E,IAAI,SAAS,SAAS,aAAa,SAAS,SAAS,WAAW,OAAO;CACvE,MAAM,UAAU,SAAS;CACzB,IAAI,YAAY,QAAQ,OAAO,YAAY,YAAY,OAAO,QAAQ,QAAQ,UAC5E,OAAO;CACT,MAAM,YAAY,QAAQ,cAAc,SAAS;CACjD,MAAM,EAAE,KAAK,YAAY,cAAc,OAAO;CAC9C,MAAM,QAAQ,QAAQ,MAAM,OAAO,KAAK,OAAO;CAC/C,IAAI,QAAQ,YAAY,YAAY,OAAO,UAAU,KAAA,KAAa,MAAM,aAAa;CACrF,IACE,OAAO,QAAQ,SAAS,YACvB,QAAQ,YAAY,QAAQ,QAAQ,YAAY,gBAEjD,OAAO;CAET,IAAI,QAAQ,UAAU,KAAA,GACpB,OACE,QAAQ,WAAW,IAAI,QAAQ,MAAM,MAAM,KAAA,KAAa,CAAC,QAAQ,MAAM,WAAW,SAAS,EAAE;CAEjG,IAAI,UAAU,KAAA,KAAa,MAAM,aAAa,WAAW,OAAO;CAChE,OACE,QAAQ,WAAW,IAAI,QAAQ,MAAM,MAAM,KAAA,KAAa,CAAC,QAAQ,MAAM,WAAW,SAAS,EAAE;AAEjG;;;;;;;;;;;;;;AAeA,MAAa,eAAe,OAAO,YAA8D;CAC/F,IAAI,QAAQ,MAAM,OAAO,OAAO;EAAE,MAAM;EAAW,GAAI,MAAM,QAAQ,OAAO;CAAG;CAC/E,MAAM,SAAS,QAAQ,MAAM,YAAY;CACzC,MAAM,QACJ,WAAW,KAAA,IAAY,KAAA,qBAAY,IAAI,KAAK,KAAK,MAAM,MAAM,IAAI,aAAa,EAAA,CAAE,YAAY;CAC9F,MAAM,QAAQ,SAAS;CACvB,IAAI,UAAU;CACd,MAAM,OAAO,QAAQ,SAAS,KAC5B;EAAE,QAAQ;EAAS,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM;CAAG,GAC7D,EACE,iBAAiB;EACf,WAAW;CACb,EACF,CACF;CACA,WAAW,MAAM,YAAY,MAC3B,IAAI,OAAO,SAAS,QAAQ,GAAG,MAAM,aAAa,SAAS,QAAQ,OAAO,UAAU,KAAK;CAE3F,IAAI,MAAM,aAAa,GAAG,OAAO,KAAA;CACjC,OAAO;EACL,MAAM;EACN,UAAU,MAAM;EAChB,GAAG,QAAQ,MAAM,OAAO;EACxB,QAAQ,MAAM;EACd,SAAS,MAAM;EACf;EACA,SAAS;EACT,iBAAiB,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,KAAK;EACzC,QAAQ,MAAM;CAChB;AACF"}
|