polite-crawler 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -3
- package/build/cli.mjs +10 -3
- package/build/cli.mjs.map +1 -1
- package/build/index.d.mts +15 -5
- package/build/index.mjs +4 -4
- package/build/mcp.d.mts +1 -1
- package/build/mcp.mjs +7 -5
- package/build/mcp.mjs.map +1 -1
- package/build/{open-CJR4b44v.mjs → open-DYZ7KL2A.mjs} +130 -23
- package/build/open-DYZ7KL2A.mjs.map +1 -0
- package/build/{open-CZqwY7GH.d.mts → open-eHcriTds.d.mts} +53 -15
- package/build/{politeness-C5bZ1OKM.d.mts → politeness-yI3xBEx9.d.mts} +95 -2
- package/build/politeness.d.mts +2 -2
- package/build/politeness.mjs +102 -1
- package/build/politeness.mjs.map +1 -1
- package/build/{version-rVx8_y1_.mjs → version-no12qgos.mjs} +2 -2
- package/build/{version-rVx8_y1_.mjs.map → version-no12qgos.mjs.map} +1 -1
- package/package.json +1 -1
- package/record.schema.json +5 -0
- package/build/open-CJR4b44v.mjs.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"open-DYZ7KL2A.mjs","names":["systemLookup","messageOf","RENDERED_MARK","messageOf"],"sources":["../src/blobs.ts","../src/egress.ts","../src/record.ts","../src/crawler.ts","../src/fold.ts","../src/docs.ts","../src/hoststate.ts","../src/log.ts","../src/pdf.ts","../src/runtime.ts","../src/open.ts"],"sourcesContent":["import { createHash, randomUUID } from \"node:crypto\";\nimport type { Stats } from \"node:fs\";\nimport { mkdir, open, readFile, rename, rm, stat } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\n\n/**\n * A blob store keeps the bytes of each response, and the hash of the bytes is the key. The crawler\n * needs only these two operations, so an application can give its own store.\n */\nexport interface BlobStore {\n put(bytes: Uint8Array): Promise<{ readonly hash: string }>;\n get(hash: string): Promise<Uint8Array | undefined>;\n}\n\n/** These options change a file blob store. */\nexport interface FileBlobStoreOptions {\n /** When this value is true (the default), a blob is on the disk (`fsync`) before `put` finishes. */\n readonly sync?: boolean | undefined;\n}\n\nconst HASH = /^[0-9a-f]{64}$/;\n\n/** A rename can fail with these codes when a different writer puts the same blob at the same time. */\nconst RACE_CODES: ReadonlySet<string> = new Set([\n \"EPERM\",\n \"EEXIST\",\n \"EBUSY\",\n \"EACCES\",\n \"ENOTEMPTY\",\n]);\n\nconst blobPath = (root: string, hash: string): string =>\n join(root, hash.slice(0, 2), hash.slice(2, 4), hash);\n\nconst hashOf = (bytes: Uint8Array): string => createHash(\"sha256\").update(bytes).digest(\"hex\");\n\nconst codeOf = (error: unknown): string =>\n typeof error === \"object\" && error !== null && \"code\" in error ? String(error.code) : \"\";\n\nconst statOf = (path: string): Promise<Stats | undefined> => stat(path).catch(() => undefined);\n\n/** This function writes a folder to the disk, so a rename in it stays after a power cut. Windows cannot open a folder, so it skips this step. */\nconst syncFolder = async (path: string): Promise<void> => {\n if (process.platform === \"win32\") return;\n try {\n const handle = await open(path, \"r\");\n try {\n await handle.sync();\n } finally {\n await handle.close();\n }\n } catch {\n // a file system that cannot sync a folder keeps the blob in any case\n }\n};\n\n/**\n * This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash\n * of the bytes. These are its rules:\n *\n * - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is\n * written again.\n * - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.\n * A crash thus cannot leave half a blob under its name.\n * - Two writers can put the same bytes at the same time. When the rename of one fails because the\n * other was first, the put succeeds if the stored file has the same hash.\n * - The method `get` examines the hash of the bytes. A damaged file gets a new name\n * (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.\n */\nexport class FileBlobStore implements BlobStore {\n private readonly writing = new Map<\n string,\n Promise<{ readonly hash: string; readonly existed: boolean }>\n >();\n private readonly sync: boolean;\n\n constructor(\n readonly dir: string,\n options: FileBlobStoreOptions = {},\n ) {\n this.sync = options.sync ?? true;\n }\n\n put(bytes: Uint8Array): Promise<{ readonly hash: string; readonly existed: boolean }> {\n const hash = hashOf(bytes);\n const running = this.writing.get(hash);\n if (running !== undefined) return running;\n const work = this.write(hash, bytes).finally(() => this.writing.delete(hash));\n this.writing.set(hash, work);\n return work;\n }\n\n /** This method tells if a file holds exactly the bytes of a hash. */\n private static async holds(path: string, hash: string): Promise<boolean> {\n try {\n return hashOf(await readFile(path)) === hash;\n } catch {\n return false;\n }\n }\n\n private async write(\n hash: string,\n bytes: Uint8Array,\n ): Promise<{ readonly hash: string; readonly existed: boolean }> {\n const target = blobPath(this.dir, hash);\n const existing = await statOf(target);\n if (existing?.isFile() === true && existing.size === bytes.length)\n return { hash, existed: true };\n const folder = dirname(target);\n await mkdir(folder, { recursive: true, mode: 0o700 });\n const temporary = `${target}.${randomUUID()}.tmp`;\n try {\n const handle = await open(temporary, \"wx\", 0o600);\n try {\n await handle.writeFile(bytes);\n if (this.sync) await handle.sync();\n } finally {\n await handle.close();\n }\n try {\n await rename(temporary, target);\n } catch (error) {\n // a different writer put the same bytes first, and Windows does not replace a file in use\n if (!RACE_CODES.has(codeOf(error)) || !(await FileBlobStore.holds(target, hash)))\n throw error;\n return { hash, existed: true };\n }\n if (this.sync) await syncFolder(folder);\n return { hash, existed: false };\n } finally {\n await rm(temporary, { force: true }).catch(() => undefined);\n }\n }\n\n async get(hash: string): Promise<Uint8Array | undefined> {\n if (!HASH.test(hash)) return undefined;\n const target = blobPath(this.dir, hash);\n let bytes: Buffer;\n try {\n bytes = await readFile(target);\n } catch {\n return undefined;\n }\n if (hashOf(bytes) !== hash) {\n // a torn or damaged file is never the body: it moves away, and the next put writes it again\n await rename(target, `${target}.${randomUUID()}.damaged`).catch(() => undefined);\n return undefined;\n }\n return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n }\n}\n","import type { LookupAddress, LookupAllOptions, LookupOptions } from \"node:dns\";\nimport { lookup as systemLookup } from \"node:dns\";\nimport type { LookupFunction } from \"node:net\";\nimport { isIP } from \"node:net\";\n\n/**\n * The egress policy keeps the crawler on the public internet. A page or an agent can ask for an\n * internal address. Examples are a loopback service, a router on the LAN and the metadata\n * service of a cloud host. The crawler refuses these addresses by default.\n *\n * - The crawler fetches only `http:` and `https:` URLs.\n * - The check uses the address after DNS. The `lookup` hook of the connection checks the address\n * that the socket then uses, so a DNS answer that changes cannot pass the check.\n * - The crawler examines a literal IP host directly, in each spelling that the URL parser reads.\n * - The crawler examines each redirect hop again.\n *\n * A refusal is a skipped fetch with a clear note. It does not throw an error.\n */\n\n/** These options change the egress policy. With no options, the crawler fetches only public addresses. */\nexport interface EgressOptions {\n /** The value true lets the crawler fetch loopback, private and link-local addresses. The default is false. */\n readonly allowPrivate?: boolean | undefined;\n /**\n * These hosts are permitted although their address is not public. An entry is a host name, a\n * name pattern (`*.corp.example`), an IP address or a CIDR range (`10.0.0.0/8`).\n */\n readonly allowHosts?: readonly string[] | undefined;\n}\n\n/** The connection hook gives this error when the policy refuses each address of a host. */\nexport class EgressRefused extends Error {\n readonly code: string = \"EGRESS_REFUSED\";\n constructor(\n readonly address: string,\n readonly kind: string,\n ) {\n super(egressNote(address, kind));\n this.name = \"EgressRefused\";\n }\n}\n\n/** This function gives the note of a refusal, with the address and the kind of address. */\nexport const egressNote = (address: string, kind: string): string =>\n `egress policy: ${address} is ${/^[aeiou]/i.test(kind) ? \"an\" : \"a\"} ${kind} address; set allowPrivate to fetch it`;\n\ntype Bytes = readonly number[];\n\n/**\n * This function reads an IPv4 address the way the URL parser reads a host. The address has one to\n * four parts, and each part is decimal, octal (`0177`) or hexadecimal (`0x7f`). Thus\n * `2130706433` and `0x7f.1` both give 127.0.0.1. A text that is not an IPv4 address gives no value.\n */\nexport const parseIPv4 = (text: string): Bytes | undefined => {\n const parts = text.split(\".\");\n if (parts.length > 1 && parts[parts.length - 1] === \"\") parts.pop();\n if (parts.length === 0 || parts.length > 4) return undefined;\n const numbers: number[] = [];\n for (const part of parts) {\n let value: number;\n if (/^0[xX][0-9a-fA-F]*$/.test(part))\n value = part.length === 2 ? 0 : Number.parseInt(part.slice(2), 16);\n else if (/^0[0-7]+$/.test(part)) value = Number.parseInt(part.slice(1), 8);\n else if (/^(0|[1-9][0-9]*)$/.test(part)) value = Number(part);\n else return undefined;\n numbers.push(value);\n }\n const last = numbers.pop()!;\n if (numbers.some((value) => value > 255)) return undefined;\n if (last >= 256 ** (4 - numbers.length)) return undefined;\n let rest = last;\n const tail: number[] = [];\n for (let index = numbers.length; index < 4; index += 1) {\n tail.unshift(rest % 256);\n rest = Math.floor(rest / 256);\n }\n return [...numbers, ...tail];\n};\n\n/** This function reads an IPv6 address, with or without brackets and a zone. It gives 16 bytes. */\nexport const parseIPv6 = (text: string): Bytes | undefined => {\n let value = text.startsWith(\"[\") && text.endsWith(\"]\") ? text.slice(1, -1) : text;\n const zone = value.indexOf(\"%\");\n if (zone !== -1) value = value.slice(0, zone);\n if (isIP(value) !== 6) return undefined;\n const lastColon = value.lastIndexOf(\":\");\n const tail = value.slice(lastColon + 1);\n if (tail.includes(\".\")) {\n const v4 = parseIPv4(tail);\n if (v4 === undefined) return undefined;\n const high = (v4[0]! << 8) | v4[1]!;\n const low = (v4[2]! << 8) | v4[3]!;\n value = `${value.slice(0, lastColon + 1)}${high.toString(16)}:${low.toString(16)}`;\n }\n const [head = \"\", rest] = value.split(\"::\");\n const groupsOf = (part: string): number[] =>\n part.length === 0 ? [] : part.split(\":\").map((group) => Number.parseInt(group, 16));\n const front = groupsOf(head);\n const back = rest === undefined ? [] : groupsOf(rest);\n const groups = [...front, ...Array<number>(8 - front.length - back.length).fill(0), ...back];\n if (groups.length !== 8) return undefined;\n return groups.flatMap((group) => [group >> 8, group & 0xff]);\n};\n\nconst v4 = (text: string): Bytes => parseIPv4(text)!;\nconst v6 = (text: string): Bytes => parseIPv6(text)!;\n\n/** This function tells if the first `bits` bits of two addresses agree. */\nconst inPrefix = (address: Bytes, prefix: Bytes, bits: number): boolean => {\n if (address.length !== prefix.length) return false;\n for (let index = 0; index < address.length; index += 1) {\n const take = Math.max(0, Math.min(8, bits - index * 8));\n if (take === 0) return true;\n const mask = (0xff << (8 - take)) & 0xff;\n if ((address[index]! & mask) !== (prefix[index]! & mask)) return false;\n }\n return true;\n};\n\n// the more specific ranges come first, so a metadata address does not read as link-local\nconst V4_RANGES: ReadonlyArray<readonly [Bytes, number, string]> = [\n [v4(\"169.254.169.254\"), 32, \"metadata\"],\n [v4(\"168.63.129.16\"), 32, \"metadata\"],\n [v4(\"100.100.100.200\"), 32, \"metadata\"],\n [v4(\"0.0.0.0\"), 8, \"unspecified\"],\n [v4(\"10.0.0.0\"), 8, \"private\"],\n [v4(\"100.64.0.0\"), 10, \"carrier-grade NAT\"],\n [v4(\"127.0.0.0\"), 8, \"loopback\"],\n [v4(\"169.254.0.0\"), 16, \"link-local\"],\n [v4(\"172.16.0.0\"), 12, \"private\"],\n [v4(\"192.0.0.0\"), 24, \"reserved\"],\n [v4(\"192.0.2.0\"), 24, \"documentation\"],\n [v4(\"192.168.0.0\"), 16, \"private\"],\n [v4(\"198.18.0.0\"), 15, \"benchmark\"],\n [v4(\"198.51.100.0\"), 24, \"documentation\"],\n [v4(\"203.0.113.0\"), 24, \"documentation\"],\n [v4(\"224.0.0.0\"), 4, \"multicast\"],\n [v4(\"240.0.0.0\"), 4, \"reserved\"],\n];\n\nconst V6_RANGES: ReadonlyArray<readonly [Bytes, number, string]> = [\n [v6(\"fd00:ec2::254\"), 128, \"metadata\"],\n [v6(\"::\"), 128, \"unspecified\"],\n [v6(\"::1\"), 128, \"loopback\"],\n [v6(\"64:ff9b:1::\"), 48, \"reserved\"],\n [v6(\"100::\"), 64, \"reserved\"],\n [v6(\"2001::\"), 32, \"reserved\"],\n [v6(\"2001:db8::\"), 32, \"documentation\"],\n [v6(\"fc00::\"), 7, \"unique local\"],\n [v6(\"fe80::\"), 10, \"link-local\"],\n [v6(\"fec0::\"), 10, \"private\"],\n [v6(\"ff00::\"), 8, \"multicast\"],\n];\n\nconst kindOfV4 = (address: Bytes): string | undefined =>\n V4_RANGES.find(([prefix, bits]) => inPrefix(address, prefix, bits))?.[2];\n\nconst kindOfV6 = (address: Bytes): string | undefined => {\n const zeroes = (from: number, to: number): boolean =>\n address.slice(from, to).every((byte) => byte === 0);\n const embedded = (from: number): string | undefined => kindOfV4(address.slice(from, from + 4));\n // an IPv4 address in an IPv6 form has the kind of the IPv4 address\n if (zeroes(0, 10) && address[10] === 0xff && address[11] === 0xff) return embedded(12);\n if (inPrefix(address, v6(\"64:ff9b::\"), 96)) return embedded(12);\n if (address[0] === 0x20 && address[1] === 0x02) return embedded(2);\n const listed = V6_RANGES.find(([prefix, bits]) => inPrefix(address, prefix, bits))?.[2];\n if (listed !== undefined) return listed;\n // the old IPv4-compatible form (::a.b.c.d) and the rest of ::/8 are not public\n if (zeroes(0, 12)) return embedded(12) ?? \"reserved\";\n if (address[0] === 0) return \"reserved\";\n return undefined;\n};\n\n/**\n * This function gives the kind of an address that is not public, for example `loopback` or\n * `private`. It reads each IPv4 spelling and each IPv6 form. A public address, or a text that is not\n * an address, gives no value.\n */\nexport const addressKind = (address: string): string | undefined => {\n const four = parseIPv4(address);\n if (four !== undefined) return kindOfV4(four);\n const six = parseIPv6(address);\n return six === undefined ? undefined : kindOfV6(six);\n};\n\nconst nameOf = (host: string): string => host.trim().toLowerCase().replace(/\\.$/, \"\");\n\n/** These names have a fixed meaning. They do not go to DNS. */\nconst NAME_KINDS: ReadonlyArray<readonly [RegExp, string]> = [\n [/(^|\\.)localhost$/, \"loopback\"],\n [/^metadata\\.google\\.internal$/, \"metadata\"],\n [/^metadata\\.goog$/, \"metadata\"],\n];\n\nconst literalOf = (host: string): Bytes | undefined => parseIPv6(host) ?? parseIPv4(host);\n\nconst inRange = (entry: string, address: Bytes): boolean => {\n const slash = entry.indexOf(\"/\");\n const base = literalOf(slash === -1 ? entry : entry.slice(0, slash));\n if (base === undefined || base.length !== address.length) return false;\n const bits = slash === -1 ? base.length * 8 : Number(entry.slice(slash + 1));\n return (\n Number.isInteger(bits) && bits >= 0 && bits <= base.length * 8 && inPrefix(address, base, bits)\n );\n};\n\n/** This function tells if an entry of `allowHosts` permits this host name or this address. */\nconst permitted = (policy: EgressOptions, host: string, address: string | undefined): boolean => {\n if (policy.allowPrivate === true) return true;\n const name = nameOf(host);\n const bytes = address === undefined ? literalOf(name) : literalOf(address);\n for (const raw of policy.allowHosts ?? []) {\n const entry = nameOf(raw);\n if (entry === name) return true;\n if (entry.startsWith(\"*.\") && name.endsWith(entry.slice(1))) return true;\n if (bytes !== undefined && inRange(entry, bytes)) return true;\n }\n return false;\n};\n\n/** This function gives a short form of an address for a note: IPv6 in brackets, IPv4 with dots. */\nconst shown = (host: string): string => {\n const bytes = literalOf(host);\n if (bytes === undefined) return host;\n if (bytes.length === 4) return bytes.join(\".\");\n const groups: string[] = [];\n for (let index = 0; index < 16; index += 2)\n groups.push(((bytes[index]! << 8) | bytes[index + 1]!).toString(16));\n return groups.join(\":\").replace(/(^|:)0(:0)+(:|$)/, \"::\");\n};\n\n/**\n * This function examines a host before DNS: a literal IP address in any spelling, and the names\n * with a fixed meaning (`localhost`). It gives the note of a refusal, or no value. A host name that\n * needs DNS gets its check in the connection hook.\n */\nexport const egressRefusal = (policy: EgressOptions, host: string): string | undefined => {\n if (policy.allowPrivate === true) return undefined;\n const name = nameOf(host);\n const literal = literalOf(name);\n if (literal !== undefined) {\n const kind = addressKind(name);\n return kind === undefined || permitted(policy, name, undefined)\n ? undefined\n : egressNote(shown(name), kind);\n }\n for (const [pattern, kind] of NAME_KINDS) {\n if (pattern.test(name) && !permitted(policy, name, undefined)) return egressNote(name, kind);\n }\n return undefined;\n};\n\n/** This is the form of `dns.lookup` that the egress hook uses. Tests give a resolver of their own. */\nexport type Resolve = (\n host: string,\n options: LookupAllOptions,\n callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void,\n) => void;\n\nconst resolveAll: Resolve = (host, options, callback) => systemLookup(host, options, callback);\n\n/** This function keeps the addresses that the policy permits. When it keeps none, it gives the refusal. */\nconst screen = (\n policy: EgressOptions,\n host: string,\n addresses: readonly LookupAddress[],\n): { readonly usable: LookupAddress[]; readonly refusal: EgressRefused | undefined } => {\n const usable: LookupAddress[] = [];\n let refusal: EgressRefused | undefined;\n for (const entry of addresses) {\n const kind = addressKind(entry.address);\n if (kind === undefined || permitted(policy, host, entry.address)) usable.push(entry);\n else refusal ??= new EgressRefused(shown(entry.address), kind);\n }\n return {\n usable,\n refusal: usable.length === 0 ? (refusal ?? new EgressRefused(host, \"unresolved\")) : undefined,\n };\n};\n\n/**\n * This function gives a `lookup` hook for the connection options of an undici `Agent`. The hook\n * resolves the name, removes each address that the policy refuses, and gives the rest to the\n * socket. When no address stays, the connection fails with `EgressRefused`.\n */\nexport const egressLookup =\n (policy: EgressOptions, resolve: Resolve = resolveAll): LookupFunction =>\n (host: string, options: LookupOptions, callback) => {\n resolve(host, { ...options, all: true }, (error, addresses) => {\n if (error !== null) {\n callback(error, []);\n return;\n }\n const { usable, refusal } =\n policy.allowPrivate === true\n ? { usable: addresses, refusal: undefined }\n : screen(policy, host, addresses);\n if (refusal !== undefined) {\n callback(refusal, []);\n return;\n }\n if (options.all === true) callback(null, usable);\n else callback(null, usable[0]!.address, usable[0]!.family);\n });\n };\n\n/**\n * This function examines a URL with DNS, for a client that does its own DNS, for example the\n * browser. It refuses a host when one of its addresses is not public. A name that DNS cannot\n * resolve gets no refusal, because the client cannot connect to it either. The function gives\n * the note of a refusal, or no value.\n *\n * The client resolves the name again, so a DNS answer that changes between the two lookups can\n * pass. The connection hook (`egressLookup`) does not have this gap.\n */\nexport const checkEgress = async (\n policy: EgressOptions,\n url: URL,\n resolve: Resolve = resolveAll,\n): Promise<string | undefined> => {\n if (policy.allowPrivate === true) return undefined;\n const before = egressRefusal(policy, url.hostname);\n if (before !== undefined) return before;\n const name = nameOf(url.hostname);\n if (literalOf(name) !== undefined || permitted(policy, name, undefined)) return undefined;\n const addresses = await new Promise<LookupAddress[] | Error>((settle) =>\n resolve(name, { all: true }, (error, found) => settle(error ?? found)),\n );\n if (addresses instanceof Error) return undefined;\n for (const entry of addresses) {\n const kind = addressKind(entry.address);\n if (kind !== undefined && !permitted(policy, name, entry.address))\n return egressNote(shown(entry.address), kind);\n }\n return undefined;\n};\n\n/** This function finds an `EgressRefused` in an error, its cause or its group of errors. */\nexport const egressError = (error: unknown): EgressRefused | undefined => {\n for (\n let current = error, depth = 0;\n current !== undefined && current !== null && depth < 5;\n depth += 1\n ) {\n if (current instanceof EgressRefused) return current;\n if (current instanceof AggregateError) {\n const inner = current.errors.find((item: unknown) => item instanceof EgressRefused) as\n EgressRefused | undefined;\n if (inner !== undefined) return inner;\n }\n current = (current as { cause?: unknown }).cause;\n }\n return undefined;\n};\n","import { createHash } from \"node:crypto\";\n\n/**\n * A capture record is one line of the capture log. Each fetch decision makes one record, and a\n * fetch that the crawler declines makes one too.\n *\n * The record id comes from a hash of its content, so the same decision always gets the same id.\n * A citation names this id.\n *\n * The id has 48 bits: the source and the first 12 hexadecimal digits of the hash. Approximately\n * 1 million records give a chance of a collision of 0.2 percent. The package keeps this length on purpose,\n * because the ids of the earlier capture logs and their citations must stay the same. A later\n * version can make the ids of new records longer behind a record version. It must still read a\n * 12-digit id.\n */\nexport interface CaptureRecord {\n readonly id: string;\n /** The source of the record. The crawler writes `fetch`, and an application can add its own sources. */\n readonly source: string;\n readonly kind: string;\n readonly external_id?: string | undefined;\n readonly occurred_at?: string | undefined;\n readonly received_at: string;\n readonly payload: unknown;\n readonly content_hash: string;\n /** This label comes from the application. The crawler writes it as it is and does not read it. */\n readonly scope?: string | undefined;\n /**\n * These labels come from the application, for example the lineage of a request. The crawler writes\n * them as they are and does not read them. Like `scope`, they are not part of the hash.\n */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/** These are the fields of a new record. The function `makeRecord` calculates the id and the hash. */\nexport interface RecordInput {\n readonly source: string;\n readonly kind: string;\n readonly payload: unknown;\n readonly received_at: string;\n readonly external_id?: string | undefined;\n readonly occurred_at?: string | undefined;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/**\n * This function writes a value as JSON with the keys of each object in sorted order. Equal values\n * thus always give equal text, and equal text gives an equal hash.\n */\nexport const stableStringify = (value: unknown): string => {\n if (value === null || typeof value !== \"object\") {\n return JSON.stringify(value) ?? \"null\";\n }\n if (Array.isArray(value)) {\n return `[${value.map((item) => stableStringify(item)).join(\",\")}]`;\n }\n const entries = Object.entries(value as Record<string, unknown>)\n .filter(([, item]) => item !== undefined)\n .sort(([a], [b]) => a.localeCompare(b))\n .map(([key, item]) => `${JSON.stringify(key)}:${stableStringify(item)}`);\n return `{${entries.join(\",\")}}`;\n};\n\n/** This function gives the SHA-256 hash of a text as hexadecimal digits. */\nexport const sha256 = (text: string): string =>\n createHash(\"sha256\").update(text, \"utf8\").digest(\"hex\");\n\n/**\n * This function makes a capture record. The hash covers the source, the kind, the external id\n * and the payload. The id is the source and the first 12 digits of the hash. The times, `scope`\n * and `labels` are not part of the hash.\n */\nexport const makeRecord = (input: RecordInput): CaptureRecord => {\n const content_hash = sha256(\n stableStringify({\n source: input.source,\n kind: input.kind,\n external_id: input.external_id ?? null,\n payload: input.payload,\n }),\n );\n return {\n id: `${input.source}.${content_hash.slice(0, 12)}`,\n source: input.source,\n kind: input.kind,\n received_at: input.received_at,\n payload: input.payload,\n content_hash,\n ...(input.external_id !== undefined ? { external_id: input.external_id } : {}),\n ...(input.occurred_at !== undefined ? { occurred_at: input.occurred_at } : {}),\n ...(input.scope !== undefined ? { scope: input.scope } : {}),\n ...(input.labels !== undefined && Object.keys(input.labels).length > 0\n ? { labels: { ...input.labels } }\n : {}),\n };\n};\n","import type { Readable } from \"node:stream\";\nimport { Transform } from \"node:stream\";\nimport { createBrotliDecompress, createGunzip, createInflate, createInflateRaw } from \"node:zlib\";\nimport type { Dispatcher } from \"undici\";\nimport { Agent, request } from \"undici\";\nimport type { BlobStore } from \"./blobs.ts\";\nimport type { Classified, Outcome } from \"./classify.ts\";\nimport { classify, isClientError, parseHttpDate } from \"./classify.ts\";\nimport type { Variant } from \"./docs.ts\";\nimport type { EgressOptions } from \"./egress.ts\";\nimport { checkEgress, egressError, egressLookup, egressRefusal } from \"./egress.ts\";\nimport type {\n Clock,\n HostResult,\n HostStateStore,\n Paced,\n PolicyOptions,\n PolitenessPolicy,\n RestInput,\n ValidatorAnswer,\n ValidatorMemo,\n} from \"./politeness.ts\";\nimport {\n DEFAULT_POLICY,\n HostScheduler,\n MAX_STOP_MS,\n HostStopped,\n checkPolicy,\n equalJitterMs,\n policyFor,\n realClock,\n restAfter,\n validatorsToSend,\n} from \"./politeness.ts\";\nimport type { CaptureRecord } from \"./record.ts\";\nimport { makeRecord } from \"./record.ts\";\nimport type { Rendered, Renderer } from \"./render.ts\";\nimport type { RefusedReason } from \"./render.ts\";\nimport { NO_PLAYWRIGHT, RenderRefused } from \"./render.ts\";\nimport type { Robots, RobotsDecision } from \"./robots.ts\";\nimport {\n decide,\n isProductToken,\n robotsFromResponse,\n robotsFromStatus,\n signalsFor,\n unreachableRobots,\n} from \"./robots.ts\";\nimport type { AiInput, Signals } from \"./signals.ts\";\nimport { NO_SIGNALS, metaSignals, readSignals } from \"./signals.ts\";\nimport { BOT_PAGE, DEFAULT_TOKEN } from \"./version.ts\";\nimport { cacheKeyOf } from \"./urls.ts\";\n\n/**\n * The crawler is a polite fetcher. It sends one request at a time to each host, and it obeys\n * robots.txt for web pages. Its `User-Agent` header tells what it is. It asks again with the\n * validators of the stored copy, and it stops when a site refuses. Each decision goes into the\n * capture log, and a fetch that the crawler declines goes there too.\n *\n * There are two kinds of request. A documented public API (`api`) has its own terms and rate\n * limits. A web page (`page`) obeys robots.txt. The settings come from measurements:\n *\n * - The `request` function of undici on one `Agent`.\n * - A keep-alive time longer than the delay for each host. With the default of 4 seconds, each\n * polite request opens a new connection.\n * - A backoff with jitter in the queue, not the retry handler of undici. That handler limits\n * Retry-After to 30 seconds and adds no jitter.\n * - A maximum size for each body, after decompression. The crawler keeps the first part of a\n * longer body and marks it as truncated.\n *\n * A fetch and a render go through one admission. It examines the URL, the egress policy, the\n * product tokens, robots.txt, the cache and the stops of the host. Only the transport is\n * different: an HTTP request or the headless browser.\n *\n * The egress policy (`EgressOptions`) keeps the crawler on public addresses by default. The\n * politeness policy (`PolitenessPolicy`) holds each limit, and a host can have its own values.\n * The fetch and the render do not throw an error: each problem is an outcome with a note.\n */\n\n/**\n * The outcome of a fetch or a render. A `skipped` request did not go out. A render of a URL that\n * gives a file and not a page is `not-html`.\n */\nexport type FetchOutcome = Outcome | \"skipped\" | \"not-html\";\n\nexport interface FetchRequest {\n readonly url: string;\n /** The value `api` is a documented API with its own terms. The value `page` is a web page, and robots.txt applies. */\n readonly mode: \"api\" | \"page\";\n /** This recipe reads the capture again. The function `reindex` uses it to replay the capture. */\n readonly recipe: string;\n readonly accept?: string | undefined;\n /**\n * These are more request headers. The crawler does not write them into the capture. They\n * cannot replace `User-Agent`, `From`, `Accept-Encoding` or the validators. A redirect to a\n * different origin removes `Authorization`, `Cookie` and `Proxy-Authorization`.\n */\n readonly headers?: Readonly<Record<string, string>> | undefined;\n /**\n * These query parameters contain secrets. Each capture shows `REDACTED` in their place. The\n * names in `DEFAULT_SECRET_PARAMS` get the same treatment with no list.\n */\n readonly secretParams?: readonly string[] | undefined;\n /** This value is the documented pace of the source: the shortest time between two of its requests. */\n readonly minIntervalMs?: number | undefined;\n /**\n * A stored capture younger than this age answers, and the crawler sends no request. The value\n * `auto` uses the freshness that the server gave (RFC 9111): `Cache-Control`, `Expires`, or 10\n * percent of the age of `Last-Modified`.\n */\n readonly maxAgeMs?: number | \"auto\" | undefined;\n readonly maxBytes?: number | undefined;\n /**\n * These are the product tokens of an agent that the fetch acts for, for example `Claude-User`.\n * The robots.txt rules for these tokens apply too.\n */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** This name connects the captures to the job that asked for them. */\n readonly job?: string | undefined;\n /** When this signal aborts, the crawler stops the fetch. The result is then `skipped` or `error`, with a note. */\n readonly signal?: AbortSignal | undefined;\n /** This label goes into each capture record of the request. It replaces the `scope` of the crawler. */\n readonly scope?: string | undefined;\n /** These labels go into each capture record of the request, for example its lineage. They are not part of the capture id. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\nexport interface Fetched {\n /** This URL is the URL of the capture. Its secrets show as `REDACTED`. */\n readonly url: string;\n readonly status: number;\n readonly outcome: FetchOutcome;\n /** This value is true when a stored capture answered, with no request. */\n readonly fromCache: boolean;\n /** This id is the capture record behind the body. A citation names it. */\n readonly capture: string | undefined;\n readonly blob: string | undefined;\n readonly body: Uint8Array | undefined;\n readonly contentType: string | undefined;\n readonly signals: Signals;\n readonly note: string | undefined;\n /** This value is true when the body was longer than the maximum, so the crawler kept only its first part. */\n readonly truncated?: boolean | undefined;\n /** This time (ISO 8601) is the time of the fetch of the body. For a stored copy, it is the time of that copy. */\n readonly fetchedAt?: string | undefined;\n /** This label is the scope of the capture. */\n readonly scope?: string | undefined;\n /** These labels are the labels of the capture. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/** The crawler writes each record to a capture writer. The class `CaptureLog` is one. */\nexport interface CaptureWriter {\n append(record: CaptureRecord): Promise<void>;\n close?(): Promise<void>;\n}\n\n/** A cache entry is the last answer for a URL: its capture, its body and its validators. */\nexport interface CacheEntry {\n /** This text is the cache key: the canonical URL, with its secrets as `REDACTED`. */\n readonly url: string;\n readonly capture: string;\n readonly blob: string | null;\n readonly etag: string | null;\n readonly last_modified: string | null;\n readonly content_type: string | null;\n readonly fetched_at: string;\n readonly status: number;\n /** This value is what the page said about AI input at the time of the fetch. */\n readonly ai_input: AiInput;\n /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */\n readonly location?: string | null | undefined;\n /** This value is true when the stored body is only the first part of the response. */\n readonly truncated?: boolean | undefined;\n /** This time (ISO 8601) is the end of the freshness that the server gave. */\n readonly fresh_until?: string | null | undefined;\n /** This text holds the request headers that the response varies on (`Vary`), with their values. */\n readonly vary_key?: string | null | undefined;\n /** This value tells a plain response from a page that the browser rendered. The default is `plain`. */\n readonly variant?: Variant | undefined;\n}\n\n/**\n * The cache index holds the last answer for each URL. The crawler reads it before a request, and\n * it writes it after an answer. The class `DocIndex` is one. An index that keeps only the\n * necessary fields of `CacheEntry` works, and the crawler keeps the other fields in memory.\n */\nexport interface CacheIndex {\n latest(url: string, variant?: Variant): CacheEntry | undefined;\n setLatest(entry: CacheEntry): void;\n}\n\n/** An event tells a listener what the crawler does. The listener cannot change it. */\nexport type CrawlEvent =\n | {\n readonly type: \"request\";\n readonly url: string;\n readonly host: string;\n readonly transport: \"http\" | \"browser\";\n }\n | {\n readonly type: \"response\";\n readonly url: string;\n readonly host: string;\n readonly status: number;\n readonly outcome: FetchOutcome;\n readonly elapsedMs: number;\n }\n | { readonly type: \"wait\"; readonly host: string; readonly ms: number; readonly reason: string }\n | {\n readonly type: \"stop\";\n readonly host: string;\n readonly until: number;\n readonly reason: string;\n }\n | { readonly type: \"skip\"; readonly url: string; readonly note: string };\n\nexport interface CrawlerOptions {\n /** This product token is the name that robots.txt rules match. It contains only letters, `_` and `-` (RFC 9309). */\n readonly token: string;\n /** This text is the full `User-Agent` header. */\n readonly userAgent: string;\n /** This contact goes into the `From` header. Without it, the crawler sends no `From` header. */\n readonly from?: string | undefined;\n readonly log: CaptureWriter;\n readonly blobs: BlobStore;\n readonly index: CacheIndex;\n readonly clock?: Clock | undefined;\n readonly random?: (() => number) | undefined;\n /** This value is the number of hosts that get requests at one time. The default is 16. */\n readonly maxActive?: number | undefined;\n /**\n * This value tells if the crawler also obeys the robots.txt group of `PoliteCrawler`. That applies\n * when the `User-Agent` links the bot page of polite-crawler (`BOT_PAGE`) and the token is different.\n * Site owners read that page, and they write their rules for `PoliteCrawler`. The most restrictive\n * of the groups applies, as for `onBehalfOf`. The default is true. Set false for a bot page of your own.\n */\n readonly obeyDefaultToken?: boolean | undefined;\n /**\n * This value is the number of pages that the browser renders at one time. The default is 2. Each\n * render is a browser context of its own, which holds much memory. A render waits for a slot before\n * the lease of its host, so the wait does not count toward the time of the host.\n */\n readonly maxRenders?: number | undefined;\n /**\n * This undici dispatcher sends the requests. With a dispatcher of your own, the crawler checks\n * only literal addresses. Put `egressLookup` in its connection options to check the address after DNS.\n */\n readonly dispatcher?: Dispatcher | undefined;\n /** This label goes into each capture record as it is. */\n readonly scope?: string | undefined;\n /**\n * This function ranks the scopes of the requests, the most private scope highest. One fetch of a\n * robots.txt can serve requests of different scopes. Its record then takes the scope with the\n * highest rank, and the labels of each request. With no function, the first request gives the scope.\n */\n readonly scopeRank?: ((scope: string) => number) | undefined;\n /** This browser renders the pages that need JavaScript. Without it, the crawler renders no page. */\n readonly renderer?: Renderer | undefined;\n /** This policy controls the addresses that the crawler fetches. By default, the crawler fetches only public addresses. */\n readonly egress?: EgressOptions | undefined;\n /** These values change the limits of the politeness policy, for each host or for some hosts. */\n readonly politeness?: PolicyOptions | undefined;\n /** This store keeps the pace, the counts and the stops of each host. The default keeps them in memory. */\n readonly hostState?: HostStateStore | undefined;\n /** This function gets an event for each request, response, wait, stop and skip. An error in it does not stop the crawler. */\n readonly onEvent?: ((event: CrawlEvent) => void) | undefined;\n}\n\nexport interface RenderRequest {\n readonly url: string;\n readonly recipe: string;\n /** A stored render younger than this age answers. The value `auto` uses the freshness that the server gave. */\n readonly maxAgeMs?: number | \"auto\" | undefined;\n readonly onBehalfOf?: readonly string[] | undefined;\n readonly job?: string | undefined;\n /** When this signal aborts, the crawler stops the render, and the browser stops the page. */\n readonly signal?: AbortSignal | undefined;\n /** This label goes into the capture record of the render. It replaces the `scope` of the crawler. */\n readonly scope?: string | undefined;\n /** These labels go into the capture record of the render. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/**\n * This function gives the key of the cache entry of a rendered page in the form of version 0.1.\n * The index keeps the variant in a column of its own, and it reads this form too.\n */\nexport const renderedKey = (url: string): string => `${url}#rendered`;\n\n/** This value is the version of the payload of the records that the crawler writes. A record with no `schema` is version 1. */\nexport const CAPTURE_SCHEMA = 2;\n\n/** The content type of a rendered page: the browser gives its text as UTF-8. */\nexport const RENDERED_TYPE = \"text/html; charset=utf-8\";\n\n/**\n * These query parameters hold secrets in many APIs. The crawler writes `REDACTED` in place of\n * their values in each capture, in addition to the names of `secretParams`.\n */\nexport const DEFAULT_SECRET_PARAMS: readonly string[] = [\n \"access_token\",\n \"api-key\",\n \"api_key\",\n \"apikey\",\n \"auth\",\n \"auth_token\",\n \"client_secret\",\n \"id_token\",\n \"passwd\",\n \"password\",\n \"pwd\",\n \"refresh_token\",\n \"secret\",\n \"session\",\n \"session_id\",\n \"sessionid\",\n \"sig\",\n \"signature\",\n \"token\",\n \"x-amz-credential\",\n \"x-amz-security-token\",\n \"x-amz-signature\",\n \"x-goog-credential\",\n \"x-goog-signature\",\n];\n\nconst ERROR_BODY_BYTES = 64 * 1024;\nconst ROBOTS_BYTES = 512 * 1024;\nconst LAST_DATE_MS = 8.64e15;\nconst WEB_PROTOCOLS: ReadonlySet<string> = new Set([\"http:\", \"https:\"]);\nconst HEADER_TEXT = /^[\\t -~]*$/;\nconst PERMANENT_REDIRECTS: ReadonlySet<number> = new Set([301, 308]);\nconst HEURISTIC_STATUSES: ReadonlySet<number> = new Set([\n 200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501,\n]);\n/** A caller cannot set these headers. The crawler sets them, or HTTP does. */\nconst RESERVED_HEADERS: ReadonlySet<string> = new Set([\n \"user-agent\",\n \"from\",\n \"accept-encoding\",\n \"if-none-match\",\n \"if-modified-since\",\n \"if-match\",\n \"if-unmodified-since\",\n \"if-range\",\n \"range\",\n \"host\",\n \"connection\",\n \"content-length\",\n \"transfer-encoding\",\n \"te\",\n \"upgrade\",\n \"keep-alive\",\n]);\n/** A redirect to a different origin removes these headers, as the Fetch standard does. */\nconst CREDENTIAL_HEADERS: ReadonlySet<string> = new Set([\n \"authorization\",\n \"cookie\",\n \"proxy-authorization\",\n]);\n/** The crawler sends the same value of these headers in each request, so a `Vary` on them changes nothing. */\nconst CONSTANT_HEADERS: ReadonlySet<string> = new Set([\"accept-encoding\", \"user-agent\", \"from\"]);\n/**\n * These content codings change nothing in the body. A server that stored an object with\n * `aws-chunked` can send the name with the decoded body.\n */\nconst IDENTITY_CODINGS: ReadonlySet<string> = new Set([\"identity\", \"aws-chunked\"]);\n\n/**\n * This function tells if an answer is a web page that the request did not ask for. The answer is\n * HTML, and the `Accept` of the request admits no HTML (`application/json`). A login or an error\n * page in place of the data of an API is such an answer. A media range with `q=0` admits nothing.\n */\nconst unaskedPage = (accept: string | undefined, contentType: string | undefined): boolean => {\n const type = (contentType ?? \"\").split(\";\")[0]!.trim().toLowerCase();\n if (\n accept === undefined ||\n accept.trim() === \"\" ||\n (type !== \"text/html\" && type !== \"application/xhtml+xml\")\n )\n return false;\n return !accept.split(\",\").some((range) => {\n const [media = \"\", ...params] = range.split(\";\").map((part) => part.trim().toLowerCase());\n const q = params.find((param) => param.startsWith(\"q=\"));\n if (q !== undefined && Number(q.slice(2)) === 0) return false;\n return (\n media === \"*/*\" ||\n media === type ||\n (media.endsWith(\"/*\") && type.startsWith(media.slice(0, -1)))\n );\n });\n};\n\n/** This function reads the TDMRep `<meta>` tags at the start of an HTML body. */\nconst pageSignals = (body: Uint8Array | undefined, contentType: string | undefined): string[] =>\n body === undefined || !/html/i.test(contentType ?? \"\")\n ? []\n : metaSignals(new TextDecoder(\"latin1\").decode(body.subarray(0, 65_536)));\n\n/** A count of slots: `max` holders at one time at most, and the others wait in sequence. */\nclass Slots {\n private used = 0;\n private readonly waiting: Array<() => void> = [];\n\n constructor(private readonly max: number) {}\n\n /** This method gives a function that frees the slot. It rejects when the signal aborts before a slot is free. */\n async acquire(signal: AbortSignal): Promise<() => void> {\n if (this.used < this.max) {\n this.used += 1;\n return this.releaser();\n }\n await new Promise<void>((resolve, reject) => {\n const stop = (): void => {\n const at = this.waiting.indexOf(go);\n if (at !== -1) this.waiting.splice(at, 1);\n reject(signal.reason instanceof Error ? signal.reason : new Error(\"cancelled\"));\n };\n const go = (): void => {\n signal.removeEventListener(\"abort\", stop);\n resolve();\n };\n if (signal.aborted) {\n stop();\n return;\n }\n this.waiting.push(go);\n signal.addEventListener(\"abort\", stop, { once: true });\n });\n // the slot of the holder that freed it passes to this waiter, so the count stays\n return this.releaser();\n }\n\n private releaser(): () => void {\n let freed = false;\n return () => {\n if (freed) return;\n freed = true;\n const next = this.waiting.shift();\n if (next !== undefined) next();\n else this.used -= 1;\n };\n }\n}\n\n/** This function tells if an error of a renderer says that it has no browser: no `playwright-core`, or no browser on the disk. */\nconst missingBrowser = (message: string): boolean =>\n message.startsWith(NO_PLAYWRIGHT) ||\n /Executable doesn't exist|Cannot find (?:module|package) 'playwright-core'/.test(message);\n\n/** This text tells why the crawler did not fetch a page: a rule of the site, or no robots.txt to read. */\nconst robotsRefusal = (robots: Robots, by: string): string =>\n robots.unreachable === undefined\n ? `robots.txt disallows it for ${by}`\n : `robots.txt could not be read (${robots.unreachable}), so nothing on the site is fetched`;\n\nconst messageOf = (error: unknown): string =>\n error instanceof Error ? (error.message.split(/\\r?\\n/)[0] ?? \"\") : String(error);\n\nconst isoOf = (ms: number): string | undefined =>\n Number.isFinite(ms) && Math.abs(ms) < LAST_DATE_MS ? new Date(ms).toISOString() : undefined;\n\nconst stopNote = (reason: string, until: number): string => {\n const end = isoOf(until);\n return end === undefined ? `host stopped: ${reason}` : `host stopped until ${end}: ${reason}`;\n};\n\nconst secretSet = (secretParams: readonly string[]): Set<string> =>\n new Set([...DEFAULT_SECRET_PARAMS, ...secretParams].map((name) => name.toLowerCase()));\n\nconst nameOfParam = (raw: string): string => {\n try {\n return decodeURIComponent(raw.replace(/\\+/g, \" \")).toLowerCase();\n } catch {\n return raw.toLowerCase();\n }\n};\n\n/**\n * This function gives the URL of a capture. It removes a user name and a password, and it writes\n * `REDACTED` in place of the value of each secret parameter. The other parameters stay as they\n * were written.\n */\nexport const redact = (url: URL, secretParams: readonly string[] = []): string => {\n const copy = new URL(url.href);\n copy.username = \"\";\n copy.password = \"\";\n if (copy.search.length > 1) {\n const secrets = secretSet(secretParams);\n const parts = copy.search\n .slice(1)\n .split(\"&\")\n .map((part) => {\n const equals = part.indexOf(\"=\");\n if (equals === -1) return part;\n const name = part.slice(0, equals);\n return secrets.has(nameOfParam(name)) ? `${name}=REDACTED` : part;\n });\n copy.search = `?${parts.join(\"&\")}`;\n }\n return copy.href;\n};\n\n/** This function tells if a URL holds a secret: a user name, a password or a secret parameter. */\nconst holdsSecrets = (url: URL, secretParams: readonly string[]): boolean => {\n if (url.username !== \"\" || url.password !== \"\") return true;\n const secrets = secretSet(secretParams);\n return url.search\n .slice(1)\n .split(\"&\")\n .some(\n (part) => part.includes(\"=\") && secrets.has(nameOfParam(part.slice(0, part.indexOf(\"=\")))),\n );\n};\n\n/** The pace, the stops and the robots.txt of a host do not change with a trailing dot or the case of the name. */\nconst hostKey = (url: URL): string =>\n `${url.hostname.toLowerCase().replace(/\\.$/, \"\")}${url.port === \"\" ? \"\" : `:${url.port}`}`;\n\nconst originKey = (url: URL): string => `${url.protocol}//${hostKey(url)}`;\n\nconst headerOf =\n (headers: Readonly<Record<string, string | string[] | undefined>>) =>\n (name: string): string | undefined => {\n const value = headers[name.toLowerCase()];\n return Array.isArray(value) ? value.join(\", \") : value;\n };\n\nclass Lru<K, V> {\n private readonly map = new Map<K, V>();\n constructor(private readonly max: number) {}\n\n get(key: K): V | undefined {\n const value = this.map.get(key);\n if (value !== undefined) {\n this.map.delete(key);\n this.map.set(key, value);\n }\n return value;\n }\n\n peek(key: K): V | undefined {\n return this.map.get(key);\n }\n\n set(key: K, value: V): void {\n this.map.delete(key);\n this.map.set(key, value);\n if (this.map.size > this.max) this.map.delete(this.map.keys().next().value as K);\n }\n\n delete(key: K): void {\n this.map.delete(key);\n }\n\n keys(): K[] {\n return [...this.map.keys()];\n }\n\n get size(): number {\n return this.map.size;\n }\n}\n\n/** A body failed in transport (a reset, a timeout) or in decompression. */\nclass BodyError extends Error {\n constructor(\n readonly kind: \"transport\" | \"decode\",\n cause: unknown,\n ) {\n super(messageOf(cause), { cause });\n }\n}\n\n/** RFC 9110 says that `deflate` is the zlib format, but some servers send raw deflate. This stream reads both. */\nconst inflateAny = (): Transform => {\n let inner: Transform | undefined;\n const outer: Transform = new Transform({\n transform(chunk: Buffer, _encoding, done) {\n if (inner === undefined) {\n const zlib =\n chunk.length >= 2 &&\n (chunk[0]! & 0x0f) === 8 &&\n ((chunk[0]! << 8) | chunk[1]!) % 31 === 0;\n inner = zlib ? createInflate() : createInflateRaw();\n inner.on(\"data\", (data: Buffer) => outer.push(data));\n inner.on(\"error\", (error) => outer.destroy(error));\n }\n inner.write(chunk, () => done());\n },\n flush(done) {\n if (inner === undefined) {\n done();\n return;\n }\n inner.once(\"end\", () => done());\n inner.end();\n },\n destroy(error, callback) {\n inner?.destroy();\n callback(error);\n },\n });\n return outer;\n};\n\nconst decoderFor = (coding: string): Transform => {\n if (coding === \"gzip\" || coding === \"x-gzip\") return createGunzip();\n if (coding === \"br\") return createBrotliDecompress();\n if (coding === \"deflate\") return inflateAny();\n throw new BodyError(\"decode\", new Error(`unsupported content-encoding \"${coding}\"`));\n};\n\n/** This function tells if a decoder stopped because the compressed body ended early: the server cut it. */\nconst endedEarly = (error: unknown): boolean =>\n typeof error === \"object\" &&\n error !== null &&\n \"code\" in error &&\n (error.code === \"Z_BUF_ERROR\" || /unexpected end of file/i.test(messageOf(error)));\n\n/**\n * This function reads a body as a stream and decodes each content coding in sequence. It counts\n * the decoded bytes and keeps at most `maxBytes`. Thus a decompression bomb stops at the\n * maximum, and a long page keeps its first part. A body that fails gives a `BodyError`. A\n * compressed body that ends early gives the part that it decoded, marked `cut`.\n */\nconst readBody = async (\n body: Readable,\n encoding: string | undefined,\n maxBytes: number,\n): Promise<{ readonly bytes: Uint8Array; readonly truncated: boolean; readonly cut: boolean }> => {\n const codings = (encoding ?? \"\")\n .split(\",\")\n .map((coding) => coding.trim().toLowerCase())\n .filter((coding) => coding.length > 0 && !IDENTITY_CODINGS.has(coding))\n .reverse();\n // undici gives an error when a body that did not end is destroyed; the loop below reports real errors\n body.on(\"error\", () => undefined);\n let decoders: Transform[];\n try {\n decoders = codings.map(decoderFor);\n } catch (error) {\n body.destroy();\n throw error;\n }\n // a compressed stream that gives almost no output stops at this length of input\n const rawCap = maxBytes * 4 + 65_536;\n let rawBytes = 0;\n let stream: Readable = body;\n if (decoders.length > 0) {\n body.on(\"data\", (chunk: Buffer) => {\n rawBytes += chunk.length;\n });\n for (const decoder of decoders) {\n const source = stream;\n source.on(\"error\", (error) => decoder.destroy(error));\n stream = source.pipe(decoder);\n }\n }\n const chunks: Buffer[] = [];\n let size = 0;\n let truncated = false;\n let cut = false;\n try {\n for await (const chunk of stream as AsyncIterable<Buffer>) {\n const room = maxBytes - size;\n if (chunk.length > room) {\n // the chunk that crosses the maximum gives its first part\n if (room > 0) chunks.push(chunk.subarray(0, room));\n size += Math.max(0, room);\n truncated = true;\n break;\n }\n chunks.push(chunk);\n size += chunk.length;\n if (rawBytes > rawCap) {\n truncated = true;\n break;\n }\n }\n } catch (error) {\n const transport = body.errored !== null || decoders.length === 0;\n // a server that closes a compressed body early: the part before the end is still the page\n if (!transport && size > 0 && endedEarly(error)) {\n cut = true;\n truncated = true;\n } else {\n throw new BodyError(transport ? \"transport\" : \"decode\", error);\n }\n } finally {\n body.destroy();\n for (const decoder of decoders) decoder.destroy();\n }\n // one copy into a buffer of its own, so no reader sees the shared pool of Node.js\n const bytes = new Uint8Array(size);\n let at = 0;\n for (const chunk of chunks) {\n bytes.set(chunk, at);\n at += chunk.length;\n }\n return { bytes, truncated, cut };\n};\n\nconst directives = (value: string | undefined): Map<string, string> => {\n const out = new Map<string, string>();\n for (const part of (value ?? \"\").split(\",\")) {\n const equals = part.indexOf(\"=\");\n const name = (equals === -1 ? part : part.slice(0, equals)).trim().toLowerCase();\n if (name.length === 0) continue;\n const raw = equals === -1 ? \"\" : part.slice(equals + 1).trim();\n out.set(\n name,\n raw.length >= 2 && raw.startsWith('\"') && raw.endsWith('\"') ? raw.slice(1, -1) : raw,\n );\n }\n return out;\n};\n\nconst TEN_YEARS_MS = 10 * 365 * 24 * 60 * 60 * 1000;\n\n/**\n * This function gives the end of the freshness of a response, as RFC 9111 §4.2 calculates it.\n * The sources in sequence are `max-age`, `Expires`, and 10 percent of the age of\n * `Last-Modified`, with a maximum. A response with `no-store` or `no-cache` is stale at once. A\n * response with no freshness gives null.\n */\nconst freshUntil = (\n header: (name: string) => string | undefined,\n status: number,\n fetchedAt: number,\n lastModified: string | null | undefined,\n capMs: number,\n): string | null => {\n const control = directives(header(\"cache-control\"));\n if (control.has(\"no-store\") || control.has(\"no-cache\")) return isoOf(fetchedAt) ?? null;\n const ageText = header(\"age\")?.trim() ?? \"\";\n const age = /^\\d+$/.test(ageText) ? Number(ageText) * 1000 : 0;\n const date = parseHttpDate(header(\"date\")) ?? fetchedAt;\n let lifetime: number | undefined;\n const maxAge = control.get(\"max-age\");\n const expires = header(\"expires\");\n if (maxAge !== undefined && /^\\d+$/.test(maxAge)) lifetime = Number(maxAge) * 1000;\n else if (expires !== undefined) {\n // an Expires that is not a date means that the response is already stale\n const at = parseHttpDate(expires);\n lifetime = at === undefined ? 0 : at - date;\n } else if (HEURISTIC_STATUSES.has(status)) {\n const modified = parseHttpDate(lastModified ?? undefined);\n if (modified !== undefined && modified < date)\n lifetime = Math.min(capMs, (date - modified) / 10);\n else if (PERMANENT_REDIRECTS.has(status)) lifetime = capMs;\n }\n if (lifetime === undefined) return null;\n return isoOf(fetchedAt + Math.max(0, Math.min(lifetime, TEN_YEARS_MS) - age)) ?? null;\n};\n\n/** This function gives the request headers that a response varies on, with their values. A `Vary: *` gives `*`. */\nconst varyKeyOf = (\n vary: string | undefined,\n sent: Readonly<Record<string, string>>,\n): string | null => {\n if (vary === undefined) return null;\n const names = vary\n .split(\",\")\n .map((name) => name.trim().toLowerCase())\n .filter((name) => name.length > 0 && !CONSTANT_HEADERS.has(name));\n if (names.includes(\"*\")) return \"*\";\n if (names.length === 0) return null;\n return [...new Set(names)]\n .sort()\n .map((name) => `${name}=${sent[name] ?? \"\"}`)\n .join(\"\\n\");\n};\n\nconst varyMatches = (\n stored: string | null | undefined,\n sent: Readonly<Record<string, string>>,\n): boolean => {\n if (stored === undefined || stored === null) return true;\n if (stored === \"*\") return false;\n return stored.split(\"\\n\").every((line) => {\n const equals = line.indexOf(\"=\");\n return (sent[line.slice(0, equals)] ?? \"\") === line.slice(equals + 1);\n });\n};\n\nconst withoutCredentials = (\n headers: Readonly<Record<string, string>> | undefined,\n): Readonly<Record<string, string>> | undefined =>\n headers === undefined\n ? undefined\n : Object.fromEntries(\n Object.entries(headers).filter(([name]) => !CREDENTIAL_HEADERS.has(name.toLowerCase())),\n );\n\nconst invalidNumber = (\n name: string,\n value: number | \"auto\" | undefined,\n auto: boolean,\n): string | undefined => {\n if (value === undefined || (auto && value === \"auto\")) return undefined;\n return typeof value === \"number\" && value >= 0\n ? undefined\n : `invalid option: ${name} is ${String(value)}`;\n};\n\nconst closedResult = (url: string): Fetched => ({\n url,\n status: 0,\n outcome: \"skipped\",\n fromCache: false,\n capture: undefined,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note: \"crawler closed\",\n});\n\n/** One step of a fetch: the answer for one URL, and the next URL when the answer is a redirect to follow. */\ninterface Step {\n readonly fetched: Fetched;\n /** This URL is the target of the redirect, with its secrets. It stays in memory. */\n readonly next?: string | undefined;\n readonly retryAfterMs?: number | undefined;\n}\n\n/** A robots verdict is a parsed robots.txt and the capture that holds it. */\ninterface RobotsVerdict {\n readonly robots: Robots;\n readonly capture: string | undefined;\n /** When the crawler could not read robots.txt and used a stored copy, this text gives the reason. */\n readonly stale?: string | undefined;\n}\n\n/** The scope and the labels of the requests that wait for one fetch of a robots.txt. The record of the fetch reads them when it is written. */\ninterface Waiting {\n scope: string | undefined;\n labels: Record<string, string> | undefined;\n}\n\ninterface RobotsEntry {\n readonly at: number;\n readonly waiting: Waiting;\n ttlMs: number;\n verdict: Promise<RobotsVerdict>;\n value: RobotsVerdict | undefined;\n /** This value counts the unreachable verdicts in sequence, for the backoff. */\n misses: number;\n /** This verdict is the last good robots.txt, with its time. */\n good: { readonly verdict: RobotsVerdict; readonly at: number } | undefined;\n}\n\ninterface Settle {\n readonly outcome: FetchOutcome;\n readonly status: number;\n readonly retryAfterMs: number | undefined;\n /** The pace of the host: the kind of the request, its duration, the crawl delay and the documented interval. */\n readonly rest: RestInput;\n /** A 401 or a 403 on robots.txt means \"no robots.txt\" (RFC 9309), so it does not stop the host. */\n readonly robotsFile: boolean;\n /** This result replaces the result that the outcome gives. */\n readonly result?: HostResult | undefined;\n}\n\ntype RenderDone =\n | {\n readonly kind: \"rendered\";\n readonly rendered: Rendered;\n readonly classified: Classified;\n readonly elapsedMs: number;\n }\n | { readonly kind: \"failed\"; readonly note: string; readonly elapsedMs: number }\n | {\n readonly kind: \"refused\";\n readonly reason: RefusedReason;\n readonly note: string;\n /** This URL is the request that the renderer refused. */\n readonly url?: string | undefined;\n readonly elapsedMs: number;\n }\n | { readonly kind: \"cancelled\"; readonly elapsedMs: number }\n | { readonly kind: \"closed\" };\n\n/** These are the parts of a request that the admission reads. */\ninterface AdmitSpec {\n readonly transport: \"http\" | \"browser\";\n readonly url: string;\n readonly mode: \"api\" | \"page\";\n readonly base: Record<string, unknown>;\n readonly secretParams: readonly string[];\n readonly invalid: string | undefined;\n readonly onBehalfOf: readonly string[] | undefined;\n readonly scope: string | undefined;\n readonly labels: Readonly<Record<string, string>> | undefined;\n /** This function answers from the cache, when the stored copy is young enough. */\n readonly cached: (url: string) => Promise<Step | undefined>;\n}\n\n/** A request that the admission let through: the facts that the transport needs. */\ninterface Admitted {\n readonly target: URL;\n readonly url: string;\n readonly host: string;\n readonly policy: PolitenessPolicy;\n readonly tokens: readonly string[];\n readonly verdict: RobotsVerdict | undefined;\n readonly check: Record<string, unknown> | undefined;\n readonly robotsLines: readonly string[];\n readonly crawlDelaySeconds: number;\n readonly skip: (note: string, extra?: Record<string, unknown>) => Promise<Fetched>;\n}\n\nconst robotsCheck = (\n verdict: RobotsVerdict,\n tokens: readonly string[],\n decision: RobotsDecision,\n): Record<string, unknown> => ({\n capture: verdict.capture ?? null,\n tokens,\n rule:\n decision.rule === undefined\n ? null\n : {\n line: decision.rule.line ?? null,\n pattern: decision.rule.pattern,\n allow: decision.rule.allow,\n },\n group_line: decision.groups?.[0]?.line ?? null,\n ...(verdict.robots.unreachable !== undefined ? { unreachable: verdict.robots.unreachable } : {}),\n ...(verdict.stale !== undefined ? { stale: verdict.stale } : {}),\n});\n\nconst raceAbort = <T>(work: Promise<T>, signal: AbortSignal): Promise<T> => {\n if (signal.aborted) {\n work.catch(() => undefined);\n return Promise.reject(signal.reason);\n }\n return new Promise<T>((resolve, reject) => {\n const cancel = (): void => reject(signal.reason);\n signal.addEventListener(\"abort\", cancel, { once: true });\n work.then(\n (value) => {\n signal.removeEventListener(\"abort\", cancel);\n resolve(value);\n },\n (error: unknown) => {\n signal.removeEventListener(\"abort\", cancel);\n reject(error);\n },\n );\n });\n};\n\n/** The result of reading robots.txt: the rules, the capture that holds them, and the wait that the server asked for. */\nexport interface RobotsRead {\n readonly robots: Robots;\n readonly capture: string | undefined;\n readonly retryAfterMs: number | undefined;\n}\n\nexport class Crawler {\n private readonly clock: Clock;\n private readonly random: () => number;\n private readonly scheduler: HostScheduler;\n private readonly dispatcher: Dispatcher;\n private readonly ownsDispatcher: boolean;\n private readonly egress: EgressOptions;\n private readonly robots = new Lru<string, RobotsEntry>(4096);\n /** The fields of a cache entry that the index does not keep, and the raw target of a redirect. */\n private readonly meta = new Lru<\n string,\n { readonly entry: CacheEntry; readonly location?: string | undefined }\n >(10_000);\n private readonly inFlight = new Map<string, Promise<Step>>();\n private readonly running = new Set<Promise<unknown>>();\n private readonly closing = new AbortController();\n private closed = false;\n private closingDone: Promise<void> | undefined;\n /** This note tells why the renderer has no browser. After the first page that finds it, the crawler renders nothing. */\n private renderOff: string | undefined;\n /** The renders that run at one time (`maxRenders`). */\n private readonly renders: Slots;\n /** These tokens apply because the `User-Agent` links the bot page (`obeyDefaultToken`). */\n private readonly pageTokens: readonly string[];\n\n constructor(private readonly options: CrawlerOptions) {\n if (!isProductToken(options.token)) {\n throw new Error(\n `the token \"${options.token}\" is not a product token: use only letters, \"_\" and \"-\" (RFC 9309)`,\n );\n }\n for (const [name, value] of [\n [\"userAgent\", options.userAgent],\n [\"from\", options.from],\n ] as const) {\n if (value !== undefined && !HEADER_TEXT.test(value)) {\n throw new Error(\n `${name} \"${value}\" has a character that an HTTP header cannot hold: use printable ASCII`,\n );\n }\n }\n checkPolicy(options.politeness);\n this.clock = options.clock ?? realClock;\n this.random = options.random ?? Math.random;\n this.egress = options.egress ?? {};\n this.scheduler = new HostScheduler(this.clock, options.maxActive ?? 16, {\n store: options.hostState,\n });\n // the bot page tells site owners to write rules for PoliteCrawler, so an application that links it obeys them\n this.pageTokens =\n options.obeyDefaultToken !== false &&\n options.token.toLowerCase() !== DEFAULT_TOKEN.toLowerCase() &&\n options.userAgent.includes(BOT_PAGE)\n ? [DEFAULT_TOKEN]\n : [];\n const maxRenders = options.maxRenders ?? 2;\n if (!Number.isInteger(maxRenders) || maxRenders < 1) {\n throw new RangeError(`maxRenders must be an integer of 1 or more, not ${String(maxRenders)}`);\n }\n this.renders = new Slots(maxRenders);\n this.ownsDispatcher = options.dispatcher === undefined;\n this.dispatcher =\n options.dispatcher ??\n new Agent({\n // past the per-host delay, or every polite request reconnects\n keepAliveTimeout: 60_000,\n keepAliveMaxTimeout: 120_000,\n allowH2: true,\n // the egress policy checks each address after DNS, at the time of the connection\n connect: { timeout: 10_000, lookup: egressLookup(this.egress) },\n headersTimeout: 30_000,\n bodyTimeout: 60_000,\n });\n }\n\n private iso(): string {\n return new Date(this.clock.now()).toISOString();\n }\n\n private policyOf(url: URL): PolitenessPolicy {\n return policyFor(this.options.politeness, url.hostname);\n }\n\n private emit(event: CrawlEvent): void {\n try {\n this.options.onEvent?.(event);\n } catch {\n // a listener that fails does not change the crawl\n }\n }\n\n private track<T>(work: Promise<T>): Promise<T> {\n this.running.add(work);\n void work.then(\n () => this.running.delete(work),\n () => this.running.delete(work),\n );\n return work;\n }\n\n /** This method gives the validator memo of a host. The memo is a hint, so an error of its store gives no memo. */\n private memoOf(host: string): ValidatorMemo | undefined {\n try {\n return this.scheduler.store.validators?.(host);\n } catch {\n return undefined;\n }\n }\n\n private stopHost(host: string, until: number, reason: string): void {\n this.scheduler.store.stop(host, until, reason);\n this.emit({ type: \"stop\", host, until, reason });\n }\n\n private async record(\n kind: \"capture\" | \"revisit\" | \"skipped\",\n payload: Record<string, unknown>,\n url: string,\n labels: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<string | undefined> {\n // a closed crawler writes nothing, so its log stays closed\n if (this.closed) return undefined;\n const at = this.iso();\n const scope = labels.scope ?? this.options.scope;\n const record = makeRecord({\n source: \"fetch\",\n kind,\n payload: { schema: CAPTURE_SCHEMA, ...payload, fetched_at: at },\n received_at: at,\n occurred_at: at,\n external_id: url,\n ...(scope !== undefined ? { scope } : {}),\n ...(labels.labels !== undefined ? { labels: labels.labels } : {}),\n });\n await this.options.log.append(record);\n return record.id;\n }\n\n private async skip(\n base: Record<string, unknown>,\n url: string,\n note: string,\n extra: Record<string, unknown> = {},\n labels: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<Fetched> {\n this.emit({ type: \"skip\", url, note });\n return {\n url,\n status: 0,\n outcome: \"skipped\",\n fromCache: false,\n capture: await this.record(\"skipped\", { ...base, url, note, ...extra }, url, labels),\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note,\n };\n }\n\n /** This method reads the cache entry of a URL, with the fields that the crawler keeps in memory. */\n private lookup(url: string, variant: Variant = \"plain\"): CacheEntry | undefined {\n const key = cacheKeyOf(url);\n const row = this.options.index.latest(key, variant);\n if (row === undefined) return undefined;\n const meta = this.meta.get(`${variant} ${key}`);\n if (meta === undefined || meta.entry.capture !== row.capture) return row;\n const kept = Object.fromEntries(Object.entries(row).filter(([, value]) => value !== undefined));\n return { ...meta.entry, ...kept } as CacheEntry;\n }\n\n private store(entry: CacheEntry, location?: string): void {\n const variant = entry.variant ?? \"plain\";\n const keyed = { ...entry, url: cacheKeyOf(entry.url), variant };\n this.options.index.setLatest(keyed);\n this.meta.set(`${variant} ${keyed.url}`, {\n entry: keyed,\n ...(location !== undefined ? { location } : {}),\n });\n }\n\n /** This method gives the target of a stored redirect, when the crawler can follow it. */\n private followable(entry: CacheEntry, secretParams: readonly string[]): string | undefined {\n const meta = this.meta.peek(`${entry.variant ?? \"plain\"} ${cacheKeyOf(entry.url)}`);\n if (meta?.location !== undefined && meta.entry.capture === entry.capture) return meta.location;\n if (entry.location === undefined || entry.location === null) return undefined;\n try {\n const target = new URL(entry.location);\n // a stored target with secrets holds REDACTED in their place, and is not the real target\n return holdsSecrets(target, secretParams) ? undefined : target.href;\n } catch {\n return undefined;\n }\n }\n\n /** This method answers from the cache, when the stored entry is young or fresh enough. */\n private async fromCache(\n url: string,\n variant: Variant,\n maxAge: number | \"auto\" | undefined,\n sent: Readonly<Record<string, string>>,\n secretParams: readonly string[],\n ): Promise<Step | undefined> {\n if (maxAge === undefined) return undefined;\n const entry = this.lookup(url, variant);\n if (entry === undefined || !varyMatches(entry.vary_key, sent)) return undefined;\n const now = this.clock.now();\n const fetchedAt = Date.parse(entry.fetched_at);\n if (!Number.isFinite(fetchedAt)) return undefined;\n const age = now - fetchedAt;\n const until =\n entry.fresh_until === undefined || entry.fresh_until === null\n ? Number.NaN\n : Date.parse(entry.fresh_until);\n const fresh = Number.isFinite(until) && until > now;\n const redirect = entry.status >= 300 && entry.status < 400;\n // an explicit age wins; `auto` uses the freshness of the server; a temporary redirect needs both\n const young =\n maxAge === \"auto\"\n ? fresh\n : redirect && !PERMANENT_REDIRECTS.has(entry.status)\n ? fresh && age < maxAge\n : age < maxAge;\n if (!young) return undefined;\n if (redirect) {\n const next = this.followable(entry, secretParams);\n if (next === undefined) return undefined;\n return {\n fetched: {\n url,\n status: entry.status,\n outcome: \"redirect\",\n fromCache: true,\n capture: entry.capture,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note: `location ${entry.location ?? redact(new URL(next), secretParams)}`,\n fetchedAt: entry.fetched_at,\n },\n next,\n };\n }\n if (entry.blob === null || entry.status < 200 || entry.status >= 300) return undefined;\n const body = await this.options.blobs.get(entry.blob);\n if (body === undefined) return undefined;\n return {\n fetched: {\n url,\n status: entry.status,\n outcome: \"ok\",\n fromCache: true,\n capture: entry.capture,\n blob: entry.blob,\n body,\n contentType: entry.content_type ?? undefined,\n signals: { ai_input: entry.ai_input ?? \"unset\", raw: [] },\n note: entry.truncated === true ? \"a truncated copy\" : undefined,\n ...(entry.truncated === true ? { truncated: true } : {}),\n fetchedAt: entry.fetched_at,\n },\n };\n }\n\n private requestHeaders(request_: FetchRequest): Record<string, string> {\n const headers: Record<string, string> = {};\n // the caller's headers go first, so the crawler's own headers replace them\n for (const [name, value] of Object.entries(request_.headers ?? {})) {\n const lower = name.toLowerCase();\n if (!RESERVED_HEADERS.has(lower)) headers[lower] = value;\n }\n headers[\"accept\"] = request_.accept ?? headers[\"accept\"] ?? \"*/*\";\n headers[\"user-agent\"] = this.options.userAgent;\n headers[\"accept-encoding\"] = \"gzip, deflate, br\";\n if (this.options.from !== undefined) headers[\"from\"] = this.options.from;\n return headers;\n }\n\n /**\n * This method records the result of a request for the host, and it gives the rest of the host.\n * The rest comes from `restAfter`. A 429 or 503 waits, an error counts toward a stop, and a\n * refusal stops the host for a time.\n */\n private settle(host: string, policy: PolitenessPolicy, step: Settle): number {\n const store = this.scheduler.store;\n const now = this.clock.now();\n const rest = restAfter(policy, step.rest);\n const delayMs = rest.ms;\n const waited = (ms: number, reason: string): number => {\n if (ms > delayMs) this.emit({ type: \"wait\", host, ms, reason });\n return ms;\n };\n if (step.result !== undefined) {\n store.record(host, step.result);\n return delayMs;\n }\n const outcome = step.outcome;\n if (outcome === \"rate-limited\" || outcome === \"unavailable\") {\n const { waits } = store.record(host, \"wait\");\n const asked = step.retryAfterMs;\n if (asked !== undefined && asked > policy.maxRetryAfterMs) {\n // a wait longer than the budget stops the host, so no request sleeps for days\n this.stopHost(\n host,\n now + Math.min(asked, MAX_STOP_MS),\n `asked to wait ${String(Math.round(asked / 1000))} s`,\n );\n return delayMs;\n }\n return waited(\n Math.max(\n delayMs,\n asked ?? equalJitterMs(waits, policy.backoffBaseMs, policy.backoffCapMs, this.random),\n ),\n asked === undefined ? \"backoff\" : \"retry-after\",\n );\n }\n if (outcome === \"error\") {\n if (isClientError(step.status)) {\n store.record(host, \"neutral\");\n return delayMs;\n }\n const { failures } = store.record(host, \"failure\");\n if (failures >= policy.maxFailures)\n this.stopHost(host, now + policy.failureStopMs, `${String(failures)} failures in a row`);\n return waited(\n Math.max(\n delayMs,\n equalJitterMs(failures, policy.backoffBaseMs, policy.backoffCapMs, this.random),\n ),\n \"backoff\",\n );\n }\n if (outcome === \"challenge\") this.stopHost(host, now + policy.challengeStopMs, outcome);\n else if (outcome === \"payment-required\")\n this.stopHost(host, now + policy.paymentStopMs, outcome);\n else if (outcome === \"blocked\" && !step.robotsFile)\n this.stopHost(host, now + policy.blockedStopMs, outcome);\n store.record(host, \"success\");\n return delayMs;\n }\n\n /**\n * This method stops the host of a URL as a challenge does: for the `challengeStopMs` of the policy.\n * The runtime uses it when a page that answered 200 is a bot check (`meta.blocked`).\n */\n reportBlocked(url: string, what: string): void {\n let target: URL;\n try {\n target = new URL(url);\n } catch {\n return;\n }\n const policy = this.policyOf(target);\n this.stopHost(\n hostKey(target),\n this.clock.now() + policy.challengeStopMs,\n `challenge (${what})`,\n );\n }\n\n /**\n * This method applies the rules that a fetch and a render share. The sequence is the URL, the\n * options, the egress policy, the product tokens and the rules for an agent. Then come the\n * cache, the stops of the host and robots.txt. It gives the answer when a rule decides, or the\n * facts that the transport needs.\n */\n private async admit(\n spec: AdmitSpec,\n extra?: (\n admitted: Omit<Admitted, \"verdict\" | \"check\" | \"robotsLines\" | \"crawlDelaySeconds\">,\n ) => Promise<Fetched | undefined>,\n ): Promise<{ readonly answer: Step } | { readonly admitted: Admitted }> {\n const marks = { scope: spec.scope, labels: spec.labels };\n let target: URL;\n try {\n target = new URL(spec.url);\n } catch {\n return {\n answer: { fetched: await this.skip(spec.base, spec.url, \"not a valid URL\", {}, marks) },\n };\n }\n if (!WEB_PROTOCOLS.has(target.protocol)) {\n return {\n answer: {\n fetched: await this.skip(\n spec.base,\n redact(target, spec.secretParams),\n \"not an http(s) URL\",\n {},\n marks,\n ),\n },\n };\n }\n // a fragment never goes to the server, so it is not part of the cache key\n target.hash = \"\";\n const url = redact(target, spec.secretParams);\n const skip = (note: string, extra_?: Record<string, unknown>): Promise<Fetched> =>\n this.skip(spec.base, url, note, extra_, marks);\n const answer = async (note: string, extra_?: Record<string, unknown>) => ({\n answer: { fetched: await skip(note, extra_) },\n });\n if (spec.invalid !== undefined) return answer(spec.invalid);\n const refused = egressRefusal(this.egress, target.hostname);\n if (refused !== undefined) return answer(refused);\n const tokens = [this.options.token, ...(spec.onBehalfOf ?? []), ...this.pageTokens];\n const badToken = tokens.find((token) => !isProductToken(token));\n if (spec.mode === \"page\" && badToken !== undefined)\n return answer(`not a valid product token: \"${badToken}\"`);\n const host = hostKey(target);\n const policy = this.policyOf(target);\n const path = `${target.pathname}${target.search}`;\n\n // the rules for an agent apply before a stored copy answers it\n let verdict: RobotsVerdict | undefined;\n if (spec.mode === \"page\") {\n verdict =\n tokens.length > 1 ? await this.robotsFor(target, policy, spec) : this.knownRobots(target);\n if (verdict !== undefined) {\n const decision = decide(verdict.robots, tokens, path);\n if (!decision.allowed) {\n return answer(robotsRefusal(verdict.robots, decision.by ?? this.options.token), {\n robots_check: robotsCheck(verdict, tokens, decision),\n });\n }\n }\n }\n const cached = await spec.cached(url);\n if (cached !== undefined) return { answer: cached };\n\n const stop = this.scheduler.stopOf(host);\n if (stop !== undefined) return answer(stopNote(stop.reason, stop.until));\n const early = await extra?.({ target, url, host, policy, tokens, skip });\n if (early !== undefined) return { answer: { fetched: early } };\n\n let robotsLines: readonly string[] = [];\n let crawlDelaySeconds = 0;\n let check: Record<string, unknown> | undefined;\n if (spec.mode === \"page\") {\n verdict ??= await this.robotsFor(target, policy, spec);\n const decision = decide(verdict.robots, tokens, path);\n check = robotsCheck(verdict, tokens, decision);\n if (!decision.allowed) {\n return answer(robotsRefusal(verdict.robots, decision.by ?? this.options.token), {\n robots_check: check,\n });\n }\n robotsLines = signalsFor(verdict.robots, tokens, path);\n crawlDelaySeconds = decision.crawlDelaySeconds ?? 0;\n }\n if (this.closed) return { answer: { fetched: closedResult(url) } };\n return {\n admitted: {\n target,\n url,\n host,\n policy,\n tokens,\n verdict,\n check,\n robotsLines,\n crawlDelaySeconds,\n skip,\n },\n };\n }\n\n /** This method fetches a URL. It follows the redirects up to the maximum, and each step is a separate decision. */\n async fetch(request_: FetchRequest): Promise<Fetched> {\n if (this.closed) return closedResult(request_.url);\n return this.track(this.follow(request_));\n }\n\n private async follow(request_: FetchRequest): Promise<Fetched> {\n const signal =\n request_.signal === undefined\n ? this.closing.signal\n : AbortSignal.any([request_.signal, this.closing.signal]);\n let url = request_.url;\n let headers = request_.headers;\n let origin: string | undefined;\n let maxRedirects = DEFAULT_POLICY.maxRedirects;\n try {\n const first = new URL(url);\n origin = first.origin;\n maxRedirects = this.policyOf(first).maxRedirects;\n } catch {\n // fetchOnce records the URL that does not parse\n }\n try {\n for (let hop = 0; ; hop += 1) {\n const step = await this.hop({ ...request_, url, headers }, signal);\n if (step.next === undefined) return step.fetched;\n if (hop >= maxRedirects) {\n return {\n ...step.fetched,\n outcome: \"error\",\n note: `more than ${String(maxRedirects)} redirects`,\n };\n }\n const next = new URL(step.next);\n // a credential for one origin does not go to a different origin\n if (next.origin !== origin) headers = withoutCredentials(headers);\n origin = next.origin;\n url = step.next;\n }\n } catch (error) {\n if (this.closed) return closedResult(request_.url);\n return {\n ...closedResult(request_.url),\n outcome: \"error\",\n note: `crawler error: ${messageOf(error)}`,\n };\n }\n }\n\n /** This method joins a request to an equal request in progress, so one URL gets one request at a time. */\n private hop(request_: FetchRequest, signal: AbortSignal): Promise<Step> {\n // a request with a signal of its own runs alone, so its cancellation stops only that request\n if (request_.signal !== undefined) return this.fetchOnce(request_, signal);\n const key = JSON.stringify([\n cacheKeyOf(request_.url.split(\"#\")[0] ?? request_.url),\n request_.mode,\n request_.recipe,\n request_.accept ?? null,\n request_.headers ?? null,\n request_.secretParams ?? null,\n request_.minIntervalMs ?? null,\n request_.maxAgeMs ?? null,\n request_.maxBytes ?? null,\n request_.onBehalfOf ?? null,\n request_.job ?? null,\n request_.scope ?? null,\n request_.labels ?? null,\n ]);\n const running = this.inFlight.get(key);\n if (running !== undefined) return running;\n const work = this.fetchOnce(request_, signal).finally(() => this.inFlight.delete(key));\n this.inFlight.set(key, work);\n return work;\n }\n\n private knownRobots(target: URL): RobotsVerdict | undefined {\n const entry = this.robots.peek(originKey(target));\n return entry !== undefined && this.clock.now() - entry.at < entry.ttlMs\n ? entry.value\n : undefined;\n }\n\n /**\n * This method gives the robots.txt of the origin of a URL. A good copy stays for the TTL of the\n * policy. An unreachable verdict stays for some minutes, with a backoff and the `Retry-After`\n * of the server. During that time the last good copy applies, when there is one (RFC 9309 §2.4).\n */\n private robotsFor(\n target: URL,\n policy: PolitenessPolicy,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n ): Promise<RobotsVerdict> {\n const key = originKey(target);\n const now = this.clock.now();\n const cached = this.robots.get(key);\n if (cached !== undefined && now - cached.at < cached.ttlMs) {\n // a request that joins a fetch in progress adds its scope and its labels to the record\n if (cached.value === undefined) this.join(cached.waiting, marks);\n return cached.verdict;\n }\n const waiting: Waiting = { scope: undefined, labels: undefined };\n this.join(waiting, marks);\n const entry: RobotsEntry = {\n at: now,\n waiting,\n ttlMs: policy.robotsTtlMs,\n verdict: Promise.resolve({ robots: unreachableRobots(\"not read yet\"), capture: undefined }),\n value: undefined,\n misses: cached?.misses ?? 0,\n good: cached?.good,\n };\n entry.verdict = this.fetchRobots(target, policy, waiting)\n .catch((error: unknown) => ({\n robots: unreachableRobots(`robots.txt failed: ${messageOf(error)}`),\n capture: undefined,\n retryAfterMs: undefined,\n }))\n .then(async (read): Promise<RobotsVerdict> => {\n let verdict: RobotsVerdict = { robots: read.robots, capture: read.capture };\n if (read.robots.unreachable === undefined) {\n entry.misses = 0;\n entry.good = { verdict, at: this.clock.now() };\n } else {\n entry.misses += 1;\n const backoff = equalJitterMs(\n entry.misses,\n policy.robotsRetryMs,\n policy.robotsRetryCapMs,\n this.random,\n );\n entry.ttlMs = Math.min(\n policy.robotsRetryCapMs,\n Math.max(read.retryAfterMs ?? 0, backoff),\n );\n const good =\n entry.good !== undefined && this.clock.now() - entry.good.at <= policy.robotsStaleMs\n ? entry.good.verdict\n : await this.storedRobots(target, policy);\n if (good !== undefined) verdict = { ...good, stale: read.robots.unreachable };\n }\n entry.value = verdict;\n return verdict;\n });\n this.robots.set(key, entry);\n return entry.verdict;\n }\n\n /**\n * This method gives the robots.txt rules that apply to a URL, as the crawler applies them. They\n * come from its memory, or from a fetch of `/robots.txt` with its redirects. A robots.txt that\n * the crawler cannot read gives rules that permit nothing, and `unreachable` gives the reason.\n */\n async readRobots(url: string): Promise<{\n readonly robots: Robots;\n readonly capture: string | undefined;\n readonly stale?: string | undefined;\n }> {\n const target = new URL(url);\n if (!WEB_PROTOCOLS.has(target.protocol))\n return { robots: unreachableRobots(\"not an http(s) URL\"), capture: undefined };\n return this.robotsFor(target, this.policyOf(target));\n }\n\n /** This method reads the stored copy of a robots.txt, when it is younger than the limit of the policy. */\n private async storedRobots(\n target: URL,\n policy: PolitenessPolicy,\n ): Promise<RobotsVerdict | undefined> {\n try {\n const row = this.lookup(redact(new URL(\"/robots.txt\", target.origin)));\n if (row === undefined || row.blob === null || row.status < 200 || row.status >= 300)\n return undefined;\n if (this.clock.now() - Date.parse(row.fetched_at) > policy.robotsStaleMs) return undefined;\n const body = await this.options.blobs.get(row.blob);\n if (body === undefined) return undefined;\n const robots = robotsFromResponse(200, row.content_type, body);\n return robots.unreachable === undefined ? { robots, capture: row.capture } : undefined;\n } catch {\n return undefined;\n }\n }\n\n /** This method adds the scope and the labels of one request to the requests that wait for a robots.txt. The most private scope wins. */\n private join(\n waiting: Waiting,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n },\n ): void {\n const scope = marks.scope ?? this.options.scope;\n const rank = this.options.scopeRank;\n if (waiting.scope === undefined) waiting.scope = scope;\n else if (scope !== undefined && rank !== undefined && rank(scope) > rank(waiting.scope))\n waiting.scope = scope;\n if (marks.labels !== undefined) waiting.labels = { ...marks.labels, ...waiting.labels };\n }\n\n private async fetchRobots(\n target: URL,\n policy: PolitenessPolicy,\n waiting: Waiting,\n ): Promise<RobotsRead> {\n let url = new URL(\"/robots.txt\", target.origin);\n const unreachable = (\n reason: string,\n capture: string | undefined,\n retryAfterMs?: number,\n ): RobotsRead => ({\n robots: unreachableRobots(reason),\n capture,\n retryAfterMs,\n });\n for (let hop = 0; hop <= policy.maxRedirects; hop += 1) {\n const step = await this.fetchOnce(\n {\n url: url.href,\n mode: \"api\",\n recipe: \"robots.txt\",\n accept: \"text/plain\",\n maxBytes: ROBOTS_BYTES,\n minIntervalMs: policy.apiIntervalMs,\n // the record reads the requests that wait at the time that it is written, not at the start\n get scope() {\n return waiting.scope;\n },\n get labels() {\n return waiting.labels;\n },\n },\n this.closing.signal,\n );\n const result = step.fetched;\n if (result.outcome === \"redirect\") {\n if (step.next === undefined)\n return unreachable(result.note ?? \"a redirect with no Location header\", result.capture);\n url = new URL(step.next);\n continue;\n }\n if (result.outcome === \"skipped\" || result.status === 0)\n return unreachable(result.note ?? \"no response\", result.capture);\n // a body that the crawler cannot read is not an empty robots.txt: the site stays closed\n if (result.outcome === \"error\" && result.status >= 200 && result.status < 300)\n return unreachable(result.note ?? \"the body could not be read\", result.capture);\n // a body that the server cut can have lost a rule\n if (result.note?.startsWith(\"the compressed body ended early\") === true)\n return unreachable(result.note, result.capture);\n if (result.outcome === \"not-modified\") {\n return result.body === undefined\n ? unreachable(\"status 304, and no stored copy\", result.capture)\n : {\n robots: robotsFromResponse(200, result.contentType ?? null, result.body),\n capture: result.capture,\n retryAfterMs: undefined,\n };\n }\n const robots =\n result.status >= 200 && result.status < 300\n ? robotsFromResponse(\n result.status,\n result.contentType ?? null,\n result.body ?? new Uint8Array(),\n )\n : robotsFromStatus(result.status, undefined);\n return { robots, capture: result.capture, retryAfterMs: step.retryAfterMs };\n }\n return unreachable(`more than ${String(policy.maxRedirects)} redirects`, undefined);\n }\n\n private async fetchOnce(request_: FetchRequest, signal: AbortSignal): Promise<Step> {\n const base = {\n recipe: request_.recipe,\n mode: request_.mode,\n ...(request_.job ? { job: request_.job } : {}),\n };\n const secretParams = request_.secretParams ?? [];\n const sent = this.requestHeaders(request_);\n // read when a record is written: more requests can wait for the fetch of a robots.txt by then\n const marks = {\n get scope() {\n return request_.scope;\n },\n get labels() {\n return request_.labels;\n },\n };\n const admission = await this.admit({\n transport: \"http\",\n url: request_.url,\n mode: request_.mode,\n base,\n secretParams,\n invalid:\n invalidNumber(\"minIntervalMs\", request_.minIntervalMs, false) ??\n invalidNumber(\"maxBytes\", request_.maxBytes, false) ??\n invalidNumber(\"maxAgeMs\", request_.maxAgeMs, true),\n onBehalfOf: request_.onBehalfOf,\n scope: request_.scope,\n labels: request_.labels,\n cached: (url) => this.fromCache(url, \"plain\", request_.maxAgeMs, sent, secretParams),\n });\n if (\"answer\" in admission) return admission.answer;\n const { target, url, host, policy, check, robotsLines, crawlDelaySeconds, skip } =\n admission.admitted;\n const maxBytes = request_.maxBytes ?? policy.maxBytes;\n const rest = (elapsedMs: number): RestInput => ({\n kind: request_.mode,\n elapsedMs,\n crawlDelaySeconds,\n minIntervalMs: request_.minIntervalMs,\n });\n const robotsFile = request_.recipe === \"robots.txt\";\n\n try {\n return await this.scheduler.run(\n host,\n async (): Promise<Paced<Step>> => {\n // an equal request that waited in the queue can find a fresh copy now\n const again = await this.fromCache(url, \"plain\", request_.maxAgeMs, sent, secretParams);\n if (again !== undefined) return { value: again, delayMs: 0 };\n const latest = this.lookup(url);\n // validators go only with a stored body of the same variant, so a 304 always has a body\n const usable =\n latest !== undefined && latest.truncated !== true && varyMatches(latest.vary_key, sent)\n ? latest\n : undefined;\n let stored: Uint8Array | undefined;\n if (usable?.blob && (usable.etag || usable.last_modified))\n stored = await this.options.blobs.get(usable.blob);\n const headers: Record<string, string> = { ...sent };\n // the memo of the host can leave out an ETag that never matches; robots.txt is often a\n // static file with validators of its own, so it keeps the two and teaches the memo nothing\n const send =\n stored === undefined || usable === undefined\n ? { etag: false, lastModified: false }\n : validatorsToSend(\n robotsFile ? undefined : this.memoOf(host),\n { etag: Boolean(usable.etag), lastModified: Boolean(usable.last_modified) },\n this.clock.now(),\n );\n if (send.etag && usable?.etag) headers[\"if-none-match\"] = usable.etag;\n if (send.lastModified && usable?.last_modified)\n headers[\"if-modified-since\"] = usable.last_modified;\n const validators = [\"if-none-match\", \"if-modified-since\"].filter(\n (name) => headers[name] !== undefined,\n );\n const started = this.clock.now();\n const timeout = AbortSignal.timeout(policy.requestTimeoutMs);\n const abort = AbortSignal.any([timeout, signal]);\n const cancelled = (): boolean => request_.signal?.aborted === true;\n const failedNote = (prefix: string, error: unknown): string =>\n timeout.aborted\n ? `${prefix}: timed out after ${String(Math.round(policy.requestTimeoutMs / 1000))} s`\n : `${prefix}: ${messageOf(error)}`;\n\n this.emit({ type: \"request\", url, host, transport: \"http\" });\n let response: Dispatcher.ResponseData;\n try {\n response = await request(target, {\n method: \"GET\",\n headers,\n dispatcher: this.dispatcher,\n signal: abort,\n });\n } catch (error) {\n const egress = egressError(error);\n if (egress !== undefined)\n return { value: { fetched: await skip(egress.message) }, delayMs: 0 };\n if (this.closed) return { value: { fetched: closedResult(url) }, delayMs: 0 };\n const elapsedMs = this.clock.now() - started;\n const note = cancelled() ? \"cancelled by the caller\" : failedNote(\"no response\", error);\n const delayMs = this.settle(host, policy, {\n outcome: \"error\",\n status: 0,\n retryAfterMs: undefined,\n rest: rest(elapsedMs),\n robotsFile,\n ...(cancelled() ? { result: \"neutral\" as const } : {}),\n });\n this.emit({ type: \"response\", url, host, status: 0, outcome: \"error\", elapsedMs });\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n status: 0,\n outcome: \"error\",\n note,\n elapsed_ms: elapsedMs,\n ...(check ? { robots_check: check } : {}),\n },\n url,\n marks,\n );\n return {\n value: {\n fetched: {\n url,\n status: 0,\n outcome: \"error\",\n fromCache: false,\n capture,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note,\n fetchedAt: new Date(started).toISOString(),\n ...marks,\n },\n },\n delayMs,\n };\n }\n\n const header = headerOf(\n response.headers as Record<string, string | string[] | undefined>,\n );\n const ttfbMs = this.clock.now() - started;\n const classified = classify(response.statusCode, header, this.clock.now());\n let outcome: Outcome = classified.outcome;\n let note = classified.note;\n let body: Uint8Array | undefined;\n let blob: string | undefined;\n let truncated = false;\n let cut = false;\n let next: string | undefined;\n let result: HostResult | undefined;\n if (outcome === \"ok\") {\n try {\n const read = await readBody(response.body, header(\"content-encoding\"), maxBytes);\n body = read.bytes;\n truncated = read.truncated;\n cut = read.cut;\n if (read.cut)\n note = `the compressed body ended early, so the crawler keeps the ${String(read.bytes.length)} bytes before the end`;\n else if (truncated) note = `truncated at ${String(maxBytes)} bytes`;\n } catch (error) {\n if (this.closed) return { value: { fetched: closedResult(url) }, delayMs: 0 };\n outcome = \"error\";\n if (cancelled()) {\n note = \"cancelled by the caller\";\n result = \"neutral\";\n } else {\n const decode = error instanceof BodyError && error.kind === \"decode\";\n note = failedNote(decode ? \"undecodable body\" : \"body failed\", error);\n }\n }\n if (body !== undefined) {\n try {\n blob = (await this.options.blobs.put(body)).hash;\n } catch (error) {\n // a full disk is not a fault of the host\n outcome = \"error\";\n note = `blob store: ${messageOf(error)}`;\n body = undefined;\n result = \"neutral\";\n }\n }\n } else {\n try {\n await readBody(response.body, undefined, ERROR_BODY_BYTES);\n } catch {\n // the body of an error answer is not kept\n }\n }\n const elapsedMs = this.clock.now() - started;\n if (outcome === \"not-modified\") {\n if (stored !== undefined && usable?.blob) {\n body = stored;\n blob = usable.blob;\n } else note = \"not modified, but the crawler has no stored copy\";\n }\n let location: string | undefined;\n if (outcome === \"redirect\") {\n const raw = header(\"location\");\n if (raw === undefined) note = \"a redirect with no Location header\";\n else {\n try {\n const resolved = new URL(raw, target);\n next = resolved.href;\n location = redact(resolved, secretParams);\n note = `location ${location}`;\n } catch {\n outcome = \"error\";\n note = \"bad redirect target\";\n }\n }\n }\n\n const revisit = outcome === \"not-modified\" && usable !== undefined;\n let signals = readSignals(robotsLines, header, pageSignals(body, header(\"content-type\")));\n if (revisit) {\n // a 304 updates the stored answer and does not replace it (RFC 9111 §4.3.4)\n const merged: AiInput =\n signals.ai_input === \"no\" || usable.ai_input === \"no\"\n ? \"no\"\n : signals.ai_input !== \"unset\"\n ? signals.ai_input\n : usable.ai_input;\n signals = { ...signals, ai_input: merged };\n }\n const contentType =\n header(\"content-type\") ?? (revisit ? (usable.content_type ?? undefined) : undefined);\n // a web page in place of the data of an API (a login or an error page) is not the stored copy\n const unasked =\n request_.mode === \"api\" &&\n outcome === \"ok\" &&\n unaskedPage(request_.accept, contentType);\n if (unasked) {\n const said = `the server sent ${contentType ?? \"\"}, and the request accepts ${request_.accept ?? \"\"}, so the crawler keeps no stored copy of it`;\n note = note === undefined ? said : `${note}; ${said}`;\n }\n const etag = header(\"etag\") ?? (revisit ? usable.etag : null) ?? null;\n const lastModified =\n header(\"last-modified\") ?? (revisit ? usable.last_modified : null) ?? null;\n const fetchedAt = this.clock.now();\n const freshness = freshUntil(\n header,\n response.statusCode,\n fetchedAt,\n lastModified,\n policy.heuristicFreshnessCapMs,\n );\n const varyKey =\n header(\"vary\") !== undefined\n ? varyKeyOf(header(\"vary\"), sent)\n : revisit\n ? (usable.vary_key ?? null)\n : null;\n const cacheHeaders = Object.fromEntries(\n [\"cache-control\", \"expires\", \"age\", \"date\", \"vary\"].flatMap((name) => {\n const value = header(name);\n return value === undefined ? [] : [[name.replace(\"-\", \"_\"), value]];\n }),\n );\n this.emit({\n type: \"response\",\n url,\n host,\n status: response.statusCode,\n outcome,\n elapsedMs,\n });\n const capture = await this.record(\n revisit ? \"revisit\" : \"capture\",\n {\n ...base,\n url,\n status: response.statusCode,\n outcome,\n elapsed_ms: elapsedMs,\n ttfb_ms: ttfbMs,\n content_type: contentType ?? null,\n bytes: body?.length ?? 0,\n blob: blob ?? null,\n etag,\n last_modified: lastModified,\n ...(validators.length > 0 ? { validators } : {}),\n ...(revisit ? { revisit_of: usable.capture } : {}),\n robots: request_.mode === \"page\" ? \"allowed\" : \"not-checked\",\n ...(check !== undefined ? { robots_check: check } : {}),\n signals,\n ...(truncated ? { truncated: true } : {}),\n ...(cut ? { cut: true } : {}),\n ...(unasked ? { unasked: true } : {}),\n ...cacheHeaders,\n ...(freshness !== null ? { fresh_until: freshness } : {}),\n ...(varyKey !== null ? { vary_key: varyKey } : {}),\n ...(location !== undefined ? { location } : {}),\n ...(note !== undefined ? { note } : {}),\n },\n url,\n marks,\n );\n const fetchedIso = new Date(fetchedAt).toISOString();\n // a body that the server cut is not a copy to keep: the next request asks for the whole body\n if (\n capture !== undefined &&\n (outcome === \"ok\" || outcome === \"not-modified\") &&\n blob !== undefined &&\n !cut &&\n !unasked\n ) {\n this.store({\n url,\n capture,\n blob,\n // a truncated body keeps no validators, so a later request gets the whole body again\n etag: truncated ? null : etag,\n last_modified: truncated ? null : lastModified,\n content_type: contentType ?? null,\n fetched_at: fetchedIso,\n // an index that a replay built can hold 304 as the status of the stored body\n status: revisit ? (usable.status === 304 ? 200 : usable.status) : response.statusCode,\n ai_input: signals.ai_input,\n truncated: truncated ? true : undefined,\n fresh_until: freshness,\n vary_key: varyKey,\n });\n }\n if (\n capture !== undefined &&\n outcome === \"redirect\" &&\n next !== undefined &&\n (PERMANENT_REDIRECTS.has(response.statusCode) || freshness !== null)\n ) {\n this.store(\n {\n url,\n capture,\n blob: null,\n etag: null,\n last_modified: null,\n content_type: null,\n fetched_at: fetchedIso,\n status: response.statusCode,\n ai_input: \"unset\",\n location: location ?? null,\n fresh_until: freshness,\n vary_key: varyKey,\n },\n next,\n );\n }\n // the memo of the host learns which validator gets a 304 (a 200 with the stored body is a miss)\n const answer: ValidatorAnswer | undefined =\n validators.length === 0 || capture === undefined || robotsFile\n ? undefined\n : outcome === \"not-modified\"\n ? \"not-modified\"\n : outcome === \"ok\" && blob !== undefined && !cut\n ? blob === usable?.blob\n ? \"same\"\n : \"changed\"\n : undefined;\n if (answer !== undefined) {\n try {\n this.scheduler.store.noteValidators?.(\n host,\n {\n etag: headers[\"if-none-match\"] !== undefined,\n lastModified: headers[\"if-modified-since\"] !== undefined,\n answer,\n },\n fetchedAt,\n );\n } catch {\n // the memo is a hint: an error of its store does not fail the fetch\n }\n }\n const delayMs = this.settle(host, policy, {\n outcome,\n status: response.statusCode,\n retryAfterMs: classified.retryAfterMs,\n rest: rest(elapsedMs),\n robotsFile,\n ...(result !== undefined ? { result } : {}),\n });\n return {\n value: {\n fetched: {\n url,\n status: response.statusCode,\n outcome,\n fromCache: false,\n capture,\n blob,\n body,\n contentType,\n signals,\n note,\n ...(truncated ? { truncated: true } : {}),\n fetchedAt: fetchedIso,\n ...marks,\n },\n next,\n retryAfterMs: classified.retryAfterMs,\n },\n delayMs,\n };\n },\n signal,\n );\n } catch (error) {\n if (error instanceof HostStopped)\n return { fetched: await skip(stopNote(error.reason, error.until)) };\n if (this.closed) return { fetched: closedResult(url) };\n if (request_.signal?.aborted === true)\n return { fetched: await skip(\"cancelled by the caller\") };\n throw error;\n }\n }\n\n /**\n * This method gives the function that decides each request of a rendered page. It refuses an\n * address that the egress policy refuses, and a URL of the page's host that robots.txt\n * disallows. For a different host, it uses a robots.txt that the crawler holds, when there is one.\n */\n private allowRequestFor(\n page: URL,\n robots: Robots,\n tokens: readonly string[],\n signal: AbortSignal,\n tally: { allowed: number; refused: number; readonly hosts: Set<string> },\n ): (url: string) => Promise<boolean> {\n const pageOrigin = originKey(page);\n const checked = new Map<string, Promise<string | undefined>>();\n const refuse = (): boolean => {\n tally.refused += 1;\n return false;\n };\n return async (raw: string): Promise<boolean> => {\n if (signal.aborted) return false;\n let target: URL;\n try {\n target = new URL(raw);\n } catch {\n return refuse();\n }\n if (\n target.protocol === \"data:\" ||\n target.protocol === \"blob:\" ||\n target.protocol === \"about:\"\n )\n return true;\n const web = WEB_PROTOCOLS.has(target.protocol);\n if (!web && target.protocol !== \"ws:\" && target.protocol !== \"wss:\") return refuse();\n let check = checked.get(target.hostname);\n if (check === undefined) {\n check = checkEgress(this.egress, target);\n checked.set(target.hostname, check);\n }\n if ((await check) !== undefined) return refuse();\n if (web) {\n const rules = originKey(target) === pageOrigin ? robots : this.knownRobots(target)?.robots;\n if (\n rules !== undefined &&\n !decide(rules, tokens, `${target.pathname}${target.search}`).allowed\n )\n return refuse();\n }\n tally.allowed += 1;\n if (tally.hosts.size < 50) tally.hosts.add(hostKey(target));\n return true;\n };\n }\n\n /**\n * This method tells why the browser did not go on to a URL that the page moved to: the egress\n * policy or robots.txt. For a different site, it uses a robots.txt that the crawler holds. It\n * gives undefined for the page itself, or for a URL that it cannot read.\n */\n private async movedRefusal(\n raw: string,\n page: URL,\n robots: Robots,\n tokens: readonly string[],\n ): Promise<{ readonly note: string; readonly url: string } | undefined> {\n let target: URL;\n try {\n target = new URL(raw);\n } catch {\n return undefined;\n }\n target.hash = \"\";\n if (target.href === page.href) return undefined;\n const url = redact(target);\n const moved = `the page moved to ${url}`;\n if (!WEB_PROTOCOLS.has(target.protocol))\n return { note: `${moved}, which is not an http(s) URL`, url };\n const egress = await checkEgress(this.egress, target);\n if (egress !== undefined) return { note: `${moved}: ${egress}`, url };\n const rules = originKey(target) === originKey(page) ? robots : this.knownRobots(target)?.robots;\n if (rules !== undefined) {\n const decision = decide(rules, tokens, `${target.pathname}${target.search}`);\n if (!decision.allowed)\n return {\n note: `${moved}: ${robotsRefusal(rules, decision.by ?? this.options.token)}`,\n url,\n };\n }\n return undefined;\n }\n\n /**\n * This method renders a page in the headless browser and captures what the page shows. The rules\n * of a fetch apply:\n *\n * - the egress policy and robots.txt, for the page, its final URL and each request of the page\n * - one request at a time for each host\n * - the backoff after a 429 or a 503\n * - a capture for each decision\n *\n * A render costs the site more than a fetch, so the host rests longer after one. A render that\n * the browser refuses (`RenderRefused`) does not count against the host.\n */\n async render(request_: RenderRequest): Promise<Fetched> {\n if (this.closed) return closedResult(request_.url);\n return this.track(\n this.renderOnce(request_).catch((error: unknown) =>\n this.closed\n ? closedResult(request_.url)\n : {\n ...closedResult(request_.url),\n outcome: \"error\" as const,\n note: `crawler error: ${messageOf(error)}`,\n },\n ),\n );\n }\n\n private async renderOnce(request_: RenderRequest): Promise<Fetched> {\n const base = {\n recipe: request_.recipe,\n mode: \"page\" as const,\n rendered: true,\n ...(request_.job ? { job: request_.job } : {}),\n };\n const marks = { scope: request_.scope, labels: request_.labels };\n const renderer = this.options.renderer;\n const admission = await this.admit(\n {\n transport: \"browser\",\n url: request_.url,\n mode: \"page\",\n base,\n secretParams: [],\n invalid: invalidNumber(\"maxAgeMs\", request_.maxAgeMs, true),\n onBehalfOf: request_.onBehalfOf,\n scope: request_.scope,\n labels: request_.labels,\n cached: async (url) => {\n const cached = await this.fromCache(url, \"rendered\", request_.maxAgeMs, {}, []);\n if (cached === undefined || cached.fetched.body === undefined) return undefined;\n return { fetched: { ...cached.fetched, url, contentType: RENDERED_TYPE } };\n },\n },\n async ({ target, skip }) => {\n if (renderer === undefined) return skip(\"no browser configured for rendering\");\n if (this.renderOff !== undefined) return skip(this.renderOff);\n // the browser does its own DNS, so the check here resolves the name too\n const resolved = await checkEgress(this.egress, target);\n return resolved === undefined ? undefined : skip(resolved);\n },\n );\n if (\"answer\" in admission) return admission.answer.fetched;\n const { target, url, host, policy, tokens, check, crawlDelaySeconds, skip } =\n admission.admitted;\n const verdict = admission.admitted.verdict!;\n const signal =\n request_.signal === undefined\n ? this.closing.signal\n : AbortSignal.any([request_.signal, this.closing.signal]);\n const tally = { allowed: 0, refused: 0, hosts: new Set<string>() };\n const allowRequest = this.allowRequestFor(target, verdict.robots, tokens, signal, tally);\n const rest = (elapsedMs: number): RestInput => ({\n kind: \"render\",\n elapsedMs,\n crawlDelaySeconds,\n });\n\n // the wait for a slot of the browser comes before the lease of the host, so the host does not wait for it\n let release: () => void;\n try {\n release = await this.renders.acquire(signal);\n } catch {\n if (this.closed) return closedResult(url);\n return skip(\"cancelled by the caller\");\n }\n let done: RenderDone;\n try {\n done = await this.scheduler.run(\n host,\n async (): Promise<Paced<RenderDone>> => {\n const started = this.clock.now();\n this.emit({ type: \"request\", url, host, transport: \"browser\" });\n try {\n const rendered = await raceAbort(\n renderer!.render(target.href, {\n userAgent: this.options.userAgent,\n timeoutMs: policy.renderTimeoutMs,\n ...(this.options.from !== undefined\n ? { headers: { from: this.options.from } }\n : {}),\n allowRequest,\n maxRedirects: policy.maxRedirects,\n signal,\n }),\n signal,\n );\n const header = (name: string): string | undefined =>\n rendered.headers[name.toLowerCase()];\n const classified: Classified =\n rendered.status === null\n ? { outcome: \"ok\", retryAfterMs: undefined, note: undefined }\n : classify(rendered.status, header, this.clock.now());\n const elapsedMs = this.clock.now() - started;\n const delayMs = this.settle(host, policy, {\n outcome: classified.outcome,\n status: rendered.status ?? 200,\n retryAfterMs: classified.retryAfterMs,\n rest: rest(elapsedMs),\n robotsFile: false,\n });\n return { value: { kind: \"rendered\", rendered, classified, elapsedMs }, delayMs };\n } catch (error) {\n const elapsedMs = this.clock.now() - started;\n if (this.closed) return { value: { kind: \"closed\" }, delayMs: 0 };\n if (request_.signal?.aborted === true) {\n return {\n value: { kind: \"cancelled\", elapsedMs },\n delayMs: restAfter(policy, rest(0)).ms,\n };\n }\n // a renderer of an application that has no browser is no fault of the host either\n const refused =\n error instanceof RenderRefused\n ? error\n : missingBrowser(messageOf(error))\n ? new RenderRefused(\"browser\", `rendering needs a browser: ${messageOf(error)}`)\n : undefined;\n if (refused !== undefined) {\n if (refused.reason === \"browser\") this.renderOff = refused.message;\n // a refused navigation, a file, a scheme or no browser is no fault of the host\n const delayMs = this.settle(host, policy, {\n outcome: \"skipped\",\n status: 0,\n retryAfterMs: undefined,\n rest: rest(0),\n robotsFile: false,\n result: \"neutral\",\n });\n return {\n value: {\n kind: \"refused\",\n reason: refused.reason,\n note: refused.message,\n url: refused.url,\n elapsedMs,\n },\n delayMs,\n };\n }\n const delayMs = this.settle(host, policy, {\n outcome: \"error\",\n status: 0,\n retryAfterMs: undefined,\n rest: rest(elapsedMs),\n robotsFile: false,\n });\n return {\n value: { kind: \"failed\", note: `render failed: ${messageOf(error)}`, elapsedMs },\n delayMs,\n };\n }\n },\n signal,\n );\n } catch (error) {\n if (error instanceof HostStopped) return skip(stopNote(error.reason, error.until));\n if (this.closed) return closedResult(url);\n if (request_.signal?.aborted === true) return skip(\"cancelled by the caller\");\n throw error;\n } finally {\n release();\n }\n\n if (done.kind === \"closed\" || this.closed) return closedResult(url);\n const subrequests = { allowed: tally.allowed, refused: tally.refused, hosts: [...tally.hosts] };\n if (done.kind === \"refused\") {\n this.emit({\n type: \"response\",\n url,\n host,\n status: 0,\n outcome: done.reason === \"download\" ? \"not-html\" : \"skipped\",\n elapsedMs: done.elapsedMs,\n });\n if (done.reason !== \"download\") {\n const moved =\n done.url === undefined\n ? undefined\n : await this.movedRefusal(done.url, target, verdict.robots, tokens);\n return skip(moved?.note ?? done.note, {\n robots_check: check,\n subrequests,\n ...(moved === undefined ? {} : { final_url: moved.url }),\n });\n }\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n status: 0,\n outcome: \"not-html\",\n note: done.note,\n elapsed_ms: done.elapsedMs,\n robots_check: check,\n subrequests,\n },\n url,\n marks,\n );\n return { ...closedResult(url), outcome: \"not-html\", capture, note: done.note, ...marks };\n }\n if (done.kind === \"failed\" || done.kind === \"cancelled\") {\n const note = done.kind === \"failed\" ? done.note : \"cancelled by the caller\";\n this.emit({\n type: \"response\",\n url,\n host,\n status: 0,\n outcome: \"error\",\n elapsedMs: done.elapsedMs,\n });\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n status: 0,\n outcome: \"error\",\n note,\n elapsed_ms: done.elapsedMs,\n robots_check: check,\n subrequests,\n },\n url,\n marks,\n );\n return {\n url,\n status: 0,\n outcome: \"error\",\n fromCache: false,\n capture,\n blob: undefined,\n body: undefined,\n contentType: undefined,\n signals: NO_SIGNALS,\n note,\n ...marks,\n };\n }\n\n const { rendered, classified, elapsedMs } = done;\n const header = (name: string): string | undefined => rendered.headers[name.toLowerCase()];\n // the page can move to a different URL: the same rules apply to it\n let finalTarget = target;\n let finalVerdict = verdict;\n try {\n finalTarget = new URL(rendered.finalUrl);\n finalTarget.hash = \"\";\n } catch {\n finalTarget = target;\n }\n const finalUrl = redact(finalTarget);\n const finalPath = `${finalTarget.pathname}${finalTarget.search}`;\n if (finalTarget.href !== target.href) {\n const moved = `the page moved to ${finalUrl}`;\n if (!WEB_PROTOCOLS.has(finalTarget.protocol))\n return skip(`${moved}, which is not an http(s) URL`, { final_url: finalUrl });\n const refusal = await checkEgress(this.egress, finalTarget);\n if (refusal !== undefined) return skip(`${moved}: ${refusal}`, { final_url: finalUrl });\n if (originKey(finalTarget) !== originKey(target)) {\n finalVerdict = await this.robotsFor(finalTarget, this.policyOf(finalTarget), request_);\n }\n const finalDecision = decide(finalVerdict.robots, tokens, finalPath);\n if (!finalDecision.allowed) {\n return skip(\n `${moved}: ${robotsRefusal(finalVerdict.robots, finalDecision.by ?? this.options.token)}`,\n {\n final_url: finalUrl,\n robots_check: robotsCheck(finalVerdict, tokens, finalDecision),\n },\n );\n }\n }\n let outcome: Outcome = classified.outcome;\n let note = classified.note;\n let body: Uint8Array | undefined;\n let blob: string | undefined;\n if (outcome === \"ok\") {\n body = new TextEncoder().encode(rendered.html);\n try {\n blob = (await this.options.blobs.put(body)).hash;\n } catch (error) {\n outcome = \"error\";\n note = `blob store: ${messageOf(error)}`;\n body = undefined;\n }\n }\n const signals = readSignals(\n signalsFor(finalVerdict.robots, tokens, finalPath),\n header,\n metaSignals(rendered.html),\n );\n const status = rendered.status ?? 200;\n const fetchedAt = this.clock.now();\n const freshness = freshUntil(\n header,\n status,\n fetchedAt,\n header(\"last-modified\"),\n policy.heuristicFreshnessCapMs,\n );\n this.emit({ type: \"response\", url, host, status, outcome, elapsedMs });\n const capture = await this.record(\n \"capture\",\n {\n ...base,\n url,\n final_url: finalUrl,\n status,\n outcome,\n elapsed_ms: elapsedMs,\n content_type: RENDERED_TYPE,\n bytes: body?.length ?? 0,\n blob: blob ?? null,\n robots: \"allowed\",\n robots_check: check,\n signals,\n subrequests,\n ...(freshness !== null ? { fresh_until: freshness } : {}),\n ...(note !== undefined ? { note } : {}),\n },\n url,\n marks,\n );\n const fetchedIso = new Date(fetchedAt).toISOString();\n if (capture !== undefined && blob !== undefined) {\n this.store({\n url,\n variant: \"rendered\",\n capture,\n blob,\n etag: null,\n last_modified: null,\n content_type: RENDERED_TYPE,\n fetched_at: fetchedIso,\n status,\n ai_input: signals.ai_input,\n fresh_until: freshness,\n });\n }\n return {\n url,\n status,\n outcome,\n fromCache: false,\n capture,\n blob,\n body,\n contentType: RENDERED_TYPE,\n signals,\n note,\n fetchedAt: fetchedIso,\n ...marks,\n };\n }\n\n /**\n * This method removes the stops and the failure counts of one host, or of each host. It also\n * forgets the robots.txt verdicts that came from an unreachable robots.txt. The pace of each host\n * stays. A process that runs for a long time uses it, for example after the network comes back.\n */\n resetHosts(host?: string): void {\n const name = host?.toLowerCase().replace(/\\.$/, \"\");\n this.scheduler.reset(name);\n for (const key of this.robots.keys()) {\n const entry = this.robots.peek(key);\n const unreachable =\n entry?.value?.robots.unreachable !== undefined || entry?.value?.stale !== undefined;\n if (unreachable && (name === undefined || key.endsWith(`//${name}`))) this.robots.delete(key);\n }\n }\n\n /**\n * This method closes the crawler. Work in the queue ends as `skipped` with the note\n * `crawler closed`, and the crawler writes no capture after the close. Then it closes the\n * browser, the log and its own dispatcher. A second close waits for the first.\n */\n close(): Promise<void> {\n if (this.closingDone === undefined) {\n this.closed = true;\n this.closing.abort(new Error(\"crawler closed\"));\n this.closingDone = (async () => {\n await Promise.allSettled(this.running);\n await this.options.renderer?.close();\n await this.options.log.close?.();\n if (this.ownsDispatcher) await this.dispatcher.close();\n })();\n }\n return this.closingDone;\n }\n}\n","/**\n * This module makes two forms of a text equal when a reader thinks that they are equal. It has two\n * modes:\n *\n * - `quote`: for the verification of a quote. It applies NFKC (a ligature `fi` is `fi`, a\n * fullwidth `A` is `A`, an ellipsis `…` is `...`) to each character and its combining marks,\n * thus NFC and NFD are equal. It removes soft hyphens and zero-width characters. It makes\n * typographic quotes, dashes and the minus sign ASCII, makes each run of spaces one space, and\n * makes the letters lowercase. Accents stay, so \"résumé\" is not \"resume\".\n * - `search`: for the search index. It also removes the accents and the other nonspacing marks,\n * and it folds the case fully (`ß` is `ss`). It puts a space around each Chinese, Japanese and\n * Thai character, because these scripts write no space between words.\n *\n * Each character of the folded text knows the start and the end of its source in the original\n * text. Thus a match in the folded text gives exact positions in the original text.\n */\n\nexport type FoldMode = \"quote\" | \"search\";\n\n/** A folded text, with the position in the original text of each UTF-16 unit. */\nexport interface Folded {\n readonly text: string;\n /** This value is the start, in the original text, of the source of each unit. */\n readonly start: Int32Array;\n /** This value is the end, in the original text, of the source of each unit. */\n readonly end: Int32Array;\n}\n\nconst MARK = /\\p{M}/u;\nconst NONSPACING = /[\\p{Mn}\\p{Me}]/gu;\nconst IGNORABLE = /^\\p{Default_Ignorable_Code_Point}$/u;\nconst SPACE = /^\\s$/u;\nconst SEGMENTED =\n /[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Thai}\\p{Script=Lao}\\p{Script=Khmer}\\p{Script=Myanmar}]/u;\n/** These characters become ASCII: typographic quotes, dashes and the minus sign. */\nconst TYPOGRAPHIC: ReadonlyMap<number, number> = new Map([\n [0x2018, 0x27],\n [0x2019, 0x27],\n [0x201a, 0x27],\n [0x201b, 0x27],\n [0x2032, 0x27],\n [0x201c, 0x22],\n [0x201d, 0x22],\n [0x201e, 0x22],\n [0x201f, 0x22],\n [0x2033, 0x22],\n [0x2010, 0x2d],\n [0x2011, 0x2d],\n [0x2012, 0x2d],\n [0x2013, 0x2d],\n [0x2014, 0x2d],\n [0x2015, 0x2d],\n [0x2212, 0x2d],\n [0xfe58, 0x2d],\n [0xfe63, 0x2d],\n]);\n\nconst isAsciiSpace = (code: number): boolean => code === 0x20 || (code >= 0x09 && code <= 0x0d);\n\n/**\n * This function folds a text, and it keeps the position in the original text of each unit of the\n * result. Its time is linear. An ASCII character with no combining mark after it takes a short path.\n */\nexport const foldText = (text: string, mode: FoldMode): Folded => {\n let codes = new Uint16Array(Math.max(16, text.length + 16));\n let starts = new Int32Array(codes.length);\n let ends = new Int32Array(codes.length);\n let size = 0;\n let space = true;\n const grow = (need: number): void => {\n if (size + need <= codes.length) return;\n const length = Math.max(codes.length * 2, size + need);\n const nextCodes = new Uint16Array(length);\n nextCodes.set(codes.subarray(0, size));\n const nextStarts = new Int32Array(length);\n nextStarts.set(starts.subarray(0, size));\n const nextEnds = new Int32Array(length);\n nextEnds.set(ends.subarray(0, size));\n codes = nextCodes;\n starts = nextStarts;\n ends = nextEnds;\n };\n const push = (code: number, from: number, to: number): void => {\n grow(1);\n codes[size] = code;\n starts[size] = from;\n ends[size] = to;\n size += 1;\n };\n const pushSpace = (from: number, to: number): void => {\n if (space) return;\n push(0x20, from, to);\n space = true;\n };\n const length = text.length;\n let index = 0;\n while (index < length) {\n const code = text.charCodeAt(index);\n const next = index + 1 < length ? text.charCodeAt(index + 1) : 0;\n // the short path: an ASCII character that no combining mark follows\n if (code < 0x80 && next < 0x300) {\n if (isAsciiSpace(code)) pushSpace(index, index + 1);\n else {\n push(code >= 0x41 && code <= 0x5a ? code + 32 : code, index, index + 1);\n space = false;\n }\n index += 1;\n continue;\n }\n // a cluster: one code point and the combining marks after it\n const point = text.codePointAt(index)!;\n let end = index + (point > 0xffff ? 2 : 1);\n while (end < length) {\n const mark = text.codePointAt(end)!;\n if (!MARK.test(String.fromCodePoint(mark))) break;\n end += mark > 0xffff ? 2 : 1;\n }\n const cluster = text.slice(index, end);\n const first = String.fromCodePoint(point);\n if (IGNORABLE.test(first) && end === index + first.length) {\n index = end;\n continue;\n }\n if (SPACE.test(first)) {\n pushSpace(index, end);\n index = end;\n continue;\n }\n let folded =\n mode === \"quote\"\n ? cluster.normalize(\"NFKC\").toLowerCase()\n : cluster\n .normalize(\"NFKD\")\n .replace(NONSPACING, \"\")\n .normalize(\"NFC\")\n .toUpperCase()\n .toLowerCase();\n if (mode === \"search\" && SEGMENTED.test(first)) folded = ` ${folded} `;\n for (let at = 0; at < folded.length; at += 1) {\n let unit = folded.charCodeAt(at);\n unit = TYPOGRAPHIC.get(unit) ?? unit;\n if (unit === 0x20 || (unit < 0x80 && isAsciiSpace(unit)) || unit === 0xa0) {\n pushSpace(index, end);\n continue;\n }\n push(unit, index, end);\n space = false;\n }\n index = end;\n }\n if (size > 0 && codes[size - 1] === 0x20) size -= 1;\n let result = \"\";\n for (let at = 0; at < size; at += 8192) {\n result += String.fromCharCode(...codes.subarray(at, Math.min(size, at + 8192)));\n }\n return { text: result, start: starts.subarray(0, size), end: ends.subarray(0, size) };\n};\n\n/** This function gives the folded text alone, for a text with no need of positions. */\nexport const fold = (text: string, mode: FoldMode): string => foldText(text, mode).text;\n\nexport interface Verified {\n readonly found: boolean;\n /** These values are the character positions in the text of the document, when the quote is there. */\n readonly start: number | null;\n readonly end: number | null;\n /** This text is the exact quote as stored, with some text before and after it. */\n readonly excerpt: string | null;\n /** This value is the number of places where the quote is, with no overlap. */\n readonly count: number;\n}\n\nconst NOT_FOUND: Verified = { found: false, start: null, end: null, excerpt: null, count: 0 };\n\n/** This function moves a position off the second half of a surrogate pair, so a slice does not cut a character. */\nconst atBoundary = (text: string, position: number, forward: boolean): number => {\n const code = text.charCodeAt(position);\n if (position > 0 && position < text.length && code >= 0xdc00 && code <= 0xdfff) {\n return forward ? position + 1 : position - 1;\n }\n return position;\n};\n\n/**\n * This function finds a quote in a text, in the `quote` mode of `foldText`. It ignores case, runs\n * of spaces, typographic punctuation, soft hyphens, zero-width characters and the differences of\n * NFKC, NFC and NFD. The positions are in the original text.\n */\nexport const locate = (text: string, quote: string): Verified => {\n const needle = fold(quote, \"quote\");\n if (needle.length === 0) return NOT_FOUND;\n const haystack = foldText(text, \"quote\");\n const hit = haystack.text.indexOf(needle);\n if (hit === -1) return NOT_FOUND;\n let count = 0;\n for (let at = hit; at !== -1; at = haystack.text.indexOf(needle, at + needle.length)) count += 1;\n const start = haystack.start[hit]!;\n const end = haystack.end[hit + needle.length - 1]!;\n const from = atBoundary(text, Math.max(0, start - 80), false);\n const to = atBoundary(text, Math.min(text.length, end + 80), true);\n return {\n found: true,\n start,\n end,\n excerpt: `${from > 0 ? \"…\" : \"\"}${text.slice(from, to)}${to < text.length ? \"…\" : \"\"}`,\n count,\n };\n};\n\n/** A token character of the search index: a letter, a digit, a private-use character or a spacing mark. */\nconst TOKEN_SPLIT = /[^\\p{L}\\p{N}\\p{Co}\\p{Mc}]+/u;\n\n/** This function gives the tokens of a text, as the search index reads them. */\nexport const tokensOf = (text: string): string[] =>\n fold(text, \"search\")\n .split(TOKEN_SPLIT)\n .filter((token) => token.length > 0);\n\n/** A query for the search index, in the two forms that `DocIndex.find` uses. */\nexport interface FtsQuery {\n /** Each part must match: the parts joined with AND. */\n readonly all: string;\n /** One part is enough: the parts joined with OR. */\n readonly any: string;\n /** These are the parts of `any`, each as FTS5 syntax. */\n readonly anyParts: readonly string[];\n /** These are the first tokens of the parts. A snippet marks them. */\n readonly terms: readonly string[];\n /** This value is the number of parts. */\n readonly parts: number;\n}\n\nconst MAX_PARTS = 16;\n\n/**\n * These English words are in almost every document. In the form that needs one part, they match\n * each document and cost the most time. Thus that form leaves them out when other words remain.\n */\nconst STOP_WORDS: ReadonlySet<string> = new Set([\n \"a\",\n \"an\",\n \"and\",\n \"are\",\n \"as\",\n \"at\",\n \"be\",\n \"by\",\n \"for\",\n \"from\",\n \"how\",\n \"in\",\n \"is\",\n \"it\",\n \"of\",\n \"on\",\n \"or\",\n \"that\",\n \"the\",\n \"this\",\n \"to\",\n \"was\",\n \"what\",\n \"when\",\n \"where\",\n \"which\",\n \"who\",\n \"with\",\n]);\n\n/**\n * This function reads a search query. The syntax:\n *\n * - A word is a term. The index folds case, accents and Unicode forms, so `cafe` finds `Café`. It\n * reduces English words to their stems, so `baking` finds `bake`.\n * - Text in double quotes is a phrase: its words must be next to each other.\n * - A `*` at the end of a word asks for a prefix: `bake*` finds `bakery`.\n * - A word of Chinese, Japanese or Thai is a phrase of its characters, so it matches in the text.\n * - Other punctuation is not syntax. The words `AND`, `OR`, `NOT` and `NEAR` are words.\n *\n * The function gives no query for a text with no word. It reads 16 parts or fewer. Each part is\n * quoted, so the text of a user cannot become FTS5 syntax. The form that needs one part (`any`)\n * leaves out the common English words (`STOP_WORDS`) when other words remain.\n */\nexport const parseQuery = (q: string): FtsQuery | undefined => {\n const parts: Array<{ readonly text: string; readonly term: string; readonly stop: boolean }> = [];\n for (const match of q.matchAll(/\"([^\"]*)\"(\\*?)|([^\\s\"]+)/g)) {\n if (parts.length >= MAX_PARTS) break;\n const phrase = match[1] !== undefined;\n const raw = phrase ? match[1]! : match[3]!;\n const prefix = phrase ? match[2] === \"*\" : raw.endsWith(\"*\");\n const tokens = tokensOf(phrase ? raw : raw.replace(/\\*+$/, \"\"));\n if (tokens.length === 0) continue;\n parts.push({\n text: `\"${tokens.join(\" \")}\"${prefix ? \" *\" : \"\"}`,\n term: tokens[0]!,\n stop: !phrase && !prefix && tokens.length === 1 && STOP_WORDS.has(tokens[0]!),\n });\n }\n if (parts.length === 0) return undefined;\n const words = parts.filter((part) => !part.stop);\n const any = words.length > 0 ? words : parts;\n return {\n all: parts.map((part) => part.text).join(\" AND \"),\n any: any.map((part) => part.text).join(\" OR \"),\n anyParts: any.map((part) => part.text),\n terms: any.map((part) => part.term),\n parts: parts.length,\n };\n};\n\n/** This function gives the FTS5 query that matches one part or more of a query. It is the form of version 0.1. */\nexport const ftsQuery = (q: string): string | undefined => parseQuery(q)?.any;\n\n/**\n * This function gives a short part of a text around the first match of a term, with the match in\n * square brackets. With no match, it gives the start of the text.\n */\nexport const snippetOf = (text: string, terms: readonly string[], width = 160): string => {\n if (text.length === 0) return \"\";\n let start = -1;\n let end = -1;\n const lower = text.toLowerCase();\n if (lower.length === text.length) {\n for (const term of terms) {\n const at = lower.indexOf(term);\n if (at !== -1 && (start === -1 || at < start)) {\n start = at;\n end = at + term.length;\n }\n }\n }\n if (start === -1) {\n const folded = foldText(text, \"search\");\n for (const term of terms) {\n const at = folded.text.indexOf(term);\n if (at !== -1) {\n const from = folded.start[at]!;\n if (start === -1 || from < start) {\n start = from;\n end = folded.end[at + term.length - 1]!;\n }\n }\n }\n }\n if (start === -1) {\n const cut = atBoundary(text, Math.min(text.length, width), false);\n return `${text.slice(0, cut)}${cut < text.length ? \"…\" : \"\"}`;\n }\n const from = atBoundary(text, Math.max(0, start - Math.floor(width / 3)), false);\n const to = atBoundary(text, Math.min(text.length, end + Math.ceil((width * 2) / 3)), true);\n const clean = (part: string): string => part.replace(/\\s+/g, \" \");\n return `${from > 0 ? \"…\" : \"\"}${clean(text.slice(from, start))}[${clean(text.slice(start, end))}]${clean(text.slice(end, to))}${to < text.length ? \"…\" : \"\"}`;\n};\n","import { createHash } from \"node:crypto\";\nimport { mkdirSync } from \"node:fs\";\nimport { dirname } from \"node:path\";\nimport type { StatementSync } from \"node:sqlite\";\nimport { DatabaseSync } from \"node:sqlite\";\nimport { fold, parseQuery, snippetOf } from \"./fold.ts\";\nimport type { AiInput } from \"./signals.ts\";\nimport { cacheKeyOf } from \"./urls.ts\";\n\nexport type { FtsQuery, Verified } from \"./fold.ts\";\nexport { ftsQuery, locate, parseQuery } from \"./fold.ts\";\n\n/**\n * The reading index holds each document that the crawler extracts. A reader can search it with\n * SQLite FTS5 (BM25), read a range of characters, and find an exact quote. The index is a\n * projection: delete the file, and `reindex` builds it again from the capture log and the blobs.\n *\n * A document is what a reader cites, not what the crawler fetched. One search response of an API\n * can hold many papers, so one capture can give many documents. One paper from two sources is one\n * document with two captures. The ids are stable and easy to read (`doi:10.1/x`, `pmid:123`).\n *\n * Each capture of a document is a version. The index keeps each version, and it shows one version\n * as the current document. These are the rules:\n *\n * - For one recipe, the latest capture by fetch time wins. A page that changes thus shows its new\n * text, also when the new text is shorter.\n * - A rendered capture wins over a plain capture of the same page. It stays current while the\n * plain body stays the same. After the plain body changes, the plain capture wins until a new render.\n * - When two recipes give one document, the longer text wins (`merge: \"richest\"`, the default).\n * With `merge: \"latest\"` on each version, the newer text wins. The metadata of the other recipes\n * fills the keys that the current version does not have.\n * - The AI input of a recipe is what its newest capture says. A \"no\" of one recipe wins over the\n * others. Thus a \"no\" stops when a later capture of the same page does not say it.\n *\n * The file has a schema version (`INDEX_SCHEMA`). A file of a different version opens empty, and\n * `stale` is true: the caller rebuilds it with `reindex`. Each connection waits for the lock of a\n * different process (`busy_timeout`), and each write is one transaction (`BEGIN IMMEDIATE`).\n */\n\n/** This value is the version of the schema of the index file. Version 0.1 of the package wrote files of version 0. */\nexport const INDEX_SCHEMA = 2;\n\n/** This value is the maximum number of capture ids that a document gives. The field `capture_count` gives the total. */\nexport const MAX_CAPTURES = 10;\n\n/** The search index reads this many characters of each text, at most. */\nconst MAX_INDEX_CHARS = 200_000;\n\n/** This is one document, as an extractor gives it. */\nexport interface ExtractedDoc {\n readonly doc: string;\n readonly kind: string;\n readonly title: string;\n /** This text is the readable text. For a paper, it is the title and the abstract. */\n readonly text: string;\n readonly url: string | null;\n readonly meta: Readonly<Record<string, unknown>>;\n /** This rule chooses between the texts of two recipes. The default is `richest`: the longer text. */\n readonly merge?: \"latest\" | \"richest\" | undefined;\n /** When the extractor cut the text, this text is the whole text. The function `verify` looks for a quote here too. */\n readonly full_text?: string | undefined;\n}\n\n/** This is one version of a document: the document as one capture gave it. */\nexport interface DocInput extends ExtractedDoc {\n /** These ids are the capture records that this version came from. The first id names the version. */\n readonly captures: readonly string[];\n readonly ai_input?: AiInput | undefined;\n /** This id is the capture of the version. The default is the first id of `captures`. */\n readonly capture?: string | undefined;\n /** This time (ISO 8601) is the time of the fetch. The default is the time of the upsert. */\n readonly fetched_at?: string | undefined;\n readonly blob?: string | null | undefined;\n /** This value is the recipe that read the capture. A version with no recipe has the recipe `\"\"`. */\n readonly recipe?: string | undefined;\n /** This value is true when the headless browser made the capture. */\n readonly rendered?: boolean | undefined;\n /** This label comes from the capture. The index writes it as it is and does not read it. */\n readonly scope?: string | undefined;\n /** These labels come from the capture. The index writes them as they are and does not read them. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/** This is the current version of a document. */\nexport interface Doc {\n readonly doc: string;\n readonly kind: string;\n readonly title: string;\n readonly text: string;\n readonly url: string | null;\n readonly meta: Readonly<Record<string, unknown>>;\n /** These ids are the captures behind the current text, the newest first, `MAX_CAPTURES` or fewer. */\n readonly captures: readonly string[];\n /** This value is the number of versions of the document. */\n readonly capture_count: number;\n /** This id is the capture of the current text. */\n readonly capture: string;\n readonly blob: string | null;\n /** This time (ISO 8601) is the fetch time of the current text. */\n readonly fetched_at: string;\n readonly recipe: string;\n readonly rendered: boolean;\n readonly ai_input: AiInput;\n /** This value is true when the extractor cut the text, and the index keeps the whole text for `verify`. */\n readonly cut: boolean;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n readonly updated_at: string;\n}\n\n/** This is one version of a document, with no text. */\nexport interface DocVersion {\n readonly doc: string;\n readonly capture: string;\n readonly recipe: string;\n readonly rendered: boolean;\n readonly fetched_at: string;\n readonly blob: string | null;\n readonly kind: string;\n readonly title: string;\n readonly url: string | null;\n readonly ai_input: AiInput;\n /** This value is the length of the text of the version. */\n readonly chars: number;\n /** This value is true when the index keeps a whole text that is longer than the text. */\n readonly cut: boolean;\n}\n\n/** This is the text of one version, with its metadata. */\nexport interface VersionText extends DocVersion {\n readonly text: string;\n /** This text is the whole text when the extractor cut `text`. */\n readonly full_text: string | undefined;\n readonly meta: Readonly<Record<string, unknown>>;\n}\n\n/**\n * This value chooses the ranking of a search:\n *\n * - `exact` (the default): BM25 ranks each document that matches.\n * - `fast`: the form of the query that needs one part leaves out the common parts. A common part\n * is in more than `FAST_RANK_SHARE` of the documents. The rarest part stays when each part is common.\n *\n * The documents that hold each part come first in the two modes, with the same rank. In the mode\n * `fast`, the rest of the results leaves out a document that holds only common parts. The common\n * parts also do not add to the score of the rest. Thus `fast` is faster for a long question, and\n * its results can be different. The README gives the measured difference.\n */\nexport type RankMode = \"exact\" | \"fast\";\n\n/** In the mode `fast`, a part is common when more than this share of the documents hold it. */\nexport const FAST_RANK_SHARE = 0.2;\n\n/** This function gives the rank mode of an option. A value that is not a rank mode throws a `RangeError`. */\nconst rankModeOf = (value: unknown): RankMode => {\n if (value === undefined || value === \"exact\") return \"exact\";\n if (value === \"fast\") return \"fast\";\n throw new RangeError(\n `rank must be \"exact\" or \"fast\", not ${JSON.stringify(value) ?? \"undefined\"}`,\n );\n};\n\n/** These are the options of a search of the index. */\nexport interface FindIndexOptions {\n /** This value is the maximum number of results. The index gives 1000 or fewer. */\n readonly limit: number;\n /** This value keeps only the documents of one kind, for example `page` or `pdf`. */\n readonly kind?: string | null | undefined;\n /** This value chooses the ranking. The default is `exact`. */\n readonly rank?: RankMode | undefined;\n}\n\nexport interface Found {\n readonly doc: string;\n readonly kind: string;\n readonly title: string;\n readonly url: string | null;\n readonly snippet: string;\n readonly score: number;\n readonly ai_input: AiInput;\n /** This time (ISO 8601) is the fetch time of the current text. */\n readonly fetched_at: string;\n /** This id is the capture of the current text. */\n readonly capture: string;\n}\n\n/** The variant of a cache entry: the plain response, or the page that the browser rendered. */\nexport type Variant = \"plain\" | \"rendered\";\n\nexport interface Latest {\n readonly url: string;\n readonly capture: string;\n readonly blob: string | null;\n readonly etag: string | null;\n readonly last_modified: string | null;\n readonly content_type: string | null;\n readonly fetched_at: string;\n readonly status: number;\n /** This value is what the page said about AI input at the time of the fetch. */\n readonly ai_input: AiInput;\n /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */\n readonly location?: string | null | undefined;\n /** This value is true when the stored body is only the first part of the response. */\n readonly truncated?: boolean | undefined;\n /** This time (ISO 8601) is the end of the freshness that the server gave. */\n readonly fresh_until?: string | null | undefined;\n /** This text holds the request headers that the response varies on (`Vary`), with their values. */\n readonly vary_key?: string | null | undefined;\n /** The default is `plain`. A key that ends in `#rendered` (the form of version 0.1) is `rendered`. */\n readonly variant?: Variant | undefined;\n}\n\n/** These are the counts of the index. */\nexport interface Counts {\n readonly docs: number;\n readonly urls: number;\n}\n\n/** A rebuild of the index writes a new generation of the tables. The old generation answers until the commit. */\nexport interface IndexBuild {\n /** This method adds a version to the new generation. The build writes the versions in batches. */\n upsert(input: DocInput, now: string): void;\n /** This method sets a cache entry in the new generation, when it is newer than the stored entry. */\n setLatest(entry: Latest): void;\n /** This method reads a cache entry of the new generation. */\n latest(url: string, variant?: Variant): Latest | undefined;\n /** This method writes the waiting versions in one transaction. */\n flush(): void;\n /**\n * This method copies the versions of each recipe that is not in `recipes` from the current\n * generation. A runtime with no extractor for a recipe thus keeps the documents of that recipe.\n * It gives the number of documents that it copied.\n */\n carry(recipes: ReadonlySet<string>): number;\n /**\n * This method makes the new generation current. It first copies the rows that a different writer\n * added to the current generation since `since`. Then it removes the old generation.\n */\n commit(since: string): Counts;\n /** This method removes the new generation. The current generation stays. */\n abort(): void;\n}\n\ninterface Names {\n readonly docs: string;\n readonly fts: string;\n readonly versions: string;\n readonly texts: string;\n readonly latest: string;\n}\n\nconst namesOf = (generation: number): Names => ({\n docs: `docs_${String(generation)}`,\n fts: `fts_${String(generation)}`,\n versions: `versions_${String(generation)}`,\n texts: `texts_${String(generation)}`,\n latest: `latest_${String(generation)}`,\n});\n\nconst GENERATION_TABLE = /^(docs|fts|versions|texts|latest)_(\\d+)$/;\n\n/** A build that started this long ago and did not finish belongs to a process that stopped. */\nconst ABANDONED_MS = 24 * 60 * 60 * 1000;\n\n/** The FTS5 tokenizer: folded text, English stems, and spacing marks kept in the words. */\nconst TOKENIZER = `tokenize = \"porter unicode61 remove_diacritics 0 categories 'L* N* Co Mc'\"`;\n\nconst ddlOf = (n: Names): string => `\nCREATE TABLE IF NOT EXISTS ${n.texts} (\n sha TEXT PRIMARY KEY,\n body TEXT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS ${n.versions} (\n doc TEXT NOT NULL,\n capture TEXT NOT NULL,\n recipe TEXT NOT NULL,\n rendered INTEGER NOT NULL,\n fetched_at TEXT NOT NULL,\n blob TEXT,\n kind TEXT NOT NULL,\n title TEXT NOT NULL,\n url TEXT,\n meta_sha TEXT NOT NULL,\n text_sha TEXT NOT NULL,\n text_chars INTEGER NOT NULL,\n full_sha TEXT,\n ai_input TEXT NOT NULL,\n merge TEXT NOT NULL,\n scope TEXT,\n labels TEXT,\n PRIMARY KEY (doc, capture)\n);\nCREATE INDEX IF NOT EXISTS ${n.versions}_time ON ${n.versions} (doc, recipe, rendered, fetched_at);\nCREATE INDEX IF NOT EXISTS ${n.versions}_capture ON ${n.versions} (capture);\nCREATE INDEX IF NOT EXISTS ${n.versions}_fetched ON ${n.versions} (fetched_at);\nCREATE INDEX IF NOT EXISTS ${n.versions}_recipe ON ${n.versions} (recipe);\nCREATE TABLE IF NOT EXISTS ${n.docs} (\n doc TEXT PRIMARY KEY,\n kind TEXT NOT NULL,\n title TEXT NOT NULL,\n url TEXT,\n meta TEXT NOT NULL,\n text_sha TEXT NOT NULL,\n full_sha TEXT,\n captures TEXT NOT NULL,\n capture_count INTEGER NOT NULL,\n capture TEXT NOT NULL,\n blob TEXT,\n fetched_at TEXT NOT NULL,\n recipe TEXT NOT NULL,\n rendered INTEGER NOT NULL,\n ai_input TEXT NOT NULL,\n scope TEXT,\n labels TEXT,\n updated_at TEXT NOT NULL\n);\nCREATE VIRTUAL TABLE IF NOT EXISTS ${n.fts} USING fts5(\n title, text, content = '', contentless_delete = 1, ${TOKENIZER}\n);\nCREATE TABLE IF NOT EXISTS ${n.latest} (\n url TEXT NOT NULL,\n variant TEXT NOT NULL,\n capture TEXT NOT NULL,\n blob TEXT,\n etag TEXT,\n last_modified TEXT,\n content_type TEXT,\n fetched_at TEXT NOT NULL,\n status INTEGER NOT NULL,\n ai_input TEXT NOT NULL,\n location TEXT,\n truncated INTEGER,\n fresh_until TEXT,\n vary_key TEXT,\n PRIMARY KEY (url, variant)\n);\nCREATE INDEX IF NOT EXISTS ${n.latest}_fetched ON ${n.latest} (fetched_at);\n`;\n\ninterface VersionRow {\n doc: string;\n capture: string;\n recipe: string;\n rendered: number;\n fetched_at: string;\n blob: string | null;\n kind: string;\n title: string;\n url: string | null;\n meta_sha: string;\n text_sha: string;\n text_chars: number;\n full_sha: string | null;\n ai_input: string;\n merge: string;\n scope: string | null;\n labels: string | null;\n}\n\ninterface DocRow {\n rowid: number;\n doc: string;\n kind: string;\n title: string;\n url: string | null;\n meta: string;\n text_sha: string;\n full_sha: string | null;\n captures: string;\n capture_count: number;\n capture: string;\n blob: string | null;\n fetched_at: string;\n recipe: string;\n rendered: number;\n ai_input: string;\n scope: string | null;\n labels: string | null;\n updated_at: string;\n}\n\ninterface LatestRow {\n url: string;\n variant: string;\n capture: string;\n blob: string | null;\n etag: string | null;\n last_modified: string | null;\n content_type: string | null;\n fetched_at: string;\n status: number;\n ai_input: string;\n location: string | null;\n truncated: number | null;\n fresh_until: string | null;\n vary_key: string | null;\n}\n\nconst RENDERED_MARK = \"#rendered\";\n\n/** This function splits a cache key into its URL and its variant. A key in the form of version 0.1 (`…#rendered`) is a rendered variant. */\nconst keyOf = (\n url: string,\n variant: Variant | undefined,\n): { readonly url: string; readonly variant: Variant } => {\n if (url.endsWith(RENDERED_MARK)) {\n return { url: cacheKeyOf(url.slice(0, -RENDERED_MARK.length)), variant: \"rendered\" };\n }\n return { url: cacheKeyOf(url), variant: variant ?? \"plain\" };\n};\n\nconst sha = (text: string): string => createHash(\"sha256\").update(text, \"utf8\").digest(\"hex\");\n\nconst parseJson = <T>(text: string | null | undefined, fallback: T): T => {\n if (text === null || text === undefined) return fallback;\n try {\n return JSON.parse(text) as T;\n } catch {\n return fallback;\n }\n};\n\nconst toLatest = (row: LatestRow): Latest => ({\n url: row.url,\n capture: row.capture,\n blob: row.blob,\n etag: row.etag,\n last_modified: row.last_modified,\n content_type: row.content_type,\n fetched_at: row.fetched_at,\n status: row.status,\n ai_input: row.ai_input as AiInput,\n location: row.location,\n truncated: row.truncated === 1 ? true : undefined,\n fresh_until: row.fresh_until,\n vary_key: row.vary_key,\n variant: row.variant as Variant,\n});\n\nconst toVersion = (row: VersionRow): DocVersion => ({\n doc: row.doc,\n capture: row.capture,\n recipe: row.recipe,\n rendered: row.rendered === 1,\n fetched_at: row.fetched_at,\n blob: row.blob,\n kind: row.kind,\n title: row.title,\n url: row.url,\n ai_input: row.ai_input as AiInput,\n chars: row.text_chars,\n cut: row.full_sha !== null,\n});\n\nconst messageOf = (error: unknown): string =>\n error instanceof Error ? error.message : String(error);\n\n/**\n * A reading index keeps the documents, their versions, the search and the cache entries. `DocIndex`\n * is the SQLite index. A runtime takes any object with these methods, for example a search index\n * of a different kind that wraps a `DocIndex`.\n */\nexport interface ReadingIndex {\n /** This value is true when the index opened empty and a rebuild did not fill it yet. */\n readonly stale: boolean;\n get(doc: string): Doc | undefined;\n upsert(input: DocInput, now: string): Doc;\n versions(doc: string, options?: { readonly limit?: number }): DocVersion[];\n versionText(doc: string, capture?: string): VersionText | undefined;\n earlierTexts(doc: string, options?: { readonly limit?: number }): VersionText[];\n docsOfCapture(capture: string): Doc[];\n /** This method tells if a capture gave a version of a document. */\n hasVersion(capture: string): boolean;\n find(q: string, options: FindIndexOptions): Found[];\n rankWithin(\n q: string,\n docs: readonly string[],\n options?: { readonly rank?: RankMode | undefined },\n ): Array<{ readonly doc: string; readonly score: number }>;\n latest(url: string, variant?: Variant): Latest | undefined;\n setLatest(entry: Latest): void;\n /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */\n newestFetch(): string | undefined;\n counts(): Counts;\n beginRebuild(options?: { readonly batch?: number }): IndexBuild;\n}\n\n/** These are the options of a reading index. */\nexport interface DocIndexOptions {\n /** This value is the time in milliseconds that a write waits for the lock of a different process. The default is 10 seconds. */\n readonly busyTimeoutMs?: number | undefined;\n}\n\nexport class DocIndex implements ReadingIndex {\n private readonly db: DatabaseSync;\n private readonly statements = new Map<string, StatementSync>();\n private depth = 0;\n private closed = false;\n private empty: boolean;\n\n /**\n * This value is true when the file had a different schema, or no tables, so the index opened\n * empty. A store with captures then needs `reindex`. The commit of a rebuild makes it false.\n */\n get stale(): boolean {\n return this.empty;\n }\n\n constructor(\n readonly path: string,\n options: DocIndexOptions = {},\n ) {\n if (path !== \":memory:\") mkdirSync(dirname(path), { recursive: true, mode: 0o700 });\n this.db = new DatabaseSync(path);\n // the wait for the lock of a different process comes first, before any statement that writes\n const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 10_000));\n this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);\n try {\n this.db.exec(\n \"CREATE VIRTUAL TABLE temp.polite_fts5_probe USING fts5(x, content = '', contentless_delete = 1); DROP TABLE temp.polite_fts5_probe;\",\n );\n } catch (cause) {\n this.db.close();\n throw new Error(\n `polite-crawler needs SQLite FTS5 with contentless_delete, and the SQLite of Node.js ${process.version} does not have it: use Node.js 22.19 or later`,\n { cause },\n );\n }\n if (path !== \":memory:\") this.db.exec(\"PRAGMA journal_mode = WAL;\");\n // the index is a projection, so a power cut can cost its last transactions and nothing else\n this.db.exec(\"PRAGMA synchronous = NORMAL;\");\n this.empty = this.write(() => this.prepareSchema());\n }\n\n /** This method makes the tables of the schema. It gives true when the index opened empty. */\n private prepareSchema(): boolean {\n const version = (this.db.prepare(\"PRAGMA user_version\").get() as { user_version: number })\n .user_version;\n const tables = this.db\n .prepare(\"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'\")\n .all() as Array<{ name: string }>;\n if (version === INDEX_SCHEMA && tables.some((table) => table.name === \"index_meta\")) {\n this.dropAbandoned();\n return false;\n }\n // an index of a different schema is a projection: it opens empty, and reindex fills it\n const virtual = this.db\n .prepare(\n \"SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'\",\n )\n .all() as Array<{ name: string }>;\n for (const { name } of virtual) this.db.exec(`DROP TABLE IF EXISTS \"${name}\"`);\n const rest = this.db\n .prepare(\"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'\")\n .all() as Array<{ name: string }>;\n for (const { name } of rest) this.db.exec(`DROP TABLE IF EXISTS \"${name}\"`);\n this.db.exec(\"CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)\");\n this.db.exec(ddlOf(namesOf(1)));\n this.statements.clear();\n this.setMeta(\"generation\", \"1\");\n this.setMeta(\"next_generation\", \"2\");\n this.db.exec(`PRAGMA user_version = ${String(INDEX_SCHEMA)}`);\n return true;\n }\n\n private setMeta(key: string, value: string): void {\n this.stmt(\n \"INSERT INTO index_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value\",\n ).run(key, value);\n }\n\n private getMeta(key: string): string | undefined {\n return (\n this.stmt(\"SELECT value FROM index_meta WHERE key = ?\").get(key) as\n { value: string } | undefined\n )?.value;\n }\n\n private stmt(sql: string): StatementSync {\n let statement = this.statements.get(sql);\n if (statement === undefined) {\n statement = this.db.prepare(sql);\n this.statements.set(sql, statement);\n }\n return statement;\n }\n\n /**\n * This method does a write in one transaction. A transaction in progress takes the write in,\n * also a transaction that the caller opened on the connection.\n */\n private write<T>(work: () => T): T {\n if (this.closed) throw new Error(\"the index is closed\");\n if (this.depth > 0 || this.db.isTransaction) {\n this.depth += 1;\n try {\n return work();\n } finally {\n this.depth -= 1;\n }\n }\n this.db.exec(\"BEGIN IMMEDIATE\");\n this.depth = 1;\n try {\n const result = work();\n this.db.exec(\"COMMIT\");\n return result;\n } catch (error) {\n if (this.db.isTransaction) this.db.exec(\"ROLLBACK\");\n throw error;\n } finally {\n this.depth = 0;\n }\n }\n\n private generation(): number {\n return Number(this.getMeta(\"generation\") ?? \"1\");\n }\n\n /**\n * This method does a step on the tables of the current generation. A rebuild of a different\n * process can remove those tables in the meantime. Then the method reads the generation again\n * and does the step once more.\n */\n private live<T>(step: (names: Names) => T): T {\n if (this.closed) throw new Error(\"the index is closed\");\n try {\n return step(namesOf(this.generation()));\n } catch (error) {\n if (!/no such table/i.test(messageOf(error)) || this.depth > 0) throw error;\n this.statements.clear();\n return step(namesOf(this.generation()));\n }\n }\n\n /** This method removes the tables of a generation. */\n private dropGeneration(generation: number): void {\n const n = namesOf(generation);\n for (const table of [n.fts, n.docs, n.versions, n.texts, n.latest]) {\n this.db.exec(`DROP TABLE IF EXISTS ${table}`);\n }\n this.db.prepare(\"DELETE FROM index_meta WHERE key = ?\").run(`building:${String(generation)}`);\n this.statements.clear();\n }\n\n /** This method removes the generations of builds that a stopped process left. */\n private dropAbandoned(): void {\n const current = this.generation();\n const generations = new Set<number>();\n for (const { name } of this.db\n .prepare(\"SELECT name FROM sqlite_master WHERE type = 'table'\")\n .all() as Array<{ name: string }>) {\n const match = GENERATION_TABLE.exec(name);\n if (match !== null) generations.add(Number(match[2]));\n }\n for (const generation of generations) {\n if (generation === current) continue;\n const started = this.getMeta(`building:${String(generation)}`);\n const age =\n started === undefined ? Number.POSITIVE_INFINITY : Date.now() - Date.parse(started);\n if (!(age < ABANDONED_MS)) this.dropGeneration(generation);\n }\n }\n\n // ── texts and versions ────────────────────────────────────────────────────\n\n private putText(n: Names, body: string): string {\n const key = sha(body);\n this.stmt(`INSERT OR IGNORE INTO ${n.texts} (sha, body) VALUES (?, ?)`).run(key, body);\n return key;\n }\n\n private textOf(n: Names, key: string | null): string | undefined {\n if (key === null) return undefined;\n return (\n this.stmt(`SELECT body FROM ${n.texts} WHERE sha = ?`).get(key) as\n { body: string } | undefined\n )?.body;\n }\n\n private putVersion(n: Names, input: DocInput, now: string): void {\n const full =\n input.full_text !== undefined && input.full_text !== input.text\n ? this.putText(n, input.full_text)\n : null;\n this.stmt(\n `INSERT INTO ${n.versions} (doc, capture, recipe, rendered, fetched_at, blob, kind, title, url,\n meta_sha, text_sha, text_chars, full_sha, ai_input, merge, scope, labels)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(doc, capture) DO UPDATE SET recipe = excluded.recipe, rendered = excluded.rendered,\n fetched_at = excluded.fetched_at, blob = excluded.blob, kind = excluded.kind, title = excluded.title,\n url = excluded.url, meta_sha = excluded.meta_sha, text_sha = excluded.text_sha,\n text_chars = excluded.text_chars, full_sha = excluded.full_sha, ai_input = excluded.ai_input,\n merge = excluded.merge, scope = excluded.scope, labels = excluded.labels`,\n ).run(\n input.doc,\n input.capture ?? input.captures[0] ?? \"\",\n input.recipe ?? \"\",\n input.rendered === true ? 1 : 0,\n input.fetched_at ?? now,\n input.blob ?? null,\n input.kind,\n input.title,\n input.url,\n this.putText(n, JSON.stringify(input.meta ?? {})),\n this.putText(n, input.text),\n input.text.length,\n full,\n input.ai_input ?? \"unset\",\n input.merge ?? \"richest\",\n input.scope ?? null,\n input.labels === undefined ? null : JSON.stringify(input.labels),\n );\n }\n\n private newest(n: Names, doc: string, recipe: string, rendered: number): VersionRow | undefined {\n return this.stmt(\n `SELECT * FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = ?\n ORDER BY fetched_at DESC, capture DESC LIMIT 1`,\n ).get(doc, recipe, rendered) as VersionRow | undefined;\n }\n\n /** This method chooses the current version of a document from its versions, and writes it with its search entry. */\n private recompute(n: Names, doc: string, now: string): Doc | undefined {\n const groups = this.stmt(\n `SELECT DISTINCT recipe, rendered FROM ${n.versions} WHERE doc = ?`,\n ).all(doc) as Array<{ recipe: string; rendered: number }>;\n const before = this.stmt(`SELECT rowid, text_sha, title FROM ${n.docs} WHERE doc = ?`).get(\n doc,\n ) as { rowid: number; text_sha: string; title: string } | undefined;\n if (groups.length === 0) {\n if (before !== undefined) {\n this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);\n this.stmt(`DELETE FROM ${n.docs} WHERE doc = ?`).run(doc);\n }\n return undefined;\n }\n const byRecipe = new Map<string, { plain?: VersionRow; rendered?: VersionRow }>();\n for (const group of groups) {\n const row = this.newest(n, doc, group.recipe, group.rendered);\n if (row === undefined) continue;\n const entry = byRecipe.get(group.recipe) ?? {};\n if (group.rendered === 1) entry.rendered = row;\n else entry.plain = row;\n byRecipe.set(group.recipe, entry);\n }\n interface Candidate {\n readonly row: VersionRow;\n readonly ai: AiInput;\n readonly others: readonly VersionRow[];\n }\n const candidates: Candidate[] = [];\n for (const [recipe, { plain, rendered }] of byRecipe) {\n let pick: VersionRow;\n if (rendered === undefined) pick = plain!;\n else if (plain === undefined || rendered.fetched_at >= plain.fetched_at) pick = rendered;\n else {\n // the plain body that the render read: the render stays current while that body stays\n const base = this.stmt(\n `SELECT blob FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = 0 AND fetched_at <= ?\n ORDER BY fetched_at DESC, capture DESC LIMIT 1`,\n ).get(doc, recipe, rendered.fetched_at) as { blob: string | null } | undefined;\n pick =\n base !== undefined && base.blob !== null && base.blob === plain.blob ? rendered : plain;\n }\n const newest =\n plain === undefined\n ? rendered!\n : rendered === undefined || plain.fetched_at > rendered.fetched_at\n ? plain\n : rendered;\n const others = [plain, rendered].filter(\n (row): row is VersionRow => row !== undefined && row !== pick,\n );\n candidates.push({ row: pick, ai: newest.ai_input as AiInput, others });\n }\n let winner = candidates[0]!;\n if (candidates.length > 1) {\n const latest = candidates.every((candidate) => candidate.row.merge === \"latest\");\n for (const candidate of candidates.slice(1)) {\n const a = candidate.row;\n const b = winner.row;\n const better = latest\n ? a.fetched_at > b.fetched_at\n : a.text_chars > b.text_chars ||\n (a.text_chars === b.text_chars && a.fetched_at > b.fetched_at);\n if (better) winner = candidate;\n }\n }\n const row = winner.row;\n // a render can stay current after a newer plain capture: a no of its own capture still holds its text back\n const aiInput: AiInput =\n row.ai_input === \"no\" || candidates.some((candidate) => candidate.ai === \"no\")\n ? \"no\"\n : winner.ai !== \"unset\"\n ? winner.ai\n : candidates.some((candidate) => candidate.ai === \"yes\")\n ? \"yes\"\n : \"unset\";\n let meta = parseJson<Record<string, unknown>>(this.textOf(n, row.meta_sha), {});\n for (const candidate of candidates) {\n // a different source of the same document fills the keys that the current version lacks\n if (candidate !== winner) {\n meta = {\n ...parseJson<Record<string, unknown>>(this.textOf(n, candidate.row.meta_sha), {}),\n ...meta,\n };\n }\n }\n const same = this.stmt(\n `SELECT capture FROM ${n.versions} WHERE doc = ? AND text_sha = ?\n ORDER BY fetched_at DESC, capture DESC LIMIT ?`,\n ).all(doc, row.text_sha, MAX_CAPTURES) as Array<{ capture: string }>;\n const captures = [\n ...new Set([\n row.capture,\n ...same.map((entry) => entry.capture),\n ...winner.others.map((other) => other.capture),\n ...candidates\n .filter((candidate) => candidate !== winner)\n .map((candidate) => candidate.row.capture),\n ]),\n ].slice(0, MAX_CAPTURES);\n const count = (\n this.stmt(`SELECT count(*) AS n FROM ${n.versions} WHERE doc = ?`).get(doc) as { n: number }\n ).n;\n const url =\n row.url ??\n candidates.map((candidate) => candidate.row.url).find((value) => value !== null) ??\n null;\n const saved = this.stmt(\n `INSERT INTO ${n.docs} (doc, kind, title, url, meta, text_sha, full_sha, captures, capture_count, capture,\n blob, fetched_at, recipe, rendered, ai_input, scope, labels, updated_at)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(doc) DO UPDATE SET kind = excluded.kind, title = excluded.title, url = excluded.url,\n meta = excluded.meta, text_sha = excluded.text_sha, full_sha = excluded.full_sha,\n captures = excluded.captures, capture_count = excluded.capture_count, capture = excluded.capture,\n blob = excluded.blob, fetched_at = excluded.fetched_at, recipe = excluded.recipe,\n rendered = excluded.rendered, ai_input = excluded.ai_input, scope = excluded.scope,\n labels = excluded.labels, updated_at = excluded.updated_at\n RETURNING rowid`,\n ).get(\n doc,\n row.kind,\n row.title,\n url,\n JSON.stringify(meta),\n row.text_sha,\n row.full_sha,\n JSON.stringify(captures),\n count,\n row.capture,\n row.blob,\n row.fetched_at,\n row.recipe,\n row.rendered,\n aiInput,\n row.scope,\n row.labels,\n now,\n ) as { rowid: number };\n const text = this.textOf(n, row.text_sha) ?? \"\";\n if (before === undefined || before.text_sha !== row.text_sha || before.title !== row.title) {\n if (before !== undefined) this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);\n this.stmt(`INSERT INTO ${n.fts} (rowid, title, text) VALUES (?, ?, ?)`).run(\n saved.rowid,\n fold(row.title, \"search\"),\n fold(text.length > MAX_INDEX_CHARS ? text.slice(0, MAX_INDEX_CHARS) : text, \"search\"),\n );\n }\n return this.docOf(n, doc, text);\n }\n\n private docOf(n: Names, doc: string, knownText?: string): Doc | undefined {\n const row = this.stmt(`SELECT rowid, * FROM ${n.docs} WHERE doc = ?`).get(doc) as\n DocRow | undefined;\n if (row === undefined) return undefined;\n return this.toDoc(n, row, knownText);\n }\n\n private toDoc(n: Names, row: DocRow, knownText?: string): Doc {\n const labels = parseJson<Record<string, string> | undefined>(row.labels, undefined);\n return {\n doc: row.doc,\n kind: row.kind,\n title: row.title,\n text: knownText ?? this.textOf(n, row.text_sha) ?? \"\",\n url: row.url,\n meta: parseJson<Record<string, unknown>>(row.meta, {}),\n captures: parseJson<string[]>(row.captures, []),\n capture_count: row.capture_count,\n capture: row.capture,\n blob: row.blob,\n fetched_at: row.fetched_at,\n recipe: row.recipe,\n rendered: row.rendered === 1,\n ai_input: row.ai_input as AiInput,\n cut: row.full_sha !== null,\n ...(row.scope !== null ? { scope: row.scope } : {}),\n ...(labels !== undefined ? { labels } : {}),\n updated_at: row.updated_at,\n };\n }\n\n private versionTextOf(n: Names, row: VersionRow): VersionText {\n return {\n ...toVersion(row),\n text: this.textOf(n, row.text_sha) ?? \"\",\n full_text: this.textOf(n, row.full_sha),\n meta: parseJson<Record<string, unknown>>(this.textOf(n, row.meta_sha), {}),\n };\n }\n\n // ── the public methods ────────────────────────────────────────────────────\n\n get(doc: string): Doc | undefined {\n return this.live((n) => this.docOf(n, doc));\n }\n\n /**\n * This method adds one version of a document, and it gives the current document. The rules of\n * the class choose the current version. An upsert of a known capture replaces that version.\n */\n upsert(input: DocInput, now: string): Doc {\n return this.live((n) =>\n this.write(() => {\n this.putVersion(n, input, now);\n return this.recompute(n, input.doc, now)!;\n }),\n );\n }\n\n /** This method gives the versions of a document, the newest first. */\n versions(doc: string, options: { readonly limit?: number } = {}): DocVersion[] {\n const limit = Math.max(1, Math.floor(options.limit ?? 100));\n return this.live((n) =>\n (\n this.stmt(\n `SELECT * FROM ${n.versions} WHERE doc = ? ORDER BY fetched_at DESC, capture DESC LIMIT ?`,\n ).all(doc, limit) as unknown as VersionRow[]\n ).map(toVersion),\n );\n }\n\n /**\n * This method gives the text of one version of a document. With no capture, it gives the current\n * version. It also gives the text of a held-back version: the caller applies `isHeldBack` to the\n * `ai_input` of the version.\n */\n versionText(doc: string, capture?: string): VersionText | undefined {\n return this.live((n) => {\n const id = capture ?? this.docOf(n, doc)?.capture;\n if (id === undefined) return undefined;\n const row = this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND capture = ?`).get(\n doc,\n id,\n ) as VersionRow | undefined;\n return row === undefined ? undefined : this.versionTextOf(n, row);\n });\n }\n\n /**\n * This method gives the texts of the earlier versions of a document, the newest first. Each\n * text comes one time, with the newest version that has it. It skips the current text. It also\n * gives held-back texts: the caller applies `isHeldBack` to the `ai_input` of each version.\n */\n earlierTexts(doc: string, options: { readonly limit?: number } = {}): VersionText[] {\n const limit = Math.max(1, Math.floor(options.limit ?? 50));\n return this.live((n) => {\n const head = this.stmt(`SELECT text_sha FROM ${n.docs} WHERE doc = ?`).get(doc) as\n { text_sha: string } | undefined;\n if (head === undefined) return [];\n const rows = this.stmt(\n `SELECT * FROM ${n.versions} WHERE doc = ? AND text_sha != ? ORDER BY fetched_at DESC, capture DESC`,\n ).all(doc, head.text_sha) as unknown as VersionRow[];\n const seen = new Set<string>();\n const out: VersionText[] = [];\n for (const row of rows) {\n if (seen.has(row.text_sha)) continue;\n seen.add(row.text_sha);\n out.push(this.versionTextOf(n, row));\n if (out.length >= limit) break;\n }\n return out;\n });\n }\n\n /** This method tells if a capture gave a version of a document. */\n hasVersion(capture: string): boolean {\n return this.live(\n (n) =>\n this.stmt(`SELECT 1 AS found FROM ${n.versions} WHERE capture = ? LIMIT 1`).get(capture) !==\n undefined,\n );\n }\n\n /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */\n newestFetch(): string | undefined {\n return this.live(\n (n) =>\n (\n this.stmt(`SELECT max(fetched_at) AS newest FROM ${n.latest}`).get() as\n { newest: string | null } | undefined\n )?.newest ?? undefined,\n );\n }\n\n /** This method gives the current documents that a capture gave. */\n docsOfCapture(capture: string): Doc[] {\n return this.live((n) =>\n (\n this.stmt(`SELECT DISTINCT doc FROM ${n.versions} WHERE capture = ?`).all(\n capture,\n ) as Array<{ doc: string }>\n )\n .map((row) => this.docOf(n, row.doc))\n .filter((doc): doc is Doc => doc !== undefined),\n );\n }\n\n /**\n * This method searches the documents with BM25, the best first. The syntax of the query is the\n * syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it\n * adds the documents that match one part. A document that asked not to be AI input is not in\n * the results, so a search cannot show its text. The option `rank` chooses the ranking (`RankMode`).\n */\n find(q: string, options: FindIndexOptions): Found[] {\n const rank = rankModeOf(options.rank);\n const query = parseQuery(q);\n if (query === undefined) return [];\n const limit = Math.max(1, Math.min(1000, Math.floor(options.limit) || 10));\n return this.live((n) => {\n const kindClause =\n options.kind === undefined || options.kind === null ? \"\" : \"AND d.kind = ?\";\n const sql = `SELECT d.rowid AS rowid, d.*, bm25(${n.fts}, 3.0, 1.0) AS rank\n FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid\n WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' ${kindClause}\n ORDER BY rank LIMIT ?`;\n const run = (match: string): Array<DocRow & { rank: number }> => {\n const params: Array<string | number> = [match];\n if (kindClause.length > 0) params.push(options.kind as string);\n params.push(limit);\n return this.stmt(sql).all(...params) as unknown as Array<DocRow & { rank: number }>;\n };\n const rows = run(query.all);\n if (rows.length < limit && query.parts > 1) {\n const seen = new Set(rows.map((row) => row.doc));\n for (const row of run(rank === \"fast\" ? this.rareParts(n, query.anyParts) : query.any)) {\n if (rows.length >= limit) break;\n if (!seen.has(row.doc)) rows.push(row);\n }\n }\n return rows.map((row) => ({\n doc: row.doc,\n kind: row.kind,\n title: row.title,\n url: row.url,\n snippet: snippetOf(this.textOf(n, row.text_sha) ?? \"\", query.terms),\n // bm25() is lower-is-better; flip it so a bigger score is a better match\n score: Math.round(-row.rank * 1000) / 1000,\n ai_input: row.ai_input as AiInput,\n fetched_at: row.fetched_at,\n capture: row.capture,\n }));\n });\n }\n\n /**\n * This method scores a set of documents against a query, with the best first. It leaves out a\n * document that matches no term, and a document that asked not to be AI input. It ranks the\n * results of a source again, because the order of the source is only as good as its search engine.\n * In the mode `fast`, it also leaves out a document that holds only common parts (`RankMode`).\n */\n rankWithin(\n q: string,\n docs: readonly string[],\n options: { readonly rank?: RankMode | undefined } = {},\n ): Array<{ readonly doc: string; readonly score: number }> {\n const rank = rankModeOf(options.rank);\n const query = parseQuery(q);\n if (query === undefined || docs.length === 0) return [];\n return this.live((n) =>\n (\n this.stmt(\n `SELECT d.doc AS doc, bm25(${n.fts}, 3.0, 1.0) AS rank\n FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid\n WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' AND d.doc IN (SELECT value FROM json_each(?))\n ORDER BY rank`,\n ).all(\n rank === \"fast\" ? this.rareParts(n, query.anyParts) : query.any,\n JSON.stringify(docs),\n ) as Array<{ doc: string; rank: number }>\n ).map((row) => ({ doc: row.doc, score: Math.round(-row.rank * 1000) / 1000 })),\n );\n }\n\n /**\n * This method gives the form of a query that needs one part, without the common parts (the mode\n * `fast` of `RankMode`). It counts the documents that hold each part. When each part is common,\n * it keeps the rarest part.\n */\n private rareParts(n: Names, parts: readonly string[]): string {\n if (parts.length < 2) return parts.join(\" OR \");\n const total = (this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get() as { n: number }).n;\n const counted = parts.map((part) => ({\n part,\n count: (\n this.stmt(`SELECT count(*) AS n FROM ${n.fts} WHERE ${n.fts} MATCH ?`).get(part) as {\n n: number;\n }\n ).n,\n }));\n const rare = counted.filter((one) => one.count <= FAST_RANK_SHARE * total);\n if (rare.length > 0) return rare.map((one) => one.part).join(\" OR \");\n return counted.reduce((best, one) => (one.count < best.count ? one : best)).part;\n }\n\n /** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */\n latest(url: string, variant?: Variant): Latest | undefined {\n const key = keyOf(url, variant);\n return this.live((n) => {\n const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(\n key.url,\n key.variant,\n ) as LatestRow | undefined;\n return row === undefined ? undefined : toLatest(row);\n });\n }\n\n private putLatest(n: Names, entry: Latest): void {\n const key = keyOf(entry.url, entry.variant);\n // an older answer never replaces a newer one, so two processes and a replay agree\n this.stmt(\n `INSERT INTO ${n.latest} (url, variant, capture, blob, etag, last_modified, content_type, fetched_at,\n status, ai_input, location, truncated, fresh_until, vary_key)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(url, variant) DO UPDATE SET capture = excluded.capture, blob = excluded.blob,\n etag = excluded.etag, last_modified = excluded.last_modified, content_type = excluded.content_type,\n fetched_at = excluded.fetched_at, status = excluded.status, ai_input = excluded.ai_input,\n location = excluded.location, truncated = excluded.truncated, fresh_until = excluded.fresh_until,\n vary_key = excluded.vary_key\n WHERE excluded.fetched_at >= ${n.latest}.fetched_at`,\n ).run(\n key.url,\n key.variant,\n entry.capture,\n entry.blob,\n entry.etag,\n entry.last_modified,\n entry.content_type,\n entry.fetched_at,\n entry.status,\n entry.ai_input,\n entry.location ?? null,\n entry.truncated === true ? 1 : null,\n entry.fresh_until ?? null,\n entry.vary_key ?? null,\n );\n }\n\n /** This method keeps the last answer for a URL. An entry that is older than the stored entry changes nothing. */\n setLatest(entry: Latest): void {\n this.live((n) => this.write(() => this.putLatest(n, entry)));\n }\n\n counts(): Counts {\n return this.live((n) => ({\n docs: (this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get() as { n: number }).n,\n urls: (this.stmt(`SELECT count(*) AS n FROM ${n.latest}`).get() as { n: number }).n,\n }));\n }\n\n /** This method empties the current generation. A rebuild uses `beginRebuild`, which keeps the index until its commit. */\n reset(): void {\n this.live((n) =>\n this.write(() => {\n for (const table of [n.fts, n.docs, n.versions, n.texts, n.latest]) {\n this.db.exec(`DELETE FROM ${table}`);\n }\n }),\n );\n }\n\n /**\n * This method starts a rebuild into a new generation of the tables. The current generation\n * answers each read and takes each write until the commit. A failed rebuild thus leaves the\n * index as it was.\n */\n beginRebuild(options: { readonly batch?: number } = {}): IndexBuild {\n const batchSize = Math.max(1, Math.floor(options.batch ?? 500));\n const generation = this.write(() => {\n const next = Number(this.getMeta(\"next_generation\") ?? \"2\");\n this.setMeta(\"next_generation\", String(next + 1));\n this.setMeta(`building:${String(next)}`, new Date().toISOString());\n this.db.exec(ddlOf(namesOf(next)));\n return next;\n });\n const n = namesOf(generation);\n const versions: Array<{ readonly input: DocInput; readonly now: string }> = [];\n const latest = new Map<string, Latest>();\n let open = true;\n const keyText = (url: string, variant: Variant | undefined): string => {\n const key = keyOf(url, variant);\n return `${key.variant} ${key.url}`;\n };\n const flush = (): void => {\n if (!open) throw new Error(\"the rebuild is over\");\n if (versions.length === 0 && latest.size === 0) return;\n this.write(() => {\n const touched = new Map<string, string>();\n for (const { input, now } of versions) {\n this.putVersion(n, input, now);\n touched.set(input.doc, now);\n }\n for (const entry of latest.values()) this.putLatest(n, entry);\n for (const [doc, now] of touched) this.recompute(n, doc, now);\n });\n versions.length = 0;\n latest.clear();\n };\n /** This function copies the texts of the versions of the current generation that agree with `where`. */\n const copyTexts = (current: Names, where: string, params: string[]): void => {\n for (const column of [\"text_sha\", \"meta_sha\", \"full_sha\"]) {\n this.db\n .prepare(\n `INSERT OR IGNORE INTO ${n.texts} (sha, body)\n SELECT t.sha, t.body FROM ${current.texts} t\n WHERE t.sha IN (SELECT ${column} FROM ${current.versions} WHERE ${where})`,\n )\n .run(...params);\n }\n };\n return {\n upsert: (input, now) => {\n if (!open) throw new Error(\"the rebuild is over\");\n versions.push({ input, now });\n if (versions.length >= batchSize) flush();\n },\n setLatest: (entry) => {\n if (!open) throw new Error(\"the rebuild is over\");\n const key = keyText(entry.url, entry.variant);\n const before = latest.get(key);\n if (before === undefined || entry.fetched_at >= before.fetched_at) latest.set(key, entry);\n if (latest.size >= batchSize) flush();\n },\n latest: (url, variant) => {\n const waiting = latest.get(keyText(url, variant));\n if (waiting !== undefined) return waiting;\n const key = keyOf(url, variant);\n const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(\n key.url,\n key.variant,\n ) as LatestRow | undefined;\n return row === undefined ? undefined : toLatest(row);\n },\n flush,\n carry: (recipes) => {\n flush();\n const known = JSON.stringify([...recipes]);\n const current = namesOf(this.generation());\n const docs = this.db\n .prepare(\n `SELECT DISTINCT doc FROM ${current.versions} WHERE recipe NOT IN (SELECT value FROM json_each(?))`,\n )\n .all(known) as Array<{ doc: string }>;\n const where =\n \"recipe NOT IN (SELECT value FROM json_each(?)) AND doc IN (SELECT value FROM json_each(?))\";\n for (let at = 0; at < docs.length; at += batchSize) {\n const part = JSON.stringify(docs.slice(at, at + batchSize).map((row) => row.doc));\n this.write(() => {\n copyTexts(current, where, [known, part]);\n this.db\n .prepare(\n `INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${current.versions} WHERE ${where}`,\n )\n .run(known, part);\n const now = new Date().toISOString();\n for (const doc of JSON.parse(part) as string[]) this.recompute(n, doc, now);\n });\n }\n return docs.length;\n },\n commit: (since) => {\n flush();\n open = false;\n return this.write(() => {\n const current = this.generation();\n const old = namesOf(current);\n // the rows that a different writer added during the rebuild\n copyTexts(old, \"fetched_at >= ?\", [since]);\n const added = this.db\n .prepare(`SELECT DISTINCT doc FROM ${old.versions} WHERE fetched_at >= ?`)\n .all(since) as Array<{ doc: string }>;\n this.db\n .prepare(\n `INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${old.versions} WHERE fetched_at >= ?`,\n )\n .run(since);\n const now = new Date().toISOString();\n for (const { doc } of added) this.recompute(n, doc, now);\n for (const row of this.db\n .prepare(`SELECT * FROM ${old.latest} WHERE fetched_at >= ?`)\n .all(since) as unknown as LatestRow[]) {\n this.putLatest(n, toLatest(row));\n }\n this.setMeta(\"generation\", String(generation));\n this.db\n .prepare(\"DELETE FROM index_meta WHERE key = ?\")\n .run(`building:${String(generation)}`);\n this.dropGeneration(current);\n this.empty = false;\n return {\n docs: (this.db.prepare(`SELECT count(*) AS n FROM ${n.docs}`).get() as { n: number }).n,\n urls: (this.db.prepare(`SELECT count(*) AS n FROM ${n.latest}`).get() as { n: number })\n .n,\n };\n });\n },\n abort: () => {\n open = false;\n versions.length = 0;\n latest.clear();\n if (this.closed) return;\n this.write(() => this.dropGeneration(generation));\n },\n };\n }\n\n /** This method closes the file. A second close does nothing. */\n close(): void {\n if (this.closed) return;\n this.closed = true;\n this.statements.clear();\n this.db.close();\n }\n}\n","import { randomUUID } from \"node:crypto\";\nimport { mkdirSync } from \"node:fs\";\nimport { dirname } from \"node:path\";\nimport type { StatementSync } from \"node:sqlite\";\nimport { DatabaseSync } from \"node:sqlite\";\nimport type {\n HostCounts,\n HostResult,\n HostStateStore,\n HostStop,\n ValidatorMemo,\n ValidatorReport,\n ValidatorVerdict,\n} from \"./politeness.ts\";\nimport { nextValidatorMemo } from \"./politeness.ts\";\n\n/** A stop with no end goes into the table as this value, which is past each date that JavaScript can hold. */\nconst NO_END = 9e15;\n/** A process that shares the store asks again after this wait, at most, during the lease of a different process. */\nconst SHARED_WAIT_MS = 1000;\n/** The store removes the rows of idle hosts after this many reservations. */\nconst SWEEP_EVERY = 1000;\n/**\n * This value is the version of the schema of the file (`PRAGMA user_version`). Version 0 (0.1.0)\n * has the table `hosts`. Version 1 adds the table `validators`, and it reads a file of version 0.\n */\nexport const HOST_STATE_SCHEMA = 1;\n\ninterface MemoRow {\n skip_etag: number;\n etag: string;\n last_modified: string;\n etag_misses: number;\n date_misses: number;\n since_probe: number;\n changed_at: number;\n}\n\nconst verdictOf = (value: string): ValidatorVerdict =>\n value === \"works\" || value === \"fails\" ? value : \"unknown\";\n\nconst memoOf = (row: MemoRow): ValidatorMemo => ({\n skipEtag: row.skip_etag === 1,\n etag: verdictOf(row.etag),\n lastModified: verdictOf(row.last_modified),\n etagMisses: row.etag_misses,\n dateMisses: row.date_misses,\n sinceProbe: row.since_probe,\n changedAt: row.changed_at,\n});\n\ninterface Row {\n next_at: number;\n leased_until: number;\n owner: string | null;\n failures: number;\n waits: number;\n stop_until: number | null;\n stop_reason: string | null;\n}\n\n/** These options change a SQLite host state store. */\nexport interface SqliteHostStateOptions {\n /** This value is the time in milliseconds that a step waits for the lock of a different process. The default is 10 seconds. */\n readonly busyTimeoutMs?: number | undefined;\n /** During the lease of a different process, a request asks again after this wait at most. The default is 1 second. */\n readonly pollMs?: number | undefined;\n}\n\n/**\n * This store keeps the state of each host in a SQLite file. Two or more processes on one store\n * share it, so they share the pace and the stops of each host. Thus a host gets one request at a\n * time from all of them. Each method is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock\n * of a different process (`busy_timeout`).\n *\n * The lease of a request belongs to the store that gave it. A different process that asks during\n * the lease gets a wait of one second or less, and asks again. A process that stops during a\n * request holds its lease until the lease ends (5 minutes by default).\n *\n * The table `validators` keeps the validator memo of each host (`ValidatorMemo`), so the processes\n * share it and it stays after a restart. A file of version 0.1.0 gets the table when it opens.\n *\n * The state is not a projection of the capture log. A deletion of the file loses only the pace,\n * the counts, the stops and the memos, and the crawler starts them again.\n */\nexport class SqliteHostStateStore implements HostStateStore {\n private readonly db: DatabaseSync;\n private readonly statements = new Map<string, StatementSync>();\n /** This id names the leases of this store. */\n private readonly owner = randomUUID();\n private reservations = 0;\n private closed = false;\n private readonly pollMs: number;\n\n constructor(\n readonly path: string,\n options: SqliteHostStateOptions = {},\n ) {\n if (path !== \":memory:\") mkdirSync(dirname(path), { recursive: true, mode: 0o700 });\n this.pollMs = Math.max(1, Math.min(SHARED_WAIT_MS, options.pollMs ?? SHARED_WAIT_MS));\n this.db = new DatabaseSync(path);\n const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 10_000));\n this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);\n if (path !== \":memory:\") this.db.exec(\"PRAGMA journal_mode = WAL;\");\n this.db.exec(\"PRAGMA synchronous = NORMAL;\");\n this.migrate();\n }\n\n /**\n * This method makes the tables, or adds the tables of a later version to an older file. It is one\n * transaction, so two processes that open one old file at the same time do not collide.\n */\n private migrate(): void {\n this.step(() => {\n this.db.exec(`CREATE TABLE IF NOT EXISTS hosts (\n host TEXT PRIMARY KEY,\n next_at REAL NOT NULL DEFAULT 0,\n leased_until REAL NOT NULL DEFAULT 0,\n owner TEXT,\n failures INTEGER NOT NULL DEFAULT 0,\n waits INTEGER NOT NULL DEFAULT 0,\n stop_until REAL,\n stop_reason TEXT\n )`);\n this.db.exec(`CREATE TABLE IF NOT EXISTS validators (\n host TEXT PRIMARY KEY,\n skip_etag INTEGER NOT NULL DEFAULT 0,\n etag TEXT NOT NULL DEFAULT 'unknown',\n last_modified TEXT NOT NULL DEFAULT 'unknown',\n etag_misses INTEGER NOT NULL DEFAULT 0,\n date_misses INTEGER NOT NULL DEFAULT 0,\n since_probe INTEGER NOT NULL DEFAULT 0,\n changed_at REAL NOT NULL DEFAULT 0\n )`);\n const { user_version: version } = this.db.prepare(\"PRAGMA user_version\").get() as {\n user_version: number;\n };\n // a file of a later version keeps its number\n if (version < HOST_STATE_SCHEMA)\n this.db.exec(`PRAGMA user_version = ${String(HOST_STATE_SCHEMA)}`);\n });\n }\n\n private stmt(sql: string): StatementSync {\n let statement = this.statements.get(sql);\n if (statement === undefined) {\n statement = this.db.prepare(sql);\n this.statements.set(sql, statement);\n }\n return statement;\n }\n\n /** This method does one step in one transaction. */\n private step<T>(work: () => T): T {\n if (this.closed) throw new Error(\"the host state store is closed\");\n this.db.exec(\"BEGIN IMMEDIATE\");\n try {\n const result = work();\n this.db.exec(\"COMMIT\");\n return result;\n } catch (error) {\n if (this.db.isTransaction) this.db.exec(\"ROLLBACK\");\n throw error;\n }\n }\n\n private row(host: string): Row {\n this.stmt(\"INSERT OR IGNORE INTO hosts (host) VALUES (?)\").run(host);\n return this.stmt(\n \"SELECT next_at, leased_until, owner, failures, waits, stop_until, stop_reason FROM hosts WHERE host = ?\",\n ).get(host) as unknown as Row;\n }\n\n /** This method removes the hosts that have nothing to remember: no rest, no lease, no counts and no stop. */\n private sweep(now: number): void {\n this.stmt(\n `DELETE FROM hosts WHERE next_at <= ? AND leased_until <= ? AND failures = 0 AND waits = 0\n AND (stop_until IS NULL OR stop_until <= ?)`,\n ).run(now, now, now);\n }\n\n reserve(host: string, now: number, leaseMs: number): number {\n return this.step(() => {\n this.reservations += 1;\n if (this.reservations % SWEEP_EVERY === 0) this.sweep(now);\n const row = this.row(host);\n if (row.leased_until > now) {\n // the holder usually releases early, so a short wait and a new question\n return row.owner === this.owner\n ? row.leased_until - now\n : Math.min(this.pollMs, row.leased_until - now);\n }\n if (row.next_at > now) return row.next_at - now;\n this.stmt(\"UPDATE hosts SET leased_until = ?, owner = ? WHERE host = ?\").run(\n now + leaseMs,\n this.owner,\n host,\n );\n return 0;\n });\n }\n\n record(host: string, result: HostResult): HostCounts {\n return this.step(() => {\n const row = this.row(host);\n let { failures, waits } = row;\n if (result === \"success\") {\n failures = 0;\n waits = 0;\n } else if (result === \"failure\") failures += 1;\n else if (result === \"wait\") waits += 1;\n if (failures !== row.failures || waits !== row.waits) {\n this.stmt(\"UPDATE hosts SET failures = ?, waits = ? WHERE host = ?\").run(\n failures,\n waits,\n host,\n );\n }\n return { failures, waits };\n });\n }\n\n release(host: string, nextAt: number): void {\n this.step(() => {\n this.row(host);\n this.stmt(\n `UPDATE hosts SET next_at = max(next_at, ?),\n leased_until = CASE WHEN owner = ? THEN 0 ELSE leased_until END,\n owner = CASE WHEN owner = ? THEN NULL ELSE owner END\n WHERE host = ?`,\n ).run(nextAt, this.owner, this.owner, host);\n });\n }\n\n defer(host: string, until: number): void {\n this.step(() => {\n this.row(host);\n this.stmt(\"UPDATE hosts SET next_at = max(next_at, ?) WHERE host = ?\").run(until, host);\n });\n }\n\n stop(host: string, until: number, reason: string): void {\n const end = Number.isFinite(until) ? until : NO_END;\n this.step(() => {\n this.row(host);\n this.stmt(\n `UPDATE hosts SET stop_until = ?, stop_reason = ?\n WHERE host = ? AND (stop_until IS NULL OR stop_until < ?)`,\n ).run(end, reason, host, end);\n });\n }\n\n stopped(host: string, now: number): HostStop | undefined {\n if (this.closed) throw new Error(\"the host state store is closed\");\n const row = this.stmt(\"SELECT stop_until, stop_reason FROM hosts WHERE host = ?\").get(host) as\n { stop_until: number | null; stop_reason: string | null } | undefined;\n if (row === undefined || row.stop_until === null || row.stop_until <= now) return undefined;\n return {\n until: row.stop_until >= NO_END ? Number.POSITIVE_INFINITY : row.stop_until,\n reason: row.stop_reason ?? \"stopped\",\n };\n }\n\n reset(host?: string): void {\n this.step(() => {\n if (host === undefined) {\n this.stmt(\n \"UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL\",\n ).run();\n } else {\n this.stmt(\n \"UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL WHERE host = ?\",\n ).run(host);\n }\n });\n }\n\n validators(host: string): ValidatorMemo | undefined {\n if (this.closed) throw new Error(\"the host state store is closed\");\n const row = this.stmt(\n `SELECT skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at\n FROM validators WHERE host = ?`,\n ).get(host) as MemoRow | undefined;\n return row === undefined ? undefined : memoOf(row);\n }\n\n noteValidators(host: string, report: ValidatorReport, now: number): ValidatorMemo {\n return this.step(() => {\n const memo = nextValidatorMemo(this.validators(host), report, now);\n this.stmt(\n `INSERT INTO validators (host, skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(host) DO UPDATE SET skip_etag = excluded.skip_etag, etag = excluded.etag,\n last_modified = excluded.last_modified, etag_misses = excluded.etag_misses,\n date_misses = excluded.date_misses, since_probe = excluded.since_probe,\n changed_at = excluded.changed_at`,\n ).run(\n host,\n memo.skipEtag ? 1 : 0,\n memo.etag,\n memo.lastModified,\n memo.etagMisses,\n memo.dateMisses,\n memo.sinceProbe,\n memo.changedAt,\n );\n return memo;\n });\n }\n\n /** This value is the number of hosts that the store holds. */\n get size(): number {\n return (this.stmt(\"SELECT count(*) AS n FROM hosts\").get() as { n: number }).n;\n }\n\n /** This method closes the file. A second close does nothing. */\n close(): void {\n if (this.closed) return;\n this.closed = true;\n this.statements.clear();\n this.db.close();\n }\n}\n","import { createReadStream, existsSync } from \"node:fs\";\nimport type { FileHandle } from \"node:fs/promises\";\nimport { mkdir, open, readdir } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\nimport type { CaptureRecord } from \"./record.ts\";\n\n/** A scan filter selects the records of one source, one kind, or the records after one time. */\nexport interface CaptureFilter {\n readonly source?: string | undefined;\n readonly kind?: string | undefined;\n /** This value is an ISO 8601 time. The scan gives the records received at this time or later. */\n readonly since?: string | undefined;\n}\n\n/** A damaged line is a line of the log that is not a record: a torn append, or bytes that a crash left. */\nexport interface DamagedLine {\n /** This value is the path of the file. */\n readonly path: string;\n /** This value is the number of the line in the file, from 1. */\n readonly line: number;\n}\n\n/** These options change a scan. */\nexport interface ScanOptions {\n /** The scan skips a damaged line, and it gives the line to this function. */\n readonly onDamaged?: ((damaged: DamagedLine) => void) | undefined;\n}\n\n/** The read side of the capture log is what `reindex` replays. */\nexport interface CaptureReader {\n scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;\n}\n\n/** These options change a capture log. */\nexport interface CaptureLogOptions {\n /**\n * When this value is true (the default), the log writes each batch of appends to the disk\n * (`fdatasync`) before the appends finish. Thus a power cut does not lose a record that the\n * crawler reported.\n */\n readonly sync?: boolean | undefined;\n}\n\ninterface Pending {\n readonly path: string;\n readonly line: string;\n readonly resolve: () => void;\n readonly reject: (error: unknown) => void;\n}\n\nconst MONTH_FILE = /^(\\d{2})(?:\\.[^.]+)?\\.ndjson$/;\nconst YEAR_DIR = /^\\d{4}$/;\nconst NEWLINE = 0x0a;\n\n/** This function writes the whole buffer. A write to a file can write less than the buffer, and the loop writes the rest. */\nconst writeAll = async (handle: FileHandle, bytes: Buffer): Promise<void> => {\n let at = 0;\n while (at < bytes.length) {\n const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);\n at += bytesWritten;\n }\n};\n\n/**\n * This function reads a file line by line, as bytes, so a long file does not become one string. It\n * gives each line and tells if a line break ends it.\n */\nasync function* linesOf(\n path: string,\n): AsyncGenerator<{ readonly text: string; readonly complete: boolean }> {\n const decoder = new TextDecoder(\"utf-8\");\n let rest: Buffer | undefined;\n for await (const chunk of createReadStream(path) as AsyncIterable<Buffer>) {\n const data = rest === undefined ? chunk : Buffer.concat([rest, chunk]);\n rest = undefined;\n let start = 0;\n for (;;) {\n const end = data.indexOf(NEWLINE, start);\n if (end === -1) break;\n yield { text: decoder.decode(data.subarray(start, end)), complete: true };\n start = end + 1;\n }\n if (start < data.length) rest = Buffer.from(data.subarray(start));\n }\n if (rest !== undefined && rest.length > 0) yield { text: decoder.decode(rest), complete: false };\n}\n\nconst monthKey = (year: string, file: string): string =>\n `${year}-${MONTH_FILE.exec(file)?.[1] ?? \"00\"}`;\n\n/**\n * The capture log keeps each fetch decision as one line of JSON in\n * `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.\n *\n * These are the rules of the log:\n *\n * - The log appends a batch of records with one write, and it writes the batch to the disk before\n * the appends finish.\n * - An append always starts on a new line. After a crash leaves a partial line, the next append\n * starts after a line break. The partial line stays as a damaged line, and the record is good.\n * - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.\n * - The log does not remove or change a line.\n *\n * Two processes can append to one log, because each batch is one write to a file in append mode.\n */\nexport class CaptureLog implements CaptureReader {\n private readonly handles = new Map<string, FileHandle>();\n private readonly pending: Pending[] = [];\n private draining: Promise<void> | undefined;\n private readonly sync: boolean;\n\n constructor(\n readonly dir: string,\n options: CaptureLogOptions = {},\n ) {\n this.sync = options.sync ?? true;\n }\n\n private pathOf(record: CaptureRecord): string {\n return join(\n this.dir,\n record.source,\n record.received_at.slice(0, 4),\n `${record.received_at.slice(5, 7)}.ndjson`,\n );\n }\n\n private async handleFor(path: string): Promise<FileHandle> {\n const known = this.handles.get(path);\n if (known !== undefined) return known;\n // the folders and the files are for the owner only, where the system has modes\n await mkdir(dirname(path), { recursive: true, mode: 0o700 });\n const handle = await open(path, \"a+\", 0o600);\n this.handles.set(path, handle);\n return handle;\n }\n\n /** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */\n private static async needsBreak(handle: FileHandle): Promise<boolean> {\n const { size } = await handle.stat();\n if (size === 0) return false;\n const last = Buffer.alloc(1);\n await handle.read(last, 0, 1, size - 1);\n return last[0] !== NEWLINE;\n }\n\n append(record: CaptureRecord): Promise<void> {\n let line: string;\n try {\n line = `${JSON.stringify(record)}\\n`;\n } catch (error) {\n return Promise.reject(error instanceof Error ? error : new Error(String(error)));\n }\n return new Promise<void>((resolve, reject) => {\n this.pending.push({ path: this.pathOf(record), line, resolve, reject });\n this.kick();\n });\n }\n\n /** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */\n private kick(): void {\n if (this.draining !== undefined) return;\n this.draining = this.drain().finally(() => {\n this.draining = undefined;\n if (this.pending.length > 0) this.kick();\n });\n }\n\n /** This method writes the waiting records in batches: one write and one sync for each file. */\n private async drain(): Promise<void> {\n while (this.pending.length > 0) {\n const batch = this.pending.splice(0);\n const byPath = new Map<string, Pending[]>();\n for (const item of batch) {\n const list = byPath.get(item.path);\n if (list === undefined) byPath.set(item.path, [item]);\n else list.push(item);\n }\n for (const [path, items] of byPath) {\n try {\n const handle = await this.handleFor(path);\n const lead = (await CaptureLog.needsBreak(handle)) ? \"\\n\" : \"\";\n await writeAll(\n handle,\n Buffer.from(`${lead}${items.map((item) => item.line).join(\"\")}`, \"utf8\"),\n );\n if (this.sync) await handle.datasync();\n for (const item of items) item.resolve();\n } catch (error) {\n // a handle that failed is opened again for the next append\n const handle = this.handles.get(path);\n this.handles.delete(path);\n await handle?.close().catch(() => undefined);\n for (const item of items) item.reject(error);\n }\n }\n }\n }\n\n /** This method waits until each append in progress is on the disk. */\n async flush(): Promise<void> {\n while (this.draining !== undefined) await this.draining;\n }\n\n /** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */\n private async filesOf(source: string, since: string | undefined): Promise<string[]> {\n const sourceDir = join(this.dir, source);\n if (!existsSync(sourceDir)) return [];\n const from = since?.slice(0, 7);\n const years = (await readdir(sourceDir, { withFileTypes: true }))\n .filter((entry) => entry.isDirectory() && YEAR_DIR.test(entry.name))\n .map((entry) => entry.name)\n .sort();\n const files: string[] = [];\n for (const year of years) {\n if (from !== undefined && year < from.slice(0, 4)) continue;\n const months = (await readdir(join(sourceDir, year)))\n .filter((name) => MONTH_FILE.test(name))\n .sort();\n for (const month of months) {\n if (from !== undefined && monthKey(year, month) < from) continue;\n files.push(join(sourceDir, year, month));\n }\n }\n return files;\n }\n\n /** This method reads the records of one source in the sequence of the files. */\n private async *readSource(\n source: string,\n filter: CaptureFilter | undefined,\n options: ScanOptions | undefined,\n ): AsyncGenerator<CaptureRecord> {\n for (const path of await this.filesOf(source, filter?.since)) {\n let number = 0;\n for await (const { text, complete } of linesOf(path)) {\n number += 1;\n const trimmed = text.trim();\n if (trimmed.length === 0) continue;\n let row: CaptureRecord;\n try {\n row = JSON.parse(trimmed) as CaptureRecord;\n if (typeof row !== \"object\" || row === null || typeof row.received_at !== \"string\") {\n throw new Error(\"not a record\");\n }\n } catch {\n // the append of a different process can be in progress at the end of the file\n if (complete) options?.onDamaged?.({ path, line: number });\n continue;\n }\n if (filter?.kind !== undefined && row.kind !== filter.kind) continue;\n if (filter?.since !== undefined && row.received_at < filter.since) continue;\n yield row;\n }\n }\n }\n\n /**\n * This method reads the records that agree with the filter. It first waits for the appends that\n * are in progress. It reads each file as a stream, so its memory does not grow with the log.\n *\n * The records of one source come in the sequence of the files. The records of two or more sources\n * merge in the sequence of `received_at`.\n */\n async *scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord> {\n await this.flush();\n const sources =\n filter?.source !== undefined\n ? [filter.source]\n : existsSync(this.dir)\n ? (await readdir(this.dir, { withFileTypes: true }))\n .filter((entry) => entry.isDirectory())\n .map((entry) => entry.name)\n .sort()\n : [];\n if (sources.length === 1) {\n yield* this.readSource(sources[0]!, filter, options);\n return;\n }\n // a merge of the sources: the next record is the earliest head\n const streams = sources.map((source) =>\n this.readSource(source, filter, options)[Symbol.asyncIterator](),\n );\n const heads: Array<CaptureRecord | undefined> = [];\n for (const stream of streams)\n heads.push((await stream.next()).value as CaptureRecord | undefined);\n for (;;) {\n let pick = -1;\n for (const [index, head] of heads.entries()) {\n if (head !== undefined && (pick === -1 || head.received_at < heads[pick]!.received_at)) {\n pick = index;\n }\n }\n if (pick === -1) return;\n yield heads[pick]!;\n heads[pick] = (await streams[pick]!.next()).value as CaptureRecord | undefined;\n }\n }\n\n /** This method writes the appends in progress, then closes the files. A later append opens them again. */\n async close(): Promise<void> {\n await this.flush();\n const handles = [...this.handles.values()];\n this.handles.clear();\n for (const handle of handles) await handle.close();\n }\n}\n","import { constants, inflateSync } from \"node:zlib\";\nimport { getDocumentProxy, getMeta } from \"unpdf\";\n\n/**\n * The crawler reads the text of a PDF. Restaurant menus are often PDFs: 29 of 231 surveyed\n * restaurant sites link one. A menu that the crawler reads one time stays searchable. A paper in a\n * PDF reads the same way. A scanned PDF has no text, so the result is empty, not a guess.\n *\n * A small file can cost much work: a stream of 150 KB can inflate to 100 MB. Thus the reader:\n *\n * - Inflates the compressed streams first, with a limit, and refuses a file that inflates past it.\n * - Reads one page at a time, and lets other work run between the pages.\n * - Stops at a page limit, a text limit and a time limit, and says so (`truncated`). The time limit\n * stops a page in the middle too, and the text limit cuts the text of a page.\n */\nexport interface PdfText {\n /** This is the title in the file, or null when the file names no real title (\"Microsoft Word - menu.docx\"). */\n readonly title: string | null;\n readonly text: string;\n readonly pages: number;\n /** This value is true when a limit stopped the reader before the last page. */\n readonly truncated: boolean;\n /** This note tells why the reader stopped or refused the file, or it is null. */\n readonly note: string | null;\n}\n\n/** These are the limits of one read. */\nexport interface PdfOptions {\n /** The reader reads no more pages than this. The default is 500. */\n readonly maxPages?: number | undefined;\n /** The reader stops when the text has this many characters, in the middle of a page too. The default is 200,000. */\n readonly maxChars?: number | undefined;\n /** The reader stops after this time in milliseconds, in the middle of a page too. The default is 20,000. */\n readonly timeBudgetMs?: number | undefined;\n /** The reader refuses a file whose streams inflate to more bytes than this. The default is 8 MB. */\n readonly maxInflatedBytes?: number | undefined;\n}\n\nconst MAX_PAGES = 500;\nconst MAX_CHARS = 200_000;\nconst TIME_BUDGET_MS = 20_000;\n/**\n * The text streams of 35 stored menus and papers inflate to 0.6 MB or less. The parser of pdf.js\n * reads a page with no pause. Thus a page of 8 MB of text operators stops the process for\n * approximately one second.\n */\nconst MAX_INFLATED_BYTES = 8 * 1024 * 1024;\n\n/** This function finds a PDF from its first bytes or its content type. Some servers put bytes before `%PDF-`. */\nexport const isPdf = (body: Uint8Array, contentType: string | null): boolean => {\n if (/\\bpdf\\b/i.test(contentType ?? \"\")) return true;\n const end = Math.min(body.length - 4, 1024);\n for (let index = 0; index < end; index += 1) {\n if (\n body[index] === 0x25 &&\n body[index + 1] === 0x50 &&\n body[index + 2] === 0x44 &&\n body[index + 3] === 0x46 &&\n body[index + 4] === 0x2d\n ) {\n return true;\n }\n }\n return false;\n};\n\n/** This function gives one line of text for each line of the PDF. It removes the empty lines. */\nconst tidy = (page: string): string =>\n page\n .split(/\\r?\\n/)\n .map((line) => line.replace(/\\s+/g, \" \").trim())\n .filter((line) => line.length > 0)\n .join(\"\\n\");\n\n/** A word processor writes the name of its file as the title (\"Microsoft Word - menu-final-v3.docx\"). */\nconst JUNK_TITLE =\n /^(?:microsoft (?:word|powerpoint|excel) - |untitled\\b|document\\d*$|presentation\\d*$)|\\.(?:docx?|pdf|pptx?|xlsx?|indd|ai|psd|rtf|odt|pages|txt)$/i;\n\nconst latin1 = (bytes: Uint8Array, start: number, end: number): string => {\n let out = \"\";\n for (let index = Math.max(0, start); index < end && index < bytes.length; index += 1) {\n out += String.fromCharCode(bytes[index]!);\n }\n return out;\n};\n\nconst indexOf = (bytes: Uint8Array, needle: string, from: number): number => {\n const first = needle.charCodeAt(0);\n for (\n let index = bytes.indexOf(first, from);\n index !== -1;\n index = bytes.indexOf(first, index + 1)\n ) {\n let match = true;\n for (let at = 1; at < needle.length; at += 1) {\n if (bytes[index + at] !== needle.charCodeAt(at)) {\n match = false;\n break;\n }\n }\n if (match) return index;\n }\n return -1;\n};\n\n/**\n * This function adds up the bytes that the compressed streams of a file inflate to, and stops at\n * the limit. It tries each stream: a stream in another format fails at its first bytes. An image\n * stream does not count, because the text reader does not decode it. The function gives true when\n * the file stays inside the limit.\n */\nconst inflatesWithin = (bytes: Uint8Array, limit: number): boolean => {\n let total = 0;\n let from = 0;\n for (;;) {\n const keyword = indexOf(bytes, \"stream\", from);\n if (keyword === -1) return true;\n from = keyword + 6;\n // \"endstream\" holds \"stream\" too\n if (latin1(bytes, keyword - 3, keyword) === \"end\") continue;\n const next = bytes[keyword + 6];\n if (next !== 0x0a && next !== 0x0d) continue;\n const dictionary = latin1(bytes, keyword - 1024, keyword);\n const head = dictionary.slice(Math.max(0, dictionary.lastIndexOf(\"<<\")));\n if (/\\/Subtype\\s*\\/Image\\b/.test(head)) continue;\n let start = keyword + 7;\n if (next === 0x0d && bytes[start] === 0x0a) start += 1;\n const declared = /\\/Length\\s+(\\d+)(?!\\s+\\d+\\s+R)/.exec(head);\n const stop = indexOf(bytes, \"endstream\", start);\n const end =\n declared !== null && start + Number(declared[1]) <= bytes.length\n ? start + Number(declared[1])\n : stop === -1\n ? bytes.length\n : stop;\n try {\n total += inflateSync(bytes.subarray(start, end), {\n maxOutputLength: Math.max(1, limit - total + 1),\n finishFlush: constants.Z_SYNC_FLUSH,\n }).length;\n } catch (error) {\n // a stream past the limit, or damaged data that pdf.js reads as far as it can\n if (error instanceof RangeError) return false;\n }\n if (total > limit) return false;\n from = Math.max(from, end);\n }\n};\n\n/** A page of pdf.js gives its text as items. A line ends where an item says so. */\ninterface TextItem {\n readonly str?: string;\n readonly hasEOL?: boolean;\n}\n\n/** This function lets the other work of the process run, for example another request. */\nconst pause = (): Promise<void> => new Promise((resolve) => setImmediate(resolve));\n\n/** This function gives the value of a promise, or undefined when the time ends first. */\nconst within = async <T>(promise: Promise<T>, ms: number): Promise<T | undefined> => {\n let timer: ReturnType<typeof setTimeout> | undefined;\n const late = new Promise<undefined>((resolve) => {\n timer = setTimeout(() => resolve(undefined), Math.max(0, ms));\n });\n try {\n return await Promise.race([promise, late]);\n } finally {\n clearTimeout(timer);\n }\n};\n\n/** This function keeps the first characters of a text, and does not cut a surrogate pair. */\nconst cutText = (text: string, length: number): string => {\n let end = Math.max(0, length);\n const code = text.charCodeAt(end);\n if (end > 0 && end < text.length && code >= 0xdc00 && code <= 0xdfff) end -= 1;\n return text.slice(0, end).trimEnd();\n};\n\nexport const readPdf = async (bytes: Uint8Array, options: PdfOptions = {}): Promise<PdfText> => {\n const maxPages = options.maxPages ?? MAX_PAGES;\n const maxChars = options.maxChars ?? MAX_CHARS;\n const budgetMs = options.timeBudgetMs ?? TIME_BUDGET_MS;\n const maxInflated = options.maxInflatedBytes ?? MAX_INFLATED_BYTES;\n if (!inflatesWithin(bytes, maxInflated)) {\n return {\n title: null,\n text: \"\",\n pages: 0,\n truncated: true,\n note: `the streams of the file inflate past ${String(Math.round(maxInflated / 1024 / 1024))} MB`,\n };\n }\n // pdf.js may take over the buffer it is handed, and this one is the caller's\n const pdf = await getDocumentProxy(new Uint8Array(bytes));\n try {\n const started = Date.now();\n const texts: string[] = [];\n let chars = 0;\n let note: string | null = null;\n const last = Math.min(pdf.numPages, maxPages);\n if (pdf.numPages > maxPages) note = `the reader stops after ${String(maxPages)} pages`;\n const late = (number: number): string =>\n `the reader stops after ${String(Math.round(budgetMs / 1000))} seconds, at page ${String(number - 1)}`;\n for (let number = 1; number <= last; number += 1) {\n if (number > 1) {\n await pause();\n if (Date.now() - started > budgetMs) {\n note = late(number);\n break;\n }\n }\n const page = await pdf.getPage(number);\n // pdf.js reads a content stream with no pause, so this limit stops only a page that waits (for\n // a font, for example). The limit of the inflated bytes keeps one page near one second\n const content = await within(page.getTextContent(), budgetMs - (Date.now() - started));\n if (content === undefined) {\n note = `the reader stops after ${String(Math.round(budgetMs / 1000))} seconds, in page ${String(number)}`;\n break;\n }\n const text = tidy(\n (content.items as readonly TextItem[])\n .filter((item) => item.str !== undefined)\n .map((item) => `${item.str ?? \"\"}${item.hasEOL === true ? \"\\n\" : \"\"}`)\n .join(\"\"),\n );\n page.cleanup();\n if (text.length > 0) {\n // the text and the blank lines between the pages so far, and the room that is left\n if (text.length > maxChars - chars) {\n texts.push(cutText(text, maxChars - chars));\n note = `the reader stops at ${String(maxChars)} characters, at page ${String(number)}`;\n break;\n }\n texts.push(text);\n chars += text.length + 2;\n }\n if (chars >= maxChars && number < pdf.numPages) {\n note = `the reader stops at ${String(maxChars)} characters, at page ${String(number)}`;\n break;\n }\n }\n const info = (await getMeta(pdf).catch(() => undefined))?.info;\n const named: unknown = info?.[\"Title\"];\n const title = typeof named === \"string\" ? named.replace(/\\s+/g, \" \").trim() : \"\";\n return {\n title: title.length > 0 && !JUNK_TITLE.test(title) ? title : null,\n text: texts.join(\"\\n\\n\"),\n pages: pdf.numPages,\n truncated: note !== null,\n note,\n };\n } finally {\n // the loading task owns the document and its worker\n await pdf.loadingTask.destroy();\n }\n};\n","import type { BlobStore } from \"./blobs.ts\";\nimport type { Crawler, FetchRequest, Fetched, RenderRequest } from \"./crawler.ts\";\nimport type {\n Doc,\n DocInput,\n ExtractedDoc,\n Found,\n Latest,\n RankMode,\n ReadingIndex,\n Variant,\n} from \"./docs.ts\";\nimport { locate } from \"./docs.ts\";\nimport type { PageExtract } from \"./html.ts\";\nimport { extractPage, typesOf } from \"./html.ts\";\nimport type { CaptureLog, CaptureReader } from \"./log.ts\";\nimport { isPdf, readPdf } from \"./pdf.ts\";\nimport type { CaptureRecord } from \"./record.ts\";\nimport { makeRecord } from \"./record.ts\";\nimport type { AiInput } from \"./signals.ts\";\nimport { decodeBody } from \"./text.ts\";\nimport { canonicalUrl } from \"./urls.ts\";\n\n/**\n * An extractor changes one captured body into the documents that it holds. It has no side effects,\n * so `reindex` can replay each capture through it and build the index again with no request. It can\n * answer later, because the crawler reads a PDF asynchronously.\n *\n * An extractor that throws does not stop a fetch, a render or a rebuild. The runtime records the\n * error and gives no document for that capture.\n */\nexport type Extractor = (\n body: Uint8Array,\n capture: { readonly url: string; readonly contentType: string | null },\n) => ReadonlyArray<ExtractedDoc> | Promise<ReadonlyArray<ExtractedDoc>>;\n\n/** This function gives the document id of a web page. Each page has one id. */\nexport const pageDocId = (url: string): string => `url:${canonicalUrl(url)}`;\n\n/** The JSON-LD that a document keeps, in characters of JSON. */\nconst MAX_JSONLD_CHARS = 30_000;\n/** These properties make a JSON-LD record large, and they say little about the place: they go first when a record is too large. */\nconst BULKY = [\n \"review\",\n \"reviews\",\n \"comment\",\n \"hasMenu\",\n \"hasMenuSection\",\n \"hasMenuItem\",\n \"menu\",\n \"photo\",\n \"image\",\n \"subjectOf\",\n \"makesOffer\",\n \"hasOfferCatalog\",\n];\n/** These properties say where and when a place is open. A record with one of them is kept first. */\nconst PLACE_KEYS = [\"openingHours\", \"openingHoursSpecification\", \"address\", \"geo\", \"telephone\"];\n\n/**\n * This function keeps the JSON-LD records that fit the limit. A record with hours, an address or a\n * position goes first. A large record loses its reviews and its menu before it is left out. A\n * record that does not fit does not stop the records after it.\n */\nconst keepJsonLd = (\n records: readonly Record<string, unknown>[],\n): { readonly kept: Record<string, unknown>[]; readonly dropped: number } => {\n const place = (record: Record<string, unknown>): boolean =>\n PLACE_KEYS.some((key) => record[key] !== undefined);\n const ordered = [...records.filter(place), ...records.filter((record) => !place(record))];\n const kept: Record<string, unknown>[] = [];\n let dropped = 0;\n let room = MAX_JSONLD_CHARS;\n for (const record of ordered) {\n let size = JSON.stringify(record).length;\n let candidate = record;\n if (size > room) {\n candidate = Object.fromEntries(\n Object.entries(record).filter(([key]) => !BULKY.includes(key)),\n );\n size = JSON.stringify(candidate).length;\n }\n if (size > room) {\n dropped += 1;\n continue;\n }\n kept.push(candidate);\n room -= size;\n }\n return { kept, dropped };\n};\n\n/**\n * This extractor is the built-in recipe for a web page, plain or rendered. It keeps the visible\n * text, the JSON-LD (whole objects, up to a limit) and the links. It decodes the body in the\n * encoding that the page declares, with `decodeBody`. It does not throw: a page that it cannot read\n * gives a document that names the error. When the reader cut the text, the document keeps the whole\n * text in `full_text`, so `verify` finds a quote past the cut.\n */\nexport const pageExtractor: Extractor = (body, capture) => {\n if (isPdf(body, capture.contentType)) return pdfDocs(body, capture.url);\n if (capture.contentType !== null && !/html|xml/i.test(capture.contentType)) return [];\n const doc = pageDocId(capture.url);\n let page: PageExtract;\n try {\n page = extractPage(decodeBody(body, capture.contentType), capture.url);\n } catch (error) {\n // one page that breaks the reader must not break the fetch, or each later reindex\n return [\n {\n doc,\n kind: \"page\",\n title: capture.url,\n text: capture.url,\n url: capture.url,\n meta: {\n extract_error: error instanceof Error ? error.message.slice(0, 300) : String(error),\n },\n },\n ];\n }\n const jsonld = keepJsonLd(page.jsonld);\n return [\n {\n doc,\n kind: \"page\",\n title: page.title,\n text: page.text.length > 0 ? page.text : page.title,\n url: capture.url,\n meta: {\n description: page.description,\n lang: page.lang,\n jsonld_types: [...new Set(page.jsonld.flatMap(typesOf))],\n jsonld: jsonld.kept,\n ...(jsonld.dropped > 0 ? { jsonld_dropped: jsonld.dropped } : {}),\n links: page.links.slice(0, 100),\n visible_chars: page.visibleChars,\n embedded: page.embedded,\n needs_render: page.needsRender,\n render_reason: page.renderReason,\n canonical: page.canonical,\n redirect: page.redirect,\n blocked: page.blocked,\n ...(page.notes.length > 0 ? { notes: page.notes } : {}),\n },\n ...(page.fullText !== undefined ? { full_text: page.fullText } : {}),\n },\n ];\n};\n\nconst MAX_PDF_TEXT_CHARS = 200_000;\n\n/**\n * This function changes a PDF into one document of its text. A PDF with no text, a scan for\n * example, gives a document of its title only, and the document says that it needs OCR. A damaged\n * file gives no document.\n */\nconst pdfDocs = async (body: Uint8Array, url: string): Promise<ReadonlyArray<ExtractedDoc>> => {\n let read;\n try {\n read = await readPdf(body, { maxChars: MAX_PDF_TEXT_CHARS });\n } catch {\n return [];\n }\n const file = new URL(url).pathname.split(\"/\").pop() ?? \"\";\n let named = file;\n try {\n named = decodeURIComponent(file);\n } catch {\n // a malformed escape: the name as written\n }\n const title = read.title ?? (named || url);\n const notes = {\n ...(read.truncated ? { truncated: true } : {}),\n ...(read.note !== null ? { note: read.note } : {}),\n };\n if (read.text.length === 0) {\n return [\n {\n doc: pageDocId(url),\n kind: \"pdf\",\n title,\n text: title,\n url,\n meta: {\n format: \"pdf\",\n pages: read.pages,\n visible_chars: 0,\n // a file that a budget stopped is not a scan\n ocr_needed: read.note === null,\n ...notes,\n },\n },\n ];\n }\n return [\n {\n doc: pageDocId(url),\n kind: \"pdf\",\n title,\n text: read.text.slice(0, MAX_PDF_TEXT_CHARS),\n url,\n meta: { format: \"pdf\", pages: read.pages, visible_chars: read.text.length, ...notes },\n },\n ];\n};\n\nexport class Extractors {\n private readonly byRecipe = new Map<string, Extractor>([[\"page\", pageExtractor]]);\n\n register(recipe: string, extractor: Extractor): void {\n if (this.byRecipe.has(recipe)) throw new Error(`recipe ${recipe} already has an extractor`);\n this.byRecipe.set(recipe, extractor);\n }\n\n get(recipe: string): Extractor | undefined {\n return this.byRecipe.get(recipe);\n }\n\n /** This method gives the names of the recipes that have an extractor. */\n recipes(): string[] {\n return [...this.byRecipe.keys()];\n }\n}\n\nexport interface CrawlRuntime {\n readonly crawler: Crawler;\n /** This index keeps the documents and the cache entries. `DocIndex` is the SQLite index. */\n readonly index: ReadingIndex;\n readonly blobs: BlobStore;\n /** This reader is the read side of the capture log. The function `reindex` replays it. */\n readonly captures: CaptureReader;\n /** This log is the write side of the capture log. */\n readonly log: CaptureLog;\n readonly extractors: Extractors;\n readonly now: () => string;\n}\n\n/** This is the payload of a capture record, as the crawler writes it. */\nexport interface CapturePayload {\n readonly url: string;\n readonly recipe: string;\n readonly mode: \"api\" | \"page\";\n readonly status?: number;\n readonly outcome?: string;\n readonly blob?: string | null;\n readonly content_type?: string | null;\n readonly etag?: string | null;\n readonly last_modified?: string | null;\n readonly fetched_at: string;\n readonly signals?: { readonly ai_input?: AiInput };\n /** This value is true when the headless browser made the capture, not a plain request. */\n readonly rendered?: boolean;\n /** This value is true when the body is only the first part of the response. */\n readonly truncated?: boolean;\n /** This value is true when the server cut a compressed body. The capture is not a copy to keep. */\n readonly cut?: boolean;\n /** This value is true when an API answered with a web page that the request did not accept. The capture is not a copy to keep. */\n readonly unasked?: boolean;\n /** This time (ISO 8601) is the end of the freshness that the server gave. Captures before version 0.2 do not have it. */\n readonly fresh_until?: string | null;\n /** This text holds the request headers that the response varies on. Captures before version 0.2 do not have it. */\n readonly vary_key?: string | null;\n /** This URL is the target of a redirect, with its secrets as `REDACTED`. */\n readonly location?: string | null;\n readonly note?: string;\n}\n\n/** The structural fields of the metadata of a page: they hold no text of the page. A held-back page shows only these. */\nexport const STRUCTURAL_META: readonly string[] = [\n \"jsonld_types\",\n \"jsonld_dropped\",\n \"visible_chars\",\n \"lang\",\n \"embedded\",\n \"needs_render\",\n \"render_reason\",\n \"canonical\",\n \"redirect\",\n \"blocked\",\n \"format\",\n \"pages\",\n \"ocr_needed\",\n \"truncated\",\n];\n\n/** The note of a document that is held back. */\nexport const HELD_BACK =\n \"This page asked not to be used as AI input, so its text is held back. Open the URL to read it, or read again with userProvided when a person gave this URL directly (the crawler logs that).\";\n\n/**\n * This function tells if a document is held back: the page asked not to be AI input, and no person\n * gave its URL directly. The only way to read a held-back document is `userProvided`.\n */\nexport const isHeldBack = (doc: { readonly ai_input: AiInput }, userProvided = false): boolean =>\n doc.ai_input === \"no\" && !userProvided;\n\n/** This function gives the metadata that a held-back document shows: only the structural fields, which hold no text of the page. */\nexport const heldBackMeta = (meta: Readonly<Record<string, unknown>>): Record<string, unknown> =>\n Object.fromEntries(Object.entries(meta).filter(([key]) => STRUCTURAL_META.includes(key)));\n\nconst messageOf = (error: unknown): string =>\n error instanceof Error ? (error.message.split(/\\r?\\n/)[0] ?? \"\") : String(error);\n\n/** This function tells which bot check or firewall sent its page in place of the content, or gives undefined. */\nconst blockedBy = (doc: ExtractedDoc): string | undefined => {\n const blocked = doc.meta[\"blocked\"];\n return typeof blocked === \"string\" && blocked.length > 0 ? blocked : undefined;\n};\n\n/** The result of `fetchAndIndex` and `renderAndIndex`. */\nexport interface Indexed {\n readonly fetched: Fetched;\n readonly docs: readonly Doc[];\n /** This text tells why the capture gave no document: the error of the extractor, or a bot check. */\n readonly error?: string | undefined;\n /** This value names the bot check or the firewall that sent its page in place of the content. */\n readonly blocked?: string | undefined;\n}\n\n/** This function reads one capture with its extractor, and adds each document to the index as a version of the capture. */\nconst indexCapture = async (\n runtime: CrawlRuntime,\n fetched: Fetched,\n recipe: string,\n rendered: boolean,\n): Promise<Omit<Indexed, \"fetched\">> => {\n const capture = fetched.capture;\n if (fetched.body === undefined || capture === undefined) return { docs: [] };\n // a stored capture that the index already read gives its documents with no new reading\n if (fetched.fromCache) {\n const known = runtime.index.docsOfCapture(capture);\n if (known.length > 0) return { docs: known };\n }\n const extractor = runtime.extractors.get(recipe);\n if (extractor === undefined) return { docs: [] };\n let extracted: ReadonlyArray<ExtractedDoc>;\n try {\n extracted = await extractor(fetched.body, {\n url: fetched.url,\n contentType: rendered ? \"text/html; charset=utf-8\" : (fetched.contentType ?? null),\n });\n } catch (error) {\n return { docs: [], error: `extract failed (${recipe}): ${messageOf(error)}` };\n }\n const blocked = extracted.map(blockedBy).find((name) => name !== undefined);\n if (blocked !== undefined) {\n // a bot check is not the content: the index keeps the earlier text, and the host stops\n runtime.crawler.reportBlocked(fetched.url, blocked);\n return {\n docs: [],\n blocked,\n error: `the page is a bot check (${blocked}), not the content, so the crawler keeps nothing of it and stops the host`,\n };\n }\n const now = runtime.now();\n const docs = extracted.map((doc) =>\n runtime.index.upsert(\n {\n ...doc,\n captures: [capture],\n capture,\n fetched_at: fetched.fetchedAt ?? now,\n blob: fetched.blob ?? null,\n recipe,\n rendered,\n ai_input: fetched.signals.ai_input,\n ...(fetched.scope !== undefined ? { scope: fetched.scope } : {}),\n ...(fetched.labels !== undefined ? { labels: fetched.labels } : {}),\n },\n now,\n ),\n );\n return { docs };\n};\n\n/**\n * This function fetches one URL and adds its documents to the index. Each recipe uses it. A fetch\n * from the cache gives the documents that the index has for that capture. Thus a repeated\n * question sends no request and reads nothing again. An extractor that throws gives no document and an\n * `error`, and the fetch still succeeds.\n */\nexport const fetchAndIndex = async (\n runtime: CrawlRuntime,\n request: FetchRequest,\n): Promise<Indexed> => {\n const fetched = await runtime.crawler.fetch(request);\n if (fetched.outcome !== \"ok\" && fetched.outcome !== \"not-modified\") return { fetched, docs: [] };\n return { fetched, ...(await indexCapture(runtime, fetched, request.recipe, false)) };\n};\n\n/** This function renders a page in the headless browser and adds what the page shows to the index. */\nexport const renderAndIndex = async (\n runtime: CrawlRuntime,\n request: RenderRequest,\n): Promise<Indexed> => {\n const fetched = await runtime.crawler.render(request);\n if (fetched.outcome !== \"ok\") return { fetched, docs: [] };\n return { fetched, ...(await indexCapture(runtime, fetched, request.recipe, true)) };\n};\n\nexport interface PageResult {\n readonly url: string;\n readonly doc: string | null;\n readonly outcome: string;\n readonly status: number;\n /** This value is the title of the page. A held-back page shows no title. */\n readonly title: string | null;\n readonly chars: number;\n readonly rendered: boolean;\n /** These are the schema.org types that the JSON-LD of the page declares. */\n readonly structured: readonly string[];\n readonly links: number;\n /** These ids are the captures behind the current text, the newest first, 10 or fewer. */\n readonly captures: readonly string[];\n /** This id is the capture of the text that the document shows. A citation names it. */\n readonly capture: string | null;\n /** This value is the number of captures of the document. */\n readonly capture_count: number;\n /** This time (ISO 8601) is the fetch time of the text that the document shows. */\n readonly fetched_at: string | null;\n /** This value is true when the stored copy answered, and the crawler sent no request. */\n readonly from_cache: boolean;\n /** This value is true when the page asked not to be AI input. Its title and text are then held back. */\n readonly held_back: boolean;\n readonly note: string | null;\n}\n\n/** These options control one page. */\nexport interface PageOptions {\n readonly render: \"auto\" | \"never\" | \"always\";\n readonly maxAgeMs?: number | \"auto\" | undefined;\n readonly onBehalfOf?: readonly string[] | undefined;\n readonly job?: string | undefined;\n /** When this signal aborts, the crawler stops the fetch and the render. */\n readonly signal?: AbortSignal | undefined;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\nconst HTML_TYPE = /html|xml/i;\n\n/**\n * This function reads one web page. It first sends a plain request. It opens the browser only when\n * the page shows almost nothing without JavaScript, or when the caller asks for it. It renders\n * only an HTML page: not a PDF, a different file or a bot check. A page that sends the reader on\n * with a `<meta http-equiv=\"refresh\">` gets one more plain fetch, of its target.\n */\nexport const fetchPage = async (\n runtime: CrawlRuntime,\n url: string,\n options: PageOptions,\n): Promise<PageResult> => {\n const plainRequest = (target: string): FetchRequest => ({\n url: target,\n mode: \"page\",\n recipe: \"page\",\n accept: \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.5\",\n maxAgeMs: options.maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n job: options.job,\n signal: options.signal,\n scope: options.scope,\n labels: options.labels,\n });\n let plain = await fetchAndIndex(runtime, plainRequest(url));\n const notes: string[] = [];\n if (plain.error !== undefined) notes.push(plain.error);\n const target = plain.docs[0]?.meta[\"redirect\"];\n if (typeof target === \"string\" && target.length > 0 && plain.blocked === undefined) {\n const followed = await fetchAndIndex(runtime, plainRequest(target));\n notes.push(\n `the page sends the reader on to ${followed.fetched.url}, so the crawler fetched that page`,\n );\n if (followed.error !== undefined) notes.push(followed.error);\n plain = followed;\n }\n let docs = plain.docs;\n let rendered = false;\n let fromCache = plain.fetched.fromCache;\n const reachable = plain.fetched.outcome === \"ok\" || plain.fetched.outcome === \"not-modified\";\n const first = docs[0];\n const html =\n first?.kind === \"page\" &&\n (plain.fetched.contentType === undefined || HTML_TYPE.test(plain.fetched.contentType));\n const wanted =\n options.render === \"always\" ||\n (options.render === \"auto\" && first?.meta[\"needs_render\"] === true);\n if (reachable && wanted) {\n if (plain.blocked !== undefined) notes.push(\"the crawler does not render a bot check\");\n else if (first === undefined || !html) {\n notes.push(\"the crawler renders only an HTML page, not a PDF or a different file\");\n } else {\n // a render of an earlier body is not the page now: the render is fresh when the plain body changed\n const current = runtime.index.get(first.doc);\n const stale = current !== undefined && !current.rendered && current.capture_count > 1;\n const shown = await renderAndIndex(runtime, {\n url: plain.fetched.url,\n recipe: \"page\",\n maxAgeMs: stale ? 0 : options.maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n job: options.job,\n signal: options.signal,\n scope: options.scope,\n labels: options.labels,\n });\n if (shown.docs.length > 0) {\n docs = shown.docs;\n rendered = true;\n fromCache = fromCache && shown.fetched.fromCache;\n } else {\n notes.push(shown.error ?? shown.fetched.note ?? \"the render gave no document\");\n }\n }\n }\n const doc = docs[0] === undefined ? undefined : (runtime.index.get(docs[0].doc) ?? docs[0]);\n const held = doc !== undefined && isHeldBack(doc);\n const types = doc?.meta[\"jsonld_types\"];\n const links = doc?.meta[\"links\"];\n if (held) notes.push(HELD_BACK);\n const note = [plain.fetched.note, ...notes].filter(\n (part): part is string => part !== undefined && part.length > 0,\n );\n return {\n url,\n doc: doc?.doc ?? null,\n outcome: plain.fetched.outcome,\n status: plain.fetched.status,\n title: doc === undefined || held ? null : doc.title,\n chars: doc?.text.length ?? 0,\n rendered: doc?.rendered ?? rendered,\n structured: Array.isArray(types) ? (types as string[]) : [],\n links: Array.isArray(links) ? links.length : 0,\n captures: doc === undefined ? [] : [...doc.captures],\n capture: doc?.capture ?? null,\n capture_count: doc?.capture_count ?? 0,\n fetched_at: doc?.fetched_at ?? plain.fetched.fetchedAt ?? null,\n from_cache: fromCache,\n held_back: held,\n note: note.length > 0 ? note.join(\"; \") : null,\n };\n};\n\n// ── reading, verifying and finding ──────────────────────────────────────────\n\n/** This result is a part of one stored document. */\nexport interface ReadResult {\n readonly doc: string;\n readonly kind: string;\n /** This value is the title of the document. A held-back document shows no title. */\n readonly title: string;\n readonly url: string | null;\n /** This value is the metadata of the document. A held-back document shows only the structural fields. */\n readonly meta: Readonly<Record<string, unknown>>;\n readonly from: number;\n readonly to: number;\n readonly total: number;\n readonly text: string;\n readonly captures: readonly string[];\n /** This id is the capture of the text that the result reads. */\n readonly capture: string;\n /** This time (ISO 8601) is the fetch time of the text that the result reads. */\n readonly fetched_at: string;\n readonly ai_input: AiInput;\n /** This value is true when the page asked not to be AI input. The text is then empty. */\n readonly held_back: boolean;\n readonly note: string | null;\n}\n\n/** This result tells if a quote is in a stored document, and where. */\nexport interface VerifyResult {\n readonly doc: string;\n /** This value is true when the quote is in the text. A held-back document gives false. */\n readonly found: boolean;\n /** These values are the positions of the quote in the text of `capture`. */\n readonly start: number | null;\n readonly end: number | null;\n readonly excerpt: string | null;\n readonly url: string | null;\n readonly captures: readonly string[];\n /** This id is the capture whose text holds the quote, or the capture that the result examined. */\n readonly capture: string | null;\n /** This hash is the blob of that capture. */\n readonly blob: string | null;\n /** This time (ISO 8601) is the fetch time of that capture. */\n readonly fetched_at: string | null;\n /** This value is true when only an earlier version of the document holds the quote. */\n readonly stale: boolean;\n /** This value is true when the quote is in the whole text of a page that the reader cut. The positions are then in that whole text. */\n readonly in_full_text: boolean;\n /** This value is false when the examined text is not complete (a PDF that the reader stopped). A quote that is not found can then be in the file. */\n readonly complete: boolean;\n /** This value is the number of places where the quote is. */\n readonly count: number;\n readonly held_back: boolean;\n readonly note: string | null;\n}\n\n/** These options control a read. */\nexport interface ReadOptions {\n /** This value is the first character to read. The default is 0. */\n readonly from?: number | undefined;\n /** This value is the number of characters to read. The default is 4000. */\n readonly length?: number | undefined;\n /** This value is true when a person gave this URL directly. It is the only way to read a held-back document. The crawler logs it. */\n readonly userProvided?: boolean | undefined;\n /** This id reads the version of one capture, not the current version. */\n readonly capture?: string | undefined;\n}\n\n/** These options control a verification. */\nexport interface VerifyOptions {\n /** This id examines the version of one capture only. A citation of that capture thus stays true after a new fetch. */\n readonly capture?: string | undefined;\n /** This value is true when a person gave this URL directly. It is the only way to verify a quote in a held-back document. */\n readonly userProvided?: boolean | undefined;\n}\n\nconst knownDoc = (runtime: CrawlRuntime, id: string): Doc => {\n const doc = runtime.index.get(id);\n if (doc === undefined)\n throw new Error(`no stored document ${id}: find one with find(), or fetch its page`);\n return doc;\n};\n\nconst wholeNumber = (name: string, value: number | undefined, fallback: number): number => {\n if (value === undefined) return fallback;\n if (!Number.isFinite(value) || value < 0) {\n throw new RangeError(`${name} must be a number of 0 or more, not ${String(value)}`);\n }\n return Math.floor(value);\n};\n\n/** This function moves a position off the second half of a surrogate pair, so a slice does not cut a character. */\nconst onBoundary = (text: string, position: number): number => {\n const code = text.charCodeAt(position);\n return position > 0 && position < text.length && code >= 0xdc00 && code <= 0xdfff\n ? position - 1\n : position;\n};\n\n/**\n * This function writes an `override` record to the capture log. A person gave the URL of a\n * held-back document directly, and the hold stops for one action (`read` or `verify`). The record\n * takes the scope and the labels of `marks`, or else those of the document, as its captures do.\n */\nexport const recordOverride = async (\n runtime: CrawlRuntime,\n doc: {\n readonly doc: string;\n readonly url: string | null;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n },\n action: string,\n marks: {\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n } = {},\n): Promise<void> => {\n const at = runtime.now();\n const scope = marks.scope ?? doc.scope;\n const labels = marks.labels ?? doc.labels;\n await runtime.log.append(\n makeRecord({\n source: \"fetch\",\n kind: \"override\",\n payload: { doc: doc.doc, url: doc.url, action, reason: \"a person provided this URL\", at },\n received_at: at,\n external_id: doc.url ?? doc.doc,\n ...(scope !== undefined ? { scope } : {}),\n ...(labels !== undefined ? { labels } : {}),\n }),\n );\n};\n\n/**\n * This function reads a range of characters of one stored document. It applies the rule for a\n * held-back document: no title, no text, and only the structural metadata, unless `userProvided`.\n * The positions do not cut a character in two.\n */\nexport const readDocument = async (\n runtime: CrawlRuntime,\n id: string,\n options: ReadOptions = {},\n): Promise<ReadResult> => {\n const from = wholeNumber(\"from\", options.from, 0);\n const length = wholeNumber(\"length\", options.length, 4000);\n const doc = knownDoc(runtime, id);\n const version =\n options.capture === undefined ? undefined : runtime.index.versionText(id, options.capture);\n if (options.capture !== undefined && version === undefined) {\n throw new Error(`the document ${id} has no version from the capture ${options.capture}`);\n }\n const aiInput: AiInput =\n doc.ai_input === \"no\" || version?.ai_input === \"no\"\n ? \"no\"\n : (version?.ai_input ?? doc.ai_input);\n const userProvided = options.userProvided ?? false;\n const held = isHeldBack({ ai_input: aiInput }, userProvided);\n if (aiInput === \"no\" && userProvided) await recordOverride(runtime, doc, \"read\");\n const text = version?.text ?? doc.text;\n const start = onBoundary(text, Math.min(from, text.length));\n const end = held ? start : onBoundary(text, Math.min(text.length, start + length));\n const meta = version?.meta ?? doc.meta;\n return {\n doc: doc.doc,\n kind: version?.kind ?? doc.kind,\n title: held ? \"\" : (version?.title ?? doc.title),\n url: version?.url ?? doc.url,\n meta: held ? heldBackMeta(meta) : meta,\n from: start,\n to: end,\n total: text.length,\n text: held ? \"\" : text.slice(start, end),\n captures: doc.captures,\n capture: version?.capture ?? doc.capture,\n fetched_at: version?.fetched_at ?? doc.fetched_at,\n ai_input: aiInput,\n held_back: held,\n note: held ? HELD_BACK : null,\n };\n};\n\n/**\n * This function examines the versions of a document for a quote, and writes nothing. It also tells\n * if `userProvided` opened a held-back version, so that `verifyQuote` logs the override.\n */\nconst examineQuote = (\n runtime: CrawlRuntime,\n id: string,\n quote: string,\n options: VerifyOptions,\n): { readonly result: VerifyResult; readonly override: boolean } => {\n const doc = knownDoc(runtime, id);\n const pinned =\n options.capture === undefined ? undefined : runtime.index.versionText(id, options.capture);\n if (options.capture !== undefined && pinned === undefined) {\n throw new Error(`the document ${id} has no version from the capture ${options.capture}`);\n }\n const userProvided = options.userProvided ?? false;\n const base = { doc: doc.doc, url: doc.url, captures: doc.captures };\n const aiInput: AiInput = doc.ai_input === \"no\" || pinned?.ai_input === \"no\" ? \"no\" : doc.ai_input;\n if (isHeldBack({ ai_input: aiInput }, userProvided)) {\n const result: VerifyResult = {\n ...base,\n found: false,\n start: null,\n end: null,\n excerpt: null,\n capture: null,\n blob: null,\n fetched_at: null,\n stale: false,\n in_full_text: false,\n complete: false,\n count: 0,\n held_back: true,\n note: HELD_BACK,\n };\n return { result, override: false };\n }\n const current = pinned ?? runtime.index.versionText(id);\n const candidates =\n pinned !== undefined\n ? [pinned]\n : [...(current === undefined ? [] : [current]), ...runtime.index.earlierTexts(id)];\n // a version that asked not to be AI input stays held back after the page drops the signal\n const held = candidates.filter((version) => version.ai_input === \"no\").length;\n const override = userProvided && (aiInput === \"no\" || held > 0);\n for (const [index, version] of candidates.entries()) {\n if (isHeldBack(version, userProvided)) continue;\n let hit = locate(version.text, quote);\n let inFull = false;\n if (!hit.found && version.full_text !== undefined) {\n hit = locate(version.full_text, quote);\n inFull = hit.found;\n }\n if (!hit.found) continue;\n const stale = pinned === undefined && index > 0;\n const result: VerifyResult = {\n ...base,\n found: true,\n start: hit.start,\n end: hit.end,\n excerpt: hit.excerpt,\n capture: version.capture,\n blob: version.blob,\n fetched_at: version.fetched_at,\n stale,\n in_full_text: inFull,\n complete: true,\n count: hit.count,\n held_back: false,\n note: stale ? \"only an earlier capture of the document holds the quote\" : null,\n };\n return { result, override };\n }\n const incomplete =\n current !== undefined && current.meta[\"truncated\"] === true && current.full_text === undefined;\n // the note does not depend on the quote, so it tells nothing about the held-back text\n const notes = [\n ...(incomplete\n ? [\n \"the reader stopped before the end of the file, so the quote can be in the part that it did not read\",\n ]\n : []),\n ...(held > 0 && !userProvided\n ? [`the check did not examine the versions that asked not to be AI input (${String(held)})`]\n : []),\n ];\n const result: VerifyResult = {\n ...base,\n found: false,\n start: null,\n end: null,\n excerpt: null,\n capture: current?.capture ?? null,\n blob: current?.blob ?? null,\n fetched_at: current?.fetched_at ?? null,\n stale: false,\n in_full_text: false,\n complete: !incomplete,\n count: 0,\n held_back: false,\n note: notes.length > 0 ? notes.join(\"; \") : null,\n };\n return { result, override };\n};\n\n/**\n * This function finds a quote in a stored document, with `locate`, and writes nothing. It is the\n * synchronous part of `verifyQuote`. A held-back document gives `found: false` and no positions,\n * unless `userProvided`, and this function does not log the override. With no capture, it does not\n * examine a version that asked not to be AI input, also when a later capture dropped the signal.\n */\nexport const checkQuote = (\n runtime: CrawlRuntime,\n id: string,\n quote: string,\n options: VerifyOptions = {},\n): VerifyResult => examineQuote(runtime, id, quote, options).result;\n\n/**\n * This function finds a quote in a stored document, with `locate`. It tells which capture holds the\n * quote, with its blob, its fetch time and the positions:\n *\n * - With `capture`, it examines the version of that capture only.\n * - With no capture, it examines the current text first. When the quote is not there, it examines\n * the earlier texts, the newest first. A match there has `stale: true`.\n * - When the reader cut a text, it also examines the whole text.\n *\n * A held-back document gives `found: false` and no positions, so a quote cannot test its text,\n * unless `userProvided`. The same rule applies to each version. A version that asked not to be AI\n * input stays held back after a later capture drops the signal. The crawler logs an override when\n * `userProvided` opens a held-back version.\n */\nexport const verifyQuote = async (\n runtime: CrawlRuntime,\n id: string,\n quote: string,\n options: VerifyOptions = {},\n): Promise<VerifyResult> => {\n const { result, override } = examineQuote(runtime, id, quote, options);\n if (override) await recordOverride(runtime, knownDoc(runtime, id), \"verify\");\n return result;\n};\n\n/**\n * This function searches the stored documents. The syntax of the query is the syntax of\n * `parseQuery`. A held-back document is not in the results. The option `rank` chooses the\n * ranking (`RankMode`), and the default is `exact`.\n */\nexport const findDocuments = async (\n runtime: CrawlRuntime,\n q: string,\n options: {\n readonly limit?: number | undefined;\n readonly kind?: string | null | undefined;\n readonly rank?: RankMode | undefined;\n } = {},\n): Promise<Found[]> => {\n await Promise.resolve();\n return runtime.index.find(q, {\n limit: options.limit ?? 10,\n kind: options.kind ?? null,\n ...(options.rank !== undefined ? { rank: options.rank } : {}),\n });\n};\n\n// ── the rebuild ─────────────────────────────────────────────────────────────\n\n/** The counts of a rebuild. */\nexport interface ReindexResult {\n /** This value is the number of captures that the rebuild read again. */\n readonly captures: number;\n readonly docs: number;\n readonly urls: number;\n /** This value is the number of captures whose extractor threw. */\n readonly failed: number;\n /** This value is the number of captures whose blob is missing or damaged. */\n readonly missing: number;\n /** This value is the number of damaged lines of the log that the rebuild skipped. */\n readonly damaged: number;\n /** This value is the number of documents of recipes with no extractor, kept from the old index. */\n readonly carried: number;\n /** These recipes have captures in the log and no extractor in this runtime. */\n readonly unknown_recipes: readonly string[];\n /** These are the first errors of the extractors. */\n readonly errors: ReadonlyArray<{ readonly capture: string; readonly error: string }>;\n}\n\nconst RENDERED_MARK = \"#rendered\";\nconst MAX_ERRORS = 20;\n\n/** This function gives the URL of a capture and its variant. A URL of version 0.1 can end in `#rendered`. */\nconst captureTarget = (\n payload: CapturePayload,\n): { readonly url: string; readonly variant: Variant } => {\n if (payload.url.endsWith(RENDERED_MARK)) {\n return { url: payload.url.slice(0, -RENDERED_MARK.length), variant: \"rendered\" };\n }\n return { url: payload.url, variant: payload.rendered === true ? \"rendered\" : \"plain\" };\n};\n\n/** A replay writes into a new generation of the index (a rebuild), or into the index itself (the tail of the log). */\ninterface ReplayTarget {\n upsert(input: DocInput, now: string): unknown;\n setLatest(entry: Latest): void;\n latest(url: string, variant?: Variant): Latest | undefined;\n}\n\n/** These are the counts of a replay. */\ninterface ReplayTally {\n captures: number;\n failed: number;\n missing: number;\n readonly unknown: Set<string>;\n readonly errors: Array<{ readonly capture: string; readonly error: string }>;\n}\n\nconst newTally = (): ReplayTally => ({\n captures: 0,\n failed: 0,\n missing: 0,\n unknown: new Set(),\n errors: [],\n});\n\n/** This function replays the cache entry of one capture, as the crawler made it at the time of the fetch. */\nconst replayLatest = (\n build: ReplayTarget,\n envelope: CaptureRecord,\n payload: CapturePayload,\n): void => {\n const { url, variant } = captureTarget(payload);\n const fetchedAt = payload.fetched_at ?? envelope.received_at;\n if (payload.outcome === \"redirect\") {\n const location =\n payload.location ??\n (typeof payload.note === \"string\" && payload.note.startsWith(\"location \")\n ? payload.note.slice(9)\n : null);\n const permanent = payload.status === 301 || payload.status === 308;\n if (location !== null && (permanent || typeof payload.fresh_until === \"string\")) {\n build.setLatest({\n url,\n variant,\n capture: envelope.id,\n blob: null,\n etag: null,\n last_modified: null,\n content_type: null,\n fetched_at: fetchedAt,\n status: payload.status ?? 301,\n ai_input: \"unset\",\n location,\n fresh_until: payload.fresh_until ?? null,\n vary_key: payload.vary_key ?? null,\n });\n }\n return;\n }\n // a body that the server cut, or a web page in place of the data of an API, was never the stored copy\n if (payload.cut === true || payload.unasked === true) return;\n const previous = build.latest(url, variant);\n const revisit = envelope.kind === \"revisit\" || payload.outcome === \"not-modified\";\n const truncated = payload.truncated === true;\n // a 304 updates the stored answer: its status stays, and its validators fill the gaps\n const etag = truncated ? null : (payload.etag ?? (revisit ? (previous?.etag ?? null) : null));\n const lastModified = truncated\n ? null\n : (payload.last_modified ?? (revisit ? (previous?.last_modified ?? null) : null));\n const kept = revisit ? (previous?.content_type ?? undefined) : undefined;\n const contentType =\n payload.content_type ?? kept ?? (variant === \"rendered\" ? \"text/html; charset=utf-8\" : null);\n const status = revisit\n ? previous === undefined || previous.status < 200 || previous.status >= 300\n ? 200\n : previous.status\n : (payload.status ?? 200);\n build.setLatest({\n url,\n variant,\n capture: envelope.id,\n blob: payload.blob ?? null,\n etag,\n last_modified: lastModified,\n content_type: contentType,\n fetched_at: fetchedAt,\n status,\n ai_input: payload.signals?.ai_input ?? \"unset\",\n ...(truncated ? { truncated: true } : {}),\n fresh_until: payload.fresh_until ?? null,\n vary_key: payload.vary_key ?? (revisit ? (previous?.vary_key ?? null) : null),\n });\n};\n\n/** This function replays one record of the log: its cache entry, and the documents that its extractor gives. */\nconst replayRecord = async (\n runtime: CrawlRuntime,\n target: ReplayTarget,\n envelope: CaptureRecord,\n tally: ReplayTally,\n): Promise<void> => {\n if (envelope.kind !== \"capture\" && envelope.kind !== \"revisit\") return;\n const payload = envelope.payload as CapturePayload | null;\n if (payload === null || typeof payload !== \"object\" || typeof payload.url !== \"string\") return;\n if (payload.outcome === \"redirect\") {\n replayLatest(target, envelope, payload);\n return;\n }\n if (typeof payload.blob !== \"string\") return;\n if (payload.outcome !== \"ok\" && payload.outcome !== \"not-modified\") return;\n tally.captures += 1;\n replayLatest(target, envelope, payload);\n const extractor = runtime.extractors.get(payload.recipe);\n if (extractor === undefined) {\n // the crawler reads robots.txt for itself; it gives no document\n if (payload.recipe !== \"robots.txt\") tally.unknown.add(payload.recipe);\n return;\n }\n const body = await runtime.blobs.get(payload.blob);\n if (body === undefined) {\n tally.missing += 1;\n return;\n }\n const { url, variant } = captureTarget(payload);\n const latest = target.latest(url, variant);\n let extracted: ReadonlyArray<ExtractedDoc>;\n try {\n extracted = await extractor(body, {\n url,\n contentType:\n variant === \"rendered\"\n ? \"text/html; charset=utf-8\"\n : (payload.content_type ?? latest?.content_type ?? null),\n });\n } catch (error) {\n tally.failed += 1;\n if (tally.errors.length < MAX_ERRORS)\n tally.errors.push({ capture: envelope.id, error: messageOf(error) });\n return;\n }\n // a bot check is not the content, so it is not a version of the document\n if (extracted.some((doc) => blockedBy(doc) !== undefined)) return;\n for (const doc of extracted) {\n target.upsert(\n {\n ...doc,\n captures: [envelope.id],\n capture: envelope.id,\n fetched_at: payload.fetched_at ?? envelope.received_at,\n blob: payload.blob,\n recipe: payload.recipe,\n rendered: variant === \"rendered\",\n ai_input: payload.signals?.ai_input ?? \"unset\",\n ...(envelope.scope !== undefined ? { scope: envelope.scope } : {}),\n ...(envelope.labels !== undefined ? { labels: envelope.labels } : {}),\n },\n envelope.received_at,\n );\n }\n};\n\n/**\n * This function builds the reading index again from the capture log and the blobs. The index is a\n * projection, and this function is the proof that its deletion loses nothing. These are its rules:\n *\n * - It builds a new generation of the tables and commits it in batches. The old index answers\n * until the end, and a failed rebuild leaves it as it was.\n * - It reads the log as a stream, so its memory does not grow with the log.\n * - It counts and skips a damaged line, a missing blob and an extractor that throws.\n * - The documents of a recipe that has no extractor in this runtime stay, from the old index.\n * - A capture that a different writer adds during the rebuild goes into the new index.\n */\nexport const reindex = async (\n runtime: CrawlRuntime,\n options: { readonly batch?: number | undefined } = {},\n): Promise<ReindexResult> => {\n const since = runtime.now();\n const build = runtime.index.beginRebuild(\n options.batch === undefined ? {} : { batch: options.batch },\n );\n const tally = newTally();\n let damaged = 0;\n try {\n const scan = runtime.captures.scan(\n { source: \"fetch\" },\n {\n onDamaged: () => {\n damaged += 1;\n },\n },\n );\n for await (const envelope of scan) await replayRecord(runtime, build, envelope, tally);\n const carried = build.carry(new Set(runtime.extractors.recipes()));\n const counts = build.commit(since);\n return {\n captures: tally.captures,\n ...counts,\n failed: tally.failed,\n missing: tally.missing,\n damaged,\n carried,\n unknown_recipes: [...tally.unknown].sort(),\n errors: tally.errors,\n };\n } catch (error) {\n build.abort();\n throw error;\n }\n};\n\n/** The result of `catchUpIndex`. */\nexport interface CatchUpResult extends ReindexResult {\n /**\n * The value `rebuild` tells that the index was stale, and the function built it again from the\n * whole log. The value `tail` tells that it read only the captures that the index missed.\n */\n readonly mode: \"rebuild\" | \"tail\";\n}\n\n/** The tail starts this long before the newest cache entry. Two writers and their clocks can differ. */\nconst TAIL_SLACK_MS = 10 * 60_000;\n\n/** This function tells if the index missed a record of the log: no cache entry as new as it, or no version of its documents. */\nconst missed = (runtime: CrawlRuntime, envelope: CaptureRecord): boolean => {\n if (envelope.kind !== \"capture\" && envelope.kind !== \"revisit\") return false;\n const payload = envelope.payload as CapturePayload | null;\n if (payload === null || typeof payload !== \"object\" || typeof payload.url !== \"string\")\n return false;\n const fetchedAt = payload.fetched_at ?? envelope.received_at;\n const { url, variant } = captureTarget(payload);\n const entry = runtime.index.latest(url, variant);\n if (payload.outcome === \"redirect\") return entry === undefined || entry.fetched_at < fetchedAt;\n if (\n typeof payload.blob !== \"string\" ||\n (payload.outcome !== \"ok\" && payload.outcome !== \"not-modified\")\n )\n return false;\n if (entry === undefined || entry.fetched_at < fetchedAt) return true;\n return (\n runtime.extractors.get(payload.recipe) !== undefined && !runtime.index.hasVersion(envelope.id)\n );\n};\n\n/**\n * This function brings the reading index up to the capture log, with no request. `openCrawler`\n * uses it at open. An application that uses `createRuntime` uses it the same way:\n *\n * - A stale index (a new schema, or a deleted file) is built again with `reindex`.\n * - An index that is behind the log gets the captures that it missed. A process can stop between\n * the log and the index, and SQLite can lose its last transactions after a power cut. The\n * function reads the log from a short time before the newest cache entry.\n *\n * It gives the counts, or undefined when the index missed nothing.\n */\nexport const catchUpIndex = async (runtime: CrawlRuntime): Promise<CatchUpResult | undefined> => {\n if (runtime.index.stale) return { mode: \"rebuild\", ...(await reindex(runtime)) };\n const newest = runtime.index.newestFetch();\n const since =\n newest === undefined ? undefined : new Date(Date.parse(newest) - TAIL_SLACK_MS).toISOString();\n const tally = newTally();\n let damaged = 0;\n const scan = runtime.captures.scan(\n { source: \"fetch\", ...(since === undefined ? {} : { since }) },\n {\n onDamaged: () => {\n damaged += 1;\n },\n },\n );\n for await (const envelope of scan) {\n if (missed(runtime, envelope)) await replayRecord(runtime, runtime.index, envelope, tally);\n }\n if (tally.captures === 0) return undefined;\n return {\n mode: \"tail\",\n captures: tally.captures,\n ...runtime.index.counts(),\n failed: tally.failed,\n missing: tally.missing,\n damaged,\n carried: 0,\n unknown_recipes: [...tally.unknown].sort(),\n errors: tally.errors,\n };\n};\n","import { existsSync, lstatSync, mkdirSync, realpathSync, statSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport type { BlobStore } from \"./blobs.ts\";\nimport { checkedRenderer } from \"./browser.ts\";\nimport { FileBlobStore } from \"./blobs.ts\";\nimport type { CrawlEvent } from \"./crawler.ts\";\nimport { Crawler } from \"./crawler.ts\";\nimport type { DocVersion, Found, RankMode, ReadingIndex } from \"./docs.ts\";\nimport { DocIndex } from \"./docs.ts\";\nimport type { EgressOptions } from \"./egress.ts\";\nimport { SqliteHostStateStore } from \"./hoststate.ts\";\nimport { CaptureLog } from \"./log.ts\";\nimport type { Clock, HostStateStore, PolicyOptions } from \"./politeness.ts\";\nimport { MemoryHostStateStore, realClock } from \"./politeness.ts\";\nimport type { Renderer } from \"./render.ts\";\nimport { playwrightRenderer } from \"./render.ts\";\nimport type {\n CrawlRuntime,\n Extractor,\n PageResult,\n ReadOptions,\n ReadResult,\n ReindexResult,\n VerifyOptions,\n VerifyResult,\n} from \"./runtime.ts\";\nimport {\n Extractors,\n catchUpIndex,\n fetchPage,\n findDocuments,\n readDocument,\n reindex,\n verifyQuote,\n} from \"./runtime.ts\";\nimport { BOT_PAGE, DEFAULT_TOKEN, VERSION } from \"./version.ts\";\n\nexport type { ReadOptions, ReadResult, VerifyOptions, VerifyResult } from \"./runtime.ts\";\n\nexport { BOT_PAGE, DEFAULT_TOKEN } from \"./version.ts\";\n\n/**\n * This function gives the default `User-Agent` header. The header names the crawler, its version\n * and the page about it, so a site owner can find out what made each request.\n */\nexport const defaultUserAgent = (contact?: string): string =>\n `${DEFAULT_TOKEN}/${VERSION} (+${BOT_PAGE}${contact === undefined ? \"\" : `; ${contact}`})`;\n\n/** A mailbox is `name@host`, or a name and `<name@host>`. A URL is not a mailbox, also when it holds an `@`. */\nconst MAILBOX = /^(?:[^\\s<>@:/]+@[^\\s<>@/]+|[^<>]*<[^\\s<>@:/]+@[^\\s<>@/]+>)$/;\n\n/**\n * This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,\n * section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in\n * the `User-Agent` only.\n */\nexport const isMailbox = (contact: string): boolean => MAILBOX.test(contact.trim());\n\n/** A duration is a count of milliseconds, or a text, for example `\"30m\"`, `\"24h\"` or `\"7d\"`. */\nexport type Duration = number | string;\n\nconst UNITS: Readonly<Record<string, number>> = {\n ms: 1,\n s: 1000,\n m: 60_000,\n h: 3_600_000,\n d: 86_400_000,\n w: 604_800_000,\n};\n\n/**\n * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,\n * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`\n * is zero. The function throws an error for a text that it cannot read.\n */\nexport const durationMs = (duration: Duration): number => {\n if (typeof duration === \"number\") {\n if (!Number.isFinite(duration) || duration < 0)\n throw new Error(`not a duration: ${String(duration)}`);\n return duration;\n }\n if (/^\\s*0+(\\.0+)?\\s*$/.test(duration)) return 0;\n const match = /^\\s*(\\d+(?:\\.\\d+)?)\\s*(ms|s|m|h|d|w)\\s*$/i.exec(duration);\n const unit = match?.[2];\n const factor = unit === undefined || unit === \"M\" ? undefined : UNITS[unit.toLowerCase()];\n if (match === null || factor === undefined) {\n throw new Error(\n `not a duration: \"${duration}\" (write a number and ms, s, m, h, d or w; m is minutes)`,\n );\n }\n return Number(match[1]) * factor;\n};\n\n/**\n * These options make a runtime. A store needs a log, a blob store and an index. Give `dir`, and\n * the runtime makes the three parts in it, or give the parts. A part that you give replaces the\n * part in `dir`.\n */\nexport interface RuntimeOptions {\n /** This folder keeps the capture log (`captures/`), the bodies (`blobs/`), the index (`index.sqlite`) and the state of each host (`hosts.sqlite`). */\n readonly dir?: string | undefined;\n /** This log replaces `<dir>/captures`. */\n readonly log?: CaptureLog | undefined;\n /** This blob store replaces `<dir>/blobs`. */\n readonly blobs?: BlobStore | undefined;\n /** This index replaces `<dir>/index.sqlite`. A runtime takes any `ReadingIndex`. */\n readonly index?: ReadingIndex | undefined;\n /**\n * This store keeps the pace and the stops of each host. The default with `dir` is a SQLite file\n * (`hosts.sqlite`), so the processes on one store share the pace. With no `dir`, it is in memory.\n */\n readonly hostState?: HostStateStore | undefined;\n /** This token is the name that robots.txt rules match. The default is `PoliteCrawler`. */\n readonly token?: string | undefined;\n /** This text is the full `User-Agent` header. The default names the crawler and its page. */\n readonly userAgent?: string | undefined;\n /**\n * This address goes into the `User-Agent` header. A mailbox (`ops@example.com`) also goes into the\n * `From` header, and a URL does not. The crawler sends no contact without it.\n */\n readonly contact?: string | undefined;\n /**\n * This browser renders the pages that need JavaScript. The value `false` turns rendering off. The\n * default checks one time that `playwright-core` and its browser are installed. With no browser,\n * each page that needs JavaScript gets a note at once, with no wait.\n */\n readonly renderer?: Renderer | false | undefined;\n /** This value is the number of sites that the crawler fetches from at one time. The default is 16. */\n readonly maxActive?: number | undefined;\n /** This value is the number of pages that the browser renders at one time. The default is 2: each render holds a browser context. */\n readonly maxRenders?: number | undefined;\n /** This `undici` dispatcher sends the requests. Tests give a mock agent, and a proxy is possible. */\n readonly dispatcher?: Dispatcher | undefined;\n /** This policy controls the addresses that the crawler fetches. By default, it fetches only public addresses. */\n readonly egress?: EgressOptions | undefined;\n /** These values change the limits of the politeness policy, for each host or for some hosts. */\n readonly politeness?: PolicyOptions | undefined;\n /** This clock gives the time to the crawler and to the index. Tests give a clock that they control. */\n readonly clock?: Clock | undefined;\n /** This function gives the random numbers of the jitter. The default is `Math.random`. */\n readonly random?: (() => number) | undefined;\n /** This label goes into each capture record. */\n readonly scope?: string | undefined;\n /** With the value false, the crawler does not also obey the group of `PoliteCrawler` when its `User-Agent` links the bot page (`CrawlerOptions.obeyDefaultToken`). */\n readonly obeyDefaultToken?: boolean | undefined;\n /** This function ranks the scopes, the most private highest. A shared fetch of a robots.txt records the scope with the highest rank (`CrawlerOptions.scopeRank`). */\n readonly scopeRank?: ((scope: string) => number) | undefined;\n /** This function gets an event for each request, response, wait, stop and skip. */\n readonly onEvent?: ((event: CrawlEvent) => void) | undefined;\n /** These extractors read the captures of the recipes of an application. The recipe `page` is built in. */\n readonly extractors?: Readonly<Record<string, Extractor>> | undefined;\n}\n\n/** A runtime that a store made. It owns the parts that it made, and it closes them. */\nexport interface StoreRuntime extends CrawlRuntime {\n readonly hostState: HostStateStore;\n /** This method closes the crawler, the browser, the log, the index and the host state store that the runtime made. A second close does nothing. */\n close(): Promise<void>;\n}\n\n/** The parts of a store that can be a symbolic link to a different place only by mistake or by attack. */\nconst STORE_PARTS = [\"captures\", \"blobs\", \"index.sqlite\", \"hosts.sqlite\"];\n\n/**\n * This function makes the folder of a store for the owner only, where the system has modes. It\n * refuses a store folder of a different user, and a part of the store that is a symbolic link.\n * The store folder itself can be a link, for example to a larger disk.\n */\nconst guardStore = (dir: string): void => {\n mkdirSync(dir, { recursive: true, mode: 0o700 });\n const real = lstatSync(dir).isSymbolicLink() ? realpathSync(dir) : dir;\n const info = statSync(real);\n if (!info.isDirectory()) throw new Error(`the store ${dir} is not a folder`);\n if (\n process.platform !== \"win32\" &&\n typeof process.getuid === \"function\" &&\n info.uid !== process.getuid()\n ) {\n throw new Error(`the store ${dir} belongs to a different user: use a folder of your own`);\n }\n for (const part of STORE_PARTS) {\n let link = false;\n try {\n link = lstatSync(join(dir, part)).isSymbolicLink();\n } catch {\n // a part that is not there yet\n }\n if (link)\n throw new Error(\n `${part} in the store ${dir} is a symbolic link: the crawler does not follow it`,\n );\n }\n};\n\n/**\n * This function makes a runtime. It is the one place that puts the log, the blob store, the index,\n * the host state, the crawler and the extractors together. The function `openCrawler` uses it.\n * An application that needs the parts uses it too. The runtime and the crawler share one clock.\n */\nexport const createRuntime = (options: RuntimeOptions): StoreRuntime => {\n const dir = options.dir;\n if (\n dir === undefined &&\n (options.log === undefined || options.blobs === undefined || options.index === undefined)\n ) {\n throw new Error(\"a runtime needs dir, or a log, a blob store and an index\");\n }\n const contact = options.contact;\n if (contact !== undefined && !/^[\\t -~]*$/.test(contact)) {\n throw new Error(\n `contact \"${contact}\" has a character that an HTTP header cannot hold: use printable ASCII`,\n );\n }\n if (dir !== undefined) guardStore(dir);\n const owned: Array<() => Promise<void> | void> = [];\n const log = options.log ?? new CaptureLog(join(dir!, \"captures\"));\n const blobs = options.blobs ?? new FileBlobStore(join(dir!, \"blobs\"));\n let index = options.index;\n if (index === undefined) {\n const made = new DocIndex(join(dir!, \"index.sqlite\"));\n owned.push(() => made.close());\n index = made;\n }\n let hostState = options.hostState;\n if (hostState === undefined) {\n if (dir === undefined) hostState = new MemoryHostStateStore();\n else {\n const made = new SqliteHostStateStore(join(dir, \"hosts.sqlite\"));\n owned.push(() => made.close());\n hostState = made;\n }\n }\n const clock = options.clock ?? realClock;\n // the default browser is checked one time: with none, each page gets its note at once\n const renderer =\n options.renderer === false\n ? undefined\n : (options.renderer ?? checkedRenderer(playwrightRenderer()));\n const crawler = new Crawler({\n token: options.token ?? DEFAULT_TOKEN,\n userAgent: options.userAgent ?? defaultUserAgent(contact),\n from: contact !== undefined && isMailbox(contact) ? contact.trim() : undefined,\n log,\n blobs,\n index,\n renderer,\n maxActive: options.maxActive,\n maxRenders: options.maxRenders,\n dispatcher: options.dispatcher,\n egress: options.egress,\n politeness: options.politeness,\n hostState,\n clock,\n random: options.random,\n scope: options.scope,\n scopeRank: options.scopeRank,\n obeyDefaultToken: options.obeyDefaultToken,\n onEvent: options.onEvent,\n });\n const extractors = new Extractors();\n for (const [recipe, extractor] of Object.entries(options.extractors ?? {})) {\n extractors.register(recipe, extractor);\n }\n let closing: Promise<void> | undefined;\n return {\n crawler,\n index,\n blobs,\n captures: log,\n log,\n extractors,\n hostState,\n now: () => new Date(clock.now()).toISOString(),\n close() {\n closing ??= (async () => {\n // the crawler closes the browser and the log\n await crawler.close();\n for (const close of owned) await close();\n })();\n return closing;\n },\n };\n};\n\n/** These options open a crawler on a folder. Only `dir` is necessary. */\nexport interface OpenOptions extends Omit<RuntimeOptions, \"dir\" | \"log\" | \"blobs\" | \"index\"> {\n /** This folder keeps the capture log, the response bodies, the search index and the state of each host. */\n readonly dir: string;\n /** These agents get their robots.txt rules applied to each fetch, for example `Claude-User`. */\n readonly onBehalfOf?: readonly string[] | undefined;\n}\n\n/** These options control one fetch. */\nexport interface FetchOptions {\n /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */\n readonly render?: \"auto\" | \"never\" | \"always\" | undefined;\n /**\n * A stored copy younger than this age answers, and the crawler sends no request. The default is\n * `24h`. The value `auto` uses the freshness that the server gave. The value `0s` asks again.\n */\n readonly maxAge?: Duration | undefined;\n /** These agents get their robots.txt rules applied to this fetch, in addition to the defaults. */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** When this signal aborts, the crawler stops the fetch and the render. */\n readonly signal?: AbortSignal | undefined;\n /** This label goes into the capture records of this fetch. */\n readonly scope?: string | undefined;\n /** These labels go into the capture records of this fetch. They are not part of the capture ids. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/** These options control a search. */\nexport interface FindOptions {\n /** This value is the maximum number of results. The default is 10. */\n readonly limit?: number | undefined;\n /** This value keeps only the documents of one kind, for example `page` or `pdf`. */\n readonly kind?: string | null | undefined;\n /**\n * This value chooses the ranking. The default `exact` ranks each match with BM25. `fast` leaves\n * the common words out of the second part of the search, so a long question is faster (`RankMode`).\n */\n readonly rank?: RankMode | undefined;\n}\n\n/** A polite crawler fetches each source one time, keeps it, and answers the next questions from the disk. */\nexport interface PoliteCrawler {\n /** The runtime gives the parts to an application that needs more control. */\n readonly runtime: StoreRuntime;\n /** This method fetches a page, reads it and adds it to the index. */\n fetch(url: string, options?: FetchOptions): Promise<PageResult>;\n /**\n * This method searches the stored documents. It sends no request. A held-back document is not\n * in the results. The syntax of the query is the syntax of `parseQuery`.\n */\n find(q: string, options?: FindOptions): Promise<Found[]>;\n /** This method reads a range of characters of one stored document, the current version or the version of one capture. */\n read(doc: string, options?: ReadOptions): Promise<ReadResult>;\n /**\n * This method finds a quote in a stored document. It ignores case, spaces, typographic\n * punctuation and Unicode forms. It tells which capture holds the quote.\n */\n verify(doc: string, quote: string, options?: VerifyOptions): Promise<VerifyResult>;\n /** This method gives the versions of a stored document, the newest first. A held-back version shows no title. */\n versions(doc: string, options?: { readonly limit?: number }): Promise<DocVersion[]>;\n /**\n * This method builds the index again from the capture log and the stored bodies, with no request.\n * When the crawler rebuilt a stale index at open, the first reindex gives the counts of that\n * rebuild, unless a fetch came before it.\n */\n reindex(): Promise<ReindexResult>;\n /** This method closes the browser, the log files and the index. A second close does nothing. */\n close(): Promise<void>;\n}\n\n/**\n * This function opens a crawler on a folder. The folder keeps the capture log (`captures/`), the\n * response bodies (`blobs/`), the search index (`index.sqlite`) and the state of each host\n * (`hosts.sqlite`). The index is a projection: `reindex` makes it again from the log and the bodies.\n * At open, `catchUpIndex` rebuilds an index of an earlier version, and gives an index that is\n * behind the log the captures that it missed.\n *\n * Each method gives a promise, and none throws before its promise. After `close`, each method\n * rejects.\n */\nexport const openCrawler = (options: OpenOptions): PoliteCrawler => {\n const runtime = createRuntime(options);\n let closed = false;\n // the rebuild at open, which the first reindex() gives when no fetch came before it\n let rebuilt: ReindexResult | undefined;\n const ready: Promise<unknown> = existsSync(join(options.dir, \"captures\"))\n ? catchUpIndex(runtime).then(\n (done) => {\n if (done?.mode === \"rebuild\") {\n const { mode, ...counts } = done;\n rebuilt = mode === \"rebuild\" ? counts : undefined;\n }\n },\n () => undefined,\n )\n : Promise.resolve();\n const live = async (): Promise<void> => {\n if (closed) throw new Error(\"the crawler is closed\");\n await ready;\n if (closed) throw new Error(\"the crawler is closed\");\n };\n return {\n runtime,\n async fetch(url, fetchOptions = {}) {\n await live();\n rebuilt = undefined;\n const maxAge = fetchOptions.maxAge ?? \"24h\";\n return fetchPage(runtime, url, {\n render: fetchOptions.render ?? \"auto\",\n maxAgeMs: maxAge === \"auto\" ? \"auto\" : durationMs(maxAge),\n onBehalfOf: [...(options.onBehalfOf ?? []), ...(fetchOptions.onBehalfOf ?? [])],\n signal: fetchOptions.signal,\n scope: fetchOptions.scope,\n labels: fetchOptions.labels,\n });\n },\n async find(q, findOptions = {}) {\n await live();\n return findDocuments(runtime, q, findOptions);\n },\n async read(id, readOptions = {}) {\n await live();\n return readDocument(runtime, id, readOptions);\n },\n async verify(id, quote, verifyOptions = {}) {\n await live();\n return verifyQuote(runtime, id, quote, verifyOptions);\n },\n async versions(id, versionOptions = {}) {\n await live();\n // a held-back version shows no title, as a read of its capture does\n const held = runtime.index.get(id)?.ai_input === \"no\";\n return runtime.index\n .versions(id, versionOptions)\n .map((version) =>\n held || version.ai_input === \"no\" ? { ...version, title: \"\" } : version,\n );\n },\n async reindex() {\n await live();\n // the index that the open rebuilt is the index that a second replay of the log makes\n const done = rebuilt;\n rebuilt = undefined;\n return done ?? reindex(runtime);\n },\n async close() {\n closed = true;\n await ready;\n await runtime.close();\n },\n };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAoBA,MAAM,OAAO;;AAGb,MAAM,6BAAkC,IAAI,IAAI;CAC9C;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,YAAY,MAAc,SAC9B,KAAK,MAAM,KAAK,MAAM,GAAG,CAAC,GAAG,KAAK,MAAM,GAAG,CAAC,GAAG,IAAI;AAErD,MAAM,UAAU,UAA8B,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK;AAE7F,MAAM,UAAU,UACd,OAAO,UAAU,YAAY,UAAU,QAAQ,UAAU,QAAQ,OAAO,MAAM,IAAI,IAAI;AAExF,MAAM,UAAU,SAA6C,KAAK,IAAI,CAAC,CAAC,YAAY,KAAA,CAAS;;AAG7F,MAAM,aAAa,OAAO,SAAgC;CACxD,IAAI,QAAQ,aAAa,SAAS;CAClC,IAAI;EACF,MAAM,SAAS,MAAM,KAAK,MAAM,GAAG;EACnC,IAAI;GACF,MAAM,OAAO,KAAK;EACpB,UAAU;GACR,MAAM,OAAO,MAAM;EACrB;CACF,QAAQ,CAER;AACF;;;;;;;;;;;;;;AAeA,IAAa,gBAAb,MAAa,cAAmC;CAQnC;CAPX,0BAA2B,IAAI,IAG7B;CACF;CAEA,YACE,KACA,UAAgC,CAAC,GACjC;EAFS,KAAA,MAAA;EAGT,KAAK,OAAO,QAAQ,QAAQ;CAC9B;CAEA,IAAI,OAAkF;EACpF,MAAM,OAAO,OAAO,KAAK;EACzB,MAAM,UAAU,KAAK,QAAQ,IAAI,IAAI;EACrC,IAAI,YAAY,KAAA,GAAW,OAAO;EAClC,MAAM,OAAO,KAAK,MAAM,MAAM,KAAK,CAAC,CAAC,cAAc,KAAK,QAAQ,OAAO,IAAI,CAAC;EAC5E,KAAK,QAAQ,IAAI,MAAM,IAAI;EAC3B,OAAO;CACT;;CAGA,aAAqB,MAAM,MAAc,MAAgC;EACvE,IAAI;GACF,OAAO,OAAO,MAAM,SAAS,IAAI,CAAC,MAAM;EAC1C,QAAQ;GACN,OAAO;EACT;CACF;CAEA,MAAc,MACZ,MACA,OAC+D;EAC/D,MAAM,SAAS,SAAS,KAAK,KAAK,IAAI;EACtC,MAAM,WAAW,MAAM,OAAO,MAAM;EACpC,IAAI,UAAU,OAAO,MAAM,QAAQ,SAAS,SAAS,MAAM,QACzD,OAAO;GAAE;GAAM,SAAS;EAAK;EAC/B,MAAM,SAAS,QAAQ,MAAM;EAC7B,MAAM,MAAM,QAAQ;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EACpD,MAAM,YAAY,GAAG,OAAO,GAAG,WAAW,EAAE;EAC5C,IAAI;GACF,MAAM,SAAS,MAAM,KAAK,WAAW,MAAM,GAAK;GAChD,IAAI;IACF,MAAM,OAAO,UAAU,KAAK;IAC5B,IAAI,KAAK,MAAM,MAAM,OAAO,KAAK;GACnC,UAAU;IACR,MAAM,OAAO,MAAM;GACrB;GACA,IAAI;IACF,MAAM,OAAO,WAAW,MAAM;GAChC,SAAS,OAAO;IAEd,IAAI,CAAC,WAAW,IAAI,OAAO,KAAK,CAAC,KAAK,CAAE,MAAM,cAAc,MAAM,QAAQ,IAAI,GAC5E,MAAM;IACR,OAAO;KAAE;KAAM,SAAS;IAAK;GAC/B;GACA,IAAI,KAAK,MAAM,MAAM,WAAW,MAAM;GACtC,OAAO;IAAE;IAAM,SAAS;GAAM;EAChC,UAAU;GACR,MAAM,GAAG,WAAW,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;EAC5D;CACF;CAEA,MAAM,IAAI,MAA+C;EACvD,IAAI,CAAC,KAAK,KAAK,IAAI,GAAG,OAAO,KAAA;EAC7B,MAAM,SAAS,SAAS,KAAK,KAAK,IAAI;EACtC,IAAI;EACJ,IAAI;GACF,QAAQ,MAAM,SAAS,MAAM;EAC/B,QAAQ;GACN;EACF;EACA,IAAI,OAAO,KAAK,MAAM,MAAM;GAE1B,MAAM,OAAO,QAAQ,GAAG,OAAO,GAAG,WAAW,EAAE,SAAS,CAAC,CAAC,YAAY,KAAA,CAAS;GAC/E;EACF;EACA,OAAO,IAAI,WAAW,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;CACxE;AACF;;;;ACxHA,IAAa,gBAAb,cAAmC,MAAM;CAG5B;CACA;CAHX,OAAwB;CACxB,YACE,SACA,MACA;EACA,MAAM,WAAW,SAAS,IAAI,CAAC;EAHtB,KAAA,UAAA;EACA,KAAA,OAAA;EAGT,KAAK,OAAO;CACd;AACF;;AAGA,MAAa,cAAc,SAAiB,SAC1C,kBAAkB,QAAQ,MAAM,YAAY,KAAK,IAAI,IAAI,OAAO,IAAI,GAAG,KAAK;;;;;;AAS9E,MAAa,aAAa,SAAoC;CAC5D,MAAM,QAAQ,KAAK,MAAM,GAAG;CAC5B,IAAI,MAAM,SAAS,KAAK,MAAM,MAAM,SAAS,OAAO,IAAI,MAAM,IAAI;CAClE,IAAI,MAAM,WAAW,KAAK,MAAM,SAAS,GAAG,OAAO,KAAA;CACnD,MAAM,UAAoB,CAAC;CAC3B,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI;EACJ,IAAI,sBAAsB,KAAK,IAAI,GACjC,QAAQ,KAAK,WAAW,IAAI,IAAI,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE;OAC9D,IAAI,YAAY,KAAK,IAAI,GAAG,QAAQ,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC;OACpE,IAAI,oBAAoB,KAAK,IAAI,GAAG,QAAQ,OAAO,IAAI;OACvD,OAAO,KAAA;EACZ,QAAQ,KAAK,KAAK;CACpB;CACA,MAAM,OAAO,QAAQ,IAAI;CACzB,IAAI,QAAQ,MAAM,UAAU,QAAQ,GAAG,GAAG,OAAO,KAAA;CACjD,IAAI,QAAQ,QAAQ,IAAI,QAAQ,SAAS,OAAO,KAAA;CAChD,IAAI,OAAO;CACX,MAAM,OAAiB,CAAC;CACxB,KAAK,IAAI,QAAQ,QAAQ,QAAQ,QAAQ,GAAG,SAAS,GAAG;EACtD,KAAK,QAAQ,OAAO,GAAG;EACvB,OAAO,KAAK,MAAM,OAAO,GAAG;CAC9B;CACA,OAAO,CAAC,GAAG,SAAS,GAAG,IAAI;AAC7B;;AAGA,MAAa,aAAa,SAAoC;CAC5D,IAAI,QAAQ,KAAK,WAAW,GAAG,KAAK,KAAK,SAAS,GAAG,IAAI,KAAK,MAAM,GAAG,EAAE,IAAI;CAC7E,MAAM,OAAO,MAAM,QAAQ,GAAG;CAC9B,IAAI,SAAS,IAAI,QAAQ,MAAM,MAAM,GAAG,IAAI;CAC5C,IAAI,KAAK,KAAK,MAAM,GAAG,OAAO,KAAA;CAC9B,MAAM,YAAY,MAAM,YAAY,GAAG;CACvC,MAAM,OAAO,MAAM,MAAM,YAAY,CAAC;CACtC,IAAI,KAAK,SAAS,GAAG,GAAG;EACtB,MAAM,KAAK,UAAU,IAAI;EACzB,IAAI,OAAO,KAAA,GAAW,OAAO,KAAA;EAC7B,MAAM,OAAQ,GAAG,MAAO,IAAK,GAAG;EAChC,MAAM,MAAO,GAAG,MAAO,IAAK,GAAG;EAC/B,QAAQ,GAAG,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,KAAK,SAAS,EAAE,EAAE,GAAG,IAAI,SAAS,EAAE;CACjF;CACA,MAAM,CAAC,OAAO,IAAI,QAAQ,MAAM,MAAM,IAAI;CAC1C,MAAM,YAAY,SAChB,KAAK,WAAW,IAAI,CAAC,IAAI,KAAK,MAAM,GAAG,CAAC,CAAC,KAAK,UAAU,OAAO,SAAS,OAAO,EAAE,CAAC;CACpF,MAAM,QAAQ,SAAS,IAAI;CAC3B,MAAM,OAAO,SAAS,KAAA,IAAY,CAAC,IAAI,SAAS,IAAI;CACpD,MAAM,SAAS;EAAC,GAAG;EAAO,GAAG,MAAc,IAAI,MAAM,SAAS,KAAK,MAAM,CAAC,CAAC,KAAK,CAAC;EAAG,GAAG;CAAI;CAC3F,IAAI,OAAO,WAAW,GAAG,OAAO,KAAA;CAChC,OAAO,OAAO,SAAS,UAAU,CAAC,SAAS,GAAG,QAAQ,GAAI,CAAC;AAC7D;AAEA,MAAM,MAAM,SAAwB,UAAU,IAAI;AAClD,MAAM,MAAM,SAAwB,UAAU,IAAI;;AAGlD,MAAM,YAAY,SAAgB,QAAe,SAA0B;CACzE,IAAI,QAAQ,WAAW,OAAO,QAAQ,OAAO;CAC7C,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;EACtD,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,OAAO,QAAQ,CAAC,CAAC;EACtD,IAAI,SAAS,GAAG,OAAO;EACvB,MAAM,OAAQ,OAAS,IAAI,OAAS;EACpC,KAAK,QAAQ,SAAU,WAAW,OAAO,SAAU,OAAO,OAAO;CACnE;CACA,OAAO;AACT;AAGA,MAAM,YAA6D;CACjE;EAAC,GAAG,iBAAiB;EAAG;EAAI;CAAU;CACtC;EAAC,GAAG,eAAe;EAAG;EAAI;CAAU;CACpC;EAAC,GAAG,iBAAiB;EAAG;EAAI;CAAU;CACtC;EAAC,GAAG,SAAS;EAAG;EAAG;CAAa;CAChC;EAAC,GAAG,UAAU;EAAG;EAAG;CAAS;CAC7B;EAAC,GAAG,YAAY;EAAG;EAAI;CAAmB;CAC1C;EAAC,GAAG,WAAW;EAAG;EAAG;CAAU;CAC/B;EAAC,GAAG,aAAa;EAAG;EAAI;CAAY;CACpC;EAAC,GAAG,YAAY;EAAG;EAAI;CAAS;CAChC;EAAC,GAAG,WAAW;EAAG;EAAI;CAAU;CAChC;EAAC,GAAG,WAAW;EAAG;EAAI;CAAe;CACrC;EAAC,GAAG,aAAa;EAAG;EAAI;CAAS;CACjC;EAAC,GAAG,YAAY;EAAG;EAAI;CAAW;CAClC;EAAC,GAAG,cAAc;EAAG;EAAI;CAAe;CACxC;EAAC,GAAG,aAAa;EAAG;EAAI;CAAe;CACvC;EAAC,GAAG,WAAW;EAAG;EAAG;CAAW;CAChC;EAAC,GAAG,WAAW;EAAG;EAAG;CAAU;AACjC;AAEA,MAAM,YAA6D;CACjE;EAAC,GAAG,eAAe;EAAG;EAAK;CAAU;CACrC;EAAC,GAAG,IAAI;EAAG;EAAK;CAAa;CAC7B;EAAC,GAAG,KAAK;EAAG;EAAK;CAAU;CAC3B;EAAC,GAAG,aAAa;EAAG;EAAI;CAAU;CAClC;EAAC,GAAG,OAAO;EAAG;EAAI;CAAU;CAC5B;EAAC,GAAG,QAAQ;EAAG;EAAI;CAAU;CAC7B;EAAC,GAAG,YAAY;EAAG;EAAI;CAAe;CACtC;EAAC,GAAG,QAAQ;EAAG;EAAG;CAAc;CAChC;EAAC,GAAG,QAAQ;EAAG;EAAI;CAAY;CAC/B;EAAC,GAAG,QAAQ;EAAG;EAAI;CAAS;CAC5B;EAAC,GAAG,QAAQ;EAAG;EAAG;CAAW;AAC/B;AAEA,MAAM,YAAY,YAChB,UAAU,MAAM,CAAC,QAAQ,UAAU,SAAS,SAAS,QAAQ,IAAI,CAAC,CAAC,GAAG;AAExE,MAAM,YAAY,YAAuC;CACvD,MAAM,UAAU,MAAc,OAC5B,QAAQ,MAAM,MAAM,EAAE,CAAC,CAAC,OAAO,SAAS,SAAS,CAAC;CACpD,MAAM,YAAY,SAAqC,SAAS,QAAQ,MAAM,MAAM,OAAO,CAAC,CAAC;CAE7F,IAAI,OAAO,GAAG,EAAE,KAAK,QAAQ,QAAQ,OAAQ,QAAQ,QAAQ,KAAM,OAAO,SAAS,EAAE;CACrF,IAAI,SAAS,SAAS,GAAG,WAAW,GAAG,EAAE,GAAG,OAAO,SAAS,EAAE;CAC9D,IAAI,QAAQ,OAAO,MAAQ,QAAQ,OAAO,GAAM,OAAO,SAAS,CAAC;CACjE,MAAM,SAAS,UAAU,MAAM,CAAC,QAAQ,UAAU,SAAS,SAAS,QAAQ,IAAI,CAAC,CAAC,GAAG;CACrF,IAAI,WAAW,KAAA,GAAW,OAAO;CAEjC,IAAI,OAAO,GAAG,EAAE,GAAG,OAAO,SAAS,EAAE,KAAK;CAC1C,IAAI,QAAQ,OAAO,GAAG,OAAO;AAE/B;;;;;;AAOA,MAAa,eAAe,YAAwC;CAClE,MAAM,OAAO,UAAU,OAAO;CAC9B,IAAI,SAAS,KAAA,GAAW,OAAO,SAAS,IAAI;CAC5C,MAAM,MAAM,UAAU,OAAO;CAC7B,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,SAAS,GAAG;AACrD;AAEA,MAAM,UAAU,SAAyB,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;;AAGpF,MAAM,aAAuD;CAC3D,CAAC,oBAAoB,UAAU;CAC/B,CAAC,gCAAgC,UAAU;CAC3C,CAAC,oBAAoB,UAAU;AACjC;AAEA,MAAM,aAAa,SAAoC,UAAU,IAAI,KAAK,UAAU,IAAI;AAExF,MAAM,WAAW,OAAe,YAA4B;CAC1D,MAAM,QAAQ,MAAM,QAAQ,GAAG;CAC/B,MAAM,OAAO,UAAU,UAAU,KAAK,QAAQ,MAAM,MAAM,GAAG,KAAK,CAAC;CACnE,IAAI,SAAS,KAAA,KAAa,KAAK,WAAW,QAAQ,QAAQ,OAAO;CACjE,MAAM,OAAO,UAAU,KAAK,KAAK,SAAS,IAAI,OAAO,MAAM,MAAM,QAAQ,CAAC,CAAC;CAC3E,OACE,OAAO,UAAU,IAAI,KAAK,QAAQ,KAAK,QAAQ,KAAK,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI;AAElG;;AAGA,MAAM,aAAa,QAAuB,MAAc,YAAyC;CAC/F,IAAI,OAAO,iBAAiB,MAAM,OAAO;CACzC,MAAM,OAAO,OAAO,IAAI;CACxB,MAAM,QAAQ,YAAY,KAAA,IAAY,UAAU,IAAI,IAAI,UAAU,OAAO;CACzE,KAAK,MAAM,OAAO,OAAO,cAAc,CAAC,GAAG;EACzC,MAAM,QAAQ,OAAO,GAAG;EACxB,IAAI,UAAU,MAAM,OAAO;EAC3B,IAAI,MAAM,WAAW,IAAI,KAAK,KAAK,SAAS,MAAM,MAAM,CAAC,CAAC,GAAG,OAAO;EACpE,IAAI,UAAU,KAAA,KAAa,QAAQ,OAAO,KAAK,GAAG,OAAO;CAC3D;CACA,OAAO;AACT;;AAGA,MAAM,SAAS,SAAyB;CACtC,MAAM,QAAQ,UAAU,IAAI;CAC5B,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,MAAM,WAAW,GAAG,OAAO,MAAM,KAAK,GAAG;CAC7C,MAAM,SAAmB,CAAC;CAC1B,KAAK,IAAI,QAAQ,GAAG,QAAQ,IAAI,SAAS,GACvC,OAAO,MAAO,MAAM,UAAW,IAAK,MAAM,QAAQ,GAAA,CAAK,SAAS,EAAE,CAAC;CACrE,OAAO,OAAO,KAAK,GAAG,CAAC,CAAC,QAAQ,oBAAoB,IAAI;AAC1D;;;;;;AAOA,MAAa,iBAAiB,QAAuB,SAAqC;CACxF,IAAI,OAAO,iBAAiB,MAAM,OAAO,KAAA;CACzC,MAAM,OAAO,OAAO,IAAI;CAExB,IADgB,UAAU,IAChB,MAAM,KAAA,GAAW;EACzB,MAAM,OAAO,YAAY,IAAI;EAC7B,OAAO,SAAS,KAAA,KAAa,UAAU,QAAQ,MAAM,KAAA,CAAS,IAC1D,KAAA,IACA,WAAW,MAAM,IAAI,GAAG,IAAI;CAClC;CACA,KAAK,MAAM,CAAC,SAAS,SAAS,YAC5B,IAAI,QAAQ,KAAK,IAAI,KAAK,CAAC,UAAU,QAAQ,MAAM,KAAA,CAAS,GAAG,OAAO,WAAW,MAAM,IAAI;AAG/F;AASA,MAAM,cAAuB,MAAM,SAAS,aAAaA,OAAa,MAAM,SAAS,QAAQ;;AAG7F,MAAM,UACJ,QACA,MACA,cACsF;CACtF,MAAM,SAA0B,CAAC;CACjC,IAAI;CACJ,KAAK,MAAM,SAAS,WAAW;EAC7B,MAAM,OAAO,YAAY,MAAM,OAAO;EACtC,IAAI,SAAS,KAAA,KAAa,UAAU,QAAQ,MAAM,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK;OAC9E,YAAY,IAAI,cAAc,MAAM,MAAM,OAAO,GAAG,IAAI;CAC/D;CACA,OAAO;EACL;EACA,SAAS,OAAO,WAAW,IAAK,WAAW,IAAI,cAAc,MAAM,YAAY,IAAK,KAAA;CACtF;AACF;;;;;;AAOA,MAAa,gBACV,QAAuB,UAAmB,gBAC1C,MAAc,SAAwB,aAAa;CAClD,QAAQ,MAAM;EAAE,GAAG;EAAS,KAAK;CAAK,IAAI,OAAO,cAAc;EAC7D,IAAI,UAAU,MAAM;GAClB,SAAS,OAAO,CAAC,CAAC;GAClB;EACF;EACA,MAAM,EAAE,QAAQ,YACd,OAAO,iBAAiB,OACpB;GAAE,QAAQ;GAAW,SAAS,KAAA;EAAU,IACxC,OAAO,QAAQ,MAAM,SAAS;EACpC,IAAI,YAAY,KAAA,GAAW;GACzB,SAAS,SAAS,CAAC,CAAC;GACpB;EACF;EACA,IAAI,QAAQ,QAAQ,MAAM,SAAS,MAAM,MAAM;OAC1C,SAAS,MAAM,OAAO,EAAE,CAAE,SAAS,OAAO,EAAE,CAAE,MAAM;CAC3D,CAAC;AACH;;;;;;;;;;AAWF,MAAa,cAAc,OACzB,QACA,KACA,UAAmB,eACa;CAChC,IAAI,OAAO,iBAAiB,MAAM,OAAO,KAAA;CACzC,MAAM,SAAS,cAAc,QAAQ,IAAI,QAAQ;CACjD,IAAI,WAAW,KAAA,GAAW,OAAO;CACjC,MAAM,OAAO,OAAO,IAAI,QAAQ;CAChC,IAAI,UAAU,IAAI,MAAM,KAAA,KAAa,UAAU,QAAQ,MAAM,KAAA,CAAS,GAAG,OAAO,KAAA;CAChF,MAAM,YAAY,MAAM,IAAI,SAAkC,WAC5D,QAAQ,MAAM,EAAE,KAAK,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS,KAAK,CAAC,CACvE;CACA,IAAI,qBAAqB,OAAO,OAAO,KAAA;CACvC,KAAK,MAAM,SAAS,WAAW;EAC7B,MAAM,OAAO,YAAY,MAAM,OAAO;EACtC,IAAI,SAAS,KAAA,KAAa,CAAC,UAAU,QAAQ,MAAM,MAAM,OAAO,GAC9D,OAAO,WAAW,MAAM,MAAM,OAAO,GAAG,IAAI;CAChD;AAEF;;AAGA,MAAa,eAAe,UAA8C;CACxE,KACE,IAAI,UAAU,OAAO,QAAQ,GAC7B,YAAY,KAAA,KAAa,YAAY,QAAQ,QAAQ,GACrD,SAAS,GACT;EACA,IAAI,mBAAmB,eAAe,OAAO;EAC7C,IAAI,mBAAmB,gBAAgB;GACrC,MAAM,QAAQ,QAAQ,OAAO,MAAM,SAAkB,gBAAgB,aAAa;GAElF,IAAI,UAAU,KAAA,GAAW,OAAO;EAClC;EACA,UAAW,QAAgC;CAC7C;AAEF;;;;;;;AC/SA,MAAa,mBAAmB,UAA2B;CACzD,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO,KAAK,UAAU,KAAK,KAAK;CAElC,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,IAAI,MAAM,KAAK,SAAS,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE;CAMlE,OAAO,IAJS,OAAO,QAAQ,KAAgC,CAAC,CAC7D,QAAQ,GAAG,UAAU,SAAS,KAAA,CAAS,CAAC,CACxC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,CACtC,KAAK,CAAC,KAAK,UAAU,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG,gBAAgB,IAAI,GACrD,CAAC,CAAC,KAAK,GAAG,EAAE;AAC/B;;AAGA,MAAa,UAAU,SACrB,WAAW,QAAQ,CAAC,CAAC,OAAO,MAAM,MAAM,CAAC,CAAC,OAAO,KAAK;;;;;;AAOxD,MAAa,cAAc,UAAsC;CAC/D,MAAM,eAAe,OACnB,gBAAgB;EACd,QAAQ,MAAM;EACd,MAAM,MAAM;EACZ,aAAa,MAAM,eAAe;EAClC,SAAS,MAAM;CACjB,CAAC,CACH;CACA,OAAO;EACL,IAAI,GAAG,MAAM,OAAO,GAAG,aAAa,MAAM,GAAG,EAAE;EAC/C,QAAQ,MAAM;EACd,MAAM,MAAM;EACZ,aAAa,MAAM;EACnB,SAAS,MAAM;EACf;EACA,GAAI,MAAM,gBAAgB,KAAA,IAAY,EAAE,aAAa,MAAM,YAAY,IAAI,CAAC;EAC5E,GAAI,MAAM,gBAAgB,KAAA,IAAY,EAAE,aAAa,MAAM,YAAY,IAAI,CAAC;EAC5E,GAAI,MAAM,UAAU,KAAA,IAAY,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;EAC1D,GAAI,MAAM,WAAW,KAAA,KAAa,OAAO,KAAK,MAAM,MAAM,CAAC,CAAC,SAAS,IACjE,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,EAAE,IAC9B,CAAC;CACP;AACF;;;;;;;ACgMA,MAAa,eAAe,QAAwB,GAAG,IAAI;;AAG3D,MAAa,iBAAiB;;AAG9B,MAAa,gBAAgB;;;;;AAM7B,MAAa,wBAA2C;CACtD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,MAAM,mBAAmB;AACzB,MAAM,eAAe;AACrB,MAAM,eAAe;AACrB,MAAM,gCAAqC,IAAI,IAAI,CAAC,SAAS,QAAQ,CAAC;AACtE,MAAM,cAAc;AACpB,MAAM,sCAA2C,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC;AACnE,MAAM,qCAA0C,IAAI,IAAI;CACtD;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;CAAK;AACzD,CAAC;;AAED,MAAM,mCAAwC,IAAI,IAAI;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;AAED,MAAM,qCAA0C,IAAI,IAAI;CACtD;CACA;CACA;AACF,CAAC;;AAED,MAAM,mCAAwC,IAAI,IAAI;CAAC;CAAmB;CAAc;AAAM,CAAC;;;;;AAK/F,MAAM,mCAAwC,IAAI,IAAI,CAAC,YAAY,aAAa,CAAC;;;;;;AAOjF,MAAM,eAAe,QAA4B,gBAA6C;CAC5F,MAAM,QAAQ,eAAe,GAAA,CAAI,MAAM,GAAG,CAAC,CAAC,EAAE,CAAE,KAAK,CAAC,CAAC,YAAY;CACnE,IACE,WAAW,KAAA,KACX,OAAO,KAAK,MAAM,MACjB,SAAS,eAAe,SAAS,yBAElC,OAAO;CACT,OAAO,CAAC,OAAO,MAAM,GAAG,CAAC,CAAC,MAAM,UAAU;EACxC,MAAM,CAAC,QAAQ,IAAI,GAAG,UAAU,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC;EACxF,MAAM,IAAI,OAAO,MAAM,UAAU,MAAM,WAAW,IAAI,CAAC;EACvD,IAAI,MAAM,KAAA,KAAa,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,GAAG,OAAO;EACxD,OACE,UAAU,SACV,UAAU,QACT,MAAM,SAAS,IAAI,KAAK,KAAK,WAAW,MAAM,MAAM,GAAG,EAAE,CAAC;CAE/D,CAAC;AACH;;AAGA,MAAM,eAAe,MAA8B,gBACjD,SAAS,KAAA,KAAa,CAAC,QAAQ,KAAK,eAAe,EAAE,IACjD,CAAC,IACD,YAAY,IAAI,YAAY,QAAQ,CAAC,CAAC,OAAO,KAAK,SAAS,GAAG,KAAM,CAAC,CAAC;;AAG5E,IAAM,QAAN,MAAY;CAImB;CAH7B,OAAe;CACf,UAA8C,CAAC;CAE/C,YAAY,KAA8B;EAAb,KAAA,MAAA;CAAc;;CAG3C,MAAM,QAAQ,QAA0C;EACtD,IAAI,KAAK,OAAO,KAAK,KAAK;GACxB,KAAK,QAAQ;GACb,OAAO,KAAK,SAAS;EACvB;EACA,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,aAAmB;IACvB,MAAM,KAAK,KAAK,QAAQ,QAAQ,EAAE;IAClC,IAAI,OAAO,IAAI,KAAK,QAAQ,OAAO,IAAI,CAAC;IACxC,OAAO,OAAO,kBAAkB,QAAQ,OAAO,yBAAS,IAAI,MAAM,WAAW,CAAC;GAChF;GACA,MAAM,WAAiB;IACrB,OAAO,oBAAoB,SAAS,IAAI;IACxC,QAAQ;GACV;GACA,IAAI,OAAO,SAAS;IAClB,KAAK;IACL;GACF;GACA,KAAK,QAAQ,KAAK,EAAE;GACpB,OAAO,iBAAiB,SAAS,MAAM,EAAE,MAAM,KAAK,CAAC;EACvD,CAAC;EAED,OAAO,KAAK,SAAS;CACvB;CAEA,WAA+B;EAC7B,IAAI,QAAQ;EACZ,aAAa;GACX,IAAI,OAAO;GACX,QAAQ;GACR,MAAM,OAAO,KAAK,QAAQ,MAAM;GAChC,IAAI,SAAS,KAAA,GAAW,KAAK;QACxB,KAAK,QAAQ;EACpB;CACF;AACF;;AAGA,MAAM,kBAAkB,YACtB,QAAQ,WAAA,6PAAwB,KAChC,4EAA4E,KAAK,OAAO;;AAG1F,MAAM,iBAAiB,QAAgB,OACrC,OAAO,gBAAgB,KAAA,IACnB,+BAA+B,OAC/B,iCAAiC,OAAO,YAAY;AAE1D,MAAMC,eAAa,UACjB,iBAAiB,QAAS,MAAM,QAAQ,MAAM,OAAO,CAAC,CAAC,MAAM,KAAM,OAAO,KAAK;AAEjF,MAAM,SAAS,OACb,OAAO,SAAS,EAAE,KAAK,KAAK,IAAI,EAAE,IAAI,eAAe,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY,IAAI,KAAA;AAEpF,MAAM,YAAY,QAAgB,UAA0B;CAC1D,MAAM,MAAM,MAAM,KAAK;CACvB,OAAO,QAAQ,KAAA,IAAY,iBAAiB,WAAW,sBAAsB,IAAI,IAAI;AACvF;AAEA,MAAM,aAAa,iBACjB,IAAI,IAAI,CAAC,GAAG,uBAAuB,GAAG,YAAY,CAAC,CAAC,KAAK,SAAS,KAAK,YAAY,CAAC,CAAC;AAEvF,MAAM,eAAe,QAAwB;CAC3C,IAAI;EACF,OAAO,mBAAmB,IAAI,QAAQ,OAAO,GAAG,CAAC,CAAC,CAAC,YAAY;CACjE,QAAQ;EACN,OAAO,IAAI,YAAY;CACzB;AACF;;;;;;AAOA,MAAa,UAAU,KAAU,eAAkC,CAAC,MAAc;CAChF,MAAM,OAAO,IAAI,IAAI,IAAI,IAAI;CAC7B,KAAK,WAAW;CAChB,KAAK,WAAW;CAChB,IAAI,KAAK,OAAO,SAAS,GAAG;EAC1B,MAAM,UAAU,UAAU,YAAY;EAUtC,KAAK,SAAS,IATA,KAAK,OAChB,MAAM,CAAC,CAAC,CACR,MAAM,GAAG,CAAC,CACV,KAAK,SAAS;GACb,MAAM,SAAS,KAAK,QAAQ,GAAG;GAC/B,IAAI,WAAW,IAAI,OAAO;GAC1B,MAAM,OAAO,KAAK,MAAM,GAAG,MAAM;GACjC,OAAO,QAAQ,IAAI,YAAY,IAAI,CAAC,IAAI,GAAG,KAAK,aAAa;EAC/D,CACoB,CAAC,CAAC,KAAK,GAAG;CAClC;CACA,OAAO,KAAK;AACd;;AAGA,MAAM,gBAAgB,KAAU,iBAA6C;CAC3E,IAAI,IAAI,aAAa,MAAM,IAAI,aAAa,IAAI,OAAO;CACvD,MAAM,UAAU,UAAU,YAAY;CACtC,OAAO,IAAI,OACR,MAAM,CAAC,CAAC,CACR,MAAM,GAAG,CAAC,CACV,MACE,SAAS,KAAK,SAAS,GAAG,KAAK,QAAQ,IAAI,YAAY,KAAK,MAAM,GAAG,KAAK,QAAQ,GAAG,CAAC,CAAC,CAAC,CAC3F;AACJ;;AAGA,MAAM,WAAW,QACf,GAAG,IAAI,SAAS,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE,IAAI,IAAI,SAAS,KAAK,KAAK,IAAI,IAAI;AAEpF,MAAM,aAAa,QAAqB,GAAG,IAAI,SAAS,IAAI,QAAQ,GAAG;AAEvE,MAAM,YACH,aACA,SAAqC;CACpC,MAAM,QAAQ,QAAQ,KAAK,YAAY;CACvC,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,KAAK,IAAI,IAAI;AACnD;AAEF,IAAM,MAAN,MAAgB;CAEe;CAD7B,sBAAuB,IAAI,IAAU;CACrC,YAAY,KAA8B;EAAb,KAAA,MAAA;CAAc;CAE3C,IAAI,KAAuB;EACzB,MAAM,QAAQ,KAAK,IAAI,IAAI,GAAG;EAC9B,IAAI,UAAU,KAAA,GAAW;GACvB,KAAK,IAAI,OAAO,GAAG;GACnB,KAAK,IAAI,IAAI,KAAK,KAAK;EACzB;EACA,OAAO;CACT;CAEA,KAAK,KAAuB;EAC1B,OAAO,KAAK,IAAI,IAAI,GAAG;CACzB;CAEA,IAAI,KAAQ,OAAgB;EAC1B,KAAK,IAAI,OAAO,GAAG;EACnB,KAAK,IAAI,IAAI,KAAK,KAAK;EACvB,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,KAAK,IAAI,OAAO,KAAK,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAU;CACjF;CAEA,OAAO,KAAc;EACnB,KAAK,IAAI,OAAO,GAAG;CACrB;CAEA,OAAY;EACV,OAAO,CAAC,GAAG,KAAK,IAAI,KAAK,CAAC;CAC5B;CAEA,IAAI,OAAe;EACjB,OAAO,KAAK,IAAI;CAClB;AACF;;AAGA,IAAM,YAAN,cAAwB,MAAM;CAEjB;CADX,YACE,MACA,OACA;EACA,MAAMA,YAAU,KAAK,GAAG,EAAE,MAAM,CAAC;EAHxB,KAAA,OAAA;CAIX;AACF;;AAGA,MAAM,mBAA8B;CAClC,IAAI;CACJ,MAAM,QAAmB,IAAI,UAAU;EACrC,UAAU,OAAe,WAAW,MAAM;GACxC,IAAI,UAAU,KAAA,GAAW;IAKvB,QAHE,MAAM,UAAU,MACf,MAAM,KAAM,QAAU,MACrB,MAAM,MAAO,IAAK,MAAM,MAAO,OAAO,IAC3B,cAAc,IAAI,iBAAiB;IAClD,MAAM,GAAG,SAAS,SAAiB,MAAM,KAAK,IAAI,CAAC;IACnD,MAAM,GAAG,UAAU,UAAU,MAAM,QAAQ,KAAK,CAAC;GACnD;GACA,MAAM,MAAM,aAAa,KAAK,CAAC;EACjC;EACA,MAAM,MAAM;GACV,IAAI,UAAU,KAAA,GAAW;IACvB,KAAK;IACL;GACF;GACA,MAAM,KAAK,aAAa,KAAK,CAAC;GAC9B,MAAM,IAAI;EACZ;EACA,QAAQ,OAAO,UAAU;GACvB,OAAO,QAAQ;GACf,SAAS,KAAK;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAEA,MAAM,cAAc,WAA8B;CAChD,IAAI,WAAW,UAAU,WAAW,UAAU,OAAO,aAAa;CAClE,IAAI,WAAW,MAAM,OAAO,uBAAuB;CACnD,IAAI,WAAW,WAAW,OAAO,WAAW;CAC5C,MAAM,IAAI,UAAU,0BAAU,IAAI,MAAM,iCAAiC,OAAO,EAAE,CAAC;AACrF;;AAGA,MAAM,cAAc,UAClB,OAAO,UAAU,YACjB,UAAU,QACV,UAAU,UACT,MAAM,SAAS,iBAAiB,0BAA0B,KAAKA,YAAU,KAAK,CAAC;;;;;;;AAQlF,MAAM,WAAW,OACf,MACA,UACA,aACgG;CAChG,MAAM,WAAW,YAAY,GAAA,CAC1B,MAAM,GAAG,CAAC,CACV,KAAK,WAAW,OAAO,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAC5C,QAAQ,WAAW,OAAO,SAAS,KAAK,CAAC,iBAAiB,IAAI,MAAM,CAAC,CAAC,CACtE,QAAQ;CAEX,KAAK,GAAG,eAAe,KAAA,CAAS;CAChC,IAAI;CACJ,IAAI;EACF,WAAW,QAAQ,IAAI,UAAU;CACnC,SAAS,OAAO;EACd,KAAK,QAAQ;EACb,MAAM;CACR;CAEA,MAAM,SAAS,WAAW,IAAI;CAC9B,IAAI,WAAW;CACf,IAAI,SAAmB;CACvB,IAAI,SAAS,SAAS,GAAG;EACvB,KAAK,GAAG,SAAS,UAAkB;GACjC,YAAY,MAAM;EACpB,CAAC;EACD,KAAK,MAAM,WAAW,UAAU;GAC9B,MAAM,SAAS;GACf,OAAO,GAAG,UAAU,UAAU,QAAQ,QAAQ,KAAK,CAAC;GACpD,SAAS,OAAO,KAAK,OAAO;EAC9B;CACF;CACA,MAAM,SAAmB,CAAC;CAC1B,IAAI,OAAO;CACX,IAAI,YAAY;CAChB,IAAI,MAAM;CACV,IAAI;EACF,WAAW,MAAM,SAAS,QAAiC;GACzD,MAAM,OAAO,WAAW;GACxB,IAAI,MAAM,SAAS,MAAM;IAEvB,IAAI,OAAO,GAAG,OAAO,KAAK,MAAM,SAAS,GAAG,IAAI,CAAC;IACjD,QAAQ,KAAK,IAAI,GAAG,IAAI;IACxB,YAAY;IACZ;GACF;GACA,OAAO,KAAK,KAAK;GACjB,QAAQ,MAAM;GACd,IAAI,WAAW,QAAQ;IACrB,YAAY;IACZ;GACF;EACF;CACF,SAAS,OAAO;EACd,MAAM,YAAY,KAAK,YAAY,QAAQ,SAAS,WAAW;EAE/D,IAAI,CAAC,aAAa,OAAO,KAAK,WAAW,KAAK,GAAG;GAC/C,MAAM;GACN,YAAY;EACd,OACE,MAAM,IAAI,UAAU,YAAY,cAAc,UAAU,KAAK;CAEjE,UAAU;EACR,KAAK,QAAQ;EACb,KAAK,MAAM,WAAW,UAAU,QAAQ,QAAQ;CAClD;CAEA,MAAM,QAAQ,IAAI,WAAW,IAAI;CACjC,IAAI,KAAK;CACT,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,IAAI,OAAO,EAAE;EACnB,MAAM,MAAM;CACd;CACA,OAAO;EAAE;EAAO;EAAW;CAAI;AACjC;AAEA,MAAM,cAAc,UAAmD;CACrE,MAAM,sBAAM,IAAI,IAAoB;CACpC,KAAK,MAAM,SAAS,SAAS,GAAA,CAAI,MAAM,GAAG,GAAG;EAC3C,MAAM,SAAS,KAAK,QAAQ,GAAG;EAC/B,MAAM,QAAQ,WAAW,KAAK,OAAO,KAAK,MAAM,GAAG,MAAM,EAAA,CAAG,KAAK,CAAC,CAAC,YAAY;EAC/E,IAAI,KAAK,WAAW,GAAG;EACvB,MAAM,MAAM,WAAW,KAAK,KAAK,KAAK,MAAM,SAAS,CAAC,CAAC,CAAC,KAAK;EAC7D,IAAI,IACF,MACA,IAAI,UAAU,KAAK,IAAI,WAAW,IAAG,KAAK,IAAI,SAAS,IAAG,IAAI,IAAI,MAAM,GAAG,EAAE,IAAI,GACnF;CACF;CACA,OAAO;AACT;AAEA,MAAM,eAAe;;;;;;;AAQrB,MAAM,cACJ,QACA,QACA,WACA,cACA,UACkB;CAClB,MAAM,UAAU,WAAW,OAAO,eAAe,CAAC;CAClD,IAAI,QAAQ,IAAI,UAAU,KAAK,QAAQ,IAAI,UAAU,GAAG,OAAO,MAAM,SAAS,KAAK;CACnF,MAAM,UAAU,OAAO,KAAK,CAAC,EAAE,KAAK,KAAK;CACzC,MAAM,MAAM,QAAQ,KAAK,OAAO,IAAI,OAAO,OAAO,IAAI,MAAO;CAC7D,MAAM,OAAO,cAAc,OAAO,MAAM,CAAC,KAAK;CAC9C,IAAI;CACJ,MAAM,SAAS,QAAQ,IAAI,SAAS;CACpC,MAAM,UAAU,OAAO,SAAS;CAChC,IAAI,WAAW,KAAA,KAAa,QAAQ,KAAK,MAAM,GAAG,WAAW,OAAO,MAAM,IAAI;MACzE,IAAI,YAAY,KAAA,GAAW;EAE9B,MAAM,KAAK,cAAc,OAAO;EAChC,WAAW,OAAO,KAAA,IAAY,IAAI,KAAK;CACzC,OAAO,IAAI,mBAAmB,IAAI,MAAM,GAAG;EACzC,MAAM,WAAW,cAAc,gBAAgB,KAAA,CAAS;EACxD,IAAI,aAAa,KAAA,KAAa,WAAW,MACvC,WAAW,KAAK,IAAI,QAAQ,OAAO,YAAY,EAAE;OAC9C,IAAI,oBAAoB,IAAI,MAAM,GAAG,WAAW;CACvD;CACA,IAAI,aAAa,KAAA,GAAW,OAAO;CACnC,OAAO,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,UAAU,YAAY,IAAI,GAAG,CAAC,KAAK;AACnF;;AAGA,MAAM,aACJ,MACA,SACkB;CAClB,IAAI,SAAS,KAAA,GAAW,OAAO;CAC/B,MAAM,QAAQ,KACX,MAAM,GAAG,CAAC,CACV,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CACxC,QAAQ,SAAS,KAAK,SAAS,KAAK,CAAC,iBAAiB,IAAI,IAAI,CAAC;CAClE,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO;CAChC,IAAI,MAAM,WAAW,GAAG,OAAO;CAC/B,OAAO,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,CAAC,CACvB,KAAK,CAAC,CACN,KAAK,SAAS,GAAG,KAAK,GAAG,KAAK,SAAS,IAAI,CAAC,CAC5C,KAAK,IAAI;AACd;AAEA,MAAM,eACJ,QACA,SACY;CACZ,IAAI,WAAW,KAAA,KAAa,WAAW,MAAM,OAAO;CACpD,IAAI,WAAW,KAAK,OAAO;CAC3B,OAAO,OAAO,MAAM,IAAI,CAAC,CAAC,OAAO,SAAS;EACxC,MAAM,SAAS,KAAK,QAAQ,GAAG;EAC/B,QAAQ,KAAK,KAAK,MAAM,GAAG,MAAM,MAAM,QAAQ,KAAK,MAAM,SAAS,CAAC;CACtE,CAAC;AACH;AAEA,MAAM,sBACJ,YAEA,YAAY,KAAA,IACR,KAAA,IACA,OAAO,YACL,OAAO,QAAQ,OAAO,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,IAAI,KAAK,YAAY,CAAC,CAAC,CACxF;AAEN,MAAM,iBACJ,MACA,OACA,SACuB;CACvB,IAAI,UAAU,KAAA,KAAc,QAAQ,UAAU,QAAS,OAAO,KAAA;CAC9D,OAAO,OAAO,UAAU,YAAY,SAAS,IACzC,KAAA,IACA,mBAAmB,KAAK,MAAM,OAAO,KAAK;AAChD;AAEA,MAAM,gBAAgB,SAA0B;CAC9C;CACA,QAAQ;CACR,SAAS;CACT,WAAW;CACX,SAAS,KAAA;CACT,MAAM,KAAA;CACN,MAAM,KAAA;CACN,aAAa,KAAA;CACb,SAAS;CACT,MAAM;AACR;AAgGA,MAAM,eACJ,SACA,QACA,cAC6B;CAC7B,SAAS,QAAQ,WAAW;CAC5B;CACA,MACE,SAAS,SAAS,KAAA,IACd,OACA;EACE,MAAM,SAAS,KAAK,QAAQ;EAC5B,SAAS,SAAS,KAAK;EACvB,OAAO,SAAS,KAAK;CACvB;CACN,YAAY,SAAS,SAAS,EAAE,EAAE,QAAQ;CAC1C,GAAI,QAAQ,OAAO,gBAAgB,KAAA,IAAY,EAAE,aAAa,QAAQ,OAAO,YAAY,IAAI,CAAC;CAC9F,GAAI,QAAQ,UAAU,KAAA,IAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAChE;AAEA,MAAM,aAAgB,MAAkB,WAAoC;CAC1E,IAAI,OAAO,SAAS;EAClB,KAAK,YAAY,KAAA,CAAS;EAC1B,OAAO,QAAQ,OAAO,OAAO,MAAM;CACrC;CACA,OAAO,IAAI,SAAY,SAAS,WAAW;EACzC,MAAM,eAAqB,OAAO,OAAO,MAAM;EAC/C,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;EACvD,KAAK,MACF,UAAU;GACT,OAAO,oBAAoB,SAAS,MAAM;GAC1C,QAAQ,KAAK;EACf,IACC,UAAmB;GAClB,OAAO,oBAAoB,SAAS,MAAM;GAC1C,OAAO,KAAK;EACd,CACF;CACF,CAAC;AACH;AASA,IAAa,UAAb,MAAqB;CAyBU;CAxB7B;CACA;CACA;CACA;CACA;CACA;CACA,SAA0B,IAAI,IAAyB,IAAI;;CAE3D,OAAwB,IAAI,IAG1B,GAAM;CACR,2BAA4B,IAAI,IAA2B;CAC3D,0BAA2B,IAAI,IAAsB;CACrD,UAA2B,IAAI,gBAAgB;CAC/C,SAAiB;CACjB;;CAEA;;CAEA;;CAEA;CAEA,YAAY,SAA0C;EAAzB,KAAA,UAAA;EAC3B,IAAI,CAAC,eAAe,QAAQ,KAAK,GAC/B,MAAM,IAAI,MACR,cAAc,QAAQ,MAAM,mEAC9B;EAEF,KAAK,MAAM,CAAC,MAAM,UAAU,CAC1B,CAAC,aAAa,QAAQ,SAAS,GAC/B,CAAC,QAAQ,QAAQ,IAAI,CACvB,GACE,IAAI,UAAU,KAAA,KAAa,CAAC,YAAY,KAAK,KAAK,GAChD,MAAM,IAAI,MACR,GAAG,KAAK,IAAI,MAAM,uEACpB;EAGJ,YAAY,QAAQ,UAAU;EAC9B,KAAK,QAAQ,QAAQ,SAAS;EAC9B,KAAK,SAAS,QAAQ,UAAU,KAAK;EACrC,KAAK,SAAS,QAAQ,UAAU,CAAC;EACjC,KAAK,YAAY,IAAI,cAAc,KAAK,OAAO,QAAQ,aAAa,IAAI,EACtE,OAAO,QAAQ,UACjB,CAAC;EAED,KAAK,aACH,QAAQ,qBAAqB,SAC7B,QAAQ,MAAM,YAAY,MAAA,gBAAoB,YAAY,KAC1D,QAAQ,UAAU,SAAA,8CAAiB,IAC/B,CAAC,aAAa,IACd,CAAC;EACP,MAAM,aAAa,QAAQ,cAAc;EACzC,IAAI,CAAC,OAAO,UAAU,UAAU,KAAK,aAAa,GAChD,MAAM,IAAI,WAAW,mDAAmD,OAAO,UAAU,GAAG;EAE9F,KAAK,UAAU,IAAI,MAAM,UAAU;EACnC,KAAK,iBAAiB,QAAQ,eAAe,KAAA;EAC7C,KAAK,aACH,QAAQ,cACR,IAAI,MAAM;GAER,kBAAkB;GAClB,qBAAqB;GACrB,SAAS;GAET,SAAS;IAAE,SAAS;IAAQ,QAAQ,aAAa,KAAK,MAAM;GAAE;GAC9D,gBAAgB;GAChB,aAAa;EACf,CAAC;CACL;CAEA,MAAsB;EACpB,OAAO,IAAI,KAAK,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,YAAY;CAChD;CAEA,SAAiB,KAA4B;EAC3C,OAAO,UAAU,KAAK,QAAQ,YAAY,IAAI,QAAQ;CACxD;CAEA,KAAa,OAAyB;EACpC,IAAI;GACF,KAAK,QAAQ,UAAU,KAAK;EAC9B,QAAQ,CAER;CACF;CAEA,MAAiB,MAA8B;EAC7C,KAAK,QAAQ,IAAI,IAAI;EACrB,KAAU,WACF,KAAK,QAAQ,OAAO,IAAI,SACxB,KAAK,QAAQ,OAAO,IAAI,CAChC;EACA,OAAO;CACT;;CAGA,OAAe,MAAyC;EACtD,IAAI;GACF,OAAO,KAAK,UAAU,MAAM,aAAa,IAAI;EAC/C,QAAQ;GACN;EACF;CACF;CAEA,SAAiB,MAAc,OAAe,QAAsB;EAClE,KAAK,UAAU,MAAM,KAAK,MAAM,OAAO,MAAM;EAC7C,KAAK,KAAK;GAAE,MAAM;GAAQ;GAAM;GAAO;EAAO,CAAC;CACjD;CAEA,MAAc,OACZ,MACA,SACA,KACA,SAGI,CAAC,GACwB;EAE7B,IAAI,KAAK,QAAQ,OAAO,KAAA;EACxB,MAAM,KAAK,KAAK,IAAI;EACpB,MAAM,QAAQ,OAAO,SAAS,KAAK,QAAQ;EAC3C,MAAM,SAAS,WAAW;GACxB,QAAQ;GACR;GACA,SAAS;IAAE,QAAA;IAAwB,GAAG;IAAS,YAAY;GAAG;GAC9D,aAAa;GACb,aAAa;GACb,aAAa;GACb,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;GACvC,GAAI,OAAO,WAAW,KAAA,IAAY,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;EACjE,CAAC;EACD,MAAM,KAAK,QAAQ,IAAI,OAAO,MAAM;EACpC,OAAO,OAAO;CAChB;CAEA,MAAc,KACZ,MACA,KACA,MACA,QAAiC,CAAC,GAClC,SAGI,CAAC,GACa;EAClB,KAAK,KAAK;GAAE,MAAM;GAAQ;GAAK;EAAK,CAAC;EACrC,OAAO;GACL;GACA,QAAQ;GACR,SAAS;GACT,WAAW;GACX,SAAS,MAAM,KAAK,OAAO,WAAW;IAAE,GAAG;IAAM;IAAK;IAAM,GAAG;GAAM,GAAG,KAAK,MAAM;GACnF,MAAM,KAAA;GACN,MAAM,KAAA;GACN,aAAa,KAAA;GACb,SAAS;GACT;EACF;CACF;;CAGA,OAAe,KAAa,UAAmB,SAAiC;EAC9E,MAAM,MAAM,WAAW,GAAG;EAC1B,MAAM,MAAM,KAAK,QAAQ,MAAM,OAAO,KAAK,OAAO;EAClD,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;EAC9B,MAAM,OAAO,KAAK,KAAK,IAAI,GAAG,QAAQ,GAAG,KAAK;EAC9C,IAAI,SAAS,KAAA,KAAa,KAAK,MAAM,YAAY,IAAI,SAAS,OAAO;EACrE,MAAM,OAAO,OAAO,YAAY,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,WAAW,UAAU,KAAA,CAAS,CAAC;EAC9F,OAAO;GAAE,GAAG,KAAK;GAAO,GAAG;EAAK;CAClC;CAEA,MAAc,OAAmB,UAAyB;EACxD,MAAM,UAAU,MAAM,WAAW;EACjC,MAAM,QAAQ;GAAE,GAAG;GAAO,KAAK,WAAW,MAAM,GAAG;GAAG;EAAQ;EAC9D,KAAK,QAAQ,MAAM,UAAU,KAAK;EAClC,KAAK,KAAK,IAAI,GAAG,QAAQ,GAAG,MAAM,OAAO;GACvC,OAAO;GACP,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;EAC/C,CAAC;CACH;;CAGA,WAAmB,OAAmB,cAAqD;EACzF,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,MAAM,WAAW,QAAQ,GAAG,WAAW,MAAM,GAAG,GAAG;EAClF,IAAI,MAAM,aAAa,KAAA,KAAa,KAAK,MAAM,YAAY,MAAM,SAAS,OAAO,KAAK;EACtF,IAAI,MAAM,aAAa,KAAA,KAAa,MAAM,aAAa,MAAM,OAAO,KAAA;EACpE,IAAI;GACF,MAAM,SAAS,IAAI,IAAI,MAAM,QAAQ;GAErC,OAAO,aAAa,QAAQ,YAAY,IAAI,KAAA,IAAY,OAAO;EACjE,QAAQ;GACN;EACF;CACF;;CAGA,MAAc,UACZ,KACA,SACA,QACA,MACA,cAC2B;EAC3B,IAAI,WAAW,KAAA,GAAW,OAAO,KAAA;EACjC,MAAM,QAAQ,KAAK,OAAO,KAAK,OAAO;EACtC,IAAI,UAAU,KAAA,KAAa,CAAC,YAAY,MAAM,UAAU,IAAI,GAAG,OAAO,KAAA;EACtE,MAAM,MAAM,KAAK,MAAM,IAAI;EAC3B,MAAM,YAAY,KAAK,MAAM,MAAM,UAAU;EAC7C,IAAI,CAAC,OAAO,SAAS,SAAS,GAAG,OAAO,KAAA;EACxC,MAAM,MAAM,MAAM;EAClB,MAAM,QACJ,MAAM,gBAAgB,KAAA,KAAa,MAAM,gBAAgB,OACrD,MACA,KAAK,MAAM,MAAM,WAAW;EAClC,MAAM,QAAQ,OAAO,SAAS,KAAK,KAAK,QAAQ;EAChD,MAAM,WAAW,MAAM,UAAU,OAAO,MAAM,SAAS;EAQvD,IAAI,EALF,WAAW,SACP,QACA,YAAY,CAAC,oBAAoB,IAAI,MAAM,MAAM,IAC/C,SAAS,MAAM,SACf,MAAM,SACF,OAAO,KAAA;EACnB,IAAI,UAAU;GACZ,MAAM,OAAO,KAAK,WAAW,OAAO,YAAY;GAChD,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;GAC/B,OAAO;IACL,SAAS;KACP;KACA,QAAQ,MAAM;KACd,SAAS;KACT,WAAW;KACX,SAAS,MAAM;KACf,MAAM,KAAA;KACN,MAAM,KAAA;KACN,aAAa,KAAA;KACb,SAAS;KACT,MAAM,YAAY,MAAM,YAAY,OAAO,IAAI,IAAI,IAAI,GAAG,YAAY;KACtE,WAAW,MAAM;IACnB;IACA;GACF;EACF;EACA,IAAI,MAAM,SAAS,QAAQ,MAAM,SAAS,OAAO,MAAM,UAAU,KAAK,OAAO,KAAA;EAC7E,MAAM,OAAO,MAAM,KAAK,QAAQ,MAAM,IAAI,MAAM,IAAI;EACpD,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;EAC/B,OAAO,EACL,SAAS;GACP;GACA,QAAQ,MAAM;GACd,SAAS;GACT,WAAW;GACX,SAAS,MAAM;GACf,MAAM,MAAM;GACZ;GACA,aAAa,MAAM,gBAAgB,KAAA;GACnC,SAAS;IAAE,UAAU,MAAM,YAAY;IAAS,KAAK,CAAC;GAAE;GACxD,MAAM,MAAM,cAAc,OAAO,qBAAqB,KAAA;GACtD,GAAI,MAAM,cAAc,OAAO,EAAE,WAAW,KAAK,IAAI,CAAC;GACtD,WAAW,MAAM;EACnB,EACF;CACF;CAEA,eAAuB,UAAgD;EACrE,MAAM,UAAkC,CAAC;EAEzC,KAAK,MAAM,CAAC,MAAM,UAAU,OAAO,QAAQ,SAAS,WAAW,CAAC,CAAC,GAAG;GAClE,MAAM,QAAQ,KAAK,YAAY;GAC/B,IAAI,CAAC,iBAAiB,IAAI,KAAK,GAAG,QAAQ,SAAS;EACrD;EACA,QAAQ,YAAY,SAAS,UAAU,QAAQ,aAAa;EAC5D,QAAQ,gBAAgB,KAAK,QAAQ;EACrC,QAAQ,qBAAqB;EAC7B,IAAI,KAAK,QAAQ,SAAS,KAAA,GAAW,QAAQ,UAAU,KAAK,QAAQ;EACpE,OAAO;CACT;;;;;;CAOA,OAAe,MAAc,QAA0B,MAAsB;EAC3E,MAAM,QAAQ,KAAK,UAAU;EAC7B,MAAM,MAAM,KAAK,MAAM,IAAI;EAE3B,MAAM,UADO,UAAU,QAAQ,KAAK,IACjB,CAAC,CAAC;EACrB,MAAM,UAAU,IAAY,WAA2B;GACrD,IAAI,KAAK,SAAS,KAAK,KAAK;IAAE,MAAM;IAAQ;IAAM;IAAI;GAAO,CAAC;GAC9D,OAAO;EACT;EACA,IAAI,KAAK,WAAW,KAAA,GAAW;GAC7B,MAAM,OAAO,MAAM,KAAK,MAAM;GAC9B,OAAO;EACT;EACA,MAAM,UAAU,KAAK;EACrB,IAAI,YAAY,kBAAkB,YAAY,eAAe;GAC3D,MAAM,EAAE,UAAU,MAAM,OAAO,MAAM,MAAM;GAC3C,MAAM,QAAQ,KAAK;GACnB,IAAI,UAAU,KAAA,KAAa,QAAQ,OAAO,iBAAiB;IAEzD,KAAK,SACH,MACA,MAAM,KAAK,IAAI,OAAO,WAAW,GACjC,iBAAiB,OAAO,KAAK,MAAM,QAAQ,GAAI,CAAC,EAAE,GACpD;IACA,OAAO;GACT;GACA,OAAO,OACL,KAAK,IACH,SACA,SAAS,cAAc,OAAO,OAAO,eAAe,OAAO,cAAc,KAAK,MAAM,CACtF,GACA,UAAU,KAAA,IAAY,YAAY,aACpC;EACF;EACA,IAAI,YAAY,SAAS;GACvB,IAAI,cAAc,KAAK,MAAM,GAAG;IAC9B,MAAM,OAAO,MAAM,SAAS;IAC5B,OAAO;GACT;GACA,MAAM,EAAE,aAAa,MAAM,OAAO,MAAM,SAAS;GACjD,IAAI,YAAY,OAAO,aACrB,KAAK,SAAS,MAAM,MAAM,OAAO,eAAe,GAAG,OAAO,QAAQ,EAAE,mBAAmB;GACzF,OAAO,OACL,KAAK,IACH,SACA,cAAc,UAAU,OAAO,eAAe,OAAO,cAAc,KAAK,MAAM,CAChF,GACA,SACF;EACF;EACA,IAAI,YAAY,aAAa,KAAK,SAAS,MAAM,MAAM,OAAO,iBAAiB,OAAO;OACjF,IAAI,YAAY,oBACnB,KAAK,SAAS,MAAM,MAAM,OAAO,eAAe,OAAO;OACpD,IAAI,YAAY,aAAa,CAAC,KAAK,YACtC,KAAK,SAAS,MAAM,MAAM,OAAO,eAAe,OAAO;EACzD,MAAM,OAAO,MAAM,SAAS;EAC5B,OAAO;CACT;;;;;CAMA,cAAc,KAAa,MAAoB;EAC7C,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,GAAG;EACtB,QAAQ;GACN;EACF;EACA,MAAM,SAAS,KAAK,SAAS,MAAM;EACnC,KAAK,SACH,QAAQ,MAAM,GACd,KAAK,MAAM,IAAI,IAAI,OAAO,iBAC1B,cAAc,KAAK,EACrB;CACF;;;;;;;CAQA,MAAc,MACZ,MACA,OAGsE;EACtE,MAAM,QAAQ;GAAE,OAAO,KAAK;GAAO,QAAQ,KAAK;EAAO;EACvD,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,KAAK,GAAG;EAC3B,QAAQ;GACN,OAAO,EACL,QAAQ,EAAE,SAAS,MAAM,KAAK,KAAK,KAAK,MAAM,KAAK,KAAK,mBAAmB,CAAC,GAAG,KAAK,EAAE,EACxF;EACF;EACA,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GACpC,OAAO,EACL,QAAQ,EACN,SAAS,MAAM,KAAK,KAClB,KAAK,MACL,OAAO,QAAQ,KAAK,YAAY,GAChC,sBACA,CAAC,GACD,KACF,EACF,EACF;EAGF,OAAO,OAAO;EACd,MAAM,MAAM,OAAO,QAAQ,KAAK,YAAY;EAC5C,MAAM,QAAQ,MAAc,WAC1B,KAAK,KAAK,KAAK,MAAM,KAAK,MAAM,QAAQ,KAAK;EAC/C,MAAM,SAAS,OAAO,MAAc,YAAsC,EACxE,QAAQ,EAAE,SAAS,MAAM,KAAK,MAAM,MAAM,EAAE,EAC9C;EACA,IAAI,KAAK,YAAY,KAAA,GAAW,OAAO,OAAO,KAAK,OAAO;EAC1D,MAAM,UAAU,cAAc,KAAK,QAAQ,OAAO,QAAQ;EAC1D,IAAI,YAAY,KAAA,GAAW,OAAO,OAAO,OAAO;EAChD,MAAM,SAAS;GAAC,KAAK,QAAQ;GAAO,GAAI,KAAK,cAAc,CAAC;GAAI,GAAG,KAAK;EAAU;EAClF,MAAM,WAAW,OAAO,MAAM,UAAU,CAAC,eAAe,KAAK,CAAC;EAC9D,IAAI,KAAK,SAAS,UAAU,aAAa,KAAA,GACvC,OAAO,OAAO,+BAA+B,SAAS,EAAE;EAC1D,MAAM,OAAO,QAAQ,MAAM;EAC3B,MAAM,SAAS,KAAK,SAAS,MAAM;EACnC,MAAM,OAAO,GAAG,OAAO,WAAW,OAAO;EAGzC,IAAI;EACJ,IAAI,KAAK,SAAS,QAAQ;GACxB,UACE,OAAO,SAAS,IAAI,MAAM,KAAK,UAAU,QAAQ,QAAQ,IAAI,IAAI,KAAK,YAAY,MAAM;GAC1F,IAAI,YAAY,KAAA,GAAW;IACzB,MAAM,WAAW,OAAO,QAAQ,QAAQ,QAAQ,IAAI;IACpD,IAAI,CAAC,SAAS,SACZ,OAAO,OAAO,cAAc,QAAQ,QAAQ,SAAS,MAAM,KAAK,QAAQ,KAAK,GAAG,EAC9E,cAAc,YAAY,SAAS,QAAQ,QAAQ,EACrD,CAAC;GAEL;EACF;EACA,MAAM,SAAS,MAAM,KAAK,OAAO,GAAG;EACpC,IAAI,WAAW,KAAA,GAAW,OAAO,EAAE,QAAQ,OAAO;EAElD,MAAM,OAAO,KAAK,UAAU,OAAO,IAAI;EACvC,IAAI,SAAS,KAAA,GAAW,OAAO,OAAO,SAAS,KAAK,QAAQ,KAAK,KAAK,CAAC;EACvE,MAAM,QAAQ,MAAM,QAAQ;GAAE;GAAQ;GAAK;GAAM;GAAQ;GAAQ;EAAK,CAAC;EACvE,IAAI,UAAU,KAAA,GAAW,OAAO,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE;EAE7D,IAAI,cAAiC,CAAC;EACtC,IAAI,oBAAoB;EACxB,IAAI;EACJ,IAAI,KAAK,SAAS,QAAQ;GACxB,YAAY,MAAM,KAAK,UAAU,QAAQ,QAAQ,IAAI;GACrD,MAAM,WAAW,OAAO,QAAQ,QAAQ,QAAQ,IAAI;GACpD,QAAQ,YAAY,SAAS,QAAQ,QAAQ;GAC7C,IAAI,CAAC,SAAS,SACZ,OAAO,OAAO,cAAc,QAAQ,QAAQ,SAAS,MAAM,KAAK,QAAQ,KAAK,GAAG,EAC9E,cAAc,MAChB,CAAC;GAEH,cAAc,WAAW,QAAQ,QAAQ,QAAQ,IAAI;GACrD,oBAAoB,SAAS,qBAAqB;EACpD;EACA,IAAI,KAAK,QAAQ,OAAO,EAAE,QAAQ,EAAE,SAAS,aAAa,GAAG,EAAE,EAAE;EACjE,OAAO,EACL,UAAU;GACR;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;EACF,EACF;CACF;;CAGA,MAAM,MAAM,UAA0C;EACpD,IAAI,KAAK,QAAQ,OAAO,aAAa,SAAS,GAAG;EACjD,OAAO,KAAK,MAAM,KAAK,OAAO,QAAQ,CAAC;CACzC;CAEA,MAAc,OAAO,UAA0C;EAC7D,MAAM,SACJ,SAAS,WAAW,KAAA,IAChB,KAAK,QAAQ,SACb,YAAY,IAAI,CAAC,SAAS,QAAQ,KAAK,QAAQ,MAAM,CAAC;EAC5D,IAAI,MAAM,SAAS;EACnB,IAAI,UAAU,SAAS;EACvB,IAAI;EACJ,IAAI,eAAe,eAAe;EAClC,IAAI;GACF,MAAM,QAAQ,IAAI,IAAI,GAAG;GACzB,SAAS,MAAM;GACf,eAAe,KAAK,SAAS,KAAK,CAAC,CAAC;EACtC,QAAQ,CAER;EACA,IAAI;GACF,KAAK,IAAI,MAAM,IAAK,OAAO,GAAG;IAC5B,MAAM,OAAO,MAAM,KAAK,IAAI;KAAE,GAAG;KAAU;KAAK;IAAQ,GAAG,MAAM;IACjE,IAAI,KAAK,SAAS,KAAA,GAAW,OAAO,KAAK;IACzC,IAAI,OAAO,cACT,OAAO;KACL,GAAG,KAAK;KACR,SAAS;KACT,MAAM,aAAa,OAAO,YAAY,EAAE;IAC1C;IAEF,MAAM,OAAO,IAAI,IAAI,KAAK,IAAI;IAE9B,IAAI,KAAK,WAAW,QAAQ,UAAU,mBAAmB,OAAO;IAChE,SAAS,KAAK;IACd,MAAM,KAAK;GACb;EACF,SAAS,OAAO;GACd,IAAI,KAAK,QAAQ,OAAO,aAAa,SAAS,GAAG;GACjD,OAAO;IACL,GAAG,aAAa,SAAS,GAAG;IAC5B,SAAS;IACT,MAAM,kBAAkBA,YAAU,KAAK;GACzC;EACF;CACF;;CAGA,IAAY,UAAwB,QAAoC;EAEtE,IAAI,SAAS,WAAW,KAAA,GAAW,OAAO,KAAK,UAAU,UAAU,MAAM;EACzE,MAAM,MAAM,KAAK,UAAU;GACzB,WAAW,SAAS,IAAI,MAAM,GAAG,CAAC,CAAC,MAAM,SAAS,GAAG;GACrD,SAAS;GACT,SAAS;GACT,SAAS,UAAU;GACnB,SAAS,WAAW;GACpB,SAAS,gBAAgB;GACzB,SAAS,iBAAiB;GAC1B,SAAS,YAAY;GACrB,SAAS,YAAY;GACrB,SAAS,cAAc;GACvB,SAAS,OAAO;GAChB,SAAS,SAAS;GAClB,SAAS,UAAU;EACrB,CAAC;EACD,MAAM,UAAU,KAAK,SAAS,IAAI,GAAG;EACrC,IAAI,YAAY,KAAA,GAAW,OAAO;EAClC,MAAM,OAAO,KAAK,UAAU,UAAU,MAAM,CAAC,CAAC,cAAc,KAAK,SAAS,OAAO,GAAG,CAAC;EACrF,KAAK,SAAS,IAAI,KAAK,IAAI;EAC3B,OAAO;CACT;CAEA,YAAoB,QAAwC;EAC1D,MAAM,QAAQ,KAAK,OAAO,KAAK,UAAU,MAAM,CAAC;EAChD,OAAO,UAAU,KAAA,KAAa,KAAK,MAAM,IAAI,IAAI,MAAM,KAAK,MAAM,QAC9D,MAAM,QACN,KAAA;CACN;;;;;;CAOA,UACE,QACA,QACA,QAGI,CAAC,GACmB;EACxB,MAAM,MAAM,UAAU,MAAM;EAC5B,MAAM,MAAM,KAAK,MAAM,IAAI;EAC3B,MAAM,SAAS,KAAK,OAAO,IAAI,GAAG;EAClC,IAAI,WAAW,KAAA,KAAa,MAAM,OAAO,KAAK,OAAO,OAAO;GAE1D,IAAI,OAAO,UAAU,KAAA,GAAW,KAAK,KAAK,OAAO,SAAS,KAAK;GAC/D,OAAO,OAAO;EAChB;EACA,MAAM,UAAmB;GAAE,OAAO,KAAA;GAAW,QAAQ,KAAA;EAAU;EAC/D,KAAK,KAAK,SAAS,KAAK;EACxB,MAAM,QAAqB;GACzB,IAAI;GACJ;GACA,OAAO,OAAO;GACd,SAAS,QAAQ,QAAQ;IAAE,QAAQ,kBAAkB,cAAc;IAAG,SAAS,KAAA;GAAU,CAAC;GAC1F,OAAO,KAAA;GACP,QAAQ,QAAQ,UAAU;GAC1B,MAAM,QAAQ;EAChB;EACA,MAAM,UAAU,KAAK,YAAY,QAAQ,QAAQ,OAAO,CAAC,CACtD,OAAO,WAAoB;GAC1B,QAAQ,kBAAkB,sBAAsBA,YAAU,KAAK,GAAG;GAClE,SAAS,KAAA;GACT,cAAc,KAAA;EAChB,EAAE,CAAC,CACF,KAAK,OAAO,SAAiC;GAC5C,IAAI,UAAyB;IAAE,QAAQ,KAAK;IAAQ,SAAS,KAAK;GAAQ;GAC1E,IAAI,KAAK,OAAO,gBAAgB,KAAA,GAAW;IACzC,MAAM,SAAS;IACf,MAAM,OAAO;KAAE;KAAS,IAAI,KAAK,MAAM,IAAI;IAAE;GAC/C,OAAO;IACL,MAAM,UAAU;IAChB,MAAM,UAAU,cACd,MAAM,QACN,OAAO,eACP,OAAO,kBACP,KAAK,MACP;IACA,MAAM,QAAQ,KAAK,IACjB,OAAO,kBACP,KAAK,IAAI,KAAK,gBAAgB,GAAG,OAAO,CAC1C;IACA,MAAM,OACJ,MAAM,SAAS,KAAA,KAAa,KAAK,MAAM,IAAI,IAAI,MAAM,KAAK,MAAM,OAAO,gBACnE,MAAM,KAAK,UACX,MAAM,KAAK,aAAa,QAAQ,MAAM;IAC5C,IAAI,SAAS,KAAA,GAAW,UAAU;KAAE,GAAG;KAAM,OAAO,KAAK,OAAO;IAAY;GAC9E;GACA,MAAM,QAAQ;GACd,OAAO;EACT,CAAC;EACH,KAAK,OAAO,IAAI,KAAK,KAAK;EAC1B,OAAO,MAAM;CACf;;;;;;CAOA,MAAM,WAAW,KAId;EACD,MAAM,SAAS,IAAI,IAAI,GAAG;EAC1B,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GACpC,OAAO;GAAE,QAAQ,kBAAkB,oBAAoB;GAAG,SAAS,KAAA;EAAU;EAC/E,OAAO,KAAK,UAAU,QAAQ,KAAK,SAAS,MAAM,CAAC;CACrD;;CAGA,MAAc,aACZ,QACA,QACoC;EACpC,IAAI;GACF,MAAM,MAAM,KAAK,OAAO,OAAO,IAAI,IAAI,eAAe,OAAO,MAAM,CAAC,CAAC;GACrE,IAAI,QAAQ,KAAA,KAAa,IAAI,SAAS,QAAQ,IAAI,SAAS,OAAO,IAAI,UAAU,KAC9E,OAAO,KAAA;GACT,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,MAAM,IAAI,UAAU,IAAI,OAAO,eAAe,OAAO,KAAA;GACjF,MAAM,OAAO,MAAM,KAAK,QAAQ,MAAM,IAAI,IAAI,IAAI;GAClD,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;GAC/B,MAAM,SAAS,mBAAmB,KAAK,IAAI,cAAc,IAAI;GAC7D,OAAO,OAAO,gBAAgB,KAAA,IAAY;IAAE;IAAQ,SAAS,IAAI;GAAQ,IAAI,KAAA;EAC/E,QAAQ;GACN;EACF;CACF;;CAGA,KACE,SACA,OAIM;EACN,MAAM,QAAQ,MAAM,SAAS,KAAK,QAAQ;EAC1C,MAAM,OAAO,KAAK,QAAQ;EAC1B,IAAI,QAAQ,UAAU,KAAA,GAAW,QAAQ,QAAQ;OAC5C,IAAI,UAAU,KAAA,KAAa,SAAS,KAAA,KAAa,KAAK,KAAK,IAAI,KAAK,QAAQ,KAAK,GACpF,QAAQ,QAAQ;EAClB,IAAI,MAAM,WAAW,KAAA,GAAW,QAAQ,SAAS;GAAE,GAAG,MAAM;GAAQ,GAAG,QAAQ;EAAO;CACxF;CAEA,MAAc,YACZ,QACA,QACA,SACqB;EACrB,IAAI,MAAM,IAAI,IAAI,eAAe,OAAO,MAAM;EAC9C,MAAM,eACJ,QACA,SACA,kBACgB;GAChB,QAAQ,kBAAkB,MAAM;GAChC;GACA;EACF;EACA,KAAK,IAAI,MAAM,GAAG,OAAO,OAAO,cAAc,OAAO,GAAG;GACtD,MAAM,OAAO,MAAM,KAAK,UACtB;IACE,KAAK,IAAI;IACT,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,eAAe,OAAO;IAEtB,IAAI,QAAQ;KACV,OAAO,QAAQ;IACjB;IACA,IAAI,SAAS;KACX,OAAO,QAAQ;IACjB;GACF,GACA,KAAK,QAAQ,MACf;GACA,MAAM,SAAS,KAAK;GACpB,IAAI,OAAO,YAAY,YAAY;IACjC,IAAI,KAAK,SAAS,KAAA,GAChB,OAAO,YAAY,OAAO,QAAQ,sCAAsC,OAAO,OAAO;IACxF,MAAM,IAAI,IAAI,KAAK,IAAI;IACvB;GACF;GACA,IAAI,OAAO,YAAY,aAAa,OAAO,WAAW,GACpD,OAAO,YAAY,OAAO,QAAQ,eAAe,OAAO,OAAO;GAEjE,IAAI,OAAO,YAAY,WAAW,OAAO,UAAU,OAAO,OAAO,SAAS,KACxE,OAAO,YAAY,OAAO,QAAQ,8BAA8B,OAAO,OAAO;GAEhF,IAAI,OAAO,MAAM,WAAW,iCAAiC,MAAM,MACjE,OAAO,YAAY,OAAO,MAAM,OAAO,OAAO;GAChD,IAAI,OAAO,YAAY,gBACrB,OAAO,OAAO,SAAS,KAAA,IACnB,YAAY,kCAAkC,OAAO,OAAO,IAC5D;IACE,QAAQ,mBAAmB,KAAK,OAAO,eAAe,MAAM,OAAO,IAAI;IACvE,SAAS,OAAO;IAChB,cAAc,KAAA;GAChB;GAUN,OAAO;IAAE,QAPP,OAAO,UAAU,OAAO,OAAO,SAAS,MACpC,mBACE,OAAO,QACP,OAAO,eAAe,MACtB,OAAO,wBAAQ,IAAI,WAAW,CAChC,IACA,iBAAiB,OAAO,QAAQ,KAAA,CAAS;IAC9B,SAAS,OAAO;IAAS,cAAc,KAAK;GAAa;EAC5E;EACA,OAAO,YAAY,aAAa,OAAO,OAAO,YAAY,EAAE,aAAa,KAAA,CAAS;CACpF;CAEA,MAAc,UAAU,UAAwB,QAAoC;EAClF,MAAM,OAAO;GACX,QAAQ,SAAS;GACjB,MAAM,SAAS;GACf,GAAI,SAAS,MAAM,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC;EAC9C;EACA,MAAM,eAAe,SAAS,gBAAgB,CAAC;EAC/C,MAAM,OAAO,KAAK,eAAe,QAAQ;EAEzC,MAAM,QAAQ;GACZ,IAAI,QAAQ;IACV,OAAO,SAAS;GAClB;GACA,IAAI,SAAS;IACX,OAAO,SAAS;GAClB;EACF;EACA,MAAM,YAAY,MAAM,KAAK,MAAM;GACjC,WAAW;GACX,KAAK,SAAS;GACd,MAAM,SAAS;GACf;GACA;GACA,SACE,cAAc,iBAAiB,SAAS,eAAe,KAAK,KAC5D,cAAc,YAAY,SAAS,UAAU,KAAK,KAClD,cAAc,YAAY,SAAS,UAAU,IAAI;GACnD,YAAY,SAAS;GACrB,OAAO,SAAS;GAChB,QAAQ,SAAS;GACjB,SAAS,QAAQ,KAAK,UAAU,KAAK,SAAS,SAAS,UAAU,MAAM,YAAY;EACrF,CAAC;EACD,IAAI,YAAY,WAAW,OAAO,UAAU;EAC5C,MAAM,EAAE,QAAQ,KAAK,MAAM,QAAQ,OAAO,aAAa,mBAAmB,SACxE,UAAU;EACZ,MAAM,WAAW,SAAS,YAAY,OAAO;EAC7C,MAAM,QAAQ,eAAkC;GAC9C,MAAM,SAAS;GACf;GACA;GACA,eAAe,SAAS;EAC1B;EACA,MAAM,aAAa,SAAS,WAAW;EAEvC,IAAI;GACF,OAAO,MAAM,KAAK,UAAU,IAC1B,MACA,YAAkC;IAEhC,MAAM,QAAQ,MAAM,KAAK,UAAU,KAAK,SAAS,SAAS,UAAU,MAAM,YAAY;IACtF,IAAI,UAAU,KAAA,GAAW,OAAO;KAAE,OAAO;KAAO,SAAS;IAAE;IAC3D,MAAM,SAAS,KAAK,OAAO,GAAG;IAE9B,MAAM,SACJ,WAAW,KAAA,KAAa,OAAO,cAAc,QAAQ,YAAY,OAAO,UAAU,IAAI,IAClF,SACA,KAAA;IACN,IAAI;IACJ,IAAI,QAAQ,SAAS,OAAO,QAAQ,OAAO,gBACzC,SAAS,MAAM,KAAK,QAAQ,MAAM,IAAI,OAAO,IAAI;IACnD,MAAM,UAAkC,EAAE,GAAG,KAAK;IAGlD,MAAM,OACJ,WAAW,KAAA,KAAa,WAAW,KAAA,IAC/B;KAAE,MAAM;KAAO,cAAc;IAAM,IACnC,iBACE,aAAa,KAAA,IAAY,KAAK,OAAO,IAAI,GACzC;KAAE,MAAM,QAAQ,OAAO,IAAI;KAAG,cAAc,QAAQ,OAAO,aAAa;IAAE,GAC1E,KAAK,MAAM,IAAI,CACjB;IACN,IAAI,KAAK,QAAQ,QAAQ,MAAM,QAAQ,mBAAmB,OAAO;IACjE,IAAI,KAAK,gBAAgB,QAAQ,eAC/B,QAAQ,uBAAuB,OAAO;IACxC,MAAM,aAAa,CAAC,iBAAiB,mBAAmB,CAAC,CAAC,QACvD,SAAS,QAAQ,UAAU,KAAA,CAC9B;IACA,MAAM,UAAU,KAAK,MAAM,IAAI;IAC/B,MAAM,UAAU,YAAY,QAAQ,OAAO,gBAAgB;IAC3D,MAAM,QAAQ,YAAY,IAAI,CAAC,SAAS,MAAM,CAAC;IAC/C,MAAM,kBAA2B,SAAS,QAAQ,YAAY;IAC9D,MAAM,cAAc,QAAgB,UAClC,QAAQ,UACJ,GAAG,OAAO,oBAAoB,OAAO,KAAK,MAAM,OAAO,mBAAmB,GAAI,CAAC,EAAE,MACjF,GAAG,OAAO,IAAIA,YAAU,KAAK;IAEnC,KAAK,KAAK;KAAE,MAAM;KAAW;KAAK;KAAM,WAAW;IAAO,CAAC;IAC3D,IAAI;IACJ,IAAI;KACF,WAAW,MAAM,QAAQ,QAAQ;MAC/B,QAAQ;MACR;MACA,YAAY,KAAK;MACjB,QAAQ;KACV,CAAC;IACH,SAAS,OAAO;KACd,MAAM,SAAS,YAAY,KAAK;KAChC,IAAI,WAAW,KAAA,GACb,OAAO;MAAE,OAAO,EAAE,SAAS,MAAM,KAAK,OAAO,OAAO,EAAE;MAAG,SAAS;KAAE;KACtE,IAAI,KAAK,QAAQ,OAAO;MAAE,OAAO,EAAE,SAAS,aAAa,GAAG,EAAE;MAAG,SAAS;KAAE;KAC5E,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;KACrC,MAAM,OAAO,UAAU,IAAI,4BAA4B,WAAW,eAAe,KAAK;KACtF,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;MACxC,SAAS;MACT,QAAQ;MACR,cAAc,KAAA;MACd,MAAM,KAAK,SAAS;MACpB;MACA,GAAI,UAAU,IAAI,EAAE,QAAQ,UAAmB,IAAI,CAAC;KACtD,CAAC;KACD,KAAK,KAAK;MAAE,MAAM;MAAY;MAAK;MAAM,QAAQ;MAAG,SAAS;MAAS;KAAU,CAAC;KACjF,MAAM,UAAU,MAAM,KAAK,OACzB,WACA;MACE,GAAG;MACH;MACA,QAAQ;MACR,SAAS;MACT;MACA,YAAY;MACZ,GAAI,QAAQ,EAAE,cAAc,MAAM,IAAI,CAAC;KACzC,GACA,KACA,KACF;KACA,OAAO;MACL,OAAO,EACL,SAAS;OACP;OACA,QAAQ;OACR,SAAS;OACT,WAAW;OACX;OACA,MAAM,KAAA;OACN,MAAM,KAAA;OACN,aAAa,KAAA;OACb,SAAS;OACT;OACA,WAAW,IAAI,KAAK,OAAO,CAAC,CAAC,YAAY;OACzC,GAAG;MACL,EACF;MACA;KACF;IACF;IAEA,MAAM,SAAS,SACb,SAAS,OACX;IACA,MAAM,SAAS,KAAK,MAAM,IAAI,IAAI;IAClC,MAAM,aAAa,SAAS,SAAS,YAAY,QAAQ,KAAK,MAAM,IAAI,CAAC;IACzE,IAAI,UAAmB,WAAW;IAClC,IAAI,OAAO,WAAW;IACtB,IAAI;IACJ,IAAI;IACJ,IAAI,YAAY;IAChB,IAAI,MAAM;IACV,IAAI;IACJ,IAAI;IACJ,IAAI,YAAY,MAAM;KACpB,IAAI;MACF,MAAM,OAAO,MAAM,SAAS,SAAS,MAAM,OAAO,kBAAkB,GAAG,QAAQ;MAC/E,OAAO,KAAK;MACZ,YAAY,KAAK;MACjB,MAAM,KAAK;MACX,IAAI,KAAK,KACP,OAAO,6DAA6D,OAAO,KAAK,MAAM,MAAM,EAAE;WAC3F,IAAI,WAAW,OAAO,gBAAgB,OAAO,QAAQ,EAAE;KAC9D,SAAS,OAAO;MACd,IAAI,KAAK,QAAQ,OAAO;OAAE,OAAO,EAAE,SAAS,aAAa,GAAG,EAAE;OAAG,SAAS;MAAE;MAC5E,UAAU;MACV,IAAI,UAAU,GAAG;OACf,OAAO;OACP,SAAS;MACX,OAEE,OAAO,WADQ,iBAAiB,aAAa,MAAM,SAAS,WACjC,qBAAqB,eAAe,KAAK;KAExE;KACA,IAAI,SAAS,KAAA,GACX,IAAI;MACF,QAAQ,MAAM,KAAK,QAAQ,MAAM,IAAI,IAAI,EAAA,CAAG;KAC9C,SAAS,OAAO;MAEd,UAAU;MACV,OAAO,eAAeA,YAAU,KAAK;MACrC,OAAO,KAAA;MACP,SAAS;KACX;IAEJ,OACE,IAAI;KACF,MAAM,SAAS,SAAS,MAAM,KAAA,GAAW,gBAAgB;IAC3D,QAAQ,CAER;IAEF,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;IACrC,IAAI,YAAY,gBAAgB;KAC9B,IAAI,WAAW,KAAA,KAAa,QAAQ,MAAM;MACxC,OAAO;MACP,OAAO,OAAO;KAChB,OAAO,OAAO;IAChB;IACA,IAAI;IACJ,IAAI,YAAY,YAAY;KAC1B,MAAM,MAAM,OAAO,UAAU;KAC7B,IAAI,QAAQ,KAAA,GAAW,OAAO;UAE5B,IAAI;MACF,MAAM,WAAW,IAAI,IAAI,KAAK,MAAM;MACpC,OAAO,SAAS;MAChB,WAAW,OAAO,UAAU,YAAY;MACxC,OAAO,YAAY;KACrB,QAAQ;MACN,UAAU;MACV,OAAO;KACT;IAEJ;IAEA,MAAM,UAAU,YAAY,kBAAkB,WAAW,KAAA;IACzD,IAAI,UAAU,YAAY,aAAa,QAAQ,YAAY,MAAM,OAAO,cAAc,CAAC,CAAC;IACxF,IAAI,SAAS;KAEX,MAAM,SACJ,QAAQ,aAAa,QAAQ,OAAO,aAAa,OAC7C,OACA,QAAQ,aAAa,UACnB,QAAQ,WACR,OAAO;KACf,UAAU;MAAE,GAAG;MAAS,UAAU;KAAO;IAC3C;IACA,MAAM,cACJ,OAAO,cAAc,MAAM,UAAW,OAAO,gBAAgB,KAAA,IAAa,KAAA;IAE5E,MAAM,UACJ,SAAS,SAAS,SAClB,YAAY,QACZ,YAAY,SAAS,QAAQ,WAAW;IAC1C,IAAI,SAAS;KACX,MAAM,OAAO,mBAAmB,eAAe,GAAG,4BAA4B,SAAS,UAAU,GAAG;KACpG,OAAO,SAAS,KAAA,IAAY,OAAO,GAAG,KAAK,IAAI;IACjD;IACA,MAAM,OAAO,OAAO,MAAM,MAAM,UAAU,OAAO,OAAO,SAAS;IACjE,MAAM,eACJ,OAAO,eAAe,MAAM,UAAU,OAAO,gBAAgB,SAAS;IACxE,MAAM,YAAY,KAAK,MAAM,IAAI;IACjC,MAAM,YAAY,WAChB,QACA,SAAS,YACT,WACA,cACA,OAAO,uBACT;IACA,MAAM,UACJ,OAAO,MAAM,MAAM,KAAA,IACf,UAAU,OAAO,MAAM,GAAG,IAAI,IAC9B,UACG,OAAO,YAAY,OACpB;IACR,MAAM,eAAe,OAAO,YAC1B;KAAC;KAAiB;KAAW;KAAO;KAAQ;IAAM,CAAC,CAAC,SAAS,SAAS;KACpE,MAAM,QAAQ,OAAO,IAAI;KACzB,OAAO,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,CAAC,KAAK,QAAQ,KAAK,GAAG,GAAG,KAAK,CAAC;IACpE,CAAC,CACH;IACA,KAAK,KAAK;KACR,MAAM;KACN;KACA;KACA,QAAQ,SAAS;KACjB;KACA;IACF,CAAC;IACD,MAAM,UAAU,MAAM,KAAK,OACzB,UAAU,YAAY,WACtB;KACE,GAAG;KACH;KACA,QAAQ,SAAS;KACjB;KACA,YAAY;KACZ,SAAS;KACT,cAAc,eAAe;KAC7B,OAAO,MAAM,UAAU;KACvB,MAAM,QAAQ;KACd;KACA,eAAe;KACf,GAAI,WAAW,SAAS,IAAI,EAAE,WAAW,IAAI,CAAC;KAC9C,GAAI,UAAU,EAAE,YAAY,OAAO,QAAQ,IAAI,CAAC;KAChD,QAAQ,SAAS,SAAS,SAAS,YAAY;KAC/C,GAAI,UAAU,KAAA,IAAY,EAAE,cAAc,MAAM,IAAI,CAAC;KACrD;KACA,GAAI,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;KACvC,GAAI,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;KAC3B,GAAI,UAAU,EAAE,SAAS,KAAK,IAAI,CAAC;KACnC,GAAG;KACH,GAAI,cAAc,OAAO,EAAE,aAAa,UAAU,IAAI,CAAC;KACvD,GAAI,YAAY,OAAO,EAAE,UAAU,QAAQ,IAAI,CAAC;KAChD,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;KAC7C,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;IACvC,GACA,KACA,KACF;IACA,MAAM,aAAa,IAAI,KAAK,SAAS,CAAC,CAAC,YAAY;IAEnD,IACE,YAAY,KAAA,MACX,YAAY,QAAQ,YAAY,mBACjC,SAAS,KAAA,KACT,CAAC,OACD,CAAC,SAED,KAAK,MAAM;KACT;KACA;KACA;KAEA,MAAM,YAAY,OAAO;KACzB,eAAe,YAAY,OAAO;KAClC,cAAc,eAAe;KAC7B,YAAY;KAEZ,QAAQ,UAAW,OAAO,WAAW,MAAM,MAAM,OAAO,SAAU,SAAS;KAC3E,UAAU,QAAQ;KAClB,WAAW,YAAY,OAAO,KAAA;KAC9B,aAAa;KACb,UAAU;IACZ,CAAC;IAEH,IACE,YAAY,KAAA,KACZ,YAAY,cACZ,SAAS,KAAA,MACR,oBAAoB,IAAI,SAAS,UAAU,KAAK,cAAc,OAE/D,KAAK,MACH;KACE;KACA;KACA,MAAM;KACN,MAAM;KACN,eAAe;KACf,cAAc;KACd,YAAY;KACZ,QAAQ,SAAS;KACjB,UAAU;KACV,UAAU,YAAY;KACtB,aAAa;KACb,UAAU;IACZ,GACA,IACF;IAGF,MAAM,SACJ,WAAW,WAAW,KAAK,YAAY,KAAA,KAAa,aAChD,KAAA,IACA,YAAY,iBACV,iBACA,YAAY,QAAQ,SAAS,KAAA,KAAa,CAAC,MACzC,SAAS,QAAQ,OACf,SACA,YACF,KAAA;IACV,IAAI,WAAW,KAAA,GACb,IAAI;KACF,KAAK,UAAU,MAAM,iBACnB,MACA;MACE,MAAM,QAAQ,qBAAqB,KAAA;MACnC,cAAc,QAAQ,yBAAyB,KAAA;MAC/C;KACF,GACA,SACF;IACF,QAAQ,CAER;IAEF,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;KACxC;KACA,QAAQ,SAAS;KACjB,cAAc,WAAW;KACzB,MAAM,KAAK,SAAS;KACpB;KACA,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;IAC3C,CAAC;IACD,OAAO;KACL,OAAO;MACL,SAAS;OACP;OACA,QAAQ,SAAS;OACjB;OACA,WAAW;OACX;OACA;OACA;OACA;OACA;OACA;OACA,GAAI,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;OACvC,WAAW;OACX,GAAG;MACL;MACA;MACA,cAAc,WAAW;KAC3B;KACA;IACF;GACF,GACA,MACF;EACF,SAAS,OAAO;GACd,IAAI,iBAAiB,aACnB,OAAO,EAAE,SAAS,MAAM,KAAK,SAAS,MAAM,QAAQ,MAAM,KAAK,CAAC,EAAE;GACpE,IAAI,KAAK,QAAQ,OAAO,EAAE,SAAS,aAAa,GAAG,EAAE;GACrD,IAAI,SAAS,QAAQ,YAAY,MAC/B,OAAO,EAAE,SAAS,MAAM,KAAK,yBAAyB,EAAE;GAC1D,MAAM;EACR;CACF;;;;;;CAOA,gBACE,MACA,QACA,QACA,QACA,OACmC;EACnC,MAAM,aAAa,UAAU,IAAI;EACjC,MAAM,0BAAU,IAAI,IAAyC;EAC7D,MAAM,eAAwB;GAC5B,MAAM,WAAW;GACjB,OAAO;EACT;EACA,OAAO,OAAO,QAAkC;GAC9C,IAAI,OAAO,SAAS,OAAO;GAC3B,IAAI;GACJ,IAAI;IACF,SAAS,IAAI,IAAI,GAAG;GACtB,QAAQ;IACN,OAAO,OAAO;GAChB;GACA,IACE,OAAO,aAAa,WACpB,OAAO,aAAa,WACpB,OAAO,aAAa,UAEpB,OAAO;GACT,MAAM,MAAM,cAAc,IAAI,OAAO,QAAQ;GAC7C,IAAI,CAAC,OAAO,OAAO,aAAa,SAAS,OAAO,aAAa,QAAQ,OAAO,OAAO;GACnF,IAAI,QAAQ,QAAQ,IAAI,OAAO,QAAQ;GACvC,IAAI,UAAU,KAAA,GAAW;IACvB,QAAQ,YAAY,KAAK,QAAQ,MAAM;IACvC,QAAQ,IAAI,OAAO,UAAU,KAAK;GACpC;GACA,IAAK,MAAM,UAAW,KAAA,GAAW,OAAO,OAAO;GAC/C,IAAI,KAAK;IACP,MAAM,QAAQ,UAAU,MAAM,MAAM,aAAa,SAAS,KAAK,YAAY,MAAM,CAAC,EAAE;IACpF,IACE,UAAU,KAAA,KACV,CAAC,OAAO,OAAO,QAAQ,GAAG,OAAO,WAAW,OAAO,QAAQ,CAAC,CAAC,SAE7D,OAAO,OAAO;GAClB;GACA,MAAM,WAAW;GACjB,IAAI,MAAM,MAAM,OAAO,IAAI,MAAM,MAAM,IAAI,QAAQ,MAAM,CAAC;GAC1D,OAAO;EACT;CACF;;;;;;CAOA,MAAc,aACZ,KACA,MACA,QACA,QACsE;EACtE,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,IAAI,GAAG;EACtB,QAAQ;GACN;EACF;EACA,OAAO,OAAO;EACd,IAAI,OAAO,SAAS,KAAK,MAAM,OAAO,KAAA;EACtC,MAAM,MAAM,OAAO,MAAM;EACzB,MAAM,QAAQ,qBAAqB;EACnC,IAAI,CAAC,cAAc,IAAI,OAAO,QAAQ,GACpC,OAAO;GAAE,MAAM,GAAG,MAAM;GAAgC;EAAI;EAC9D,MAAM,SAAS,MAAM,YAAY,KAAK,QAAQ,MAAM;EACpD,IAAI,WAAW,KAAA,GAAW,OAAO;GAAE,MAAM,GAAG,MAAM,IAAI;GAAU;EAAI;EACpE,MAAM,QAAQ,UAAU,MAAM,MAAM,UAAU,IAAI,IAAI,SAAS,KAAK,YAAY,MAAM,CAAC,EAAE;EACzF,IAAI,UAAU,KAAA,GAAW;GACvB,MAAM,WAAW,OAAO,OAAO,QAAQ,GAAG,OAAO,WAAW,OAAO,QAAQ;GAC3E,IAAI,CAAC,SAAS,SACZ,OAAO;IACL,MAAM,GAAG,MAAM,IAAI,cAAc,OAAO,SAAS,MAAM,KAAK,QAAQ,KAAK;IACzE;GACF;EACJ;CAEF;;;;;;;;;;;;;CAcA,MAAM,OAAO,UAA2C;EACtD,IAAI,KAAK,QAAQ,OAAO,aAAa,SAAS,GAAG;EACjD,OAAO,KAAK,MACV,KAAK,WAAW,QAAQ,CAAC,CAAC,OAAO,UAC/B,KAAK,SACD,aAAa,SAAS,GAAG,IACzB;GACE,GAAG,aAAa,SAAS,GAAG;GAC5B,SAAS;GACT,MAAM,kBAAkBA,YAAU,KAAK;EACzC,CACN,CACF;CACF;CAEA,MAAc,WAAW,UAA2C;EAClE,MAAM,OAAO;GACX,QAAQ,SAAS;GACjB,MAAM;GACN,UAAU;GACV,GAAI,SAAS,MAAM,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC;EAC9C;EACA,MAAM,QAAQ;GAAE,OAAO,SAAS;GAAO,QAAQ,SAAS;EAAO;EAC/D,MAAM,WAAW,KAAK,QAAQ;EAC9B,MAAM,YAAY,MAAM,KAAK,MAC3B;GACE,WAAW;GACX,KAAK,SAAS;GACd,MAAM;GACN;GACA,cAAc,CAAC;GACf,SAAS,cAAc,YAAY,SAAS,UAAU,IAAI;GAC1D,YAAY,SAAS;GACrB,OAAO,SAAS;GAChB,QAAQ,SAAS;GACjB,QAAQ,OAAO,QAAQ;IACrB,MAAM,SAAS,MAAM,KAAK,UAAU,KAAK,YAAY,SAAS,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,WAAW,KAAA,KAAa,OAAO,QAAQ,SAAS,KAAA,GAAW,OAAO,KAAA;IACtE,OAAO,EAAE,SAAS;KAAE,GAAG,OAAO;KAAS;KAAK,aAAa;IAAc,EAAE;GAC3E;EACF,GACA,OAAO,EAAE,QAAQ,WAAW;GAC1B,IAAI,aAAa,KAAA,GAAW,OAAO,KAAK,qCAAqC;GAC7E,IAAI,KAAK,cAAc,KAAA,GAAW,OAAO,KAAK,KAAK,SAAS;GAE5D,MAAM,WAAW,MAAM,YAAY,KAAK,QAAQ,MAAM;GACtD,OAAO,aAAa,KAAA,IAAY,KAAA,IAAY,KAAK,QAAQ;EAC3D,CACF;EACA,IAAI,YAAY,WAAW,OAAO,UAAU,OAAO;EACnD,MAAM,EAAE,QAAQ,KAAK,MAAM,QAAQ,QAAQ,OAAO,mBAAmB,SACnE,UAAU;EACZ,MAAM,UAAU,UAAU,SAAS;EACnC,MAAM,SACJ,SAAS,WAAW,KAAA,IAChB,KAAK,QAAQ,SACb,YAAY,IAAI,CAAC,SAAS,QAAQ,KAAK,QAAQ,MAAM,CAAC;EAC5D,MAAM,QAAQ;GAAE,SAAS;GAAG,SAAS;GAAG,uBAAO,IAAI,IAAY;EAAE;EACjE,MAAM,eAAe,KAAK,gBAAgB,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,KAAK;EACvF,MAAM,QAAQ,eAAkC;GAC9C,MAAM;GACN;GACA;EACF;EAGA,IAAI;EACJ,IAAI;GACF,UAAU,MAAM,KAAK,QAAQ,QAAQ,MAAM;EAC7C,QAAQ;GACN,IAAI,KAAK,QAAQ,OAAO,aAAa,GAAG;GACxC,OAAO,KAAK,yBAAyB;EACvC;EACA,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,KAAK,UAAU,IAC1B,MACA,YAAwC;IACtC,MAAM,UAAU,KAAK,MAAM,IAAI;IAC/B,KAAK,KAAK;KAAE,MAAM;KAAW;KAAK;KAAM,WAAW;IAAU,CAAC;IAC9D,IAAI;KACF,MAAM,WAAW,MAAM,UACrB,SAAU,OAAO,OAAO,MAAM;MAC5B,WAAW,KAAK,QAAQ;MACxB,WAAW,OAAO;MAClB,GAAI,KAAK,QAAQ,SAAS,KAAA,IACtB,EAAE,SAAS,EAAE,MAAM,KAAK,QAAQ,KAAK,EAAE,IACvC,CAAC;MACL;MACA,cAAc,OAAO;MACrB;KACF,CAAC,GACD,MACF;KACA,MAAM,UAAU,SACd,SAAS,QAAQ,KAAK,YAAY;KACpC,MAAM,aACJ,SAAS,WAAW,OAChB;MAAE,SAAS;MAAM,cAAc,KAAA;MAAW,MAAM,KAAA;KAAU,IAC1D,SAAS,SAAS,QAAQ,QAAQ,KAAK,MAAM,IAAI,CAAC;KACxD,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;KACrC,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;MACxC,SAAS,WAAW;MACpB,QAAQ,SAAS,UAAU;MAC3B,cAAc,WAAW;MACzB,MAAM,KAAK,SAAS;MACpB,YAAY;KACd,CAAC;KACD,OAAO;MAAE,OAAO;OAAE,MAAM;OAAY;OAAU;OAAY;MAAU;MAAG;KAAQ;IACjF,SAAS,OAAO;KACd,MAAM,YAAY,KAAK,MAAM,IAAI,IAAI;KACrC,IAAI,KAAK,QAAQ,OAAO;MAAE,OAAO,EAAE,MAAM,SAAS;MAAG,SAAS;KAAE;KAChE,IAAI,SAAS,QAAQ,YAAY,MAC/B,OAAO;MACL,OAAO;OAAE,MAAM;OAAa;MAAU;MACtC,SAAS,UAAU,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;KACtC;KAGF,MAAM,UACJ,iBAAiB,gBACb,QACA,eAAeA,YAAU,KAAK,CAAC,IAC7B,IAAI,cAAc,WAAW,8BAA8BA,YAAU,KAAK,GAAG,IAC7E,KAAA;KACR,IAAI,YAAY,KAAA,GAAW;MACzB,IAAI,QAAQ,WAAW,WAAW,KAAK,YAAY,QAAQ;MAE3D,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;OACxC,SAAS;OACT,QAAQ;OACR,cAAc,KAAA;OACd,MAAM,KAAK,CAAC;OACZ,YAAY;OACZ,QAAQ;MACV,CAAC;MACD,OAAO;OACL,OAAO;QACL,MAAM;QACN,QAAQ,QAAQ;QAChB,MAAM,QAAQ;QACd,KAAK,QAAQ;QACb;OACF;OACA;MACF;KACF;KACA,MAAM,UAAU,KAAK,OAAO,MAAM,QAAQ;MACxC,SAAS;MACT,QAAQ;MACR,cAAc,KAAA;MACd,MAAM,KAAK,SAAS;MACpB,YAAY;KACd,CAAC;KACD,OAAO;MACL,OAAO;OAAE,MAAM;OAAU,MAAM,kBAAkBA,YAAU,KAAK;OAAK;MAAU;MAC/E;KACF;IACF;GACF,GACA,MACF;EACF,SAAS,OAAO;GACd,IAAI,iBAAiB,aAAa,OAAO,KAAK,SAAS,MAAM,QAAQ,MAAM,KAAK,CAAC;GACjF,IAAI,KAAK,QAAQ,OAAO,aAAa,GAAG;GACxC,IAAI,SAAS,QAAQ,YAAY,MAAM,OAAO,KAAK,yBAAyB;GAC5E,MAAM;EACR,UAAU;GACR,QAAQ;EACV;EAEA,IAAI,KAAK,SAAS,YAAY,KAAK,QAAQ,OAAO,aAAa,GAAG;EAClE,MAAM,cAAc;GAAE,SAAS,MAAM;GAAS,SAAS,MAAM;GAAS,OAAO,CAAC,GAAG,MAAM,KAAK;EAAE;EAC9F,IAAI,KAAK,SAAS,WAAW;GAC3B,KAAK,KAAK;IACR,MAAM;IACN;IACA;IACA,QAAQ;IACR,SAAS,KAAK,WAAW,aAAa,aAAa;IACnD,WAAW,KAAK;GAClB,CAAC;GACD,IAAI,KAAK,WAAW,YAAY;IAC9B,MAAM,QACJ,KAAK,QAAQ,KAAA,IACT,KAAA,IACA,MAAM,KAAK,aAAa,KAAK,KAAK,QAAQ,QAAQ,QAAQ,MAAM;IACtE,OAAO,KAAK,OAAO,QAAQ,KAAK,MAAM;KACpC,cAAc;KACd;KACA,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,MAAM,IAAI;IACxD,CAAC;GACH;GACA,MAAM,UAAU,MAAM,KAAK,OACzB,WACA;IACE,GAAG;IACH;IACA,QAAQ;IACR,SAAS;IACT,MAAM,KAAK;IACX,YAAY,KAAK;IACjB,cAAc;IACd;GACF,GACA,KACA,KACF;GACA,OAAO;IAAE,GAAG,aAAa,GAAG;IAAG,SAAS;IAAY;IAAS,MAAM,KAAK;IAAM,GAAG;GAAM;EACzF;EACA,IAAI,KAAK,SAAS,YAAY,KAAK,SAAS,aAAa;GACvD,MAAM,OAAO,KAAK,SAAS,WAAW,KAAK,OAAO;GAClD,KAAK,KAAK;IACR,MAAM;IACN;IACA;IACA,QAAQ;IACR,SAAS;IACT,WAAW,KAAK;GAClB,CAAC;GAgBD,OAAO;IACL;IACA,QAAQ;IACR,SAAS;IACT,WAAW;IACX,SAAA,MApBoB,KAAK,OACzB,WACA;KACE,GAAG;KACH;KACA,QAAQ;KACR,SAAS;KACT;KACA,YAAY,KAAK;KACjB,cAAc;KACd;IACF,GACA,KACA,KACF;IAOE,MAAM,KAAA;IACN,MAAM,KAAA;IACN,aAAa,KAAA;IACb,SAAS;IACT;IACA,GAAG;GACL;EACF;EAEA,MAAM,EAAE,UAAU,YAAY,cAAc;EAC5C,MAAM,UAAU,SAAqC,SAAS,QAAQ,KAAK,YAAY;EAEvF,IAAI,cAAc;EAClB,IAAI,eAAe;EACnB,IAAI;GACF,cAAc,IAAI,IAAI,SAAS,QAAQ;GACvC,YAAY,OAAO;EACrB,QAAQ;GACN,cAAc;EAChB;EACA,MAAM,WAAW,OAAO,WAAW;EACnC,MAAM,YAAY,GAAG,YAAY,WAAW,YAAY;EACxD,IAAI,YAAY,SAAS,OAAO,MAAM;GACpC,MAAM,QAAQ,qBAAqB;GACnC,IAAI,CAAC,cAAc,IAAI,YAAY,QAAQ,GACzC,OAAO,KAAK,GAAG,MAAM,gCAAgC,EAAE,WAAW,SAAS,CAAC;GAC9E,MAAM,UAAU,MAAM,YAAY,KAAK,QAAQ,WAAW;GAC1D,IAAI,YAAY,KAAA,GAAW,OAAO,KAAK,GAAG,MAAM,IAAI,WAAW,EAAE,WAAW,SAAS,CAAC;GACtF,IAAI,UAAU,WAAW,MAAM,UAAU,MAAM,GAC7C,eAAe,MAAM,KAAK,UAAU,aAAa,KAAK,SAAS,WAAW,GAAG,QAAQ;GAEvF,MAAM,gBAAgB,OAAO,aAAa,QAAQ,QAAQ,SAAS;GACnE,IAAI,CAAC,cAAc,SACjB,OAAO,KACL,GAAG,MAAM,IAAI,cAAc,aAAa,QAAQ,cAAc,MAAM,KAAK,QAAQ,KAAK,KACtF;IACE,WAAW;IACX,cAAc,YAAY,cAAc,QAAQ,aAAa;GAC/D,CACF;EAEJ;EACA,IAAI,UAAmB,WAAW;EAClC,IAAI,OAAO,WAAW;EACtB,IAAI;EACJ,IAAI;EACJ,IAAI,YAAY,MAAM;GACpB,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,SAAS,IAAI;GAC7C,IAAI;IACF,QAAQ,MAAM,KAAK,QAAQ,MAAM,IAAI,IAAI,EAAA,CAAG;GAC9C,SAAS,OAAO;IACd,UAAU;IACV,OAAO,eAAeA,YAAU,KAAK;IACrC,OAAO,KAAA;GACT;EACF;EACA,MAAM,UAAU,YACd,WAAW,aAAa,QAAQ,QAAQ,SAAS,GACjD,QACA,YAAY,SAAS,IAAI,CAC3B;EACA,MAAM,SAAS,SAAS,UAAU;EAClC,MAAM,YAAY,KAAK,MAAM,IAAI;EACjC,MAAM,YAAY,WAChB,QACA,QACA,WACA,OAAO,eAAe,GACtB,OAAO,uBACT;EACA,KAAK,KAAK;GAAE,MAAM;GAAY;GAAK;GAAM;GAAQ;GAAS;EAAU,CAAC;EACrE,MAAM,UAAU,MAAM,KAAK,OACzB,WACA;GACE,GAAG;GACH;GACA,WAAW;GACX;GACA;GACA,YAAY;GACZ,cAAc;GACd,OAAO,MAAM,UAAU;GACvB,MAAM,QAAQ;GACd,QAAQ;GACR,cAAc;GACd;GACA;GACA,GAAI,cAAc,OAAO,EAAE,aAAa,UAAU,IAAI,CAAC;GACvD,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;EACvC,GACA,KACA,KACF;EACA,MAAM,aAAa,IAAI,KAAK,SAAS,CAAC,CAAC,YAAY;EACnD,IAAI,YAAY,KAAA,KAAa,SAAS,KAAA,GACpC,KAAK,MAAM;GACT;GACA,SAAS;GACT;GACA;GACA,MAAM;GACN,eAAe;GACf,cAAc;GACd,YAAY;GACZ;GACA,UAAU,QAAQ;GAClB,aAAa;EACf,CAAC;EAEH,OAAO;GACL;GACA;GACA;GACA,WAAW;GACX;GACA;GACA;GACA,aAAa;GACb;GACA;GACA,WAAW;GACX,GAAG;EACL;CACF;;;;;;CAOA,WAAW,MAAqB;EAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;EAClD,KAAK,UAAU,MAAM,IAAI;EACzB,KAAK,MAAM,OAAO,KAAK,OAAO,KAAK,GAAG;GACpC,MAAM,QAAQ,KAAK,OAAO,KAAK,GAAG;GAGlC,KADE,OAAO,OAAO,OAAO,gBAAgB,KAAA,KAAa,OAAO,OAAO,UAAU,KAAA,OACxD,SAAS,KAAA,KAAa,IAAI,SAAS,KAAK,MAAM,IAAI,KAAK,OAAO,OAAO,GAAG;EAC9F;CACF;;;;;;CAOA,QAAuB;EACrB,IAAI,KAAK,gBAAgB,KAAA,GAAW;GAClC,KAAK,SAAS;GACd,KAAK,QAAQ,sBAAM,IAAI,MAAM,gBAAgB,CAAC;GAC9C,KAAK,eAAe,YAAY;IAC9B,MAAM,QAAQ,WAAW,KAAK,OAAO;IACrC,MAAM,KAAK,QAAQ,UAAU,MAAM;IACnC,MAAM,KAAK,QAAQ,IAAI,QAAQ;IAC/B,IAAI,KAAK,gBAAgB,MAAM,KAAK,WAAW,MAAM;GACvD,EAAA,CAAG;EACL;EACA,OAAO,KAAK;CACd;AACF;;;ACjjFA,MAAM,OAAO;AACb,MAAM,aAAa;AACnB,MAAM,YAAY;AAClB,MAAM,QAAQ;AACd,MAAM,YACJ;;AAEF,MAAM,8BAA2C,IAAI,IAAI;CACvD,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,MAAQ,EAAI;CACb,CAAC,OAAQ,EAAI;CACb,CAAC,OAAQ,EAAI;AACf,CAAC;AAED,MAAM,gBAAgB,SAA0B,SAAS,MAAS,QAAQ,KAAQ,QAAQ;;;;;AAM1F,MAAa,YAAY,MAAc,SAA2B;CAChE,IAAI,QAAQ,IAAI,YAAY,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;CAC1D,IAAI,SAAS,IAAI,WAAW,MAAM,MAAM;CACxC,IAAI,OAAO,IAAI,WAAW,MAAM,MAAM;CACtC,IAAI,OAAO;CACX,IAAI,QAAQ;CACZ,MAAM,QAAQ,SAAuB;EACnC,IAAI,OAAO,QAAQ,MAAM,QAAQ;EACjC,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,GAAG,OAAO,IAAI;EACrD,MAAM,YAAY,IAAI,YAAY,MAAM;EACxC,UAAU,IAAI,MAAM,SAAS,GAAG,IAAI,CAAC;EACrC,MAAM,aAAa,IAAI,WAAW,MAAM;EACxC,WAAW,IAAI,OAAO,SAAS,GAAG,IAAI,CAAC;EACvC,MAAM,WAAW,IAAI,WAAW,MAAM;EACtC,SAAS,IAAI,KAAK,SAAS,GAAG,IAAI,CAAC;EACnC,QAAQ;EACR,SAAS;EACT,OAAO;CACT;CACA,MAAM,QAAQ,MAAc,MAAc,OAAqB;EAC7D,KAAK,CAAC;EACN,MAAM,QAAQ;EACd,OAAO,QAAQ;EACf,KAAK,QAAQ;EACb,QAAQ;CACV;CACA,MAAM,aAAa,MAAc,OAAqB;EACpD,IAAI,OAAO;EACX,KAAK,IAAM,MAAM,EAAE;EACnB,QAAQ;CACV;CACA,MAAM,SAAS,KAAK;CACpB,IAAI,QAAQ;CACZ,OAAO,QAAQ,QAAQ;EACrB,MAAM,OAAO,KAAK,WAAW,KAAK;EAClC,MAAM,OAAO,QAAQ,IAAI,SAAS,KAAK,WAAW,QAAQ,CAAC,IAAI;EAE/D,IAAI,OAAO,OAAQ,OAAO,KAAO;GAC/B,IAAI,aAAa,IAAI,GAAG,UAAU,OAAO,QAAQ,CAAC;QAC7C;IACH,KAAK,QAAQ,MAAQ,QAAQ,KAAO,OAAO,KAAK,MAAM,OAAO,QAAQ,CAAC;IACtE,QAAQ;GACV;GACA,SAAS;GACT;EACF;EAEA,MAAM,QAAQ,KAAK,YAAY,KAAK;EACpC,IAAI,MAAM,SAAS,QAAQ,QAAS,IAAI;EACxC,OAAO,MAAM,QAAQ;GACnB,MAAM,OAAO,KAAK,YAAY,GAAG;GACjC,IAAI,CAAC,KAAK,KAAK,OAAO,cAAc,IAAI,CAAC,GAAG;GAC5C,OAAO,OAAO,QAAS,IAAI;EAC7B;EACA,MAAM,UAAU,KAAK,MAAM,OAAO,GAAG;EACrC,MAAM,QAAQ,OAAO,cAAc,KAAK;EACxC,IAAI,UAAU,KAAK,KAAK,KAAK,QAAQ,QAAQ,MAAM,QAAQ;GACzD,QAAQ;GACR;EACF;EACA,IAAI,MAAM,KAAK,KAAK,GAAG;GACrB,UAAU,OAAO,GAAG;GACpB,QAAQ;GACR;EACF;EACA,IAAI,SACF,SAAS,UACL,QAAQ,UAAU,MAAM,CAAC,CAAC,YAAY,IACtC,QACG,UAAU,MAAM,CAAC,CACjB,QAAQ,YAAY,EAAE,CAAC,CACvB,UAAU,KAAK,CAAC,CAChB,YAAY,CAAC,CACb,YAAY;EACrB,IAAI,SAAS,YAAY,UAAU,KAAK,KAAK,GAAG,SAAS,IAAI,OAAO;EACpE,KAAK,IAAI,KAAK,GAAG,KAAK,OAAO,QAAQ,MAAM,GAAG;GAC5C,IAAI,OAAO,OAAO,WAAW,EAAE;GAC/B,OAAO,YAAY,IAAI,IAAI,KAAK;GAChC,IAAI,SAAS,MAAS,OAAO,OAAQ,aAAa,IAAI,KAAM,SAAS,KAAM;IACzE,UAAU,OAAO,GAAG;IACpB;GACF;GACA,KAAK,MAAM,OAAO,GAAG;GACrB,QAAQ;EACV;EACA,QAAQ;CACV;CACA,IAAI,OAAO,KAAK,MAAM,OAAO,OAAO,IAAM,QAAQ;CAClD,IAAI,SAAS;CACb,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,MAAM,MAChC,UAAU,OAAO,aAAa,GAAG,MAAM,SAAS,IAAI,KAAK,IAAI,MAAM,KAAK,IAAI,CAAC,CAAC;CAEhF,OAAO;EAAE,MAAM;EAAQ,OAAO,OAAO,SAAS,GAAG,IAAI;EAAG,KAAK,KAAK,SAAS,GAAG,IAAI;CAAE;AACtF;;AAGA,MAAa,QAAQ,MAAc,SAA2B,SAAS,MAAM,IAAI,CAAC,CAAC;AAanF,MAAM,YAAsB;CAAE,OAAO;CAAO,OAAO;CAAM,KAAK;CAAM,SAAS;CAAM,OAAO;AAAE;;AAG5F,MAAM,cAAc,MAAc,UAAkB,YAA6B;CAC/E,MAAM,OAAO,KAAK,WAAW,QAAQ;CACrC,IAAI,WAAW,KAAK,WAAW,KAAK,UAAU,QAAQ,SAAU,QAAQ,OACtE,OAAO,UAAU,WAAW,IAAI,WAAW;CAE7C,OAAO;AACT;;;;;;AAOA,MAAa,UAAU,MAAc,UAA4B;CAC/D,MAAM,SAAS,KAAK,OAAO,OAAO;CAClC,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,WAAW,SAAS,MAAM,OAAO;CACvC,MAAM,MAAM,SAAS,KAAK,QAAQ,MAAM;CACxC,IAAI,QAAQ,IAAI,OAAO;CACvB,IAAI,QAAQ;CACZ,KAAK,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,SAAS,KAAK,QAAQ,QAAQ,KAAK,OAAO,MAAM,GAAG,SAAS;CAC/F,MAAM,QAAQ,SAAS,MAAM;CAC7B,MAAM,MAAM,SAAS,IAAI,MAAM,OAAO,SAAS;CAC/C,MAAM,OAAO,WAAW,MAAM,KAAK,IAAI,GAAG,QAAQ,EAAE,GAAG,KAAK;CAC5D,MAAM,KAAK,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,MAAM,EAAE,GAAG,IAAI;CACjE,OAAO;EACL,OAAO;EACP;EACA;EACA,SAAS,GAAG,OAAO,IAAI,MAAM,KAAK,KAAK,MAAM,MAAM,EAAE,IAAI,KAAK,KAAK,SAAS,MAAM;EAClF;CACF;AACF;;AAGA,MAAM,cAAc;;AAGpB,MAAa,YAAY,SACvB,KAAK,MAAM,QAAQ,CAAC,CACjB,MAAM,WAAW,CAAC,CAClB,QAAQ,UAAU,MAAM,SAAS,CAAC;AAgBvC,MAAM,YAAY;;;;;AAMlB,MAAM,6BAAkC,IAAI,IAAI;CAC9C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;;;;;;;;;;AAgBD,MAAa,cAAc,MAAoC;CAC7D,MAAM,QAAyF,CAAC;CAChG,KAAK,MAAM,SAAS,EAAE,SAAS,2BAA2B,GAAG;EAC3D,IAAI,MAAM,UAAU,WAAW;EAC/B,MAAM,SAAS,MAAM,OAAO,KAAA;EAC5B,MAAM,MAAM,SAAS,MAAM,KAAM,MAAM;EACvC,MAAM,SAAS,SAAS,MAAM,OAAO,MAAM,IAAI,SAAS,GAAG;EAC3D,MAAM,SAAS,SAAS,SAAS,MAAM,IAAI,QAAQ,QAAQ,EAAE,CAAC;EAC9D,IAAI,OAAO,WAAW,GAAG;EACzB,MAAM,KAAK;GACT,MAAM,IAAI,OAAO,KAAK,GAAG,EAAE,GAAG,SAAS,OAAO;GAC9C,MAAM,OAAO;GACb,MAAM,CAAC,UAAU,CAAC,UAAU,OAAO,WAAW,KAAK,WAAW,IAAI,OAAO,EAAG;EAC9E,CAAC;CACH;CACA,IAAI,MAAM,WAAW,GAAG,OAAO,KAAA;CAC/B,MAAM,QAAQ,MAAM,QAAQ,SAAS,CAAC,KAAK,IAAI;CAC/C,MAAM,MAAM,MAAM,SAAS,IAAI,QAAQ;CACvC,OAAO;EACL,KAAK,MAAM,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,OAAO;EAChD,KAAK,IAAI,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,MAAM;EAC7C,UAAU,IAAI,KAAK,SAAS,KAAK,IAAI;EACrC,OAAO,IAAI,KAAK,SAAS,KAAK,IAAI;EAClC,OAAO,MAAM;CACf;AACF;;;;;AASA,MAAa,aAAa,MAAc,OAA0B,QAAQ,QAAgB;CACxF,IAAI,KAAK,WAAW,GAAG,OAAO;CAC9B,IAAI,QAAQ;CACZ,IAAI,MAAM;CACV,MAAM,QAAQ,KAAK,YAAY;CAC/B,IAAI,MAAM,WAAW,KAAK,QACxB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,KAAK,MAAM,QAAQ,IAAI;EAC7B,IAAI,OAAO,OAAO,UAAU,MAAM,KAAK,QAAQ;GAC7C,QAAQ;GACR,MAAM,KAAK,KAAK;EAClB;CACF;CAEF,IAAI,UAAU,IAAI;EAChB,MAAM,SAAS,SAAS,MAAM,QAAQ;EACtC,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,KAAK,OAAO,KAAK,QAAQ,IAAI;GACnC,IAAI,OAAO,IAAI;IACb,MAAM,OAAO,OAAO,MAAM;IAC1B,IAAI,UAAU,MAAM,OAAO,OAAO;KAChC,QAAQ;KACR,MAAM,OAAO,IAAI,KAAK,KAAK,SAAS;IACtC;GACF;EACF;CACF;CACA,IAAI,UAAU,IAAI;EAChB,MAAM,MAAM,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,KAAK,GAAG,KAAK;EAChE,OAAO,GAAG,KAAK,MAAM,GAAG,GAAG,IAAI,MAAM,KAAK,SAAS,MAAM;CAC3D;CACA,MAAM,OAAO,WAAW,MAAM,KAAK,IAAI,GAAG,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC,GAAG,KAAK;CAC/E,MAAM,KAAK,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,MAAM,KAAK,KAAM,QAAQ,IAAK,CAAC,CAAC,GAAG,IAAI;CACzF,MAAM,SAAS,SAAyB,KAAK,QAAQ,QAAQ,GAAG;CAChE,OAAO,GAAG,OAAO,IAAI,MAAM,KAAK,MAAM,KAAK,MAAM,MAAM,KAAK,CAAC,EAAE,GAAG,MAAM,KAAK,MAAM,OAAO,GAAG,CAAC,EAAE,GAAG,MAAM,KAAK,MAAM,KAAK,EAAE,CAAC,IAAI,KAAK,KAAK,SAAS,MAAM;AAC3J;;AClTA,MAAM,kBAAkB;;AAyGxB,MAAa,kBAAkB;;AAG/B,MAAM,cAAc,UAA6B;CAC/C,IAAI,UAAU,KAAA,KAAa,UAAU,SAAS,OAAO;CACrD,IAAI,UAAU,QAAQ,OAAO;CAC7B,MAAM,IAAI,WACR,uCAAuC,KAAK,UAAU,KAAK,KAAK,aAClE;AACF;AA2FA,MAAM,WAAW,gBAA+B;CAC9C,MAAM,QAAQ,OAAO,UAAU;CAC/B,KAAK,OAAO,OAAO,UAAU;CAC7B,UAAU,YAAY,OAAO,UAAU;CACvC,OAAO,SAAS,OAAO,UAAU;CACjC,QAAQ,UAAU,OAAO,UAAU;AACrC;AAEA,MAAM,mBAAmB;;AAGzB,MAAM,eAAe;;AAGrB,MAAM,YAAY;AAElB,MAAM,SAAS,MAAqB;6BACP,EAAE,MAAM;;;;6BAIR,EAAE,SAAS;;;;;;;;;;;;;;;;;;;;6BAoBX,EAAE,SAAS,WAAW,EAAE,SAAS;6BACjC,EAAE,SAAS,cAAc,EAAE,SAAS;6BACpC,EAAE,SAAS,cAAc,EAAE,SAAS;6BACpC,EAAE,SAAS,aAAa,EAAE,SAAS;6BACnC,EAAE,KAAK;;;;;;;;;;;;;;;;;;;;qCAoBC,EAAE,IAAI;uDACY,UAAU;;6BAEpC,EAAE,OAAO;;;;;;;;;;;;;;;;;6BAiBT,EAAE,OAAO,cAAc,EAAE,OAAO;;AA8D7D,MAAMC,kBAAgB;;AAGtB,MAAM,SACJ,KACA,YACwD;CACxD,IAAI,IAAI,SAASA,eAAa,GAC5B,OAAO;EAAE,KAAK,WAAW,IAAI,MAAM,GAAG,EAAqB,CAAC;EAAG,SAAS;CAAW;CAErF,OAAO;EAAE,KAAK,WAAW,GAAG;EAAG,SAAS,WAAW;CAAQ;AAC7D;AAEA,MAAM,OAAO,SAAyB,WAAW,QAAQ,CAAC,CAAC,OAAO,MAAM,MAAM,CAAC,CAAC,OAAO,KAAK;AAE5F,MAAM,aAAgB,MAAiC,aAAmB;CACxE,IAAI,SAAS,QAAQ,SAAS,KAAA,GAAW,OAAO;CAChD,IAAI;EACF,OAAO,KAAK,MAAM,IAAI;CACxB,QAAQ;EACN,OAAO;CACT;AACF;AAEA,MAAM,YAAY,SAA4B;CAC5C,KAAK,IAAI;CACT,SAAS,IAAI;CACb,MAAM,IAAI;CACV,MAAM,IAAI;CACV,eAAe,IAAI;CACnB,cAAc,IAAI;CAClB,YAAY,IAAI;CAChB,QAAQ,IAAI;CACZ,UAAU,IAAI;CACd,UAAU,IAAI;CACd,WAAW,IAAI,cAAc,IAAI,OAAO,KAAA;CACxC,aAAa,IAAI;CACjB,UAAU,IAAI;CACd,SAAS,IAAI;AACf;AAEA,MAAM,aAAa,SAAiC;CAClD,KAAK,IAAI;CACT,SAAS,IAAI;CACb,QAAQ,IAAI;CACZ,UAAU,IAAI,aAAa;CAC3B,YAAY,IAAI;CAChB,MAAM,IAAI;CACV,MAAM,IAAI;CACV,OAAO,IAAI;CACX,KAAK,IAAI;CACT,UAAU,IAAI;CACd,OAAO,IAAI;CACX,KAAK,IAAI,aAAa;AACxB;AAEA,MAAMC,eAAa,UACjB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAsCvD,IAAa,WAAb,MAA8C;CAgBjC;CAfX;CACA,6BAA8B,IAAI,IAA2B;CAC7D,QAAgB;CAChB,SAAiB;CACjB;;;;;CAMA,IAAI,QAAiB;EACnB,OAAO,KAAK;CACd;CAEA,YACE,MACA,UAA2B,CAAC,GAC5B;EAFS,KAAA,OAAA;EAGT,IAAI,SAAS,YAAY,UAAU,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAClF,KAAK,KAAK,IAAI,aAAa,IAAI;EAE/B,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,iBAAiB,GAAM,CAAC;EACpE,KAAK,GAAG,KAAK,yBAAyB,OAAO,IAAI,EAAE,EAAE;EACrD,IAAI;GACF,KAAK,GAAG,KACN,qIACF;EACF,SAAS,OAAO;GACd,KAAK,GAAG,MAAM;GACd,MAAM,IAAI,MACR,uFAAuF,QAAQ,QAAQ,gDACvG,EAAE,MAAM,CACV;EACF;EACA,IAAI,SAAS,YAAY,KAAK,GAAG,KAAK,4BAA4B;EAElE,KAAK,GAAG,KAAK,8BAA8B;EAC3C,KAAK,QAAQ,KAAK,YAAY,KAAK,cAAc,CAAC;CACpD;;CAGA,gBAAiC;EAC/B,MAAM,UAAW,KAAK,GAAG,QAAQ,qBAAqB,CAAC,CAAC,IAAI,CAAC,CAC1D;EACH,MAAM,SAAS,KAAK,GACjB,QAAQ,kFAAkF,CAAC,CAC3F,IAAI;EACP,IAAI,YAAA,KAA4B,OAAO,MAAM,UAAU,MAAM,SAAS,YAAY,GAAG;GACnF,KAAK,cAAc;GACnB,OAAO;EACT;EAEA,MAAM,UAAU,KAAK,GAClB,QACC,0FACF,CAAC,CACA,IAAI;EACP,KAAK,MAAM,EAAE,UAAU,SAAS,KAAK,GAAG,KAAK,yBAAyB,KAAK,EAAE;EAC7E,MAAM,OAAO,KAAK,GACf,QAAQ,kFAAkF,CAAC,CAC3F,IAAI;EACP,KAAK,MAAM,EAAE,UAAU,MAAM,KAAK,GAAG,KAAK,yBAAyB,KAAK,EAAE;EAC1E,KAAK,GAAG,KAAK,qEAAqE;EAClF,KAAK,GAAG,KAAK,MAAM,QAAQ,CAAC,CAAC,CAAC;EAC9B,KAAK,WAAW,MAAM;EACtB,KAAK,QAAQ,cAAc,GAAG;EAC9B,KAAK,QAAQ,mBAAmB,GAAG;EACnC,KAAK,GAAG,KAAK,yBAAyB,OAAA,CAAmB,GAAG;EAC5D,OAAO;CACT;CAEA,QAAgB,KAAa,OAAqB;EAChD,KAAK,KACH,yGACF,CAAC,CAAC,IAAI,KAAK,KAAK;CAClB;CAEA,QAAgB,KAAiC;EAC/C,OACE,KAAK,KAAK,4CAA4C,CAAC,CAAC,IAAI,GAAG,CAAC,EAE/D;CACL;CAEA,KAAa,KAA4B;EACvC,IAAI,YAAY,KAAK,WAAW,IAAI,GAAG;EACvC,IAAI,cAAc,KAAA,GAAW;GAC3B,YAAY,KAAK,GAAG,QAAQ,GAAG;GAC/B,KAAK,WAAW,IAAI,KAAK,SAAS;EACpC;EACA,OAAO;CACT;;;;;CAMA,MAAiB,MAAkB;EACjC,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,qBAAqB;EACtD,IAAI,KAAK,QAAQ,KAAK,KAAK,GAAG,eAAe;GAC3C,KAAK,SAAS;GACd,IAAI;IACF,OAAO,KAAK;GACd,UAAU;IACR,KAAK,SAAS;GAChB;EACF;EACA,KAAK,GAAG,KAAK,iBAAiB;EAC9B,KAAK,QAAQ;EACb,IAAI;GACF,MAAM,SAAS,KAAK;GACpB,KAAK,GAAG,KAAK,QAAQ;GACrB,OAAO;EACT,SAAS,OAAO;GACd,IAAI,KAAK,GAAG,eAAe,KAAK,GAAG,KAAK,UAAU;GAClD,MAAM;EACR,UAAU;GACR,KAAK,QAAQ;EACf;CACF;CAEA,aAA6B;EAC3B,OAAO,OAAO,KAAK,QAAQ,YAAY,KAAK,GAAG;CACjD;;;;;;CAOA,KAAgB,MAA8B;EAC5C,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,qBAAqB;EACtD,IAAI;GACF,OAAO,KAAK,QAAQ,KAAK,WAAW,CAAC,CAAC;EACxC,SAAS,OAAO;GACd,IAAI,CAAC,iBAAiB,KAAKA,YAAU,KAAK,CAAC,KAAK,KAAK,QAAQ,GAAG,MAAM;GACtE,KAAK,WAAW,MAAM;GACtB,OAAO,KAAK,QAAQ,KAAK,WAAW,CAAC,CAAC;EACxC;CACF;;CAGA,eAAuB,YAA0B;EAC/C,MAAM,IAAI,QAAQ,UAAU;EAC5B,KAAK,MAAM,SAAS;GAAC,EAAE;GAAK,EAAE;GAAM,EAAE;GAAU,EAAE;GAAO,EAAE;EAAM,GAC/D,KAAK,GAAG,KAAK,wBAAwB,OAAO;EAE9C,KAAK,GAAG,QAAQ,sCAAsC,CAAC,CAAC,IAAI,YAAY,OAAO,UAAU,GAAG;EAC5F,KAAK,WAAW,MAAM;CACxB;;CAGA,gBAA8B;EAC5B,MAAM,UAAU,KAAK,WAAW;EAChC,MAAM,8BAAc,IAAI,IAAY;EACpC,KAAK,MAAM,EAAE,UAAU,KAAK,GACzB,QAAQ,qDAAqD,CAAC,CAC9D,IAAI,GAA8B;GACnC,MAAM,QAAQ,iBAAiB,KAAK,IAAI;GACxC,IAAI,UAAU,MAAM,YAAY,IAAI,OAAO,MAAM,EAAE,CAAC;EACtD;EACA,KAAK,MAAM,cAAc,aAAa;GACpC,IAAI,eAAe,SAAS;GAC5B,MAAM,UAAU,KAAK,QAAQ,YAAY,OAAO,UAAU,GAAG;GAG7D,IAAI,GADF,YAAY,KAAA,IAAY,OAAO,oBAAoB,KAAK,IAAI,IAAI,KAAK,MAAM,OAAO,KACxE,eAAe,KAAK,eAAe,UAAU;EAC3D;CACF;CAIA,QAAgB,GAAU,MAAsB;EAC9C,MAAM,MAAM,IAAI,IAAI;EACpB,KAAK,KAAK,yBAAyB,EAAE,MAAM,2BAA2B,CAAC,CAAC,IAAI,KAAK,IAAI;EACrF,OAAO;CACT;CAEA,OAAe,GAAU,KAAwC;EAC/D,IAAI,QAAQ,MAAM,OAAO,KAAA;EACzB,OACE,KAAK,KAAK,oBAAoB,EAAE,MAAM,eAAe,CAAC,CAAC,IAAI,GAAG,CAAC,EAE9D;CACL;CAEA,WAAmB,GAAU,OAAiB,KAAmB;EAC/D,MAAM,OACJ,MAAM,cAAc,KAAA,KAAa,MAAM,cAAc,MAAM,OACvD,KAAK,QAAQ,GAAG,MAAM,SAAS,IAC/B;EACN,KAAK,KACH,eAAe,EAAE,SAAS;;;;;;;kFAQ5B,CAAC,CAAC,IACA,MAAM,KACN,MAAM,WAAW,MAAM,SAAS,MAAM,IACtC,MAAM,UAAU,IAChB,MAAM,aAAa,OAAO,IAAI,GAC9B,MAAM,cAAc,KACpB,MAAM,QAAQ,MACd,MAAM,MACN,MAAM,OACN,MAAM,KACN,KAAK,QAAQ,GAAG,KAAK,UAAU,MAAM,QAAQ,CAAC,CAAC,CAAC,GAChD,KAAK,QAAQ,GAAG,MAAM,IAAI,GAC1B,MAAM,KAAK,QACX,MACA,MAAM,YAAY,SAClB,MAAM,SAAS,WACf,MAAM,SAAS,MACf,MAAM,WAAW,KAAA,IAAY,OAAO,KAAK,UAAU,MAAM,MAAM,CACjE;CACF;CAEA,OAAe,GAAU,KAAa,QAAgB,UAA0C;EAC9F,OAAO,KAAK,KACV,iBAAiB,EAAE,SAAS;uDAE9B,CAAC,CAAC,IAAI,KAAK,QAAQ,QAAQ;CAC7B;;CAGA,UAAkB,GAAU,KAAa,KAA8B;EACrE,MAAM,SAAS,KAAK,KAClB,yCAAyC,EAAE,SAAS,eACtD,CAAC,CAAC,IAAI,GAAG;EACT,MAAM,SAAS,KAAK,KAAK,sCAAsC,EAAE,KAAK,eAAe,CAAC,CAAC,IACrF,GACF;EACA,IAAI,OAAO,WAAW,GAAG;GACvB,IAAI,WAAW,KAAA,GAAW;IACxB,KAAK,KAAK,eAAe,EAAE,IAAI,iBAAiB,CAAC,CAAC,IAAI,OAAO,KAAK;IAClE,KAAK,KAAK,eAAe,EAAE,KAAK,eAAe,CAAC,CAAC,IAAI,GAAG;GAC1D;GACA;EACF;EACA,MAAM,2BAAW,IAAI,IAA2D;EAChF,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,MAAM,KAAK,OAAO,GAAG,KAAK,MAAM,QAAQ,MAAM,QAAQ;GAC5D,IAAI,QAAQ,KAAA,GAAW;GACvB,MAAM,QAAQ,SAAS,IAAI,MAAM,MAAM,KAAK,CAAC;GAC7C,IAAI,MAAM,aAAa,GAAG,MAAM,WAAW;QACtC,MAAM,QAAQ;GACnB,SAAS,IAAI,MAAM,QAAQ,KAAK;EAClC;EAMA,MAAM,aAA0B,CAAC;EACjC,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,eAAe,UAAU;GACpD,IAAI;GACJ,IAAI,aAAa,KAAA,GAAW,OAAO;QAC9B,IAAI,UAAU,KAAA,KAAa,SAAS,cAAc,MAAM,YAAY,OAAO;QAC3E;IAEH,MAAM,OAAO,KAAK,KAChB,oBAAoB,EAAE,SAAS;2DAEjC,CAAC,CAAC,IAAI,KAAK,QAAQ,SAAS,UAAU;IACtC,OACE,SAAS,KAAA,KAAa,KAAK,SAAS,QAAQ,KAAK,SAAS,MAAM,OAAO,WAAW;GACtF;GACA,MAAM,SACJ,UAAU,KAAA,IACN,WACA,aAAa,KAAA,KAAa,MAAM,aAAa,SAAS,aACpD,QACA;GACR,MAAM,SAAS,CAAC,OAAO,QAAQ,CAAC,CAAC,QAC9B,QAA2B,QAAQ,KAAA,KAAa,QAAQ,IAC3D;GACA,WAAW,KAAK;IAAE,KAAK;IAAM,IAAI,OAAO;IAAqB;GAAO,CAAC;EACvE;EACA,IAAI,SAAS,WAAW;EACxB,IAAI,WAAW,SAAS,GAAG;GACzB,MAAM,SAAS,WAAW,OAAO,cAAc,UAAU,IAAI,UAAU,QAAQ;GAC/E,KAAK,MAAM,aAAa,WAAW,MAAM,CAAC,GAAG;IAC3C,MAAM,IAAI,UAAU;IACpB,MAAM,IAAI,OAAO;IAKjB,IAJe,SACX,EAAE,aAAa,EAAE,aACjB,EAAE,aAAa,EAAE,cAChB,EAAE,eAAe,EAAE,cAAc,EAAE,aAAa,EAAE,YAC3C,SAAS;GACvB;EACF;EACA,MAAM,MAAM,OAAO;EAEnB,MAAM,UACJ,IAAI,aAAa,QAAQ,WAAW,MAAM,cAAc,UAAU,OAAO,IAAI,IACzE,OACA,OAAO,OAAO,UACZ,OAAO,KACP,WAAW,MAAM,cAAc,UAAU,OAAO,KAAK,IACnD,QACA;EACV,IAAI,OAAO,UAAmC,KAAK,OAAO,GAAG,IAAI,QAAQ,GAAG,CAAC,CAAC;EAC9E,KAAK,MAAM,aAAa,YAEtB,IAAI,cAAc,QAChB,OAAO;GACL,GAAG,UAAmC,KAAK,OAAO,GAAG,UAAU,IAAI,QAAQ,GAAG,CAAC,CAAC;GAChF,GAAG;EACL;EAGJ,MAAM,OAAO,KAAK,KAChB,uBAAuB,EAAE,SAAS;uDAEpC,CAAC,CAAC,IAAI,KAAK,IAAI,UAAA,EAAsB;EACrC,MAAM,WAAW,CACf,mBAAG,IAAI,IAAI;GACT,IAAI;GACJ,GAAG,KAAK,KAAK,UAAU,MAAM,OAAO;GACpC,GAAG,OAAO,OAAO,KAAK,UAAU,MAAM,OAAO;GAC7C,GAAG,WACA,QAAQ,cAAc,cAAc,MAAM,CAAC,CAC3C,KAAK,cAAc,UAAU,IAAI,OAAO;EAC7C,CAAC,CACH,CAAC,CAAC,MAAM,GAAA,EAAe;EACvB,MAAM,QACJ,KAAK,KAAK,6BAA6B,EAAE,SAAS,eAAe,CAAC,CAAC,IAAI,GAAG,CAAC,CAC3E;EACF,MAAM,MACJ,IAAI,OACJ,WAAW,KAAK,cAAc,UAAU,IAAI,GAAG,CAAC,CAAC,MAAM,UAAU,UAAU,IAAI,KAC/E;EACF,MAAM,QAAQ,KAAK,KACjB,eAAe,EAAE,KAAK;;;;;;;;;uBAUxB,CAAC,CAAC,IACA,KACA,IAAI,MACJ,IAAI,OACJ,KACA,KAAK,UAAU,IAAI,GACnB,IAAI,UACJ,IAAI,UACJ,KAAK,UAAU,QAAQ,GACvB,OACA,IAAI,SACJ,IAAI,MACJ,IAAI,YACJ,IAAI,QACJ,IAAI,UACJ,SACA,IAAI,OACJ,IAAI,QACJ,GACF;EACA,MAAM,OAAO,KAAK,OAAO,GAAG,IAAI,QAAQ,KAAK;EAC7C,IAAI,WAAW,KAAA,KAAa,OAAO,aAAa,IAAI,YAAY,OAAO,UAAU,IAAI,OAAO;GAC1F,IAAI,WAAW,KAAA,GAAW,KAAK,KAAK,eAAe,EAAE,IAAI,iBAAiB,CAAC,CAAC,IAAI,OAAO,KAAK;GAC5F,KAAK,KAAK,eAAe,EAAE,IAAI,uCAAuC,CAAC,CAAC,IACtE,MAAM,OACN,KAAK,IAAI,OAAO,QAAQ,GACxB,KAAK,KAAK,SAAS,kBAAkB,KAAK,MAAM,GAAG,eAAe,IAAI,MAAM,QAAQ,CACtF;EACF;EACA,OAAO,KAAK,MAAM,GAAG,KAAK,IAAI;CAChC;CAEA,MAAc,GAAU,KAAa,WAAqC;EACxE,MAAM,MAAM,KAAK,KAAK,wBAAwB,EAAE,KAAK,eAAe,CAAC,CAAC,IAAI,GAAG;EAE7E,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;EAC9B,OAAO,KAAK,MAAM,GAAG,KAAK,SAAS;CACrC;CAEA,MAAc,GAAU,KAAa,WAAyB;EAC5D,MAAM,SAAS,UAA8C,IAAI,QAAQ,KAAA,CAAS;EAClF,OAAO;GACL,KAAK,IAAI;GACT,MAAM,IAAI;GACV,OAAO,IAAI;GACX,MAAM,aAAa,KAAK,OAAO,GAAG,IAAI,QAAQ,KAAK;GACnD,KAAK,IAAI;GACT,MAAM,UAAmC,IAAI,MAAM,CAAC,CAAC;GACrD,UAAU,UAAoB,IAAI,UAAU,CAAC,CAAC;GAC9C,eAAe,IAAI;GACnB,SAAS,IAAI;GACb,MAAM,IAAI;GACV,YAAY,IAAI;GAChB,QAAQ,IAAI;GACZ,UAAU,IAAI,aAAa;GAC3B,UAAU,IAAI;GACd,KAAK,IAAI,aAAa;GACtB,GAAI,IAAI,UAAU,OAAO,EAAE,OAAO,IAAI,MAAM,IAAI,CAAC;GACjD,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;GACzC,YAAY,IAAI;EAClB;CACF;CAEA,cAAsB,GAAU,KAA8B;EAC5D,OAAO;GACL,GAAG,UAAU,GAAG;GAChB,MAAM,KAAK,OAAO,GAAG,IAAI,QAAQ,KAAK;GACtC,WAAW,KAAK,OAAO,GAAG,IAAI,QAAQ;GACtC,MAAM,UAAmC,KAAK,OAAO,GAAG,IAAI,QAAQ,GAAG,CAAC,CAAC;EAC3E;CACF;CAIA,IAAI,KAA8B;EAChC,OAAO,KAAK,MAAM,MAAM,KAAK,MAAM,GAAG,GAAG,CAAC;CAC5C;;;;;CAMA,OAAO,OAAiB,KAAkB;EACxC,OAAO,KAAK,MAAM,MAChB,KAAK,YAAY;GACf,KAAK,WAAW,GAAG,OAAO,GAAG;GAC7B,OAAO,KAAK,UAAU,GAAG,MAAM,KAAK,GAAG;EACzC,CAAC,CACH;CACF;;CAGA,SAAS,KAAa,UAAuC,CAAC,GAAiB;EAC7E,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,GAAG,CAAC;EAC1D,OAAO,KAAK,MAAM,MAEd,KAAK,KACH,iBAAiB,EAAE,SAAS,8DAC9B,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CACjB,IAAI,SAAS,CACjB;CACF;;;;;;CAOA,YAAY,KAAa,SAA2C;EAClE,OAAO,KAAK,MAAM,MAAM;GACtB,MAAM,KAAK,WAAW,KAAK,MAAM,GAAG,GAAG,CAAC,EAAE;GAC1C,IAAI,OAAO,KAAA,GAAW,OAAO,KAAA;GAC7B,MAAM,MAAM,KAAK,KAAK,iBAAiB,EAAE,SAAS,+BAA+B,CAAC,CAAC,IACjF,KACA,EACF;GACA,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,KAAK,cAAc,GAAG,GAAG;EAClE,CAAC;CACH;;;;;;CAOA,aAAa,KAAa,UAAuC,CAAC,GAAkB;EAClF,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,EAAE,CAAC;EACzD,OAAO,KAAK,MAAM,MAAM;GACtB,MAAM,OAAO,KAAK,KAAK,wBAAwB,EAAE,KAAK,eAAe,CAAC,CAAC,IAAI,GAAG;GAE9E,IAAI,SAAS,KAAA,GAAW,OAAO,CAAC;GAChC,MAAM,OAAO,KAAK,KAChB,iBAAiB,EAAE,SAAS,wEAC9B,CAAC,CAAC,IAAI,KAAK,KAAK,QAAQ;GACxB,MAAM,uBAAO,IAAI,IAAY;GAC7B,MAAM,MAAqB,CAAC;GAC5B,KAAK,MAAM,OAAO,MAAM;IACtB,IAAI,KAAK,IAAI,IAAI,QAAQ,GAAG;IAC5B,KAAK,IAAI,IAAI,QAAQ;IACrB,IAAI,KAAK,KAAK,cAAc,GAAG,GAAG,CAAC;IACnC,IAAI,IAAI,UAAU,OAAO;GAC3B;GACA,OAAO;EACT,CAAC;CACH;;CAGA,WAAW,SAA0B;EACnC,OAAO,KAAK,MACT,MACC,KAAK,KAAK,0BAA0B,EAAE,SAAS,2BAA2B,CAAC,CAAC,IAAI,OAAO,MACvF,KAAA,CACJ;CACF;;CAGA,cAAkC;EAChC,OAAO,KAAK,MACT,MAEG,KAAK,KAAK,yCAAyC,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,EAEnE,UAAU,KAAA,CACjB;CACF;;CAGA,cAAc,SAAwB;EACpC,OAAO,KAAK,MAAM,MAEd,KAAK,KAAK,4BAA4B,EAAE,SAAS,mBAAmB,CAAC,CAAC,IACpE,OACF,CAAC,CAEA,KAAK,QAAQ,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CACpC,QAAQ,QAAoB,QAAQ,KAAA,CAAS,CAClD;CACF;;;;;;;CAQA,KAAK,GAAW,SAAoC;EAClD,MAAM,OAAO,WAAW,QAAQ,IAAI;EACpC,MAAM,QAAQ,WAAW,CAAC;EAC1B,IAAI,UAAU,KAAA,GAAW,OAAO,CAAC;EACjC,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,KAAM,KAAK,MAAM,QAAQ,KAAK,KAAK,EAAE,CAAC;EACzE,OAAO,KAAK,MAAM,MAAM;GACtB,MAAM,aACJ,QAAQ,SAAS,KAAA,KAAa,QAAQ,SAAS,OAAO,KAAK;GAC7D,MAAM,MAAM,sCAAsC,EAAE,IAAI;iBAC7C,EAAE,IAAI,QAAQ,EAAE,KAAK,kBAAkB,EAAE,IAAI;iBAC7C,EAAE,IAAI,kCAAkC,WAAW;;GAE9D,MAAM,OAAO,UAAoD;IAC/D,MAAM,SAAiC,CAAC,KAAK;IAC7C,IAAI,WAAW,SAAS,GAAG,OAAO,KAAK,QAAQ,IAAc;IAC7D,OAAO,KAAK,KAAK;IACjB,OAAO,KAAK,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,MAAM;GACrC;GACA,MAAM,OAAO,IAAI,MAAM,GAAG;GAC1B,IAAI,KAAK,SAAS,SAAS,MAAM,QAAQ,GAAG;IAC1C,MAAM,OAAO,IAAI,IAAI,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC;IAC/C,KAAK,MAAM,OAAO,IAAI,SAAS,SAAS,KAAK,UAAU,GAAG,MAAM,QAAQ,IAAI,MAAM,GAAG,GAAG;KACtF,IAAI,KAAK,UAAU,OAAO;KAC1B,IAAI,CAAC,KAAK,IAAI,IAAI,GAAG,GAAG,KAAK,KAAK,GAAG;IACvC;GACF;GACA,OAAO,KAAK,KAAK,SAAS;IACxB,KAAK,IAAI;IACT,MAAM,IAAI;IACV,OAAO,IAAI;IACX,KAAK,IAAI;IACT,SAAS,UAAU,KAAK,OAAO,GAAG,IAAI,QAAQ,KAAK,IAAI,MAAM,KAAK;IAElE,OAAO,KAAK,MAAM,CAAC,IAAI,OAAO,GAAI,IAAI;IACtC,UAAU,IAAI;IACd,YAAY,IAAI;IAChB,SAAS,IAAI;GACf,EAAE;EACJ,CAAC;CACH;;;;;;;CAQA,WACE,GACA,MACA,UAAoD,CAAC,GACI;EACzD,MAAM,OAAO,WAAW,QAAQ,IAAI;EACpC,MAAM,QAAQ,WAAW,CAAC;EAC1B,IAAI,UAAU,KAAA,KAAa,KAAK,WAAW,GAAG,OAAO,CAAC;EACtD,OAAO,KAAK,MAAM,MAEd,KAAK,KACH,6BAA6B,EAAE,IAAI;oBACzB,EAAE,IAAI,QAAQ,EAAE,KAAK,kBAAkB,EAAE,IAAI;oBAC7C,EAAE,IAAI;0BAElB,CAAC,CAAC,IACA,SAAS,SAAS,KAAK,UAAU,GAAG,MAAM,QAAQ,IAAI,MAAM,KAC5D,KAAK,UAAU,IAAI,CACrB,CAAC,CACD,KAAK,SAAS;GAAE,KAAK,IAAI;GAAK,OAAO,KAAK,MAAM,CAAC,IAAI,OAAO,GAAI,IAAI;EAAK,EAAE,CAC/E;CACF;;;;;;CAOA,UAAkB,GAAU,OAAkC;EAC5D,IAAI,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,MAAM;EAC9C,MAAM,QAAS,KAAK,KAAK,6BAA6B,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAmB;EACxF,MAAM,UAAU,MAAM,KAAK,UAAU;GACnC;GACA,OACE,KAAK,KAAK,6BAA6B,EAAE,IAAI,SAAS,EAAE,IAAI,SAAS,CAAC,CAAC,IAAI,IAAI,CAAC,CAGhF;EACJ,EAAE;EACF,MAAM,OAAO,QAAQ,QAAQ,QAAQ,IAAI,SAAS,kBAAkB,KAAK;EACzE,IAAI,KAAK,SAAS,GAAG,OAAO,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,KAAK,MAAM;EACnE,OAAO,QAAQ,QAAQ,MAAM,QAAS,IAAI,QAAQ,KAAK,QAAQ,MAAM,IAAK,CAAC,CAAC;CAC9E;;CAGA,OAAO,KAAa,SAAuC;EACzD,MAAM,MAAM,MAAM,KAAK,OAAO;EAC9B,OAAO,KAAK,MAAM,MAAM;GACtB,MAAM,MAAM,KAAK,KAAK,iBAAiB,EAAE,OAAO,+BAA+B,CAAC,CAAC,IAC/E,IAAI,KACJ,IAAI,OACN;GACA,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,SAAS,GAAG;EACrD,CAAC;CACH;CAEA,UAAkB,GAAU,OAAqB;EAC/C,MAAM,MAAM,MAAM,MAAM,KAAK,MAAM,OAAO;EAE1C,KAAK,KACH,eAAe,EAAE,OAAO;;;;;;;;sCAQQ,EAAE,OAAO,YAC3C,CAAC,CAAC,IACA,IAAI,KACJ,IAAI,SACJ,MAAM,SACN,MAAM,MACN,MAAM,MACN,MAAM,eACN,MAAM,cACN,MAAM,YACN,MAAM,QACN,MAAM,UACN,MAAM,YAAY,MAClB,MAAM,cAAc,OAAO,IAAI,MAC/B,MAAM,eAAe,MACrB,MAAM,YAAY,IACpB;CACF;;CAGA,UAAU,OAAqB;EAC7B,KAAK,MAAM,MAAM,KAAK,YAAY,KAAK,UAAU,GAAG,KAAK,CAAC,CAAC;CAC7D;CAEA,SAAiB;EACf,OAAO,KAAK,MAAM,OAAO;GACvB,MAAO,KAAK,KAAK,6BAA6B,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAmB;GAChF,MAAO,KAAK,KAAK,6BAA6B,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAmB;EACpF,EAAE;CACJ;;CAGA,QAAc;EACZ,KAAK,MAAM,MACT,KAAK,YAAY;GACf,KAAK,MAAM,SAAS;IAAC,EAAE;IAAK,EAAE;IAAM,EAAE;IAAU,EAAE;IAAO,EAAE;GAAM,GAC/D,KAAK,GAAG,KAAK,eAAe,OAAO;EAEvC,CAAC,CACH;CACF;;;;;;CAOA,aAAa,UAAuC,CAAC,GAAe;EAClE,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,GAAG,CAAC;EAC9D,MAAM,aAAa,KAAK,YAAY;GAClC,MAAM,OAAO,OAAO,KAAK,QAAQ,iBAAiB,KAAK,GAAG;GAC1D,KAAK,QAAQ,mBAAmB,OAAO,OAAO,CAAC,CAAC;GAChD,KAAK,QAAQ,YAAY,OAAO,IAAI,sBAAK,IAAI,KAAK,EAAA,CAAE,YAAY,CAAC;GACjE,KAAK,GAAG,KAAK,MAAM,QAAQ,IAAI,CAAC,CAAC;GACjC,OAAO;EACT,CAAC;EACD,MAAM,IAAI,QAAQ,UAAU;EAC5B,MAAM,WAAsE,CAAC;EAC7E,MAAM,yBAAS,IAAI,IAAoB;EACvC,IAAI,OAAO;EACX,MAAM,WAAW,KAAa,YAAyC;GACrE,MAAM,MAAM,MAAM,KAAK,OAAO;GAC9B,OAAO,GAAG,IAAI,QAAQ,GAAG,IAAI;EAC/B;EACA,MAAM,cAAoB;GACxB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qBAAqB;GAChD,IAAI,SAAS,WAAW,KAAK,OAAO,SAAS,GAAG;GAChD,KAAK,YAAY;IACf,MAAM,0BAAU,IAAI,IAAoB;IACxC,KAAK,MAAM,EAAE,OAAO,SAAS,UAAU;KACrC,KAAK,WAAW,GAAG,OAAO,GAAG;KAC7B,QAAQ,IAAI,MAAM,KAAK,GAAG;IAC5B;IACA,KAAK,MAAM,SAAS,OAAO,OAAO,GAAG,KAAK,UAAU,GAAG,KAAK;IAC5D,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS,KAAK,UAAU,GAAG,KAAK,GAAG;GAC9D,CAAC;GACD,SAAS,SAAS;GAClB,OAAO,MAAM;EACf;;EAEA,MAAM,aAAa,SAAgB,OAAe,WAA2B;GAC3E,KAAK,MAAM,UAAU;IAAC;IAAY;IAAY;GAAU,GACtD,KAAK,GACF,QACC,yBAAyB,EAAE,MAAM;yCACJ,QAAQ,MAAM;uCAChB,OAAO,QAAQ,QAAQ,SAAS,SAAS,MAAM,EAC5E,CAAC,CACA,IAAI,GAAG,MAAM;EAEpB;EACA,OAAO;GACL,SAAS,OAAO,QAAQ;IACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qBAAqB;IAChD,SAAS,KAAK;KAAE;KAAO;IAAI,CAAC;IAC5B,IAAI,SAAS,UAAU,WAAW,MAAM;GAC1C;GACA,YAAY,UAAU;IACpB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qBAAqB;IAChD,MAAM,MAAM,QAAQ,MAAM,KAAK,MAAM,OAAO;IAC5C,MAAM,SAAS,OAAO,IAAI,GAAG;IAC7B,IAAI,WAAW,KAAA,KAAa,MAAM,cAAc,OAAO,YAAY,OAAO,IAAI,KAAK,KAAK;IACxF,IAAI,OAAO,QAAQ,WAAW,MAAM;GACtC;GACA,SAAS,KAAK,YAAY;IACxB,MAAM,UAAU,OAAO,IAAI,QAAQ,KAAK,OAAO,CAAC;IAChD,IAAI,YAAY,KAAA,GAAW,OAAO;IAClC,MAAM,MAAM,MAAM,KAAK,OAAO;IAC9B,MAAM,MAAM,KAAK,KAAK,iBAAiB,EAAE,OAAO,+BAA+B,CAAC,CAAC,IAC/E,IAAI,KACJ,IAAI,OACN;IACA,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,SAAS,GAAG;GACrD;GACA;GACA,QAAQ,YAAY;IAClB,MAAM;IACN,MAAM,QAAQ,KAAK,UAAU,CAAC,GAAG,OAAO,CAAC;IACzC,MAAM,UAAU,QAAQ,KAAK,WAAW,CAAC;IACzC,MAAM,OAAO,KAAK,GACf,QACC,4BAA4B,QAAQ,SAAS,sDAC/C,CAAC,CACA,IAAI,KAAK;IACZ,MAAM,QACJ;IACF,KAAK,IAAI,KAAK,GAAG,KAAK,KAAK,QAAQ,MAAM,WAAW;KAClD,MAAM,OAAO,KAAK,UAAU,KAAK,MAAM,IAAI,KAAK,SAAS,CAAC,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC;KAChF,KAAK,YAAY;MACf,UAAU,SAAS,OAAO,CAAC,OAAO,IAAI,CAAC;MACvC,KAAK,GACF,QACC,yBAAyB,EAAE,SAAS,iBAAiB,QAAQ,SAAS,SAAS,OACjF,CAAC,CACA,IAAI,OAAO,IAAI;MAClB,MAAM,uBAAM,IAAI,KAAK,EAAA,CAAE,YAAY;MACnC,KAAK,MAAM,OAAO,KAAK,MAAM,IAAI,GAAe,KAAK,UAAU,GAAG,KAAK,GAAG;KAC5E,CAAC;IACH;IACA,OAAO,KAAK;GACd;GACA,SAAS,UAAU;IACjB,MAAM;IACN,OAAO;IACP,OAAO,KAAK,YAAY;KACtB,MAAM,UAAU,KAAK,WAAW;KAChC,MAAM,MAAM,QAAQ,OAAO;KAE3B,UAAU,KAAK,mBAAmB,CAAC,KAAK,CAAC;KACzC,MAAM,QAAQ,KAAK,GAChB,QAAQ,4BAA4B,IAAI,SAAS,uBAAuB,CAAC,CACzE,IAAI,KAAK;KACZ,KAAK,GACF,QACC,yBAAyB,EAAE,SAAS,iBAAiB,IAAI,SAAS,uBACpE,CAAC,CACA,IAAI,KAAK;KACZ,MAAM,uBAAM,IAAI,KAAK,EAAA,CAAE,YAAY;KACnC,KAAK,MAAM,EAAE,SAAS,OAAO,KAAK,UAAU,GAAG,KAAK,GAAG;KACvD,KAAK,MAAM,OAAO,KAAK,GACpB,QAAQ,iBAAiB,IAAI,OAAO,uBAAuB,CAAC,CAC5D,IAAI,KAAK,GACV,KAAK,UAAU,GAAG,SAAS,GAAG,CAAC;KAEjC,KAAK,QAAQ,cAAc,OAAO,UAAU,CAAC;KAC7C,KAAK,GACF,QAAQ,sCAAsC,CAAC,CAC/C,IAAI,YAAY,OAAO,UAAU,GAAG;KACvC,KAAK,eAAe,OAAO;KAC3B,KAAK,QAAQ;KACb,OAAO;MACL,MAAO,KAAK,GAAG,QAAQ,6BAA6B,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAmB;MACtF,MAAO,KAAK,GAAG,QAAQ,6BAA6B,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,CAClE;KACL;IACF,CAAC;GACH;GACA,aAAa;IACX,OAAO;IACP,SAAS,SAAS;IAClB,OAAO,MAAM;IACb,IAAI,KAAK,QAAQ;IACjB,KAAK,YAAY,KAAK,eAAe,UAAU,CAAC;GAClD;EACF;CACF;;CAGA,QAAc;EACZ,IAAI,KAAK,QAAQ;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,MAAM;EACtB,KAAK,GAAG,MAAM;CAChB;AACF;;;;ACtyCA,MAAM,SAAS;;AAEf,MAAM,iBAAiB;;AAEvB,MAAM,cAAc;AAiBpB,MAAM,aAAa,UACjB,UAAU,WAAW,UAAU,UAAU,QAAQ;AAEnD,MAAM,UAAU,SAAiC;CAC/C,UAAU,IAAI,cAAc;CAC5B,MAAM,UAAU,IAAI,IAAI;CACxB,cAAc,UAAU,IAAI,aAAa;CACzC,YAAY,IAAI;CAChB,YAAY,IAAI;CAChB,YAAY,IAAI;CAChB,WAAW,IAAI;AACjB;;;;;;;;;;;;;;;;;AAoCA,IAAa,uBAAb,MAA4D;CAU/C;CATX;CACA,6BAA8B,IAAI,IAA2B;;CAE7D,QAAyB,WAAW;CACpC,eAAuB;CACvB,SAAiB;CACjB;CAEA,YACE,MACA,UAAkC,CAAC,GACnC;EAFS,KAAA,OAAA;EAGT,IAAI,SAAS,YAAY,UAAU,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAClF,KAAK,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,QAAQ,UAAU,cAAc,CAAC;EACpF,KAAK,KAAK,IAAI,aAAa,IAAI;EAC/B,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,iBAAiB,GAAM,CAAC;EACpE,KAAK,GAAG,KAAK,yBAAyB,OAAO,IAAI,EAAE,EAAE;EACrD,IAAI,SAAS,YAAY,KAAK,GAAG,KAAK,4BAA4B;EAClE,KAAK,GAAG,KAAK,8BAA8B;EAC3C,KAAK,QAAQ;CACf;;;;;CAMA,UAAwB;EACtB,KAAK,WAAW;GACd,KAAK,GAAG,KAAK;;;;;;;;;QASX;GACF,KAAK,GAAG,KAAK;;;;;;;;;QASX;GACF,MAAM,EAAE,cAAc,YAAY,KAAK,GAAG,QAAQ,qBAAqB,CAAC,CAAC,IAAI;GAI7E,IAAI,UAAA,GACF,KAAK,GAAG,KAAK,yBAAyB,OAAA,CAAwB,GAAG;EACrE,CAAC;CACH;CAEA,KAAa,KAA4B;EACvC,IAAI,YAAY,KAAK,WAAW,IAAI,GAAG;EACvC,IAAI,cAAc,KAAA,GAAW;GAC3B,YAAY,KAAK,GAAG,QAAQ,GAAG;GAC/B,KAAK,WAAW,IAAI,KAAK,SAAS;EACpC;EACA,OAAO;CACT;;CAGA,KAAgB,MAAkB;EAChC,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,gCAAgC;EACjE,KAAK,GAAG,KAAK,iBAAiB;EAC9B,IAAI;GACF,MAAM,SAAS,KAAK;GACpB,KAAK,GAAG,KAAK,QAAQ;GACrB,OAAO;EACT,SAAS,OAAO;GACd,IAAI,KAAK,GAAG,eAAe,KAAK,GAAG,KAAK,UAAU;GAClD,MAAM;EACR;CACF;CAEA,IAAY,MAAmB;EAC7B,KAAK,KAAK,+CAA+C,CAAC,CAAC,IAAI,IAAI;EACnE,OAAO,KAAK,KACV,yGACF,CAAC,CAAC,IAAI,IAAI;CACZ;;CAGA,MAAc,KAAmB;EAC/B,KAAK,KACH;qDAEF,CAAC,CAAC,IAAI,KAAK,KAAK,GAAG;CACrB;CAEA,QAAQ,MAAc,KAAa,SAAyB;EAC1D,OAAO,KAAK,WAAW;GACrB,KAAK,gBAAgB;GACrB,IAAI,KAAK,eAAe,gBAAgB,GAAG,KAAK,MAAM,GAAG;GACzD,MAAM,MAAM,KAAK,IAAI,IAAI;GACzB,IAAI,IAAI,eAAe,KAErB,OAAO,IAAI,UAAU,KAAK,QACtB,IAAI,eAAe,MACnB,KAAK,IAAI,KAAK,QAAQ,IAAI,eAAe,GAAG;GAElD,IAAI,IAAI,UAAU,KAAK,OAAO,IAAI,UAAU;GAC5C,KAAK,KAAK,6DAA6D,CAAC,CAAC,IACvE,MAAM,SACN,KAAK,OACL,IACF;GACA,OAAO;EACT,CAAC;CACH;CAEA,OAAO,MAAc,QAAgC;EACnD,OAAO,KAAK,WAAW;GACrB,MAAM,MAAM,KAAK,IAAI,IAAI;GACzB,IAAI,EAAE,UAAU,UAAU;GAC1B,IAAI,WAAW,WAAW;IACxB,WAAW;IACX,QAAQ;GACV,OAAO,IAAI,WAAW,WAAW,YAAY;QACxC,IAAI,WAAW,QAAQ,SAAS;GACrC,IAAI,aAAa,IAAI,YAAY,UAAU,IAAI,OAC7C,KAAK,KAAK,yDAAyD,CAAC,CAAC,IACnE,UACA,OACA,IACF;GAEF,OAAO;IAAE;IAAU;GAAM;EAC3B,CAAC;CACH;CAEA,QAAQ,MAAc,QAAsB;EAC1C,KAAK,WAAW;GACd,KAAK,IAAI,IAAI;GACb,KAAK,KACH;;;wBAIF,CAAC,CAAC,IAAI,QAAQ,KAAK,OAAO,KAAK,OAAO,IAAI;EAC5C,CAAC;CACH;CAEA,MAAM,MAAc,OAAqB;EACvC,KAAK,WAAW;GACd,KAAK,IAAI,IAAI;GACb,KAAK,KAAK,2DAA2D,CAAC,CAAC,IAAI,OAAO,IAAI;EACxF,CAAC;CACH;CAEA,KAAK,MAAc,OAAe,QAAsB;EACtD,MAAM,MAAM,OAAO,SAAS,KAAK,IAAI,QAAQ;EAC7C,KAAK,WAAW;GACd,KAAK,IAAI,IAAI;GACb,KAAK,KACH;mEAEF,CAAC,CAAC,IAAI,KAAK,QAAQ,MAAM,GAAG;EAC9B,CAAC;CACH;CAEA,QAAQ,MAAc,KAAmC;EACvD,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,gCAAgC;EACjE,MAAM,MAAM,KAAK,KAAK,0DAA0D,CAAC,CAAC,IAAI,IAAI;EAE1F,IAAI,QAAQ,KAAA,KAAa,IAAI,eAAe,QAAQ,IAAI,cAAc,KAAK,OAAO,KAAA;EAClF,OAAO;GACL,OAAO,IAAI,cAAc,SAAS,OAAO,oBAAoB,IAAI;GACjE,QAAQ,IAAI,eAAe;EAC7B;CACF;CAEA,MAAM,MAAqB;EACzB,KAAK,WAAW;GACd,IAAI,SAAS,KAAA,GACX,KAAK,KACH,iFACF,CAAC,CAAC,IAAI;QAEN,KAAK,KACH,gGACF,CAAC,CAAC,IAAI,IAAI;EAEd,CAAC;CACH;CAEA,WAAW,MAAyC;EAClD,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,gCAAgC;EACjE,MAAM,MAAM,KAAK,KACf;wCAEF,CAAC,CAAC,IAAI,IAAI;EACV,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,OAAO,GAAG;CACnD;CAEA,eAAe,MAAc,QAAyB,KAA4B;EAChF,OAAO,KAAK,WAAW;GACrB,MAAM,OAAO,kBAAkB,KAAK,WAAW,IAAI,GAAG,QAAQ,GAAG;GACjE,KAAK,KACH;;;;;4CAMF,CAAC,CAAC,IACA,MACA,KAAK,WAAW,IAAI,GACpB,KAAK,MACL,KAAK,cACL,KAAK,YACL,KAAK,YACL,KAAK,YACL,KAAK,SACP;GACA,OAAO;EACT,CAAC;CACH;;CAGA,IAAI,OAAe;EACjB,OAAQ,KAAK,KAAK,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAmB;CAC/E;;CAGA,QAAc;EACZ,IAAI,KAAK,QAAQ;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,MAAM;EACtB,KAAK,GAAG,MAAM;CAChB;AACF;;;AChRA,MAAM,aAAa;AACnB,MAAM,WAAW;AACjB,MAAM,UAAU;;AAGhB,MAAM,WAAW,OAAO,QAAoB,UAAiC;CAC3E,IAAI,KAAK;CACT,OAAO,KAAK,MAAM,QAAQ;EACxB,MAAM,EAAE,iBAAiB,MAAM,OAAO,MAAM,OAAO,IAAI,MAAM,SAAS,EAAE;EACxE,MAAM;CACR;AACF;;;;;AAMA,gBAAgB,QACd,MACuE;CACvE,MAAM,UAAU,IAAI,YAAY,OAAO;CACvC,IAAI;CACJ,WAAW,MAAM,SAAS,iBAAiB,IAAI,GAA4B;EACzE,MAAM,OAAO,SAAS,KAAA,IAAY,QAAQ,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC;EACrE,OAAO,KAAA;EACP,IAAI,QAAQ;EACZ,SAAS;GACP,MAAM,MAAM,KAAK,QAAQ,SAAS,KAAK;GACvC,IAAI,QAAQ,IAAI;GAChB,MAAM;IAAE,MAAM,QAAQ,OAAO,KAAK,SAAS,OAAO,GAAG,CAAC;IAAG,UAAU;GAAK;GACxE,QAAQ,MAAM;EAChB;EACA,IAAI,QAAQ,KAAK,QAAQ,OAAO,OAAO,KAAK,KAAK,SAAS,KAAK,CAAC;CAClE;CACA,IAAI,SAAS,KAAA,KAAa,KAAK,SAAS,GAAG,MAAM;EAAE,MAAM,QAAQ,OAAO,IAAI;EAAG,UAAU;CAAM;AACjG;AAEA,MAAM,YAAY,MAAc,SAC9B,GAAG,KAAK,GAAG,WAAW,KAAK,IAAI,CAAC,GAAG,MAAM;;;;;;;;;;;;;;;;AAiB3C,IAAa,aAAb,MAAa,WAAoC;CAOpC;CANX,0BAA2B,IAAI,IAAwB;CACvD,UAAsC,CAAC;CACvC;CACA;CAEA,YACE,KACA,UAA6B,CAAC,GAC9B;EAFS,KAAA,MAAA;EAGT,KAAK,OAAO,QAAQ,QAAQ;CAC9B;CAEA,OAAe,QAA+B;EAC5C,OAAO,KACL,KAAK,KACL,OAAO,QACP,OAAO,YAAY,MAAM,GAAG,CAAC,GAC7B,GAAG,OAAO,YAAY,MAAM,GAAG,CAAC,EAAE,QACpC;CACF;CAEA,MAAc,UAAU,MAAmC;EACzD,MAAM,QAAQ,KAAK,QAAQ,IAAI,IAAI;EACnC,IAAI,UAAU,KAAA,GAAW,OAAO;EAEhC,MAAM,MAAM,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAC3D,MAAM,SAAS,MAAM,KAAK,MAAM,MAAM,GAAK;EAC3C,KAAK,QAAQ,IAAI,MAAM,MAAM;EAC7B,OAAO;CACT;;CAGA,aAAqB,WAAW,QAAsC;EACpE,MAAM,EAAE,SAAS,MAAM,OAAO,KAAK;EACnC,IAAI,SAAS,GAAG,OAAO;EACvB,MAAM,OAAO,OAAO,MAAM,CAAC;EAC3B,MAAM,OAAO,KAAK,MAAM,GAAG,GAAG,OAAO,CAAC;EACtC,OAAO,KAAK,OAAO;CACrB;CAEA,OAAO,QAAsC;EAC3C,IAAI;EACJ,IAAI;GACF,OAAO,GAAG,KAAK,UAAU,MAAM,EAAE;EACnC,SAAS,OAAO;GACd,OAAO,QAAQ,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC,CAAC;EACjF;EACA,OAAO,IAAI,SAAe,SAAS,WAAW;GAC5C,KAAK,QAAQ,KAAK;IAAE,MAAM,KAAK,OAAO,MAAM;IAAG;IAAM;IAAS;GAAO,CAAC;GACtE,KAAK,KAAK;EACZ,CAAC;CACH;;CAGA,OAAqB;EACnB,IAAI,KAAK,aAAa,KAAA,GAAW;EACjC,KAAK,WAAW,KAAK,MAAM,CAAC,CAAC,cAAc;GACzC,KAAK,WAAW,KAAA;GAChB,IAAI,KAAK,QAAQ,SAAS,GAAG,KAAK,KAAK;EACzC,CAAC;CACH;;CAGA,MAAc,QAAuB;EACnC,OAAO,KAAK,QAAQ,SAAS,GAAG;GAC9B,MAAM,QAAQ,KAAK,QAAQ,OAAO,CAAC;GACnC,MAAM,yBAAS,IAAI,IAAuB;GAC1C,KAAK,MAAM,QAAQ,OAAO;IACxB,MAAM,OAAO,OAAO,IAAI,KAAK,IAAI;IACjC,IAAI,SAAS,KAAA,GAAW,OAAO,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;SAC/C,KAAK,KAAK,IAAI;GACrB;GACA,KAAK,MAAM,CAAC,MAAM,UAAU,QAC1B,IAAI;IACF,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;IACxC,MAAM,OAAQ,MAAM,WAAW,WAAW,MAAM,IAAK,OAAO;IAC5D,MAAM,SACJ,QACA,OAAO,KAAK,GAAG,OAAO,MAAM,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,MAAM,CACzE;IACA,IAAI,KAAK,MAAM,MAAM,OAAO,SAAS;IACrC,KAAK,MAAM,QAAQ,OAAO,KAAK,QAAQ;GACzC,SAAS,OAAO;IAEd,MAAM,SAAS,KAAK,QAAQ,IAAI,IAAI;IACpC,KAAK,QAAQ,OAAO,IAAI;IACxB,MAAM,QAAQ,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;IAC3C,KAAK,MAAM,QAAQ,OAAO,KAAK,OAAO,KAAK;GAC7C;EAEJ;CACF;;CAGA,MAAM,QAAuB;EAC3B,OAAO,KAAK,aAAa,KAAA,GAAW,MAAM,KAAK;CACjD;;CAGA,MAAc,QAAQ,QAAgB,OAA8C;EAClF,MAAM,YAAY,KAAK,KAAK,KAAK,MAAM;EACvC,IAAI,CAAC,WAAW,SAAS,GAAG,OAAO,CAAC;EACpC,MAAM,OAAO,OAAO,MAAM,GAAG,CAAC;EAC9B,MAAM,SAAS,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC,EAAA,CAC5D,QAAQ,UAAU,MAAM,YAAY,KAAK,SAAS,KAAK,MAAM,IAAI,CAAC,CAAC,CACnE,KAAK,UAAU,MAAM,IAAI,CAAC,CAC1B,KAAK;EACR,MAAM,QAAkB,CAAC;EACzB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,SAAS,KAAA,KAAa,OAAO,KAAK,MAAM,GAAG,CAAC,GAAG;GACnD,MAAM,UAAU,MAAM,QAAQ,KAAK,WAAW,IAAI,CAAC,EAAA,CAChD,QAAQ,SAAS,WAAW,KAAK,IAAI,CAAC,CAAC,CACvC,KAAK;GACR,KAAK,MAAM,SAAS,QAAQ;IAC1B,IAAI,SAAS,KAAA,KAAa,SAAS,MAAM,KAAK,IAAI,MAAM;IACxD,MAAM,KAAK,KAAK,WAAW,MAAM,KAAK,CAAC;GACzC;EACF;EACA,OAAO;CACT;;CAGA,OAAe,WACb,QACA,QACA,SAC+B;EAC/B,KAAK,MAAM,QAAQ,MAAM,KAAK,QAAQ,QAAQ,QAAQ,KAAK,GAAG;GAC5D,IAAI,SAAS;GACb,WAAW,MAAM,EAAE,MAAM,cAAc,QAAQ,IAAI,GAAG;IACpD,UAAU;IACV,MAAM,UAAU,KAAK,KAAK;IAC1B,IAAI,QAAQ,WAAW,GAAG;IAC1B,IAAI;IACJ,IAAI;KACF,MAAM,KAAK,MAAM,OAAO;KACxB,IAAI,OAAO,QAAQ,YAAY,QAAQ,QAAQ,OAAO,IAAI,gBAAgB,UACxE,MAAM,IAAI,MAAM,cAAc;IAElC,QAAQ;KAEN,IAAI,UAAU,SAAS,YAAY;MAAE;MAAM,MAAM;KAAO,CAAC;KACzD;IACF;IACA,IAAI,QAAQ,SAAS,KAAA,KAAa,IAAI,SAAS,OAAO,MAAM;IAC5D,IAAI,QAAQ,UAAU,KAAA,KAAa,IAAI,cAAc,OAAO,OAAO;IACnE,MAAM;GACR;EACF;CACF;;;;;;;;CASA,OAAO,KAAK,QAAwB,SAAqD;EACvF,MAAM,KAAK,MAAM;EACjB,MAAM,UACJ,QAAQ,WAAW,KAAA,IACf,CAAC,OAAO,MAAM,IACd,WAAW,KAAK,GAAG,KAChB,MAAM,QAAQ,KAAK,KAAK,EAAE,eAAe,KAAK,CAAC,EAAA,CAC7C,QAAQ,UAAU,MAAM,YAAY,CAAC,CAAC,CACtC,KAAK,UAAU,MAAM,IAAI,CAAC,CAC1B,KAAK,IACR,CAAC;EACT,IAAI,QAAQ,WAAW,GAAG;GACxB,OAAO,KAAK,WAAW,QAAQ,IAAK,QAAQ,OAAO;GACnD;EACF;EAEA,MAAM,UAAU,QAAQ,KAAK,WAC3B,KAAK,WAAW,QAAQ,QAAQ,OAAO,CAAC,CAAC,OAAO,cAAc,CAAC,CACjE;EACA,MAAM,QAA0C,CAAC;EACjD,KAAK,MAAM,UAAU,SACnB,MAAM,MAAM,MAAM,OAAO,KAAK,EAAA,CAAG,KAAkC;EACrE,SAAS;GACP,IAAI,OAAO;GACX,KAAK,MAAM,CAAC,OAAO,SAAS,MAAM,QAAQ,GACxC,IAAI,SAAS,KAAA,MAAc,SAAS,MAAM,KAAK,cAAc,MAAM,KAAK,CAAE,cACxE,OAAO;GAGX,IAAI,SAAS,IAAI;GACjB,MAAM,MAAM;GACZ,MAAM,SAAS,MAAM,QAAQ,KAAK,CAAE,KAAK,EAAA,CAAG;EAC9C;CACF;;CAGA,MAAM,QAAuB;EAC3B,MAAM,KAAK,MAAM;EACjB,MAAM,UAAU,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC;EACzC,KAAK,QAAQ,MAAM;EACnB,KAAK,MAAM,UAAU,SAAS,MAAM,OAAO,MAAM;CACnD;AACF;;;AC5QA,MAAM,YAAY;AAClB,MAAM,YAAY;AAClB,MAAM,iBAAiB;;;;;;AAMvB,MAAM,qBAAqB;;AAG3B,MAAa,SAAS,MAAkB,gBAAwC;CAC9E,IAAI,WAAW,KAAK,eAAe,EAAE,GAAG,OAAO;CAC/C,MAAM,MAAM,KAAK,IAAI,KAAK,SAAS,GAAG,IAAI;CAC1C,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,SAAS,GACxC,IACE,KAAK,WAAW,MAChB,KAAK,QAAQ,OAAO,MACpB,KAAK,QAAQ,OAAO,MACpB,KAAK,QAAQ,OAAO,MACpB,KAAK,QAAQ,OAAO,IAEpB,OAAO;CAGX,OAAO;AACT;;AAGA,MAAM,QAAQ,SACZ,KACG,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAC/C,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,IAAI;;AAGd,MAAM,aACJ;AAEF,MAAM,UAAU,OAAmB,OAAe,QAAwB;CACxE,IAAI,MAAM;CACV,KAAK,IAAI,QAAQ,KAAK,IAAI,GAAG,KAAK,GAAG,QAAQ,OAAO,QAAQ,MAAM,QAAQ,SAAS,GACjF,OAAO,OAAO,aAAa,MAAM,MAAO;CAE1C,OAAO;AACT;AAEA,MAAM,WAAW,OAAmB,QAAgB,SAAyB;CAC3E,MAAM,QAAQ,OAAO,WAAW,CAAC;CACjC,KACE,IAAI,QAAQ,MAAM,QAAQ,OAAO,IAAI,GACrC,UAAU,IACV,QAAQ,MAAM,QAAQ,OAAO,QAAQ,CAAC,GACtC;EACA,IAAI,QAAQ;EACZ,KAAK,IAAI,KAAK,GAAG,KAAK,OAAO,QAAQ,MAAM,GACzC,IAAI,MAAM,QAAQ,QAAQ,OAAO,WAAW,EAAE,GAAG;GAC/C,QAAQ;GACR;EACF;EAEF,IAAI,OAAO,OAAO;CACpB;CACA,OAAO;AACT;;;;;;;AAQA,MAAM,kBAAkB,OAAmB,UAA2B;CACpE,IAAI,QAAQ;CACZ,IAAI,OAAO;CACX,SAAS;EACP,MAAM,UAAU,QAAQ,OAAO,UAAU,IAAI;EAC7C,IAAI,YAAY,IAAI,OAAO;EAC3B,OAAO,UAAU;EAEjB,IAAI,OAAO,OAAO,UAAU,GAAG,OAAO,MAAM,OAAO;EACnD,MAAM,OAAO,MAAM,UAAU;EAC7B,IAAI,SAAS,MAAQ,SAAS,IAAM;EACpC,MAAM,aAAa,OAAO,OAAO,UAAU,MAAM,OAAO;EACxD,MAAM,OAAO,WAAW,MAAM,KAAK,IAAI,GAAG,WAAW,YAAY,IAAI,CAAC,CAAC;EACvE,IAAI,wBAAwB,KAAK,IAAI,GAAG;EACxC,IAAI,QAAQ,UAAU;EACtB,IAAI,SAAS,MAAQ,MAAM,WAAW,IAAM,SAAS;EACrD,MAAM,WAAW,iCAAiC,KAAK,IAAI;EAC3D,MAAM,OAAO,QAAQ,OAAO,aAAa,KAAK;EAC9C,MAAM,MACJ,aAAa,QAAQ,QAAQ,OAAO,SAAS,EAAE,KAAK,MAAM,SACtD,QAAQ,OAAO,SAAS,EAAE,IAC1B,SAAS,KACP,MAAM,SACN;EACR,IAAI;GACF,SAAS,YAAY,MAAM,SAAS,OAAO,GAAG,GAAG;IAC/C,iBAAiB,KAAK,IAAI,GAAG,QAAQ,QAAQ,CAAC;IAC9C,aAAa,UAAU;GACzB,CAAC,CAAC,CAAC;EACL,SAAS,OAAO;GAEd,IAAI,iBAAiB,YAAY,OAAO;EAC1C;EACA,IAAI,QAAQ,OAAO,OAAO;EAC1B,OAAO,KAAK,IAAI,MAAM,GAAG;CAC3B;AACF;;AASA,MAAM,cAA6B,IAAI,SAAS,YAAY,aAAa,OAAO,CAAC;;AAGjF,MAAM,SAAS,OAAU,SAAqB,OAAuC;CACnF,IAAI;CACJ,MAAM,OAAO,IAAI,SAAoB,YAAY;EAC/C,QAAQ,iBAAiB,QAAQ,KAAA,CAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAC;CAC9D,CAAC;CACD,IAAI;EACF,OAAO,MAAM,QAAQ,KAAK,CAAC,SAAS,IAAI,CAAC;CAC3C,UAAU;EACR,aAAa,KAAK;CACpB;AACF;;AAGA,MAAM,WAAW,MAAc,WAA2B;CACxD,IAAI,MAAM,KAAK,IAAI,GAAG,MAAM;CAC5B,MAAM,OAAO,KAAK,WAAW,GAAG;CAChC,IAAI,MAAM,KAAK,MAAM,KAAK,UAAU,QAAQ,SAAU,QAAQ,OAAQ,OAAO;CAC7E,OAAO,KAAK,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ;AACpC;AAEA,MAAa,UAAU,OAAO,OAAmB,UAAsB,CAAC,MAAwB;CAC9F,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,WAAW,QAAQ,gBAAgB;CACzC,MAAM,cAAc,QAAQ,oBAAoB;CAChD,IAAI,CAAC,eAAe,OAAO,WAAW,GACpC,OAAO;EACL,OAAO;EACP,MAAM;EACN,OAAO;EACP,WAAW;EACX,MAAM,wCAAwC,OAAO,KAAK,MAAM,cAAc,OAAO,IAAI,CAAC,EAAE;CAC9F;CAGF,MAAM,MAAM,MAAM,iBAAiB,IAAI,WAAW,KAAK,CAAC;CACxD,IAAI;EACF,MAAM,UAAU,KAAK,IAAI;EACzB,MAAM,QAAkB,CAAC;EACzB,IAAI,QAAQ;EACZ,IAAI,OAAsB;EAC1B,MAAM,OAAO,KAAK,IAAI,IAAI,UAAU,QAAQ;EAC5C,IAAI,IAAI,WAAW,UAAU,OAAO,0BAA0B,OAAO,QAAQ,EAAE;EAC/E,MAAM,QAAQ,WACZ,0BAA0B,OAAO,KAAK,MAAM,WAAW,GAAI,CAAC,EAAE,oBAAoB,OAAO,SAAS,CAAC;EACrG,KAAK,IAAI,SAAS,GAAG,UAAU,MAAM,UAAU,GAAG;GAChD,IAAI,SAAS,GAAG;IACd,MAAM,MAAM;IACZ,IAAI,KAAK,IAAI,IAAI,UAAU,UAAU;KACnC,OAAO,KAAK,MAAM;KAClB;IACF;GACF;GACA,MAAM,OAAO,MAAM,IAAI,QAAQ,MAAM;GAGrC,MAAM,UAAU,MAAM,OAAO,KAAK,eAAe,GAAG,YAAY,KAAK,IAAI,IAAI,QAAQ;GACrF,IAAI,YAAY,KAAA,GAAW;IACzB,OAAO,0BAA0B,OAAO,KAAK,MAAM,WAAW,GAAI,CAAC,EAAE,oBAAoB,OAAO,MAAM;IACtG;GACF;GACA,MAAM,OAAO,KACV,QAAQ,MACN,QAAQ,SAAS,KAAK,QAAQ,KAAA,CAAS,CAAC,CACxC,KAAK,SAAS,GAAG,KAAK,OAAO,KAAK,KAAK,WAAW,OAAO,OAAO,IAAI,CAAC,CACrE,KAAK,EAAE,CACZ;GACA,KAAK,QAAQ;GACb,IAAI,KAAK,SAAS,GAAG;IAEnB,IAAI,KAAK,SAAS,WAAW,OAAO;KAClC,MAAM,KAAK,QAAQ,MAAM,WAAW,KAAK,CAAC;KAC1C,OAAO,uBAAuB,OAAO,QAAQ,EAAE,uBAAuB,OAAO,MAAM;KACnF;IACF;IACA,MAAM,KAAK,IAAI;IACf,SAAS,KAAK,SAAS;GACzB;GACA,IAAI,SAAS,YAAY,SAAS,IAAI,UAAU;IAC9C,OAAO,uBAAuB,OAAO,QAAQ,EAAE,uBAAuB,OAAO,MAAM;IACnF;GACF;EACF;EAEA,MAAM,UADQ,MAAM,QAAQ,GAAG,CAAC,CAAC,YAAY,KAAA,CAAS,EAAA,EAAI,KAAA,GAC5B;EAC9B,MAAM,QAAQ,OAAO,UAAU,WAAW,MAAM,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,IAAI;EAC9E,OAAO;GACL,OAAO,MAAM,SAAS,KAAK,CAAC,WAAW,KAAK,KAAK,IAAI,QAAQ;GAC7D,MAAM,MAAM,KAAK,MAAM;GACvB,OAAO,IAAI;GACX,WAAW,SAAS;GACpB;EACF;CACF,UAAU;EAER,MAAM,IAAI,YAAY,QAAQ;CAChC;AACF;;;;AC3NA,MAAa,aAAa,QAAwB,OAAO,aAAa,GAAG;;AAGzE,MAAM,mBAAmB;;AAEzB,MAAM,QAAQ;CACZ;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;AAEA,MAAM,aAAa;CAAC;CAAgB;CAA6B;CAAW;CAAO;AAAW;;;;;;AAO9F,MAAM,cACJ,YAC2E;CAC3E,MAAM,SAAS,WACb,WAAW,MAAM,QAAQ,OAAO,SAAS,KAAA,CAAS;CACpD,MAAM,UAAU,CAAC,GAAG,QAAQ,OAAO,KAAK,GAAG,GAAG,QAAQ,QAAQ,WAAW,CAAC,MAAM,MAAM,CAAC,CAAC;CACxF,MAAM,OAAkC,CAAC;CACzC,IAAI,UAAU;CACd,IAAI,OAAO;CACX,KAAK,MAAM,UAAU,SAAS;EAC5B,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,CAAC;EAClC,IAAI,YAAY;EAChB,IAAI,OAAO,MAAM;GACf,YAAY,OAAO,YACjB,OAAO,QAAQ,MAAM,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,SAAS,GAAG,CAAC,CAC/D;GACA,OAAO,KAAK,UAAU,SAAS,CAAC,CAAC;EACnC;EACA,IAAI,OAAO,MAAM;GACf,WAAW;GACX;EACF;EACA,KAAK,KAAK,SAAS;EACnB,QAAQ;CACV;CACA,OAAO;EAAE;EAAM;CAAQ;AACzB;;;;;;;;AASA,MAAa,iBAA4B,MAAM,YAAY;CACzD,IAAI,MAAM,MAAM,QAAQ,WAAW,GAAG,OAAO,QAAQ,MAAM,QAAQ,GAAG;CACtE,IAAI,QAAQ,gBAAgB,QAAQ,CAAC,YAAY,KAAK,QAAQ,WAAW,GAAG,OAAO,CAAC;CACpF,MAAM,MAAM,UAAU,QAAQ,GAAG;CACjC,IAAI;CACJ,IAAI;EACF,OAAO,YAAY,WAAW,MAAM,QAAQ,WAAW,GAAG,QAAQ,GAAG;CACvE,SAAS,OAAO;EAEd,OAAO,CACL;GACE;GACA,MAAM;GACN,OAAO,QAAQ;GACf,MAAM,QAAQ;GACd,KAAK,QAAQ;GACb,MAAM,EACJ,eAAe,iBAAiB,QAAQ,MAAM,QAAQ,MAAM,GAAG,GAAG,IAAI,OAAO,KAAK,EACpF;EACF,CACF;CACF;CACA,MAAM,SAAS,WAAW,KAAK,MAAM;CACrC,OAAO,CACL;EACE;EACA,MAAM;EACN,OAAO,KAAK;EACZ,MAAM,KAAK,KAAK,SAAS,IAAI,KAAK,OAAO,KAAK;EAC9C,KAAK,QAAQ;EACb,MAAM;GACJ,aAAa,KAAK;GAClB,MAAM,KAAK;GACX,cAAc,CAAC,GAAG,IAAI,IAAI,KAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;GACvD,QAAQ,OAAO;GACf,GAAI,OAAO,UAAU,IAAI,EAAE,gBAAgB,OAAO,QAAQ,IAAI,CAAC;GAC/D,OAAO,KAAK,MAAM,MAAM,GAAG,GAAG;GAC9B,eAAe,KAAK;GACpB,UAAU,KAAK;GACf,cAAc,KAAK;GACnB,eAAe,KAAK;GACpB,WAAW,KAAK;GAChB,UAAU,KAAK;GACf,SAAS,KAAK;GACd,GAAI,KAAK,MAAM,SAAS,IAAI,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;EACvD;EACA,GAAI,KAAK,aAAa,KAAA,IAAY,EAAE,WAAW,KAAK,SAAS,IAAI,CAAC;CACpE,CACF;AACF;AAEA,MAAM,qBAAqB;;;;;;AAO3B,MAAM,UAAU,OAAO,MAAkB,QAAsD;CAC7F,IAAI;CACJ,IAAI;EACF,OAAO,MAAM,QAAQ,MAAM,EAAE,UAAU,mBAAmB,CAAC;CAC7D,QAAQ;EACN,OAAO,CAAC;CACV;CACA,MAAM,OAAO,IAAI,IAAI,GAAG,CAAC,CAAC,SAAS,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;CACvD,IAAI,QAAQ;CACZ,IAAI;EACF,QAAQ,mBAAmB,IAAI;CACjC,QAAQ,CAER;CACA,MAAM,QAAQ,KAAK,UAAU,SAAS;CACtC,MAAM,QAAQ;EACZ,GAAI,KAAK,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;EAC5C,GAAI,KAAK,SAAS,OAAO,EAAE,MAAM,KAAK,KAAK,IAAI,CAAC;CAClD;CACA,IAAI,KAAK,KAAK,WAAW,GACvB,OAAO,CACL;EACE,KAAK,UAAU,GAAG;EAClB,MAAM;EACN;EACA,MAAM;EACN;EACA,MAAM;GACJ,QAAQ;GACR,OAAO,KAAK;GACZ,eAAe;GAEf,YAAY,KAAK,SAAS;GAC1B,GAAG;EACL;CACF,CACF;CAEF,OAAO,CACL;EACE,KAAK,UAAU,GAAG;EAClB,MAAM;EACN;EACA,MAAM,KAAK,KAAK,MAAM,GAAG,kBAAkB;EAC3C;EACA,MAAM;GAAE,QAAQ;GAAO,OAAO,KAAK;GAAO,eAAe,KAAK,KAAK;GAAQ,GAAG;EAAM;CACtF,CACF;AACF;AAEA,IAAa,aAAb,MAAwB;CACtB,2BAA4B,IAAI,IAAuB,CAAC,CAAC,QAAQ,aAAa,CAAC,CAAC;CAEhF,SAAS,QAAgB,WAA4B;EACnD,IAAI,KAAK,SAAS,IAAI,MAAM,GAAG,MAAM,IAAI,MAAM,UAAU,OAAO,0BAA0B;EAC1F,KAAK,SAAS,IAAI,QAAQ,SAAS;CACrC;CAEA,IAAI,QAAuC;EACzC,OAAO,KAAK,SAAS,IAAI,MAAM;CACjC;;CAGA,UAAoB;EAClB,OAAO,CAAC,GAAG,KAAK,SAAS,KAAK,CAAC;CACjC;AACF;;AA8CA,MAAa,kBAAqC;CAChD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;AAGA,MAAa,YACX;;;;;AAMF,MAAa,cAAc,KAAqC,eAAe,UAC7E,IAAI,aAAa,QAAQ,CAAC;;AAG5B,MAAa,gBAAgB,SAC3B,OAAO,YAAY,OAAO,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,gBAAgB,SAAS,GAAG,CAAC,CAAC;AAE1F,MAAM,aAAa,UACjB,iBAAiB,QAAS,MAAM,QAAQ,MAAM,OAAO,CAAC,CAAC,MAAM,KAAM,OAAO,KAAK;;AAGjF,MAAM,aAAa,QAA0C;CAC3D,MAAM,UAAU,IAAI,KAAK;CACzB,OAAO,OAAO,YAAY,YAAY,QAAQ,SAAS,IAAI,UAAU,KAAA;AACvE;;AAaA,MAAM,eAAe,OACnB,SACA,SACA,QACA,aACsC;CACtC,MAAM,UAAU,QAAQ;CACxB,IAAI,QAAQ,SAAS,KAAA,KAAa,YAAY,KAAA,GAAW,OAAO,EAAE,MAAM,CAAC,EAAE;CAE3E,IAAI,QAAQ,WAAW;EACrB,MAAM,QAAQ,QAAQ,MAAM,cAAc,OAAO;EACjD,IAAI,MAAM,SAAS,GAAG,OAAO,EAAE,MAAM,MAAM;CAC7C;CACA,MAAM,YAAY,QAAQ,WAAW,IAAI,MAAM;CAC/C,IAAI,cAAc,KAAA,GAAW,OAAO,EAAE,MAAM,CAAC,EAAE;CAC/C,IAAI;CACJ,IAAI;EACF,YAAY,MAAM,UAAU,QAAQ,MAAM;GACxC,KAAK,QAAQ;GACb,aAAa,WAAW,6BAA8B,QAAQ,eAAe;EAC/E,CAAC;CACH,SAAS,OAAO;EACd,OAAO;GAAE,MAAM,CAAC;GAAG,OAAO,mBAAmB,OAAO,KAAK,UAAU,KAAK;EAAI;CAC9E;CACA,MAAM,UAAU,UAAU,IAAI,SAAS,CAAC,CAAC,MAAM,SAAS,SAAS,KAAA,CAAS;CAC1E,IAAI,YAAY,KAAA,GAAW;EAEzB,QAAQ,QAAQ,cAAc,QAAQ,KAAK,OAAO;EAClD,OAAO;GACL,MAAM,CAAC;GACP;GACA,OAAO,4BAA4B,QAAQ;EAC7C;CACF;CACA,MAAM,MAAM,QAAQ,IAAI;CAkBxB,OAAO,EAAE,MAjBI,UAAU,KAAK,QAC1B,QAAQ,MAAM,OACZ;EACE,GAAG;EACH,UAAU,CAAC,OAAO;EAClB;EACA,YAAY,QAAQ,aAAa;EACjC,MAAM,QAAQ,QAAQ;EACtB;EACA;EACA,UAAU,QAAQ,QAAQ;EAC1B,GAAI,QAAQ,UAAU,KAAA,IAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;EAC9D,GAAI,QAAQ,WAAW,KAAA,IAAY,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;CACnE,GACA,GACF,CAEU,EAAE;AAChB;;;;;;;AAQA,MAAa,gBAAgB,OAC3B,SACA,YACqB;CACrB,MAAM,UAAU,MAAM,QAAQ,QAAQ,MAAM,OAAO;CACnD,IAAI,QAAQ,YAAY,QAAQ,QAAQ,YAAY,gBAAgB,OAAO;EAAE;EAAS,MAAM,CAAC;CAAE;CAC/F,OAAO;EAAE;EAAS,GAAI,MAAM,aAAa,SAAS,SAAS,QAAQ,QAAQ,KAAK;CAAG;AACrF;;AAGA,MAAa,iBAAiB,OAC5B,SACA,YACqB;CACrB,MAAM,UAAU,MAAM,QAAQ,QAAQ,OAAO,OAAO;CACpD,IAAI,QAAQ,YAAY,MAAM,OAAO;EAAE;EAAS,MAAM,CAAC;CAAE;CACzD,OAAO;EAAE;EAAS,GAAI,MAAM,aAAa,SAAS,SAAS,QAAQ,QAAQ,IAAI;CAAG;AACpF;AAyCA,MAAM,YAAY;;;;;;;AAQlB,MAAa,YAAY,OACvB,SACA,KACA,YACwB;CACxB,MAAM,gBAAgB,YAAkC;EACtD,KAAK;EACL,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,UAAU,QAAQ;EAClB,YAAY,QAAQ;EACpB,KAAK,QAAQ;EACb,QAAQ,QAAQ;EAChB,OAAO,QAAQ;EACf,QAAQ,QAAQ;CAClB;CACA,IAAI,QAAQ,MAAM,cAAc,SAAS,aAAa,GAAG,CAAC;CAC1D,MAAM,QAAkB,CAAC;CACzB,IAAI,MAAM,UAAU,KAAA,GAAW,MAAM,KAAK,MAAM,KAAK;CACrD,MAAM,SAAS,MAAM,KAAK,EAAE,EAAE,KAAK;CACnC,IAAI,OAAO,WAAW,YAAY,OAAO,SAAS,KAAK,MAAM,YAAY,KAAA,GAAW;EAClF,MAAM,WAAW,MAAM,cAAc,SAAS,aAAa,MAAM,CAAC;EAClE,MAAM,KACJ,mCAAmC,SAAS,QAAQ,IAAI,mCAC1D;EACA,IAAI,SAAS,UAAU,KAAA,GAAW,MAAM,KAAK,SAAS,KAAK;EAC3D,QAAQ;CACV;CACA,IAAI,OAAO,MAAM;CACjB,IAAI,WAAW;CACf,IAAI,YAAY,MAAM,QAAQ;CAC9B,MAAM,YAAY,MAAM,QAAQ,YAAY,QAAQ,MAAM,QAAQ,YAAY;CAC9E,MAAM,QAAQ,KAAK;CACnB,MAAM,OACJ,OAAO,SAAS,WACf,MAAM,QAAQ,gBAAgB,KAAA,KAAa,UAAU,KAAK,MAAM,QAAQ,WAAW;CACtF,MAAM,SACJ,QAAQ,WAAW,YAClB,QAAQ,WAAW,UAAU,OAAO,KAAK,oBAAoB;CAChE,IAAI,aAAa,QAAQ;EACvB,IAAI,MAAM,YAAY,KAAA,GAAW,MAAM,KAAK,yCAAyC;OAChF,IAAI,UAAU,KAAA,KAAa,CAAC,MAC/B,MAAM,KAAK,sEAAsE;OAC5E;GAEL,MAAM,UAAU,QAAQ,MAAM,IAAI,MAAM,GAAG;GAC3C,MAAM,QAAQ,YAAY,KAAA,KAAa,CAAC,QAAQ,YAAY,QAAQ,gBAAgB;GACpF,MAAM,QAAQ,MAAM,eAAe,SAAS;IAC1C,KAAK,MAAM,QAAQ;IACnB,QAAQ;IACR,UAAU,QAAQ,IAAI,QAAQ;IAC9B,YAAY,QAAQ;IACpB,KAAK,QAAQ;IACb,QAAQ,QAAQ;IAChB,OAAO,QAAQ;IACf,QAAQ,QAAQ;GAClB,CAAC;GACD,IAAI,MAAM,KAAK,SAAS,GAAG;IACzB,OAAO,MAAM;IACb,WAAW;IACX,YAAY,aAAa,MAAM,QAAQ;GACzC,OACE,MAAM,KAAK,MAAM,SAAS,MAAM,QAAQ,QAAQ,6BAA6B;EAEjF;CACF;CACA,MAAM,MAAM,KAAK,OAAO,KAAA,IAAY,KAAA,IAAa,QAAQ,MAAM,IAAI,KAAK,EAAE,CAAC,GAAG,KAAK,KAAK;CACxF,MAAM,OAAO,QAAQ,KAAA,KAAa,WAAW,GAAG;CAChD,MAAM,QAAQ,KAAK,KAAK;CACxB,MAAM,QAAQ,KAAK,KAAK;CACxB,IAAI,MAAM,MAAM,KAAK,SAAS;CAC9B,MAAM,OAAO,CAAC,MAAM,QAAQ,MAAM,GAAG,KAAK,CAAC,CAAC,QACzC,SAAyB,SAAS,KAAA,KAAa,KAAK,SAAS,CAChE;CACA,OAAO;EACL;EACA,KAAK,KAAK,OAAO;EACjB,SAAS,MAAM,QAAQ;EACvB,QAAQ,MAAM,QAAQ;EACtB,OAAO,QAAQ,KAAA,KAAa,OAAO,OAAO,IAAI;EAC9C,OAAO,KAAK,KAAK,UAAU;EAC3B,UAAU,KAAK,YAAY;EAC3B,YAAY,MAAM,QAAQ,KAAK,IAAK,QAAqB,CAAC;EAC1D,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,SAAS;EAC7C,UAAU,QAAQ,KAAA,IAAY,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ;EACnD,SAAS,KAAK,WAAW;EACzB,eAAe,KAAK,iBAAiB;EACrC,YAAY,KAAK,cAAc,MAAM,QAAQ,aAAa;EAC1D,YAAY;EACZ,WAAW;EACX,MAAM,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI;CAC5C;AACF;AA6EA,MAAM,YAAY,SAAuB,OAAoB;CAC3D,MAAM,MAAM,QAAQ,MAAM,IAAI,EAAE;CAChC,IAAI,QAAQ,KAAA,GACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,0CAA0C;CACrF,OAAO;AACT;AAEA,MAAM,eAAe,MAAc,OAA2B,aAA6B;CACzF,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GACrC,MAAM,IAAI,WAAW,GAAG,KAAK,sCAAsC,OAAO,KAAK,GAAG;CAEpF,OAAO,KAAK,MAAM,KAAK;AACzB;;AAGA,MAAM,cAAc,MAAc,aAA6B;CAC7D,MAAM,OAAO,KAAK,WAAW,QAAQ;CACrC,OAAO,WAAW,KAAK,WAAW,KAAK,UAAU,QAAQ,SAAU,QAAQ,QACvE,WAAW,IACX;AACN;;;;;;AAOA,MAAa,iBAAiB,OAC5B,SACA,KAMA,QACA,QAGI,CAAC,MACa;CAClB,MAAM,KAAK,QAAQ,IAAI;CACvB,MAAM,QAAQ,MAAM,SAAS,IAAI;CACjC,MAAM,SAAS,MAAM,UAAU,IAAI;CACnC,MAAM,QAAQ,IAAI,OAChB,WAAW;EACT,QAAQ;EACR,MAAM;EACN,SAAS;GAAE,KAAK,IAAI;GAAK,KAAK,IAAI;GAAK;GAAQ,QAAQ;GAA8B;EAAG;EACxF,aAAa;EACb,aAAa,IAAI,OAAO,IAAI;EAC5B,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;EACvC,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CAC3C,CAAC,CACH;AACF;;;;;;AAOA,MAAa,eAAe,OAC1B,SACA,IACA,UAAuB,CAAC,MACA;CACxB,MAAM,OAAO,YAAY,QAAQ,QAAQ,MAAM,CAAC;CAChD,MAAM,SAAS,YAAY,UAAU,QAAQ,QAAQ,GAAI;CACzD,MAAM,MAAM,SAAS,SAAS,EAAE;CAChC,MAAM,UACJ,QAAQ,YAAY,KAAA,IAAY,KAAA,IAAY,QAAQ,MAAM,YAAY,IAAI,QAAQ,OAAO;CAC3F,IAAI,QAAQ,YAAY,KAAA,KAAa,YAAY,KAAA,GAC/C,MAAM,IAAI,MAAM,gBAAgB,GAAG,mCAAmC,QAAQ,SAAS;CAEzF,MAAM,UACJ,IAAI,aAAa,QAAQ,SAAS,aAAa,OAC3C,OACC,SAAS,YAAY,IAAI;CAChC,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,OAAO,WAAW,EAAE,UAAU,QAAQ,GAAG,YAAY;CAC3D,IAAI,YAAY,QAAQ,cAAc,MAAM,eAAe,SAAS,KAAK,MAAM;CAC/E,MAAM,OAAO,SAAS,QAAQ,IAAI;CAClC,MAAM,QAAQ,WAAW,MAAM,KAAK,IAAI,MAAM,KAAK,MAAM,CAAC;CAC1D,MAAM,MAAM,OAAO,QAAQ,WAAW,MAAM,KAAK,IAAI,KAAK,QAAQ,QAAQ,MAAM,CAAC;CACjF,MAAM,OAAO,SAAS,QAAQ,IAAI;CAClC,OAAO;EACL,KAAK,IAAI;EACT,MAAM,SAAS,QAAQ,IAAI;EAC3B,OAAO,OAAO,KAAM,SAAS,SAAS,IAAI;EAC1C,KAAK,SAAS,OAAO,IAAI;EACzB,MAAM,OAAO,aAAa,IAAI,IAAI;EAClC,MAAM;EACN,IAAI;EACJ,OAAO,KAAK;EACZ,MAAM,OAAO,KAAK,KAAK,MAAM,OAAO,GAAG;EACvC,UAAU,IAAI;EACd,SAAS,SAAS,WAAW,IAAI;EACjC,YAAY,SAAS,cAAc,IAAI;EACvC,UAAU;EACV,WAAW;EACX,MAAM,OAAO,YAAY;CAC3B;AACF;;;;;AAMA,MAAM,gBACJ,SACA,IACA,OACA,YACkE;CAClE,MAAM,MAAM,SAAS,SAAS,EAAE;CAChC,MAAM,SACJ,QAAQ,YAAY,KAAA,IAAY,KAAA,IAAY,QAAQ,MAAM,YAAY,IAAI,QAAQ,OAAO;CAC3F,IAAI,QAAQ,YAAY,KAAA,KAAa,WAAW,KAAA,GAC9C,MAAM,IAAI,MAAM,gBAAgB,GAAG,mCAAmC,QAAQ,SAAS;CAEzF,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,OAAO;EAAE,KAAK,IAAI;EAAK,KAAK,IAAI;EAAK,UAAU,IAAI;CAAS;CAClE,MAAM,UAAmB,IAAI,aAAa,QAAQ,QAAQ,aAAa,OAAO,OAAO,IAAI;CACzF,IAAI,WAAW,EAAE,UAAU,QAAQ,GAAG,YAAY,GAiBhD,OAAO;EAAE,QAAA;GAfP,GAAG;GACH,OAAO;GACP,OAAO;GACP,KAAK;GACL,SAAS;GACT,SAAS;GACT,MAAM;GACN,YAAY;GACZ,OAAO;GACP,cAAc;GACd,UAAU;GACV,OAAO;GACP,WAAW;GACX,MAAM;EAEM;EAAG,UAAU;CAAM;CAEnC,MAAM,UAAU,UAAU,QAAQ,MAAM,YAAY,EAAE;CACtD,MAAM,aACJ,WAAW,KAAA,IACP,CAAC,MAAM,IACP,CAAC,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,CAAC,OAAO,GAAI,GAAG,QAAQ,MAAM,aAAa,EAAE,CAAC;CAErF,MAAM,OAAO,WAAW,QAAQ,YAAY,QAAQ,aAAa,IAAI,CAAC,CAAC;CACvE,MAAM,WAAW,iBAAiB,YAAY,QAAQ,OAAO;CAC7D,KAAK,MAAM,CAAC,OAAO,YAAY,WAAW,QAAQ,GAAG;EACnD,IAAI,WAAW,SAAS,YAAY,GAAG;EACvC,IAAI,MAAM,OAAO,QAAQ,MAAM,KAAK;EACpC,IAAI,SAAS;EACb,IAAI,CAAC,IAAI,SAAS,QAAQ,cAAc,KAAA,GAAW;GACjD,MAAM,OAAO,QAAQ,WAAW,KAAK;GACrC,SAAS,IAAI;EACf;EACA,IAAI,CAAC,IAAI,OAAO;EAChB,MAAM,QAAQ,WAAW,KAAA,KAAa,QAAQ;EAiB9C,OAAO;GAAE,QAAA;IAfP,GAAG;IACH,OAAO;IACP,OAAO,IAAI;IACX,KAAK,IAAI;IACT,SAAS,IAAI;IACb,SAAS,QAAQ;IACjB,MAAM,QAAQ;IACd,YAAY,QAAQ;IACpB;IACA,cAAc;IACd,UAAU;IACV,OAAO,IAAI;IACX,WAAW;IACX,MAAM,QAAQ,4DAA4D;GAE9D;GAAG;EAAS;CAC5B;CACA,MAAM,aACJ,YAAY,KAAA,KAAa,QAAQ,KAAK,iBAAiB,QAAQ,QAAQ,cAAc,KAAA;CAEvF,MAAM,QAAQ,CACZ,GAAI,aACA,CACE,qGACF,IACA,CAAC,GACL,GAAI,OAAO,KAAK,CAAC,eACb,CAAC,yEAAyE,OAAO,IAAI,EAAE,EAAE,IACzF,CAAC,CACP;CAiBA,OAAO;EAAE,QAAA;GAfP,GAAG;GACH,OAAO;GACP,OAAO;GACP,KAAK;GACL,SAAS;GACT,SAAS,SAAS,WAAW;GAC7B,MAAM,SAAS,QAAQ;GACvB,YAAY,SAAS,cAAc;GACnC,OAAO;GACP,cAAc;GACd,UAAU,CAAC;GACX,OAAO;GACP,WAAW;GACX,MAAM,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI;EAEhC;EAAG;CAAS;AAC5B;;;;;;;;;;;;;;;AA6BA,MAAa,cAAc,OACzB,SACA,IACA,OACA,UAAyB,CAAC,MACA;CAC1B,MAAM,EAAE,QAAQ,aAAa,aAAa,SAAS,IAAI,OAAO,OAAO;CACrE,IAAI,UAAU,MAAM,eAAe,SAAS,SAAS,SAAS,EAAE,GAAG,QAAQ;CAC3E,OAAO;AACT;;;;;;AAOA,MAAa,gBAAgB,OAC3B,SACA,GACA,UAII,CAAC,MACgB;CACrB,MAAM,QAAQ,QAAQ;CACtB,OAAO,QAAQ,MAAM,KAAK,GAAG;EAC3B,OAAO,QAAQ,SAAS;EACxB,MAAM,QAAQ,QAAQ;EACtB,GAAI,QAAQ,SAAS,KAAA,IAAY,EAAE,MAAM,QAAQ,KAAK,IAAI,CAAC;CAC7D,CAAC;AACH;AAwBA,MAAM,gBAAgB;AACtB,MAAM,aAAa;;AAGnB,MAAM,iBACJ,YACwD;CACxD,IAAI,QAAQ,IAAI,SAAS,aAAa,GACpC,OAAO;EAAE,KAAK,QAAQ,IAAI,MAAM,GAAG,EAAqB;EAAG,SAAS;CAAW;CAEjF,OAAO;EAAE,KAAK,QAAQ;EAAK,SAAS,QAAQ,aAAa,OAAO,aAAa;CAAQ;AACvF;AAkBA,MAAM,kBAA+B;CACnC,UAAU;CACV,QAAQ;CACR,SAAS;CACT,yBAAS,IAAI,IAAI;CACjB,QAAQ,CAAC;AACX;;AAGA,MAAM,gBACJ,OACA,UACA,YACS;CACT,MAAM,EAAE,KAAK,YAAY,cAAc,OAAO;CAC9C,MAAM,YAAY,QAAQ,cAAc,SAAS;CACjD,IAAI,QAAQ,YAAY,YAAY;EAClC,MAAM,WACJ,QAAQ,aACP,OAAO,QAAQ,SAAS,YAAY,QAAQ,KAAK,WAAW,WAAW,IACpE,QAAQ,KAAK,MAAM,CAAC,IACpB;EACN,MAAM,YAAY,QAAQ,WAAW,OAAO,QAAQ,WAAW;EAC/D,IAAI,aAAa,SAAS,aAAa,OAAO,QAAQ,gBAAgB,WACpE,MAAM,UAAU;GACd;GACA;GACA,SAAS,SAAS;GAClB,MAAM;GACN,MAAM;GACN,eAAe;GACf,cAAc;GACd,YAAY;GACZ,QAAQ,QAAQ,UAAU;GAC1B,UAAU;GACV;GACA,aAAa,QAAQ,eAAe;GACpC,UAAU,QAAQ,YAAY;EAChC,CAAC;EAEH;CACF;CAEA,IAAI,QAAQ,QAAQ,QAAQ,QAAQ,YAAY,MAAM;CACtD,MAAM,WAAW,MAAM,OAAO,KAAK,OAAO;CAC1C,MAAM,UAAU,SAAS,SAAS,aAAa,QAAQ,YAAY;CACnE,MAAM,YAAY,QAAQ,cAAc;CAExC,MAAM,OAAO,YAAY,OAAQ,QAAQ,SAAS,UAAW,UAAU,QAAQ,OAAQ;CACvF,MAAM,eAAe,YACjB,OACC,QAAQ,kBAAkB,UAAW,UAAU,iBAAiB,OAAQ;CAC7E,MAAM,OAAO,UAAW,UAAU,gBAAgB,KAAA,IAAa,KAAA;CAC/D,MAAM,cACJ,QAAQ,gBAAgB,SAAS,YAAY,aAAa,6BAA6B;CACzF,MAAM,SAAS,UACX,aAAa,KAAA,KAAa,SAAS,SAAS,OAAO,SAAS,UAAU,MACpE,MACA,SAAS,SACV,QAAQ,UAAU;CACvB,MAAM,UAAU;EACd;EACA;EACA,SAAS,SAAS;EAClB,MAAM,QAAQ,QAAQ;EACtB;EACA,eAAe;EACf,cAAc;EACd,YAAY;EACZ;EACA,UAAU,QAAQ,SAAS,YAAY;EACvC,GAAI,YAAY,EAAE,WAAW,KAAK,IAAI,CAAC;EACvC,aAAa,QAAQ,eAAe;EACpC,UAAU,QAAQ,aAAa,UAAW,UAAU,YAAY,OAAQ;CAC1E,CAAC;AACH;;AAGA,MAAM,eAAe,OACnB,SACA,QACA,UACA,UACkB;CAClB,IAAI,SAAS,SAAS,aAAa,SAAS,SAAS,WAAW;CAChE,MAAM,UAAU,SAAS;CACzB,IAAI,YAAY,QAAQ,OAAO,YAAY,YAAY,OAAO,QAAQ,QAAQ,UAAU;CACxF,IAAI,QAAQ,YAAY,YAAY;EAClC,aAAa,QAAQ,UAAU,OAAO;EACtC;CACF;CACA,IAAI,OAAO,QAAQ,SAAS,UAAU;CACtC,IAAI,QAAQ,YAAY,QAAQ,QAAQ,YAAY,gBAAgB;CACpE,MAAM,YAAY;CAClB,aAAa,QAAQ,UAAU,OAAO;CACtC,MAAM,YAAY,QAAQ,WAAW,IAAI,QAAQ,MAAM;CACvD,IAAI,cAAc,KAAA,GAAW;EAE3B,IAAI,QAAQ,WAAW,cAAc,MAAM,QAAQ,IAAI,QAAQ,MAAM;EACrE;CACF;CACA,MAAM,OAAO,MAAM,QAAQ,MAAM,IAAI,QAAQ,IAAI;CACjD,IAAI,SAAS,KAAA,GAAW;EACtB,MAAM,WAAW;EACjB;CACF;CACA,MAAM,EAAE,KAAK,YAAY,cAAc,OAAO;CAC9C,MAAM,SAAS,OAAO,OAAO,KAAK,OAAO;CACzC,IAAI;CACJ,IAAI;EACF,YAAY,MAAM,UAAU,MAAM;GAChC;GACA,aACE,YAAY,aACR,6BACC,QAAQ,gBAAgB,QAAQ,gBAAgB;EACzD,CAAC;CACH,SAAS,OAAO;EACd,MAAM,UAAU;EAChB,IAAI,MAAM,OAAO,SAAS,YACxB,MAAM,OAAO,KAAK;GAAE,SAAS,SAAS;GAAI,OAAO,UAAU,KAAK;EAAE,CAAC;EACrE;CACF;CAEA,IAAI,UAAU,MAAM,QAAQ,UAAU,GAAG,MAAM,KAAA,CAAS,GAAG;CAC3D,KAAK,MAAM,OAAO,WAChB,OAAO,OACL;EACE,GAAG;EACH,UAAU,CAAC,SAAS,EAAE;EACtB,SAAS,SAAS;EAClB,YAAY,QAAQ,cAAc,SAAS;EAC3C,MAAM,QAAQ;EACd,QAAQ,QAAQ;EAChB,UAAU,YAAY;EACtB,UAAU,QAAQ,SAAS,YAAY;EACvC,GAAI,SAAS,UAAU,KAAA,IAAY,EAAE,OAAO,SAAS,MAAM,IAAI,CAAC;EAChE,GAAI,SAAS,WAAW,KAAA,IAAY,EAAE,QAAQ,SAAS,OAAO,IAAI,CAAC;CACrE,GACA,SAAS,WACX;AAEJ;;;;;;;;;;;;AAaA,MAAa,UAAU,OACrB,SACA,UAAmD,CAAC,MACzB;CAC3B,MAAM,QAAQ,QAAQ,IAAI;CAC1B,MAAM,QAAQ,QAAQ,MAAM,aAC1B,QAAQ,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,QAAQ,MAAM,CAC5D;CACA,MAAM,QAAQ,SAAS;CACvB,IAAI,UAAU;CACd,IAAI;EACF,MAAM,OAAO,QAAQ,SAAS,KAC5B,EAAE,QAAQ,QAAQ,GAClB,EACE,iBAAiB;GACf,WAAW;EACb,EACF,CACF;EACA,WAAW,MAAM,YAAY,MAAM,MAAM,aAAa,SAAS,OAAO,UAAU,KAAK;EACrF,MAAM,UAAU,MAAM,MAAM,IAAI,IAAI,QAAQ,WAAW,QAAQ,CAAC,CAAC;EACjE,MAAM,SAAS,MAAM,OAAO,KAAK;EACjC,OAAO;GACL,UAAU,MAAM;GAChB,GAAG;GACH,QAAQ,MAAM;GACd,SAAS,MAAM;GACf;GACA;GACA,iBAAiB,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,KAAK;GACzC,QAAQ,MAAM;EAChB;CACF,SAAS,OAAO;EACd,MAAM,MAAM;EACZ,MAAM;CACR;AACF;;AAYA,MAAM,gBAAgB;;AAGtB,MAAM,UAAU,SAAuB,aAAqC;CAC1E,IAAI,SAAS,SAAS,aAAa,SAAS,SAAS,WAAW,OAAO;CACvE,MAAM,UAAU,SAAS;CACzB,IAAI,YAAY,QAAQ,OAAO,YAAY,YAAY,OAAO,QAAQ,QAAQ,UAC5E,OAAO;CACT,MAAM,YAAY,QAAQ,cAAc,SAAS;CACjD,MAAM,EAAE,KAAK,YAAY,cAAc,OAAO;CAC9C,MAAM,QAAQ,QAAQ,MAAM,OAAO,KAAK,OAAO;CAC/C,IAAI,QAAQ,YAAY,YAAY,OAAO,UAAU,KAAA,KAAa,MAAM,aAAa;CACrF,IACE,OAAO,QAAQ,SAAS,YACvB,QAAQ,YAAY,QAAQ,QAAQ,YAAY,gBAEjD,OAAO;CACT,IAAI,UAAU,KAAA,KAAa,MAAM,aAAa,WAAW,OAAO;CAChE,OACE,QAAQ,WAAW,IAAI,QAAQ,MAAM,MAAM,KAAA,KAAa,CAAC,QAAQ,MAAM,WAAW,SAAS,EAAE;AAEjG;;;;;;;;;;;;AAaA,MAAa,eAAe,OAAO,YAA8D;CAC/F,IAAI,QAAQ,MAAM,OAAO,OAAO;EAAE,MAAM;EAAW,GAAI,MAAM,QAAQ,OAAO;CAAG;CAC/E,MAAM,SAAS,QAAQ,MAAM,YAAY;CACzC,MAAM,QACJ,WAAW,KAAA,IAAY,KAAA,qBAAY,IAAI,KAAK,KAAK,MAAM,MAAM,IAAI,aAAa,EAAA,CAAE,YAAY;CAC9F,MAAM,QAAQ,SAAS;CACvB,IAAI,UAAU;CACd,MAAM,OAAO,QAAQ,SAAS,KAC5B;EAAE,QAAQ;EAAS,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM;CAAG,GAC7D,EACE,iBAAiB;EACf,WAAW;CACb,EACF,CACF;CACA,WAAW,MAAM,YAAY,MAC3B,IAAI,OAAO,SAAS,QAAQ,GAAG,MAAM,aAAa,SAAS,QAAQ,OAAO,UAAU,KAAK;CAE3F,IAAI,MAAM,aAAa,GAAG,OAAO,KAAA;CACjC,OAAO;EACL,MAAM;EACN,UAAU,MAAM;EAChB,GAAG,QAAQ,MAAM,OAAO;EACxB,QAAQ,MAAM;EACd,SAAS,MAAM;EACf;EACA,SAAS;EACT,iBAAiB,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,KAAK;EACzC,QAAQ,MAAM;CAChB;AACF;;;;;;;;;;;;;;;;AC1oCA,MAAa,oBAAoB,YAC/B,GAAG,cAAc,GAAG,QAAQ,KAAK,WAAW,YAAY,KAAA,IAAY,KAAK,KAAK,UAAU;;AAG1F,MAAM,UAAU;;;;;;AAOhB,MAAa,aAAa,YAA6B,QAAQ,KAAK,QAAQ,KAAK,CAAC;AAKlF,MAAM,QAA0C;CAC9C,IAAI;CACJ,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;CACH,GAAG;AACL;;;;;;AAOA,MAAa,cAAc,aAA+B;CACxD,IAAI,OAAO,aAAa,UAAU;EAChC,IAAI,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC3C,MAAM,IAAI,MAAM,mBAAmB,OAAO,QAAQ,GAAG;EACvD,OAAO;CACT;CACA,IAAI,oBAAoB,KAAK,QAAQ,GAAG,OAAO;CAC/C,MAAM,QAAQ,4CAA4C,KAAK,QAAQ;CACvE,MAAM,OAAO,QAAQ;CACrB,MAAM,SAAS,SAAS,KAAA,KAAa,SAAS,MAAM,KAAA,IAAY,MAAM,KAAK,YAAY;CACvF,IAAI,UAAU,QAAQ,WAAW,KAAA,GAC/B,MAAM,IAAI,MACR,oBAAoB,SAAS,yDAC/B;CAEF,OAAO,OAAO,MAAM,EAAE,IAAI;AAC5B;;AAsEA,MAAM,cAAc;CAAC;CAAY;CAAS;CAAgB;AAAc;;;;;;AAOxE,MAAM,cAAc,QAAsB;CACxC,UAAU,KAAK;EAAE,WAAW;EAAM,MAAM;CAAM,CAAC;CAC/C,MAAM,OAAO,UAAU,GAAG,CAAC,CAAC,eAAe,IAAI,aAAa,GAAG,IAAI;CACnE,MAAM,OAAO,SAAS,IAAI;CAC1B,IAAI,CAAC,KAAK,YAAY,GAAG,MAAM,IAAI,MAAM,aAAa,IAAI,iBAAiB;CAC3E,IACE,QAAQ,aAAa,WACrB,OAAO,QAAQ,WAAW,cAC1B,KAAK,QAAQ,QAAQ,OAAO,GAE5B,MAAM,IAAI,MAAM,aAAa,IAAI,uDAAuD;CAE1F,KAAK,MAAM,QAAQ,aAAa;EAC9B,IAAI,OAAO;EACX,IAAI;GACF,OAAO,UAAU,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe;EACnD,QAAQ,CAER;EACA,IAAI,MACF,MAAM,IAAI,MACR,GAAG,KAAK,gBAAgB,IAAI,oDAC9B;CACJ;AACF;;;;;;AAOA,MAAa,iBAAiB,YAA0C;CACtE,MAAM,MAAM,QAAQ;CACpB,IACE,QAAQ,KAAA,MACP,QAAQ,QAAQ,KAAA,KAAa,QAAQ,UAAU,KAAA,KAAa,QAAQ,UAAU,KAAA,IAE/E,MAAM,IAAI,MAAM,0DAA0D;CAE5E,MAAM,UAAU,QAAQ;CACxB,IAAI,YAAY,KAAA,KAAa,CAAC,aAAa,KAAK,OAAO,GACrD,MAAM,IAAI,MACR,YAAY,QAAQ,uEACtB;CAEF,IAAI,QAAQ,KAAA,GAAW,WAAW,GAAG;CACrC,MAAM,QAA2C,CAAC;CAClD,MAAM,MAAM,QAAQ,OAAO,IAAI,WAAW,KAAK,KAAM,UAAU,CAAC;CAChE,MAAM,QAAQ,QAAQ,SAAS,IAAI,cAAc,KAAK,KAAM,OAAO,CAAC;CACpE,IAAI,QAAQ,QAAQ;CACpB,IAAI,UAAU,KAAA,GAAW;EACvB,MAAM,OAAO,IAAI,SAAS,KAAK,KAAM,cAAc,CAAC;EACpD,MAAM,WAAW,KAAK,MAAM,CAAC;EAC7B,QAAQ;CACV;CACA,IAAI,YAAY,QAAQ;CACxB,IAAI,cAAc,KAAA,GAAW;EAC3B,IAAI,QAAQ,KAAA,GAAW,YAAY,IAAI,qBAAqB;OACvD;GACH,MAAM,OAAO,IAAI,qBAAqB,KAAK,KAAK,cAAc,CAAC;GAC/D,MAAM,WAAW,KAAK,MAAM,CAAC;GAC7B,YAAY;EACd;CACF;CACA,MAAM,QAAQ,QAAQ,SAAS;CAE/B,MAAM,WACJ,QAAQ,aAAa,QACjB,KAAA,IACC,QAAQ,YAAY,gBAAgB,mBAAmB,CAAC;CAC/D,MAAM,UAAU,IAAI,QAAQ;EAC1B,OAAO,QAAQ,SAAA;EACf,WAAW,QAAQ,aAAa,iBAAiB,OAAO;EACxD,MAAM,YAAY,KAAA,KAAa,UAAU,OAAO,IAAI,QAAQ,KAAK,IAAI,KAAA;EACrE;EACA;EACA;EACA;EACA,WAAW,QAAQ;EACnB,YAAY,QAAQ;EACpB,YAAY,QAAQ;EACpB,QAAQ,QAAQ;EAChB,YAAY,QAAQ;EACpB;EACA;EACA,QAAQ,QAAQ;EAChB,OAAO,QAAQ;EACf,WAAW,QAAQ;EACnB,kBAAkB,QAAQ;EAC1B,SAAS,QAAQ;CACnB,CAAC;CACD,MAAM,aAAa,IAAI,WAAW;CAClC,KAAK,MAAM,CAAC,QAAQ,cAAc,OAAO,QAAQ,QAAQ,cAAc,CAAC,CAAC,GACvE,WAAW,SAAS,QAAQ,SAAS;CAEvC,IAAI;CACJ,OAAO;EACL;EACA;EACA;EACA,UAAU;EACV;EACA;EACA;EACA,WAAW,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,YAAY;EAC7C,QAAQ;GACN,aAAa,YAAY;IAEvB,MAAM,QAAQ,MAAM;IACpB,KAAK,MAAM,SAAS,OAAO,MAAM,MAAM;GACzC,EAAA,CAAG;GACH,OAAO;EACT;CACF;AACF;;;;;;;;;;;AAkFA,MAAa,eAAe,YAAwC;CAClE,MAAM,UAAU,cAAc,OAAO;CACrC,IAAI,SAAS;CAEb,IAAI;CACJ,MAAM,QAA0B,WAAW,KAAK,QAAQ,KAAK,UAAU,CAAC,IACpE,aAAa,OAAO,CAAC,CAAC,MACnB,SAAS;EACR,IAAI,MAAM,SAAS,WAAW;GAC5B,MAAM,EAAE,MAAM,GAAG,WAAW;GAC5B,UAAU,SAAS,YAAY,SAAS,KAAA;EAC1C;CACF,SACM,KAAA,CACR,IACA,QAAQ,QAAQ;CACpB,MAAM,OAAO,YAA2B;EACtC,IAAI,QAAQ,MAAM,IAAI,MAAM,uBAAuB;EACnD,MAAM;EACN,IAAI,QAAQ,MAAM,IAAI,MAAM,uBAAuB;CACrD;CACA,OAAO;EACL;EACA,MAAM,MAAM,KAAK,eAAe,CAAC,GAAG;GAClC,MAAM,KAAK;GACX,UAAU,KAAA;GACV,MAAM,SAAS,aAAa,UAAU;GACtC,OAAO,UAAU,SAAS,KAAK;IAC7B,QAAQ,aAAa,UAAU;IAC/B,UAAU,WAAW,SAAS,SAAS,WAAW,MAAM;IACxD,YAAY,CAAC,GAAI,QAAQ,cAAc,CAAC,GAAI,GAAI,aAAa,cAAc,CAAC,CAAE;IAC9E,QAAQ,aAAa;IACrB,OAAO,aAAa;IACpB,QAAQ,aAAa;GACvB,CAAC;EACH;EACA,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG;GAC9B,MAAM,KAAK;GACX,OAAO,cAAc,SAAS,GAAG,WAAW;EAC9C;EACA,MAAM,KAAK,IAAI,cAAc,CAAC,GAAG;GAC/B,MAAM,KAAK;GACX,OAAO,aAAa,SAAS,IAAI,WAAW;EAC9C;EACA,MAAM,OAAO,IAAI,OAAO,gBAAgB,CAAC,GAAG;GAC1C,MAAM,KAAK;GACX,OAAO,YAAY,SAAS,IAAI,OAAO,aAAa;EACtD;EACA,MAAM,SAAS,IAAI,iBAAiB,CAAC,GAAG;GACtC,MAAM,KAAK;GAEX,MAAM,OAAO,QAAQ,MAAM,IAAI,EAAE,CAAC,EAAE,aAAa;GACjD,OAAO,QAAQ,MACZ,SAAS,IAAI,cAAc,CAAC,CAC5B,KAAK,YACJ,QAAQ,QAAQ,aAAa,OAAO;IAAE,GAAG;IAAS,OAAO;GAAG,IAAI,OAClE;EACJ;EACA,MAAM,UAAU;GACd,MAAM,KAAK;GAEX,MAAM,OAAO;GACb,UAAU,KAAA;GACV,OAAO,QAAQ,QAAQ,OAAO;EAChC;EACA,MAAM,QAAQ;GACZ,SAAS;GACT,MAAM;GACN,MAAM,QAAQ,MAAM;EACtB;CACF;AACF"}
|