polite-crawler 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/build/{cli-daemon-qS4PH2bM.mjs → cli-daemon-DkqyVxAI.mjs} +2 -2
  2. package/build/{cli-daemon-qS4PH2bM.mjs.map → cli-daemon-DkqyVxAI.mjs.map} +1 -1
  3. package/build/{cli-recipes-qbLRVThF.mjs → cli-recipes-D96ppgSX.mjs} +2 -2
  4. package/build/{cli-recipes-qbLRVThF.mjs.map → cli-recipes-D96ppgSX.mjs.map} +1 -1
  5. package/build/cli.mjs +7 -7
  6. package/build/{daemon-BXBbUrni.mjs → daemon-Bu1wpAwl.mjs} +3 -3
  7. package/build/{daemon-BXBbUrni.mjs.map → daemon-Bu1wpAwl.mjs.map} +1 -1
  8. package/build/{diff-CpKN0Jdc.mjs → diff-Dyzr2-Oi.mjs} +2 -2
  9. package/build/{diff-CpKN0Jdc.mjs.map → diff-Dyzr2-Oi.mjs.map} +1 -1
  10. package/build/{extract-DqeqiNHb.mjs → extract-DCClsy1y.mjs} +2 -2
  11. package/build/{extract-DqeqiNHb.mjs.map → extract-DCClsy1y.mjs.map} +1 -1
  12. package/build/html-DkQUAE_d.mjs.map +1 -1
  13. package/build/{html-DMgIeQfJ.d.mts → html-o5bDHjNi.d.mts} +2 -2
  14. package/build/html.d.mts +2 -2
  15. package/build/index.d.mts +55 -29
  16. package/build/index.mjs +10 -10
  17. package/build/{open-CEo22Hpu.mjs → open-DXTLJWhU.mjs} +10 -8
  18. package/build/open-DXTLJWhU.mjs.map +1 -0
  19. package/build/{recipe-B2SkepDZ.mjs → recipe-WTgRhbwY.mjs} +3 -3
  20. package/build/{recipe-B2SkepDZ.mjs.map → recipe-WTgRhbwY.mjs.map} +1 -1
  21. package/build/{recipe-product-CyZE3Ukd.mjs → recipe-product-CuUlNx4_.mjs} +4 -4
  22. package/build/recipe-product-CuUlNx4_.mjs.map +1 -0
  23. package/build/{runtime-C_2BQibH.mjs → runtime-BwOZJ71n.mjs} +2 -2
  24. package/build/runtime-BwOZJ71n.mjs.map +1 -0
  25. package/build/{version-Djva7JkN.mjs → version-CHqkjAh1.mjs} +2 -2
  26. package/build/{version-Djva7JkN.mjs.map → version-CHqkjAh1.mjs.map} +1 -1
  27. package/build/{wacz-DKk2HBmx.mjs → wacz-CBMk0bzI.mjs} +3 -3
  28. package/build/{wacz-DKk2HBmx.mjs.map → wacz-CBMk0bzI.mjs.map} +1 -1
  29. package/build/{warc-C_ew76bV.mjs → warc-D632nC5y.mjs} +2 -2
  30. package/build/{warc-C_ew76bV.mjs.map → warc-D632nC5y.mjs.map} +1 -1
  31. package/build/webbotauth-Dq_dA_u3.mjs.map +1 -1
  32. package/package.json +1 -1
  33. package/build/open-CEo22Hpu.mjs.map +0 -1
  34. package/build/recipe-product-CyZE3Ukd.mjs.map +0 -1
  35. package/build/runtime-C_2BQibH.mjs.map +0 -1
@@ -0,0 +1 @@
1
+ {"version":3,"file":"open-DXTLJWhU.mjs","names":["parseJson","whole","REFUSALS","sha256"],"sources":["../src/docs.ts","../src/sitemaps.ts","../src/discover.ts","../src/frontier.ts","../src/hoststate.ts","../src/embed.ts","../src/hybrid.ts","../src/vectors.ts","../src/watch.ts","../src/open.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { mkdirSync } from \"node:fs\";\nimport { dirname } from \"node:path\";\nimport type { StatementSync } from \"node:sqlite\";\nimport { DatabaseSync } from \"node:sqlite\";\nimport { fold, parseQuery, snippetOf } from \"./fold.ts\";\nimport type { Archived } from \"./fallback.ts\";\nimport type { AiInput } from \"./signals.ts\";\nimport { cacheKeyOf } from \"./urls.ts\";\n\nexport type { FtsQuery, Verified } from \"./fold.ts\";\nexport { ftsQuery, locate, parseQuery } from \"./fold.ts\";\n\n/**\n * The reading index holds each document that the crawler extracts. A reader can search it with\n * SQLite FTS5 (BM25), read a range of characters, and find an exact quote. The index is a\n * projection: delete the file, and `reindex` builds it again from the capture log and the blobs.\n *\n * A document is what a reader cites, not what the crawler fetched. One search response of an API\n * can hold many papers, so one capture can give many documents. One paper from two sources is one\n * document with two captures. The ids are stable and easy to read (`doi:10.1/x`, `pmid:123`).\n *\n * Each capture of a document is a version. The index keeps each version, and it shows one version\n * as the current document. These are the rules:\n *\n * - For one recipe, the latest capture by fetch time wins. A page that changes thus shows its new\n * text, also when the new text is shorter.\n * - A rendered capture wins over a plain capture of the same page. It stays current while the\n * plain body stays the same. After the plain body changes, the plain capture wins until a new render.\n * - When two recipes give one document, the longer text wins (`merge: \"richest\"`, the default).\n * With `merge: \"latest\"` on each version, the newer text wins. The metadata of the other recipes\n * fills the keys that the current version does not have.\n * - The AI input of a recipe is what its newest capture says. A \"no\" of one recipe wins over the\n * others. Thus a \"no\" stops when a later capture of the same page does not say it.\n *\n * The file has a schema version (`INDEX_SCHEMA`). A file of a different version opens empty, and\n * `stale` is true: the caller rebuilds it with `reindex`. Each connection waits for the lock of a\n * different process (`busy_timeout`), and each write is one transaction (`BEGIN IMMEDIATE`).\n */\n\n/** This value is the version of the schema of the index file. Version 0.1 of the package wrote files of version 0. */\nexport const INDEX_SCHEMA = 2;\n\n/** This value is the maximum number of capture ids that a document gives. The field `capture_count` gives the total. */\nexport const MAX_CAPTURES = 10;\n\n/** The search index reads this many characters of each text, at most. */\nconst MAX_INDEX_CHARS = 200_000;\n\n/** This is one document, as an extractor gives it. */\nexport interface ExtractedDoc {\n readonly doc: string;\n readonly kind: string;\n readonly title: string;\n /** This text is the readable text. For a paper, it is the title and the abstract. */\n readonly text: string;\n readonly url: string | null;\n readonly meta: Readonly<Record<string, unknown>>;\n /** This rule chooses between the texts of two recipes. The default is `richest`: the longer text. */\n readonly merge?: \"latest\" | \"richest\" | undefined;\n /** When the extractor cut the text, this text is the whole text. The function `verify` looks for a quote here too. */\n readonly full_text?: string | undefined;\n /**\n * The value `no` holds the document back from the AI reader, also when the capture said nothing.\n * An extractor sets it when the body states it, for example the headers in an archived response.\n *\n * @beta\n */\n readonly ai_input?: AiInput | undefined;\n}\n\n/**\n * This is one version of a document: the document as one capture gave it.\n *\n * @beta\n */\nexport interface DocInput extends ExtractedDoc {\n /** These ids are the capture records that this version came from. The first id names the version. */\n readonly captures: readonly string[];\n readonly ai_input?: AiInput | undefined;\n /** This id is the capture of the version. The default is the first id of `captures`. */\n readonly capture?: string | undefined;\n /** This time (ISO 8601) is the time of the fetch. The default is the time of the upsert. */\n readonly fetched_at?: string | undefined;\n readonly blob?: string | null | undefined;\n /** This value is the recipe that read the capture. A version with no recipe has the recipe `\"\"`. */\n readonly recipe?: string | undefined;\n /** This value is true when the headless browser made the capture. */\n readonly rendered?: boolean | undefined;\n /** This label comes from the capture. The index writes it as it is and does not read it. */\n readonly scope?: string | undefined;\n /** These labels come from the capture. The index writes them as they are and does not read them. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/** This is the current version of a document. */\nexport interface Doc {\n readonly doc: string;\n readonly kind: string;\n readonly title: string;\n readonly text: string;\n readonly url: string | null;\n readonly meta: Readonly<Record<string, unknown>>;\n /** These ids are the captures behind the current text, the newest first, `MAX_CAPTURES` or fewer. */\n readonly captures: readonly string[];\n /** This value is the number of versions of the document. */\n readonly capture_count: number;\n /** This id is the capture of the current text. */\n readonly capture: string;\n readonly blob: string | null;\n /** This time (ISO 8601) is the fetch time of the current text. */\n readonly fetched_at: string;\n readonly recipe: string;\n readonly rendered: boolean;\n readonly ai_input: AiInput;\n /** This value is true when the extractor cut the text, and the index keeps the whole text for `verify`. */\n readonly cut: boolean;\n readonly scope?: string | undefined;\n readonly labels?: Readonly<Record<string, string>> | undefined;\n readonly updated_at: string;\n}\n\n/** This is one version of a document, with no text. */\nexport interface DocVersion {\n readonly doc: string;\n readonly capture: string;\n readonly recipe: string;\n readonly rendered: boolean;\n readonly fetched_at: string;\n readonly blob: string | null;\n readonly kind: string;\n readonly title: string;\n readonly url: string | null;\n readonly ai_input: AiInput;\n /** This value is the length of the text of the version. */\n readonly chars: number;\n /** This value is true when the index keeps a whole text that is longer than the text. */\n readonly cut: boolean;\n}\n\n/** This is the text of one version, with its metadata. */\nexport interface VersionText extends DocVersion {\n readonly text: string;\n /** This text is the whole text when the extractor cut `text`. */\n readonly full_text: string | undefined;\n readonly meta: Readonly<Record<string, unknown>>;\n}\n\n/** This is the version of a document that was current at a time (`ReadingIndex.versionAt`). */\nexport interface VersionAt {\n /**\n * This version was current at the time, or it is undefined when the time is before the first\n * version. Its `ai_input` is `no` when the document was held back at that time.\n */\n readonly version: VersionText | undefined;\n /** This time (ISO 8601) is the fetch time of the first version of the document, or undefined for an unknown document. */\n readonly first: string | undefined;\n}\n\n/**\n * This value chooses the ranking of a search:\n *\n * - `exact` (the default): BM25 ranks each document that matches.\n * - `fast`: the form of the query that needs one part leaves out the common parts. A common part\n * is in more than `FAST_RANK_SHARE` of the documents. The rarest part stays when each part is common.\n *\n * The documents that hold each part come first in the two modes, with the same rank. In the mode\n * `fast`, the rest of the results leaves out a document that holds only common parts. The common\n * parts also do not add to the score of the rest. Thus `fast` is faster for a long question, and\n * its results can be different. The README gives the measured difference.\n */\nexport type RankMode = \"exact\" | \"fast\";\n\n/**\n * In the mode `fast`, a part is common when more than this share of the documents hold it.\n *\n * @beta\n */\nexport const FAST_RANK_SHARE = 0.2;\n\n/**\n * This value chooses the ranking of a search of the crawler: a `RankMode` of the index, or\n * `hybrid`. The mode `hybrid` fuses the BM25 ranking with the ranking of the vectors of an\n * embedder (`OpenOptions.embedder`). Without an embedder, it throws an error.\n */\nexport type SearchRank = RankMode | \"hybrid\";\n\n/** The text of the error of hybrid search with no embedder. */\nexport const NO_EMBEDDER =\n \"hybrid search needs an embedder: open the crawler with the option embedder (transformersEmbedder or ollamaEmbedder), or give --embedder transformers or ollama to the command line\";\n\n/** This function gives the rank mode of an option. A value that is not a rank mode throws a `RangeError`. */\nconst rankModeOf = (value: unknown): RankMode => {\n if (value === undefined || value === \"exact\") return \"exact\";\n if (value === \"fast\") return \"fast\";\n // the index has no vectors: the crawler ranks with an embedder, and the index never falls back\n if (value === \"hybrid\")\n throw new RangeError(\n 'the index ranks with \"exact\" and \"fast\" only: rank \"hybrid\" needs the vectors of an embedder, so use find or rankWithin of a crawler with the option embedder',\n );\n throw new RangeError(\n `rank must be \"exact\" or \"fast\", not ${JSON.stringify(value) ?? \"undefined\"}`,\n );\n};\n\n/**\n * These are the options of a search of the index.\n *\n * @beta\n */\nexport interface FindIndexOptions {\n /** This value is the maximum number of results. The index gives 1000 or fewer. */\n readonly limit: number;\n /** This value keeps only the documents of one kind, for example `page` or `pdf`. */\n readonly kind?: string | null | undefined;\n /** This value chooses the ranking. The default is `exact`. */\n readonly rank?: RankMode | undefined;\n}\n\n/** This is the passage of a hit of a hybrid search: the chunk of the text that is closest to the question. */\nexport interface Passage {\n /** These values are the positions of the chunk in the text of the capture of the hit, in UTF-16 units. */\n readonly start: number;\n readonly end: number;\n /** This value is the cosine of the vectors of the chunk and the question, from -1 to 1. */\n readonly similarity: number;\n}\n\nexport interface Found {\n readonly doc: string;\n readonly kind: string;\n readonly title: string;\n readonly url: string | null;\n readonly snippet: string;\n readonly score: number;\n readonly ai_input: AiInput;\n /** This time (ISO 8601) is the fetch time of the current text. */\n readonly fetched_at: string;\n /** This id is the capture of the current text. */\n readonly capture: string;\n /** This value tells that the document is an archived copy (the kind `archived`), with the archive and the date of the copy. */\n readonly archived?: Archived | undefined;\n /** This value is the passage that a hybrid search found. A search of the other modes does not give it. */\n readonly passage?: Passage | undefined;\n}\n\n/**\n * This is the current text of a document in short: its hash, with no text.\n *\n * @beta\n */\nexport interface DocDigest {\n readonly doc: string;\n readonly kind: string;\n readonly ai_input: AiInput;\n /** This value is the SHA-256 of the current text of the document, in hex. */\n readonly text_sha: string;\n}\n\n/**\n * These options choose the documents of `ReadingIndex.digests`.\n *\n * @beta\n */\nexport interface DigestOptions {\n /** These documents, in place of a list of each document. A document that is not in the index gives nothing. */\n readonly docs?: readonly string[] | undefined;\n /** In a list of each document, the documents with an id after this id, in the sequence of the ids. */\n readonly after?: string | undefined;\n /** In a list of each document, this number of documents at most. The default is 500. */\n readonly limit?: number | undefined;\n}\n\n/**\n * The variant of a cache entry: the plain response, or the page that the browser rendered.\n *\n * @beta\n */\nexport type Variant = \"plain\" | \"rendered\";\n\n/** @beta */\nexport interface Latest {\n readonly url: string;\n readonly capture: string;\n readonly blob: string | null;\n readonly etag: string | null;\n readonly last_modified: string | null;\n readonly content_type: string | null;\n readonly fetched_at: string;\n readonly status: number;\n /** This value is what the page said about AI input at the time of the fetch. */\n readonly ai_input: AiInput;\n /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */\n readonly location?: string | null | undefined;\n /** This value is true when the stored body is only the first part of the response. */\n readonly truncated?: boolean | undefined;\n /** This time (ISO 8601) is the end of the freshness that the server gave. */\n readonly fresh_until?: string | null | undefined;\n /** This text holds the request headers that the response varies on (`Vary`), with their values. */\n readonly vary_key?: string | null | undefined;\n /** The default is `plain`. A key that ends in `#rendered` (the form of version 0.1) is `rendered`. */\n readonly variant?: Variant | undefined;\n}\n\n/**\n * These are the counts of the index.\n *\n * @beta\n */\nexport interface IndexCounts {\n readonly docs: number;\n readonly urls: number;\n}\n\n/**\n * A rebuild of the index writes a new generation of the tables. The old generation answers until the commit.\n *\n * @beta\n */\nexport interface IndexBuild {\n /** This method adds a version to the new generation. The build writes the versions in batches. */\n upsert(input: DocInput, now: string): void;\n /** This method sets a cache entry in the new generation, when it is newer than the stored entry. */\n setLatest(entry: Latest): void;\n /** This method reads a cache entry of the new generation. */\n latest(url: string, variant?: Variant): Latest | undefined;\n /** This method writes the waiting versions in one transaction. */\n flush(): void;\n /**\n * This method copies the versions of each recipe that is not in `recipes` from the current\n * generation. A runtime with no extractor for a recipe thus keeps the documents of that recipe.\n * It gives the number of documents that it copied.\n */\n carry(recipes: ReadonlySet<string>): number;\n /**\n * This method makes the new generation current. It first copies the rows that a different writer\n * added to the current generation since `since`. Then it removes the old generation.\n */\n commit(since: string): IndexCounts;\n /** This method removes the new generation. The current generation stays. */\n abort(): void;\n}\n\ninterface Names {\n readonly docs: string;\n readonly fts: string;\n readonly versions: string;\n readonly texts: string;\n readonly latest: string;\n}\n\nconst namesOf = (generation: number): Names => ({\n docs: `docs_${String(generation)}`,\n fts: `fts_${String(generation)}`,\n versions: `versions_${String(generation)}`,\n texts: `texts_${String(generation)}`,\n latest: `latest_${String(generation)}`,\n});\n\nconst GENERATION_TABLE = /^(docs|fts|versions|texts|latest)_(\\d+)$/;\n\n/** A build that started this long ago and did not finish belongs to a process that stopped. */\nconst ABANDONED_MS = 24 * 60 * 60 * 1000;\n\n/** The FTS5 tokenizer: folded text, English stems, and spacing marks kept in the words. */\nconst TOKENIZER = `tokenize = \"porter unicode61 remove_diacritics 0 categories 'L* N* Co Mc'\"`;\n\nconst ddlOf = (n: Names): string => `\nCREATE TABLE IF NOT EXISTS ${n.texts} (\n sha TEXT PRIMARY KEY,\n body TEXT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS ${n.versions} (\n doc TEXT NOT NULL,\n capture TEXT NOT NULL,\n recipe TEXT NOT NULL,\n rendered INTEGER NOT NULL,\n fetched_at TEXT NOT NULL,\n blob TEXT,\n kind TEXT NOT NULL,\n title TEXT NOT NULL,\n url TEXT,\n meta_sha TEXT NOT NULL,\n text_sha TEXT NOT NULL,\n text_chars INTEGER NOT NULL,\n full_sha TEXT,\n ai_input TEXT NOT NULL,\n merge TEXT NOT NULL,\n scope TEXT,\n labels TEXT,\n PRIMARY KEY (doc, capture)\n);\nCREATE INDEX IF NOT EXISTS ${n.versions}_time ON ${n.versions} (doc, recipe, rendered, fetched_at);\nCREATE INDEX IF NOT EXISTS ${n.versions}_capture ON ${n.versions} (capture);\nCREATE INDEX IF NOT EXISTS ${n.versions}_fetched ON ${n.versions} (fetched_at);\nCREATE INDEX IF NOT EXISTS ${n.versions}_recipe ON ${n.versions} (recipe);\nCREATE TABLE IF NOT EXISTS ${n.docs} (\n doc TEXT PRIMARY KEY,\n kind TEXT NOT NULL,\n title TEXT NOT NULL,\n url TEXT,\n meta TEXT NOT NULL,\n text_sha TEXT NOT NULL,\n full_sha TEXT,\n captures TEXT NOT NULL,\n capture_count INTEGER NOT NULL,\n capture TEXT NOT NULL,\n blob TEXT,\n fetched_at TEXT NOT NULL,\n recipe TEXT NOT NULL,\n rendered INTEGER NOT NULL,\n ai_input TEXT NOT NULL,\n scope TEXT,\n labels TEXT,\n updated_at TEXT NOT NULL\n);\nCREATE VIRTUAL TABLE IF NOT EXISTS ${n.fts} USING fts5(\n title, text, content = '', contentless_delete = 1, ${TOKENIZER}\n);\nCREATE TABLE IF NOT EXISTS ${n.latest} (\n url TEXT NOT NULL,\n variant TEXT NOT NULL,\n capture TEXT NOT NULL,\n blob TEXT,\n etag TEXT,\n last_modified TEXT,\n content_type TEXT,\n fetched_at TEXT NOT NULL,\n status INTEGER NOT NULL,\n ai_input TEXT NOT NULL,\n location TEXT,\n truncated INTEGER,\n fresh_until TEXT,\n vary_key TEXT,\n PRIMARY KEY (url, variant)\n);\nCREATE INDEX IF NOT EXISTS ${n.latest}_fetched ON ${n.latest} (fetched_at);\n`;\n\ninterface VersionRow {\n doc: string;\n capture: string;\n recipe: string;\n rendered: number;\n fetched_at: string;\n blob: string | null;\n kind: string;\n title: string;\n url: string | null;\n meta_sha: string;\n text_sha: string;\n text_chars: number;\n full_sha: string | null;\n ai_input: string;\n merge: string;\n scope: string | null;\n labels: string | null;\n}\n\ninterface DocRow {\n rowid: number;\n doc: string;\n kind: string;\n title: string;\n url: string | null;\n meta: string;\n text_sha: string;\n full_sha: string | null;\n captures: string;\n capture_count: number;\n capture: string;\n blob: string | null;\n fetched_at: string;\n recipe: string;\n rendered: number;\n ai_input: string;\n scope: string | null;\n labels: string | null;\n updated_at: string;\n}\n\n/** A candidate for the current version: the version that one recipe gives. */\ninterface Candidate {\n readonly row: VersionRow;\n /** This value is what the newest capture of the recipe says about AI input. */\n readonly ai: AiInput;\n /** These rows are the other variant of the recipe (plain or rendered), when it has one. */\n readonly others: readonly VersionRow[];\n}\n\n/** The choice of the current version: the winner, the candidates of each recipe and the AI input of the document. */\ninterface Choice {\n readonly winner: Candidate;\n readonly candidates: readonly Candidate[];\n readonly aiInput: AiInput;\n}\n\ninterface LatestRow {\n url: string;\n variant: string;\n capture: string;\n blob: string | null;\n etag: string | null;\n last_modified: string | null;\n content_type: string | null;\n fetched_at: string;\n status: number;\n ai_input: string;\n location: string | null;\n truncated: number | null;\n fresh_until: string | null;\n vary_key: string | null;\n}\n\nconst RENDERED_MARK = \"#rendered\";\n\n/** This function splits a cache key into its URL and its variant. A key in the form of version 0.1 (`…#rendered`) is a rendered variant. */\nconst keyOf = (\n url: string,\n variant: Variant | undefined,\n): { readonly url: string; readonly variant: Variant } => {\n if (url.endsWith(RENDERED_MARK)) {\n return { url: cacheKeyOf(url.slice(0, -RENDERED_MARK.length)), variant: \"rendered\" };\n }\n return { url: cacheKeyOf(url), variant: variant ?? \"plain\" };\n};\n\nconst sha = (text: string): string => createHash(\"sha256\").update(text, \"utf8\").digest(\"hex\");\n\nconst parseJson = <T>(text: string | null | undefined, fallback: T): T => {\n if (text === null || text === undefined) return fallback;\n try {\n return JSON.parse(text) as T;\n } catch {\n return fallback;\n }\n};\n\nconst toLatest = (row: LatestRow): Latest => ({\n url: row.url,\n capture: row.capture,\n blob: row.blob,\n etag: row.etag,\n last_modified: row.last_modified,\n content_type: row.content_type,\n fetched_at: row.fetched_at,\n status: row.status,\n ai_input: row.ai_input as AiInput,\n location: row.location,\n truncated: row.truncated === 1 ? true : undefined,\n fresh_until: row.fresh_until,\n vary_key: row.vary_key,\n variant: row.variant as Variant,\n});\n\nconst toVersion = (row: VersionRow): DocVersion => ({\n doc: row.doc,\n capture: row.capture,\n recipe: row.recipe,\n rendered: row.rendered === 1,\n fetched_at: row.fetched_at,\n blob: row.blob,\n kind: row.kind,\n title: row.title,\n url: row.url,\n ai_input: row.ai_input as AiInput,\n chars: row.text_chars,\n cut: row.full_sha !== null,\n});\n\nconst messageOf = (error: unknown): string =>\n error instanceof Error ? error.message : String(error);\n\n/**\n * A reading index keeps the documents, their versions, the search and the cache entries. `DocIndex`\n * is the SQLite index. A runtime takes any object with these methods, for example a search index\n * of a different kind that wraps a `DocIndex`.\n *\n * @beta\n */\nexport interface ReadingIndex {\n /** This value is true when the index opened empty and a rebuild did not fill it yet. */\n readonly stale: boolean;\n get(doc: string): Doc | undefined;\n upsert(input: DocInput, now: string): Doc;\n versions(doc: string, options?: { readonly limit?: number }): DocVersion[];\n versionText(doc: string, capture?: string): VersionText | undefined;\n /**\n * This optional method gives the version that was current at a time (ISO 8601), with the rules\n * that choose the current version. An index without it gets the newest version at or before the\n * time, from `versions` and `versionText`.\n */\n versionAt?(doc: string, at: string): VersionAt;\n earlierTexts(doc: string, options?: { readonly limit?: number }): VersionText[];\n docsOfCapture(capture: string): Doc[];\n /** This method tells if a capture gave a version of a document. */\n hasVersion(capture: string): boolean;\n /**\n * This optional method gives the hash of the current text of documents, with no text. Hybrid\n * search uses it to find the documents whose text has no vectors. An index without it gives the\n * same facts from `get`, more slowly.\n */\n digests?(options: DigestOptions): DocDigest[];\n find(q: string, options: FindIndexOptions): Found[];\n rankWithin(\n q: string,\n docs: readonly string[],\n options?: { readonly rank?: RankMode | undefined },\n ): Array<{ readonly doc: string; readonly score: number }>;\n latest(url: string, variant?: Variant): Latest | undefined;\n setLatest(entry: Latest): void;\n /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */\n newestFetch(): string | undefined;\n counts(): IndexCounts;\n beginRebuild(options?: { readonly batch?: number }): IndexBuild;\n}\n\n/**\n * These are the options of a reading index.\n *\n * @beta\n */\nexport interface DocIndexOptions {\n /** This value is the time in milliseconds that a write waits for the lock of a different process. The default is 10 seconds. */\n readonly busyTimeoutMs?: number | undefined;\n}\n\n/** @beta */\nexport class DocIndex implements ReadingIndex {\n private readonly db: DatabaseSync;\n private readonly statements = new Map<string, StatementSync>();\n private depth = 0;\n private closed = false;\n private empty: boolean;\n\n /**\n * This value is true when the file had a different schema, or no tables, so the index opened\n * empty. A store with captures then needs `reindex`. The commit of a rebuild makes it false.\n */\n get stale(): boolean {\n return this.empty;\n }\n\n constructor(\n readonly path: string,\n options: DocIndexOptions = {},\n ) {\n if (path !== \":memory:\") mkdirSync(dirname(path), { recursive: true, mode: 0o700 });\n this.db = new DatabaseSync(path);\n // the wait for the lock of a different process comes first, before any statement that writes\n const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 10_000));\n this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);\n try {\n this.db.exec(\n \"CREATE VIRTUAL TABLE temp.polite_fts5_probe USING fts5(x, content = '', contentless_delete = 1); DROP TABLE temp.polite_fts5_probe;\",\n );\n } catch (cause) {\n this.db.close();\n throw new Error(\n `polite-crawler needs SQLite FTS5 with contentless_delete, and the SQLite of Node.js ${process.version} does not have it: use Node.js 22.19 or later`,\n { cause },\n );\n }\n if (path !== \":memory:\") this.db.exec(\"PRAGMA journal_mode = WAL;\");\n // the index is a projection, so a power cut can cost its last transactions and nothing else\n this.db.exec(\"PRAGMA synchronous = NORMAL;\");\n this.empty = this.write(() => this.prepareSchema());\n }\n\n /** This method makes the tables of the schema. It gives true when the index opened empty. */\n private prepareSchema(): boolean {\n const version = (this.db.prepare(\"PRAGMA user_version\").get() as { user_version: number })\n .user_version;\n const tables = this.db\n .prepare(\"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'\")\n .all() as Array<{ name: string }>;\n if (version === INDEX_SCHEMA && tables.some((table) => table.name === \"index_meta\")) {\n this.dropAbandoned();\n return false;\n }\n // an index of a different schema is a projection: it opens empty, and reindex fills it\n const virtual = this.db\n .prepare(\n \"SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'\",\n )\n .all() as Array<{ name: string }>;\n for (const { name } of virtual) this.db.exec(`DROP TABLE IF EXISTS \"${name}\"`);\n const rest = this.db\n .prepare(\"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'\")\n .all() as Array<{ name: string }>;\n for (const { name } of rest) this.db.exec(`DROP TABLE IF EXISTS \"${name}\"`);\n this.db.exec(\"CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)\");\n this.db.exec(ddlOf(namesOf(1)));\n this.statements.clear();\n this.setMeta(\"generation\", \"1\");\n this.setMeta(\"next_generation\", \"2\");\n this.db.exec(`PRAGMA user_version = ${String(INDEX_SCHEMA)}`);\n return true;\n }\n\n private setMeta(key: string, value: string): void {\n this.stmt(\n \"INSERT INTO index_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value\",\n ).run(key, value);\n }\n\n private getMeta(key: string): string | undefined {\n return (\n this.stmt(\"SELECT value FROM index_meta WHERE key = ?\").get(key) as\n { value: string } | undefined\n )?.value;\n }\n\n private stmt(sql: string): StatementSync {\n let statement = this.statements.get(sql);\n if (statement === undefined) {\n statement = this.db.prepare(sql);\n this.statements.set(sql, statement);\n }\n return statement;\n }\n\n /**\n * This method does a write in one transaction. A transaction in progress takes the write in,\n * also a transaction that the caller opened on the connection.\n */\n private write<T>(work: () => T): T {\n if (this.closed) throw new Error(\"the index is closed\");\n if (this.depth > 0 || this.db.isTransaction) {\n this.depth += 1;\n try {\n return work();\n } finally {\n this.depth -= 1;\n }\n }\n this.db.exec(\"BEGIN IMMEDIATE\");\n this.depth = 1;\n try {\n const result = work();\n this.db.exec(\"COMMIT\");\n return result;\n } catch (error) {\n if (this.db.isTransaction) this.db.exec(\"ROLLBACK\");\n throw error;\n } finally {\n this.depth = 0;\n }\n }\n\n private generation(): number {\n return Number(this.getMeta(\"generation\") ?? \"1\");\n }\n\n /**\n * This method does a step on the tables of the current generation. A rebuild of a different\n * process can remove those tables in the meantime. Then the method reads the generation again\n * and does the step once more.\n */\n private live<T>(step: (names: Names) => T): T {\n if (this.closed) throw new Error(\"the index is closed\");\n try {\n return step(namesOf(this.generation()));\n } catch (error) {\n if (!/no such table/i.test(messageOf(error)) || this.depth > 0) throw error;\n this.statements.clear();\n return step(namesOf(this.generation()));\n }\n }\n\n /** This method removes the tables of a generation. */\n private dropGeneration(generation: number): void {\n const n = namesOf(generation);\n for (const table of [n.fts, n.docs, n.versions, n.texts, n.latest]) {\n this.db.exec(`DROP TABLE IF EXISTS ${table}`);\n }\n this.db.prepare(\"DELETE FROM index_meta WHERE key = ?\").run(`building:${String(generation)}`);\n this.statements.clear();\n }\n\n /** This method removes the generations of builds that a stopped process left. */\n private dropAbandoned(): void {\n const current = this.generation();\n const generations = new Set<number>();\n for (const { name } of this.db\n .prepare(\"SELECT name FROM sqlite_master WHERE type = 'table'\")\n .all() as Array<{ name: string }>) {\n const match = GENERATION_TABLE.exec(name);\n if (match !== null) generations.add(Number(match[2]));\n }\n for (const generation of generations) {\n if (generation === current) continue;\n const started = this.getMeta(`building:${String(generation)}`);\n const age =\n started === undefined ? Number.POSITIVE_INFINITY : Date.now() - Date.parse(started);\n if (!(age < ABANDONED_MS)) this.dropGeneration(generation);\n }\n }\n\n // ── texts and versions ────────────────────────────────────────────────────\n\n private putText(n: Names, body: string): string {\n const key = sha(body);\n this.stmt(`INSERT OR IGNORE INTO ${n.texts} (sha, body) VALUES (?, ?)`).run(key, body);\n return key;\n }\n\n private textOf(n: Names, key: string | null): string | undefined {\n if (key === null) return undefined;\n return (\n this.stmt(`SELECT body FROM ${n.texts} WHERE sha = ?`).get(key) as\n { body: string } | undefined\n )?.body;\n }\n\n private putVersion(n: Names, input: DocInput, now: string): void {\n const full =\n input.full_text !== undefined && input.full_text !== input.text\n ? this.putText(n, input.full_text)\n : null;\n this.stmt(\n `INSERT INTO ${n.versions} (doc, capture, recipe, rendered, fetched_at, blob, kind, title, url,\n meta_sha, text_sha, text_chars, full_sha, ai_input, merge, scope, labels)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(doc, capture) DO UPDATE SET recipe = excluded.recipe, rendered = excluded.rendered,\n fetched_at = excluded.fetched_at, blob = excluded.blob, kind = excluded.kind, title = excluded.title,\n url = excluded.url, meta_sha = excluded.meta_sha, text_sha = excluded.text_sha,\n text_chars = excluded.text_chars, full_sha = excluded.full_sha, ai_input = excluded.ai_input,\n merge = excluded.merge, scope = excluded.scope, labels = excluded.labels`,\n ).run(\n input.doc,\n input.capture ?? input.captures[0] ?? \"\",\n input.recipe ?? \"\",\n input.rendered === true ? 1 : 0,\n input.fetched_at ?? now,\n input.blob ?? null,\n input.kind,\n input.title,\n input.url,\n this.putText(n, JSON.stringify(input.meta ?? {})),\n this.putText(n, input.text),\n input.text.length,\n full,\n input.ai_input ?? \"unset\",\n input.merge ?? \"richest\",\n input.scope ?? null,\n input.labels === undefined ? null : JSON.stringify(input.labels),\n );\n }\n\n /** This method gives the newest version of a document for one recipe and one variant, at `until` or before when it is given. */\n private newest(\n n: Names,\n doc: string,\n recipe: string,\n rendered: number,\n until: string | undefined,\n ): VersionRow | undefined {\n if (until === undefined) {\n return this.stmt(\n `SELECT * FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = ?\n ORDER BY fetched_at DESC, capture DESC LIMIT 1`,\n ).get(doc, recipe, rendered) as VersionRow | undefined;\n }\n return this.stmt(\n `SELECT * FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = ? AND fetched_at <= ?\n ORDER BY fetched_at DESC, capture DESC LIMIT 1`,\n ).get(doc, recipe, rendered, until) as VersionRow | undefined;\n }\n\n /**\n * This method chooses the current version of a document with the rules of the class. With\n * `until`, it uses only the versions that the crawler fetched at that time or before. Thus it\n * gives the version that was current at that time. It gives undefined when no version agrees.\n */\n private choose(n: Names, doc: string, until?: string): Choice | undefined {\n const groups = (\n until === undefined\n ? this.stmt(`SELECT DISTINCT recipe, rendered FROM ${n.versions} WHERE doc = ?`).all(doc)\n : this.stmt(\n `SELECT DISTINCT recipe, rendered FROM ${n.versions} WHERE doc = ? AND fetched_at <= ?`,\n ).all(doc, until)\n ) as Array<{ recipe: string; rendered: number }>;\n if (groups.length === 0) return undefined;\n const byRecipe = new Map<string, { plain?: VersionRow; rendered?: VersionRow }>();\n for (const group of groups) {\n const row = this.newest(n, doc, group.recipe, group.rendered, until);\n if (row === undefined) continue;\n const entry = byRecipe.get(group.recipe) ?? {};\n if (group.rendered === 1) entry.rendered = row;\n else entry.plain = row;\n byRecipe.set(group.recipe, entry);\n }\n const candidates: Candidate[] = [];\n for (const [recipe, { plain, rendered }] of byRecipe) {\n let pick: VersionRow;\n if (rendered === undefined) pick = plain!;\n else if (plain === undefined || rendered.fetched_at >= plain.fetched_at) pick = rendered;\n else {\n // the plain body that the render read: the render stays current while that body stays\n const base = this.stmt(\n `SELECT blob FROM ${n.versions} WHERE doc = ? AND recipe = ? AND rendered = 0 AND fetched_at <= ?\n ORDER BY fetched_at DESC, capture DESC LIMIT 1`,\n ).get(doc, recipe, rendered.fetched_at) as { blob: string | null } | undefined;\n pick =\n base !== undefined && base.blob !== null && base.blob === plain.blob ? rendered : plain;\n }\n const newest =\n plain === undefined\n ? rendered!\n : rendered === undefined || plain.fetched_at > rendered.fetched_at\n ? plain\n : rendered;\n const others = [plain, rendered].filter(\n (row): row is VersionRow => row !== undefined && row !== pick,\n );\n candidates.push({ row: pick, ai: newest.ai_input as AiInput, others });\n }\n let winner = candidates[0]!;\n if (candidates.length > 1) {\n const latest = candidates.every((candidate) => candidate.row.merge === \"latest\");\n for (const candidate of candidates.slice(1)) {\n const a = candidate.row;\n const b = winner.row;\n const better = latest\n ? a.fetched_at > b.fetched_at\n : a.text_chars > b.text_chars ||\n (a.text_chars === b.text_chars && a.fetched_at > b.fetched_at);\n if (better) winner = candidate;\n }\n }\n // a render can stay current after a newer plain capture: a no of its own capture still holds its text back\n const aiInput: AiInput =\n winner.row.ai_input === \"no\" || candidates.some((candidate) => candidate.ai === \"no\")\n ? \"no\"\n : winner.ai !== \"unset\"\n ? winner.ai\n : candidates.some((candidate) => candidate.ai === \"yes\")\n ? \"yes\"\n : \"unset\";\n return { winner, candidates, aiInput };\n }\n\n /** This method chooses the current version of a document from its versions, and writes it with its search entry. */\n private recompute(n: Names, doc: string, now: string): Doc | undefined {\n const before = this.stmt(`SELECT rowid, text_sha, title FROM ${n.docs} WHERE doc = ?`).get(\n doc,\n ) as { rowid: number; text_sha: string; title: string } | undefined;\n const choice = this.choose(n, doc);\n if (choice === undefined) {\n if (before !== undefined) {\n this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);\n this.stmt(`DELETE FROM ${n.docs} WHERE doc = ?`).run(doc);\n }\n return undefined;\n }\n const { winner, candidates, aiInput } = choice;\n const row = winner.row;\n let meta = parseJson<Record<string, unknown>>(this.textOf(n, row.meta_sha), {});\n for (const candidate of candidates) {\n // a different source of the same document fills the keys that the current version lacks\n if (candidate !== winner) {\n meta = {\n ...parseJson<Record<string, unknown>>(this.textOf(n, candidate.row.meta_sha), {}),\n ...meta,\n };\n }\n }\n const same = this.stmt(\n `SELECT capture FROM ${n.versions} WHERE doc = ? AND text_sha = ?\n ORDER BY fetched_at DESC, capture DESC LIMIT ?`,\n ).all(doc, row.text_sha, MAX_CAPTURES) as Array<{ capture: string }>;\n const captures = [\n ...new Set([\n row.capture,\n ...same.map((entry) => entry.capture),\n ...winner.others.map((other) => other.capture),\n ...candidates\n .filter((candidate) => candidate !== winner)\n .map((candidate) => candidate.row.capture),\n ]),\n ].slice(0, MAX_CAPTURES);\n const count = (\n this.stmt(`SELECT count(*) AS n FROM ${n.versions} WHERE doc = ?`).get(doc) as { n: number }\n ).n;\n const url =\n row.url ??\n candidates.map((candidate) => candidate.row.url).find((value) => value !== null) ??\n null;\n const saved = this.stmt(\n `INSERT INTO ${n.docs} (doc, kind, title, url, meta, text_sha, full_sha, captures, capture_count, capture,\n blob, fetched_at, recipe, rendered, ai_input, scope, labels, updated_at)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(doc) DO UPDATE SET kind = excluded.kind, title = excluded.title, url = excluded.url,\n meta = excluded.meta, text_sha = excluded.text_sha, full_sha = excluded.full_sha,\n captures = excluded.captures, capture_count = excluded.capture_count, capture = excluded.capture,\n blob = excluded.blob, fetched_at = excluded.fetched_at, recipe = excluded.recipe,\n rendered = excluded.rendered, ai_input = excluded.ai_input, scope = excluded.scope,\n labels = excluded.labels, updated_at = excluded.updated_at\n RETURNING rowid`,\n ).get(\n doc,\n row.kind,\n row.title,\n url,\n JSON.stringify(meta),\n row.text_sha,\n row.full_sha,\n JSON.stringify(captures),\n count,\n row.capture,\n row.blob,\n row.fetched_at,\n row.recipe,\n row.rendered,\n aiInput,\n row.scope,\n row.labels,\n now,\n ) as { rowid: number };\n const text = this.textOf(n, row.text_sha) ?? \"\";\n if (before === undefined || before.text_sha !== row.text_sha || before.title !== row.title) {\n if (before !== undefined) this.stmt(`DELETE FROM ${n.fts} WHERE rowid = ?`).run(before.rowid);\n this.stmt(`INSERT INTO ${n.fts} (rowid, title, text) VALUES (?, ?, ?)`).run(\n saved.rowid,\n fold(row.title, \"search\"),\n fold(text.length > MAX_INDEX_CHARS ? text.slice(0, MAX_INDEX_CHARS) : text, \"search\"),\n );\n }\n return this.docOf(n, doc, text);\n }\n\n private docOf(n: Names, doc: string, knownText?: string): Doc | undefined {\n const row = this.stmt(`SELECT rowid, * FROM ${n.docs} WHERE doc = ?`).get(doc) as\n DocRow | undefined;\n if (row === undefined) return undefined;\n return this.toDoc(n, row, knownText);\n }\n\n private toDoc(n: Names, row: DocRow, knownText?: string): Doc {\n const labels = parseJson<Record<string, string> | undefined>(row.labels, undefined);\n return {\n doc: row.doc,\n kind: row.kind,\n title: row.title,\n text: knownText ?? this.textOf(n, row.text_sha) ?? \"\",\n url: row.url,\n meta: parseJson<Record<string, unknown>>(row.meta, {}),\n captures: parseJson<string[]>(row.captures, []),\n capture_count: row.capture_count,\n capture: row.capture,\n blob: row.blob,\n fetched_at: row.fetched_at,\n recipe: row.recipe,\n rendered: row.rendered === 1,\n ai_input: row.ai_input as AiInput,\n cut: row.full_sha !== null,\n ...(row.scope !== null ? { scope: row.scope } : {}),\n ...(labels !== undefined ? { labels } : {}),\n updated_at: row.updated_at,\n };\n }\n\n private versionTextOf(n: Names, row: VersionRow): VersionText {\n return {\n ...toVersion(row),\n text: this.textOf(n, row.text_sha) ?? \"\",\n full_text: this.textOf(n, row.full_sha),\n meta: parseJson<Record<string, unknown>>(this.textOf(n, row.meta_sha), {}),\n };\n }\n\n // ── the public methods ────────────────────────────────────────────────────\n\n get(doc: string): Doc | undefined {\n return this.live((n) => this.docOf(n, doc));\n }\n\n /**\n * This method adds one version of a document, and it gives the current document. The rules of\n * the class choose the current version. An upsert of a known capture replaces that version.\n */\n upsert(input: DocInput, now: string): Doc {\n return this.live((n) =>\n this.write(() => {\n this.putVersion(n, input, now);\n return this.recompute(n, input.doc, now)!;\n }),\n );\n }\n\n /** This method gives the versions of a document, the newest first. */\n versions(doc: string, options: { readonly limit?: number } = {}): DocVersion[] {\n const limit = Math.max(1, Math.floor(options.limit ?? 100));\n return this.live((n) =>\n (\n this.stmt(\n `SELECT * FROM ${n.versions} WHERE doc = ? ORDER BY fetched_at DESC, capture DESC LIMIT ?`,\n ).all(doc, limit) as unknown as VersionRow[]\n ).map(toVersion),\n );\n }\n\n /**\n * This method gives the text of one version of a document. With no capture, it gives the current\n * version. It also gives the text of a held-back version: the caller applies `isHeldBack` to the\n * `ai_input` of the version.\n */\n versionText(doc: string, capture?: string): VersionText | undefined {\n return this.live((n) => {\n const id = capture ?? this.docOf(n, doc)?.capture;\n if (id === undefined) return undefined;\n const row = this.stmt(`SELECT * FROM ${n.versions} WHERE doc = ? AND capture = ?`).get(\n doc,\n id,\n ) as VersionRow | undefined;\n return row === undefined ? undefined : this.versionTextOf(n, row);\n });\n }\n\n /**\n * This method gives the version of a document that was current at a time (ISO 8601). It applies\n * the rules of the class to the versions that the crawler fetched at that time or before. Thus\n * a later capture does not change the answer. It also gives a held-back version: its `ai_input`\n * is `no` when the document was held back at that time, and the caller applies `isHeldBack`.\n */\n versionAt(doc: string, at: string): VersionAt {\n return this.live((n) => {\n const first =\n (\n this.stmt(`SELECT min(fetched_at) AS first FROM ${n.versions} WHERE doc = ?`).get(doc) as\n { first: string | null } | undefined\n )?.first ?? undefined;\n const choice = this.choose(n, doc, at);\n if (choice === undefined) return { version: undefined, first };\n const version = this.versionTextOf(n, choice.winner.row);\n return {\n version: choice.aiInput === \"no\" ? { ...version, ai_input: \"no\" } : version,\n first,\n };\n });\n }\n\n /**\n * This method gives the texts of the earlier versions of a document, the newest first. Each\n * text comes one time, with the newest version that has it. It skips the current text. It also\n * gives held-back texts: the caller applies `isHeldBack` to the `ai_input` of each version.\n */\n earlierTexts(doc: string, options: { readonly limit?: number } = {}): VersionText[] {\n const limit = Math.max(1, Math.floor(options.limit ?? 50));\n return this.live((n) => {\n const head = this.stmt(`SELECT text_sha FROM ${n.docs} WHERE doc = ?`).get(doc) as\n { text_sha: string } | undefined;\n if (head === undefined) return [];\n const rows = this.stmt(\n `SELECT * FROM ${n.versions} WHERE doc = ? AND text_sha != ? ORDER BY fetched_at DESC, capture DESC`,\n ).all(doc, head.text_sha) as unknown as VersionRow[];\n const seen = new Set<string>();\n const out: VersionText[] = [];\n for (const row of rows) {\n if (seen.has(row.text_sha)) continue;\n seen.add(row.text_sha);\n out.push(this.versionTextOf(n, row));\n if (out.length >= limit) break;\n }\n return out;\n });\n }\n\n /** This method tells if a capture gave a version of a document. */\n hasVersion(capture: string): boolean {\n return this.live(\n (n) =>\n this.stmt(`SELECT 1 AS found FROM ${n.versions} WHERE capture = ? LIMIT 1`).get(capture) !==\n undefined,\n );\n }\n\n /**\n * This method gives the hash of the current text of documents, with no text. It gives the\n * documents of `docs`, or each document after `after` in the sequence of the ids, to `limit`.\n */\n digests(options: DigestOptions = {}): DocDigest[] {\n return this.live((n) => {\n if (options.docs !== undefined) {\n if (options.docs.length === 0) return [];\n return this.stmt(\n `SELECT doc, kind, ai_input, text_sha FROM ${n.docs} WHERE doc IN (SELECT value FROM json_each(?))`,\n ).all(JSON.stringify(options.docs)) as unknown as DocDigest[];\n }\n const limit = Math.max(1, Math.floor(options.limit ?? 500));\n return this.stmt(\n `SELECT doc, kind, ai_input, text_sha FROM ${n.docs} WHERE doc > ? ORDER BY doc LIMIT ?`,\n ).all(options.after ?? \"\", limit) as unknown as DocDigest[];\n });\n }\n\n /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */\n newestFetch(): string | undefined {\n return this.live(\n (n) =>\n (\n this.stmt(`SELECT max(fetched_at) AS newest FROM ${n.latest}`).get() as\n { newest: string | null } | undefined\n )?.newest ?? undefined,\n );\n }\n\n /** This method gives the current documents that a capture gave. */\n docsOfCapture(capture: string): Doc[] {\n return this.live((n) =>\n (\n this.stmt(`SELECT DISTINCT doc FROM ${n.versions} WHERE capture = ?`).all(\n capture,\n ) as Array<{ doc: string }>\n )\n .map((row) => this.docOf(n, row.doc))\n .filter((doc): doc is Doc => doc !== undefined),\n );\n }\n\n /**\n * This method searches the documents with BM25, the best first. The syntax of the query is the\n * syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it\n * adds the documents that match one part. A document that asked not to be AI input is not in\n * the results, so a search cannot show its text. The option `rank` chooses the ranking (`RankMode`).\n */\n find(q: string, options: FindIndexOptions): Found[] {\n const rank = rankModeOf(options.rank);\n const query = parseQuery(q);\n if (query === undefined) return [];\n const limit = Math.max(1, Math.min(1000, Math.floor(options.limit) || 10));\n return this.live((n) => {\n const kindClause =\n options.kind === undefined || options.kind === null ? \"\" : \"AND d.kind = ?\";\n const sql = `SELECT d.rowid AS rowid, d.*, bm25(${n.fts}, 3.0, 1.0) AS rank\n FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid\n WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' ${kindClause}\n ORDER BY rank LIMIT ?`;\n const run = (match: string): Array<DocRow & { rank: number }> => {\n const params: Array<string | number> = [match];\n if (kindClause.length > 0) params.push(options.kind as string);\n params.push(limit);\n return this.stmt(sql).all(...params) as unknown as Array<DocRow & { rank: number }>;\n };\n const rows = run(query.all);\n if (rows.length < limit && query.parts > 1) {\n const seen = new Set(rows.map((row) => row.doc));\n for (const row of run(rank === \"fast\" ? this.rareParts(n, query.anyParts) : query.any)) {\n if (rows.length >= limit) break;\n if (!seen.has(row.doc)) rows.push(row);\n }\n }\n return rows.map((row) => ({\n doc: row.doc,\n kind: row.kind,\n title: row.title,\n url: row.url,\n snippet: snippetOf(this.textOf(n, row.text_sha) ?? \"\", query.terms),\n // bm25() is lower-is-better; flip it so a bigger score is a better match\n score: Math.round(-row.rank * 1000) / 1000,\n ai_input: row.ai_input as AiInput,\n fetched_at: row.fetched_at,\n capture: row.capture,\n }));\n });\n }\n\n /**\n * This method scores a set of documents against a query, with the best first. It leaves out a\n * document that matches no term, and a document that asked not to be AI input. It ranks the\n * results of a source again, because the order of the source is only as good as its search engine.\n * In the mode `fast`, it also leaves out a document that holds only common parts (`RankMode`).\n */\n rankWithin(\n q: string,\n docs: readonly string[],\n options: { readonly rank?: RankMode | undefined } = {},\n ): Array<{ readonly doc: string; readonly score: number }> {\n const rank = rankModeOf(options.rank);\n const query = parseQuery(q);\n if (query === undefined || docs.length === 0) return [];\n return this.live((n) =>\n (\n this.stmt(\n `SELECT d.doc AS doc, bm25(${n.fts}, 3.0, 1.0) AS rank\n FROM ${n.fts} JOIN ${n.docs} d ON d.rowid = ${n.fts}.rowid\n WHERE ${n.fts} MATCH ? AND d.ai_input != 'no' AND d.doc IN (SELECT value FROM json_each(?))\n ORDER BY rank`,\n ).all(\n rank === \"fast\" ? this.rareParts(n, query.anyParts) : query.any,\n JSON.stringify(docs),\n ) as Array<{ doc: string; rank: number }>\n ).map((row) => ({ doc: row.doc, score: Math.round(-row.rank * 1000) / 1000 })),\n );\n }\n\n /**\n * This method gives the form of a query that needs one part, without the common parts (the mode\n * `fast` of `RankMode`). It counts the documents that hold each part. When each part is common,\n * it keeps the rarest part.\n */\n private rareParts(n: Names, parts: readonly string[]): string {\n if (parts.length < 2) return parts.join(\" OR \");\n const total = (this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get() as { n: number }).n;\n const counted = parts.map((part) => ({\n part,\n count: (\n this.stmt(`SELECT count(*) AS n FROM ${n.fts} WHERE ${n.fts} MATCH ?`).get(part) as {\n n: number;\n }\n ).n,\n }));\n const rare = counted.filter((one) => one.count <= FAST_RANK_SHARE * total);\n if (rare.length > 0) return rare.map((one) => one.part).join(\" OR \");\n return counted.reduce((best, one) => (one.count < best.count ? one : best)).part;\n }\n\n /** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */\n latest(url: string, variant?: Variant): Latest | undefined {\n const key = keyOf(url, variant);\n return this.live((n) => {\n const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(\n key.url,\n key.variant,\n ) as LatestRow | undefined;\n return row === undefined ? undefined : toLatest(row);\n });\n }\n\n private putLatest(n: Names, entry: Latest): void {\n const key = keyOf(entry.url, entry.variant);\n // an older answer never replaces a newer one, so two processes and a replay agree\n this.stmt(\n `INSERT INTO ${n.latest} (url, variant, capture, blob, etag, last_modified, content_type, fetched_at,\n status, ai_input, location, truncated, fresh_until, vary_key)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(url, variant) DO UPDATE SET capture = excluded.capture, blob = excluded.blob,\n etag = excluded.etag, last_modified = excluded.last_modified, content_type = excluded.content_type,\n fetched_at = excluded.fetched_at, status = excluded.status, ai_input = excluded.ai_input,\n location = excluded.location, truncated = excluded.truncated, fresh_until = excluded.fresh_until,\n vary_key = excluded.vary_key\n WHERE excluded.fetched_at >= ${n.latest}.fetched_at`,\n ).run(\n key.url,\n key.variant,\n entry.capture,\n entry.blob,\n entry.etag,\n entry.last_modified,\n entry.content_type,\n entry.fetched_at,\n entry.status,\n entry.ai_input,\n entry.location ?? null,\n entry.truncated === true ? 1 : null,\n entry.fresh_until ?? null,\n entry.vary_key ?? null,\n );\n }\n\n /** This method keeps the last answer for a URL. An entry that is older than the stored entry changes nothing. */\n setLatest(entry: Latest): void {\n this.live((n) => this.write(() => this.putLatest(n, entry)));\n }\n\n counts(): IndexCounts {\n return this.live((n) => ({\n docs: (this.stmt(`SELECT count(*) AS n FROM ${n.docs}`).get() as { n: number }).n,\n urls: (this.stmt(`SELECT count(*) AS n FROM ${n.latest}`).get() as { n: number }).n,\n }));\n }\n\n /** This method empties the current generation. A rebuild uses `beginRebuild`, which keeps the index until its commit. */\n reset(): void {\n this.live((n) =>\n this.write(() => {\n for (const table of [n.fts, n.docs, n.versions, n.texts, n.latest]) {\n this.db.exec(`DELETE FROM ${table}`);\n }\n }),\n );\n }\n\n /**\n * This method starts a rebuild into a new generation of the tables. The current generation\n * answers each read and takes each write until the commit. A failed rebuild thus leaves the\n * index as it was.\n */\n beginRebuild(options: { readonly batch?: number } = {}): IndexBuild {\n const batchSize = Math.max(1, Math.floor(options.batch ?? 500));\n const generation = this.write(() => {\n const next = Number(this.getMeta(\"next_generation\") ?? \"2\");\n this.setMeta(\"next_generation\", String(next + 1));\n this.setMeta(`building:${String(next)}`, new Date().toISOString());\n this.db.exec(ddlOf(namesOf(next)));\n return next;\n });\n const n = namesOf(generation);\n const versions: Array<{ readonly input: DocInput; readonly now: string }> = [];\n const latest = new Map<string, Latest>();\n let open = true;\n const keyText = (url: string, variant: Variant | undefined): string => {\n const key = keyOf(url, variant);\n return `${key.variant} ${key.url}`;\n };\n const flush = (): void => {\n if (!open) throw new Error(\"the rebuild is over\");\n if (versions.length === 0 && latest.size === 0) return;\n this.write(() => {\n const touched = new Map<string, string>();\n for (const { input, now } of versions) {\n this.putVersion(n, input, now);\n touched.set(input.doc, now);\n }\n for (const entry of latest.values()) this.putLatest(n, entry);\n for (const [doc, now] of touched) this.recompute(n, doc, now);\n });\n versions.length = 0;\n latest.clear();\n };\n /** This function copies the texts of the versions of the current generation that agree with `where`. */\n const copyTexts = (current: Names, where: string, params: string[]): void => {\n for (const column of [\"text_sha\", \"meta_sha\", \"full_sha\"]) {\n this.db\n .prepare(\n `INSERT OR IGNORE INTO ${n.texts} (sha, body)\n SELECT t.sha, t.body FROM ${current.texts} t\n WHERE t.sha IN (SELECT ${column} FROM ${current.versions} WHERE ${where})`,\n )\n .run(...params);\n }\n };\n return {\n upsert: (input, now) => {\n if (!open) throw new Error(\"the rebuild is over\");\n versions.push({ input, now });\n if (versions.length >= batchSize) flush();\n },\n setLatest: (entry) => {\n if (!open) throw new Error(\"the rebuild is over\");\n const key = keyText(entry.url, entry.variant);\n const before = latest.get(key);\n if (before === undefined || entry.fetched_at >= before.fetched_at) latest.set(key, entry);\n if (latest.size >= batchSize) flush();\n },\n latest: (url, variant) => {\n const waiting = latest.get(keyText(url, variant));\n if (waiting !== undefined) return waiting;\n const key = keyOf(url, variant);\n const row = this.stmt(`SELECT * FROM ${n.latest} WHERE url = ? AND variant = ?`).get(\n key.url,\n key.variant,\n ) as LatestRow | undefined;\n return row === undefined ? undefined : toLatest(row);\n },\n flush,\n carry: (recipes) => {\n flush();\n const known = JSON.stringify([...recipes]);\n const current = namesOf(this.generation());\n const docs = this.db\n .prepare(\n `SELECT DISTINCT doc FROM ${current.versions} WHERE recipe NOT IN (SELECT value FROM json_each(?))`,\n )\n .all(known) as Array<{ doc: string }>;\n const where =\n \"recipe NOT IN (SELECT value FROM json_each(?)) AND doc IN (SELECT value FROM json_each(?))\";\n for (let at = 0; at < docs.length; at += batchSize) {\n const part = JSON.stringify(docs.slice(at, at + batchSize).map((row) => row.doc));\n this.write(() => {\n copyTexts(current, where, [known, part]);\n this.db\n .prepare(\n `INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${current.versions} WHERE ${where}`,\n )\n .run(known, part);\n const now = new Date().toISOString();\n for (const doc of JSON.parse(part) as string[]) this.recompute(n, doc, now);\n });\n }\n return docs.length;\n },\n commit: (since) => {\n flush();\n open = false;\n return this.write(() => {\n const current = this.generation();\n const old = namesOf(current);\n // the rows that a different writer added during the rebuild\n copyTexts(old, \"fetched_at >= ?\", [since]);\n const added = this.db\n .prepare(`SELECT DISTINCT doc FROM ${old.versions} WHERE fetched_at >= ?`)\n .all(since) as Array<{ doc: string }>;\n this.db\n .prepare(\n `INSERT OR IGNORE INTO ${n.versions} SELECT * FROM ${old.versions} WHERE fetched_at >= ?`,\n )\n .run(since);\n const now = new Date().toISOString();\n for (const { doc } of added) this.recompute(n, doc, now);\n for (const row of this.db\n .prepare(`SELECT * FROM ${old.latest} WHERE fetched_at >= ?`)\n .all(since) as unknown as LatestRow[]) {\n this.putLatest(n, toLatest(row));\n }\n this.setMeta(\"generation\", String(generation));\n this.db\n .prepare(\"DELETE FROM index_meta WHERE key = ?\")\n .run(`building:${String(generation)}`);\n this.dropGeneration(current);\n this.empty = false;\n return {\n docs: (this.db.prepare(`SELECT count(*) AS n FROM ${n.docs}`).get() as { n: number }).n,\n urls: (this.db.prepare(`SELECT count(*) AS n FROM ${n.latest}`).get() as { n: number })\n .n,\n };\n });\n },\n abort: () => {\n open = false;\n versions.length = 0;\n latest.clear();\n if (this.closed) return;\n this.write(() => this.dropGeneration(generation));\n },\n };\n }\n\n /** This method closes the file. A second close does nothing. */\n close(): void {\n if (this.closed) return;\n this.closed = true;\n this.statements.clear();\n this.db.close();\n }\n}\n","import { canonicalUrl } from \"./urls.ts\";\n\n/**\n * The sitemap reader reads the files that list the pages of a site. These are an XML sitemap, a\n * sitemap index, an RSS 2.0 or RSS 1.0 feed and an Atom feed. A text sitemap with one URL on each\n * line is a file of this kind too. The reader is small and strict, because the files come from the web:\n *\n * - It reads the text one time from the start to the end, and it builds no tree. Its time is\n * linear in the length of the text.\n * - It expands no entity. It knows only the five entities of XML and the character references. A\n * DTD in the file changes nothing.\n * - It stops at a maximum depth of elements, at a maximum number of entries and at the first\n * error in the markup. The entries before the stop stay.\n *\n * It imports no Node.js module, so a browser can use it.\n */\n\n/** The kind of a file: the name of its root element, or `text` for a text sitemap. */\nexport type SitemapKind = \"urlset\" | \"sitemapindex\" | \"rss\" | \"atom\" | \"rdf\" | \"text\" | \"unknown\";\n\n/** The values of `<changefreq>` in the sitemap protocol. */\nexport type Changefreq = \"always\" | \"hourly\" | \"daily\" | \"weekly\" | \"monthly\" | \"yearly\" | \"never\";\n\n/**\n * One entry of a file: a page, or a sitemap of an index.\n *\n * @beta\n */\nexport interface SitemapEntry {\n /** This URL is the canonical form of the URL of the entry. */\n readonly url: string;\n /** This time (ISO 8601) is the last change that the file gives, or null. */\n readonly lastmod: string | null;\n /** This value is the `<changefreq>` of a sitemap entry, or null. */\n readonly changefreq: Changefreq | null;\n}\n\n/**\n * The result of one file.\n *\n * @beta\n */\nexport interface SitemapRead {\n readonly kind: SitemapKind;\n /** These entries are the pages of the file, or the sitemaps of an index, in the order of the file. */\n readonly entries: readonly SitemapEntry[];\n /** This value counts the entries with no valid http(s) URL. */\n readonly invalid: number;\n /** This value is false when the reader stopped before the end of the file. */\n readonly complete: boolean;\n /** These notes tell why the reader stopped, or what it ignored. */\n readonly notes: readonly string[];\n}\n\n/**\n * These limits stop the reader on a large or a hostile file.\n *\n * @beta\n */\nexport interface SitemapLimits {\n /** This value is the maximum number of entries. The default is 50,000, the limit of the sitemap protocol. */\n readonly maxEntries?: number | undefined;\n /** This value is the maximum depth of the elements. The default is 32. */\n readonly maxDepth?: number | undefined;\n}\n\n/**\n * These are the default limits of the reader, and the maximum length of one value.\n *\n * @beta\n */\nexport const SITEMAP_LIMITS = {\n maxEntries: 50_000,\n maxDepth: 32,\n /** A value (a URL or a date) longer than this number of characters is not valid. */\n maxValue: 4096,\n} as const;\n\n/** The reader counts its steps here. A test uses the count to show that the time is linear. */\nexport interface Meter {\n steps: number;\n}\n\nconst CHANGEFREQ: ReadonlySet<string> = new Set([\n \"always\",\n \"hourly\",\n \"daily\",\n \"weekly\",\n \"monthly\",\n \"yearly\",\n \"never\",\n]);\nconst SITEMAP_NS = [\n \"http://www.sitemaps.org/schemas/sitemap/0.9\",\n \"http://www.google.com/schemas/sitemap/0.84\",\n \"http://www.google.com/schemas/sitemap/0.9\",\n];\nconst ATOM_NS = [\"http://www.w3.org/2005/Atom\"];\nconst RSS1_NS = [\"http://purl.org/rss/1.0/\"];\nconst DC_NS = [\"http://purl.org/dc/elements/1.1/\"];\nconst RDF_NS = [\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\"];\nconst XML_ENTITIES: Readonly<Record<string, string>> = {\n amp: \"&\",\n lt: \"<\",\n gt: \">\",\n quot: '\"',\n apos: \"'\",\n};\n/** A tag keeps this number of attributes at most. The reader reads the others and keeps nothing of them. */\nconst MAX_ATTRIBUTES = 64;\nconst NAME = /^[A-Za-z_][\\w.:-]*$/;\nconst W3C_DATE =\n /^\\d{4}(?:-\\d{2}(?:-\\d{2}(?:[T ]\\d{2}:\\d{2}(?::\\d{2}(?:\\.\\d+)?)?(Z|[+-]\\d{2}:?\\d{2})?)?)?)?$/;\nconst MAIL_DATE =\n /^(?:[A-Za-z]{3},?\\s+)?\\d{1,2}\\s+[A-Za-z]{3,9}\\s+\\d{2,4}\\s+\\d{1,2}:\\d{2}(?::\\d{2})?(?:\\s*(?:[A-Za-z]{1,5}|[+-]\\d{4}))?$/;\n\nconst short = (name: string | undefined): string =>\n name === undefined ? \"nothing\" : name.length > 40 ? `${name.slice(0, 40)}...` : name;\n\n/**\n * This function changes a date of a sitemap or a feed into ISO 8601. It reads the W3C form of the\n * sitemap protocol and Atom, and the mail form of RSS. A time with no zone is UTC. It gives null\n * for a text that it cannot read.\n *\n * @beta\n */\nexport const feedDate = (value: string): string | null => {\n const text = value.trim();\n if (text.length === 0 || text.length > 64) return null;\n let parse: string | undefined;\n const w3c = W3C_DATE.exec(text);\n if (w3c !== null) {\n const iso = text.replace(\" \", \"T\");\n // a time with no zone is UTC, and Date.parse reads it as local time\n parse = iso.includes(\"T\") && w3c[1] === undefined ? `${iso}Z` : iso;\n } else if (MAIL_DATE.test(text)) parse = text;\n if (parse === undefined) return null;\n const ms = Date.parse(parse);\n return Number.isFinite(ms) && Math.abs(ms) < 8.64e15 ? new Date(ms).toISOString() : null;\n};\n\n/** This function gives the canonical form of an http(s) URL, or undefined. A relative URL needs a base. */\nconst webUrl = (value: string, base: string | undefined): string | undefined => {\n const text = value.trim();\n if (text.length === 0 || text.length > SITEMAP_LIMITS.maxValue) return undefined;\n try {\n const url = base === undefined ? new URL(text) : new URL(text, base);\n if (url.protocol !== \"http:\" && url.protocol !== \"https:\") return undefined;\n return canonicalUrl(url.href);\n } catch {\n return undefined;\n }\n};\n\n/** This function decodes the five entities of XML and the character references. It gives undefined for each other entity. */\nconst decodeText = (raw: string, meter: Meter): string | undefined => {\n meter.steps += raw.length;\n if (!raw.includes(\"&\")) return raw;\n let out = \"\";\n let at = 0;\n for (;;) {\n const amp = raw.indexOf(\"&\", at);\n if (amp === -1) return out + raw.slice(at);\n out += raw.slice(at, amp);\n const semi = raw.indexOf(\";\", amp + 1);\n if (semi === -1 || semi - amp > 10) return undefined;\n const name = raw.slice(amp + 1, semi);\n let char = XML_ENTITIES[name];\n if (char === undefined && /^#(?:\\d{1,7}|x[0-9a-fA-F]{1,6})$/.test(name)) {\n const code = name[1] === \"x\" ? Number.parseInt(name.slice(2), 16) : Number(name.slice(1));\n if (code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff))\n char = String.fromCodePoint(code);\n }\n // an entity of a DTD is not expanded: the value is not valid\n if (char === undefined) return undefined;\n out += char;\n at = semi + 1;\n }\n};\n\ninterface Tag {\n readonly name: string;\n readonly attributes: ReadonlyArray<readonly [string, string]>;\n readonly selfClosing: boolean;\n /** This value is the position after the `>` of the tag. */\n readonly end: number;\n}\n\nconst isSpace = (code: number): boolean => code === 32 || code === 9 || code === 10 || code === 13;\n\n/** This function reads a start tag from the position after its `<`. It gives undefined for a tag that is not well formed. */\nconst readTag = (text: string, from: number, meter: Meter): Tag | undefined => {\n const tag = tagAt(text, from);\n // the scan of a tag moves forward only, so its steps are the characters from its start to its end\n meter.steps += (tag?.end ?? text.length) - from;\n return tag;\n};\n\nconst tagAt = (text: string, from: number): Tag | undefined => {\n const n = text.length;\n let at = from;\n while (at < n) {\n const code = text.charCodeAt(at);\n if (isSpace(code) || code === 47 || code === 62) break;\n at += 1;\n }\n const name = text.slice(from, Math.min(at, from + 256));\n if (at - from > 256 || !NAME.test(name)) return undefined;\n const attributes: Array<readonly [string, string]> = [];\n for (;;) {\n while (at < n && isSpace(text.charCodeAt(at))) at += 1;\n if (at >= n) return undefined;\n const code = text.charCodeAt(at);\n if (code === 62) return { name, attributes, selfClosing: false, end: at + 1 };\n if (code === 47) {\n return text.charCodeAt(at + 1) === 62\n ? { name, attributes, selfClosing: true, end: at + 2 }\n : undefined;\n }\n const start = at;\n while (at < n) {\n const next = text.charCodeAt(at);\n if (isSpace(next) || next === 61 || next === 62 || next === 47) break;\n at += 1;\n }\n const key = text.slice(start, Math.min(at, start + 256));\n if (at - start > 256 || !NAME.test(key)) return undefined;\n while (at < n && isSpace(text.charCodeAt(at))) at += 1;\n if (text.charCodeAt(at) !== 61) return undefined;\n at += 1;\n while (at < n && isSpace(text.charCodeAt(at))) at += 1;\n const quote = text[at];\n if (quote !== '\"' && quote !== \"'\") return undefined;\n const close = text.indexOf(quote, at + 1);\n if (close === -1) return undefined;\n if (attributes.length < MAX_ATTRIBUTES && close - at <= SITEMAP_LIMITS.maxValue)\n attributes.push([key, text.slice(at + 1, close)]);\n at = close + 1;\n }\n};\n\n/**\n * This function finds the end of a declaration (`<!DOCTYPE ...>`) with its internal subset. It\n * gives the position after the `>`, or -1, and it tells if the declaration has an internal subset.\n */\nconst declarationEnd = (\n text: string,\n from: number,\n meter: Meter,\n): { readonly end: number; readonly subset: boolean } => {\n let depth = 0;\n let quote = 0;\n let subset = false;\n for (let at = from; at < text.length; at += 1) {\n meter.steps += 1;\n const code = text.charCodeAt(at);\n if (quote !== 0) {\n if (code === quote) quote = 0;\n } else if (code === 34 || code === 39) quote = code;\n else if (code === 91) {\n depth += 1;\n subset = true;\n } else if (code === 93) depth = Math.max(0, depth - 1);\n else if (code === 62 && depth === 0) return { end: at + 1, subset };\n }\n return { end: -1, subset };\n};\n\n/** The fields of an entry that the reader keeps. */\ntype Field = \"loc\" | \"lastmod\" | \"changefreq\" | \"link\" | \"guid\" | \"date\";\n\ninterface OpenEntry {\n readonly depth: number;\n readonly base: string;\n loc: string | undefined;\n link: string | undefined;\n guid: string | undefined;\n lastmod: string | undefined;\n changefreq: string | undefined;\n about: string | undefined;\n /** This value is true when the Atom link of the entry is a link to a web page. */\n linkPage: boolean;\n}\n\ninterface OpenField {\n readonly field: Field;\n readonly depth: number;\n readonly parts: string[];\n length: number;\n bad: boolean;\n}\n\n/** This function reads a text sitemap: each line that is an absolute http(s) URL is an entry. */\nconst readLines = (text: string, maxEntries: number, meter: Meter): SitemapRead => {\n const entries: SitemapEntry[] = [];\n let invalid = 0;\n let at = 0;\n while (at < text.length) {\n let end = text.indexOf(\"\\n\", at);\n if (end === -1) end = text.length;\n meter.steps += end - at + 1;\n const line = end - at > SITEMAP_LIMITS.maxValue + 2 ? \"-\" : text.slice(at, end).trim();\n at = end + 1;\n if (line.length === 0) continue;\n const url = webUrl(line, undefined);\n if (url === undefined) invalid += 1;\n else entries.push({ url, lastmod: null, changefreq: null });\n if (entries.length >= maxEntries) {\n const complete = at >= text.length;\n return {\n kind: \"text\",\n entries,\n invalid,\n complete,\n notes: complete ? [] : [`the reader stopped at ${String(maxEntries)} entries`],\n };\n }\n }\n return { kind: \"text\", entries, invalid, complete: true, notes: [] };\n};\n\n/**\n * This function reads a sitemap, a sitemap index, a feed or a text sitemap, and it counts its\n * steps in `meter`. The function `readSitemap` uses it. A test uses it to show that the time is linear.\n */\nexport const scanSitemap = (\n text: string,\n base: string,\n limits: SitemapLimits = {},\n meter: Meter = { steps: 0 },\n): SitemapRead => {\n const maxEntries = Math.max(1, Math.floor(limits.maxEntries ?? SITEMAP_LIMITS.maxEntries));\n const maxDepth = Math.max(1, Math.floor(limits.maxDepth ?? SITEMAP_LIMITS.maxDepth));\n let at = text.charCodeAt(0) === 0xfeff ? 1 : 0;\n while (at < text.length && isSpace(text.charCodeAt(at))) at += 1;\n if (at < text.length && text[at] !== \"<\") return readLines(text.slice(at), maxEntries, meter);\n\n const n = text.length;\n const notes: string[] = [];\n const entries: SitemapEntry[] = [];\n const stack: string[] = [];\n const prefixes = new Map<string, string>();\n let kind: SitemapKind = \"unknown\";\n let invalid = 0;\n let complete = true;\n let done = false;\n let feedBase = base;\n let entry: OpenEntry | undefined;\n let field: OpenField | undefined;\n const stop = (note: string): void => {\n complete = false;\n done = true;\n notes.push(note);\n };\n\n /** This function tells if a name is a local name of one of the namespaces, or the local name with no prefix. */\n const named = (qname: string, local: string, spaces: readonly string[]): boolean => {\n const colon = qname.indexOf(\":\");\n if (colon === -1) return qname === local;\n return (\n qname.slice(colon + 1) === local && spaces.includes(prefixes.get(qname.slice(0, colon)) ?? \"\")\n );\n };\n const attribute = (tag: Tag, local: string, spaces?: readonly string[]): string | undefined => {\n for (const [key, raw] of tag.attributes) {\n if (spaces === undefined ? key === local : named(key, local, spaces) && key.includes(\":\"))\n return decodeText(raw, meter);\n }\n return undefined;\n };\n\n const fieldOf = (qname: string): Field | undefined => {\n switch (kind) {\n case \"urlset\":\n case \"sitemapindex\":\n if (named(qname, \"loc\", SITEMAP_NS)) return \"loc\";\n if (named(qname, \"lastmod\", SITEMAP_NS)) return \"lastmod\";\n return kind === \"urlset\" && named(qname, \"changefreq\", SITEMAP_NS)\n ? \"changefreq\"\n : undefined;\n case \"rss\":\n if (qname === \"link\") return \"link\";\n if (qname === \"guid\") return \"guid\";\n return qname === \"pubDate\" ? \"date\" : undefined;\n case \"atom\":\n return named(qname, \"updated\", ATOM_NS) || named(qname, \"published\", ATOM_NS)\n ? \"date\"\n : undefined;\n case \"rdf\":\n if (named(qname, \"link\", RSS1_NS)) return \"link\";\n return named(qname, \"date\", DC_NS) ? \"date\" : undefined;\n default:\n return undefined;\n }\n };\n\n const isEntry = (qname: string, depth: number): boolean => {\n switch (kind) {\n case \"urlset\":\n return depth === 2 && named(qname, \"url\", SITEMAP_NS);\n case \"sitemapindex\":\n return depth === 2 && named(qname, \"sitemap\", SITEMAP_NS);\n case \"rss\":\n return depth === 3 && qname === \"item\" && stack[1] === \"channel\";\n case \"atom\":\n return depth === 2 && named(qname, \"entry\", ATOM_NS);\n case \"rdf\":\n return depth === 2 && named(qname, \"item\", RSS1_NS);\n default:\n return false;\n }\n };\n\n const resolveBase = (tag: Tag, against: string): string => {\n const declared = attribute(tag, \"xml:base\");\n if (declared === undefined) return against;\n try {\n return new URL(declared.trim(), against).href;\n } catch {\n return against;\n }\n };\n\n const open = (tag: Tag, depth: number): void => {\n for (const [key, raw] of tag.attributes) {\n if (key.startsWith(\"xmlns:\")) {\n const uri = decodeText(raw, meter);\n if (uri !== undefined) prefixes.set(key.slice(6), uri.trim());\n }\n }\n if (depth === 1) {\n const local = tag.name.slice(tag.name.indexOf(\":\") + 1);\n if (local === \"urlset\" || local === \"sitemapindex\" || local === \"rss\") kind = local;\n else if (local === \"feed\") kind = \"atom\";\n else if (local === \"RDF\") kind = \"rdf\";\n else {\n stop(`not a sitemap or a feed: the root element is <${short(tag.name)}>`);\n return;\n }\n feedBase = resolveBase(tag, base);\n return;\n }\n if (entry === undefined) {\n if (isEntry(tag.name, depth)) {\n entry = {\n depth,\n base: kind === \"atom\" ? resolveBase(tag, feedBase) : feedBase,\n loc: undefined,\n link: undefined,\n guid: undefined,\n lastmod: undefined,\n changefreq: undefined,\n about: kind === \"rdf\" ? attribute(tag, \"about\", RDF_NS) : undefined,\n linkPage: false,\n };\n }\n return;\n }\n if (field !== undefined) {\n // a field holds text only\n field.bad = true;\n return;\n }\n if (depth !== entry.depth + 1) return;\n if (kind === \"atom\" && named(tag.name, \"link\", ATOM_NS)) {\n const rel = (attribute(tag, \"rel\") ?? \"alternate\").trim().toLowerCase();\n const href = attribute(tag, \"href\");\n const type = (attribute(tag, \"type\") ?? \"text/html\").trim().toLowerCase();\n // the first alternate link wins, and a link to a web page wins over a link to a file\n const page = type.startsWith(\"text/html\") || type.startsWith(\"application/xhtml\");\n if (\n rel === \"alternate\" &&\n href !== undefined &&\n (entry.link === undefined || (page && !entry.linkPage))\n ) {\n entry.link = href;\n entry.linkPage = page;\n }\n return;\n }\n const which = fieldOf(tag.name);\n if (which === undefined) return;\n if (which === \"guid\") {\n // a guid is a link only when it says so, or when it does not say\n const permalink = (attribute(tag, \"isPermaLink\") ?? \"true\").trim().toLowerCase();\n if (permalink === \"false\") return;\n }\n field = { field: which, depth, parts: [], length: 0, bad: false };\n };\n\n const finishEntry = (current: OpenEntry): void => {\n const raw =\n kind === \"urlset\" || kind === \"sitemapindex\"\n ? current.loc\n : (current.link ?? (kind === \"rdf\" ? current.about : current.guid));\n const url =\n raw === undefined\n ? undefined\n : webUrl(raw, kind === \"urlset\" || kind === \"sitemapindex\" ? undefined : current.base);\n if (url === undefined) {\n invalid += 1;\n return;\n }\n const freq = current.changefreq?.trim().toLowerCase();\n entries.push({\n url,\n lastmod: current.lastmod === undefined ? null : feedDate(current.lastmod),\n changefreq:\n freq !== undefined && CHANGEFREQ.has(freq) && kind === \"urlset\"\n ? (freq as Changefreq)\n : null,\n });\n if (entries.length >= maxEntries) stop(`the reader stopped at ${String(maxEntries)} entries`);\n };\n\n const close = (depth: number): void => {\n if (field !== undefined && depth === field.depth) {\n const finished = field;\n field = undefined;\n if (finished.bad || entry === undefined) return;\n const value = finished.parts.join(\"\").trim();\n // an empty field gives no value, so a later field (a guid after an empty link) can\n if (value.length === 0) return;\n if (finished.field === \"loc\") entry.loc ??= value;\n else if (finished.field === \"link\") entry.link ??= value;\n else if (finished.field === \"guid\") entry.guid ??= value;\n else if (finished.field === \"lastmod\" || finished.field === \"date\") {\n // an Atom entry gives `updated` and `published`: the later field does not replace the first\n entry.lastmod ??= value;\n } else entry.changefreq ??= value;\n return;\n }\n if (entry !== undefined && depth === entry.depth) {\n const finished = entry;\n entry = undefined;\n finishEntry(finished);\n }\n };\n\n const addText = (value: string | undefined): void => {\n if (field === undefined || field.bad) return;\n if (value === undefined || field.length + value.length > SITEMAP_LIMITS.maxValue) {\n field.bad = true;\n return;\n }\n field.parts.push(value);\n field.length += value.length;\n };\n\n while (at < n && !done) {\n meter.steps += 1;\n const lt = text.indexOf(\"<\", at);\n const end = lt === -1 ? n : lt;\n if (end > at) {\n meter.steps += end - at;\n if (field !== undefined && !field.bad) {\n addText(\n end - at > SITEMAP_LIMITS.maxValue ? undefined : decodeText(text.slice(at, end), meter),\n );\n }\n }\n if (lt === -1) break;\n at = lt;\n if (text.startsWith(\"<!--\", at)) {\n const endOf = text.indexOf(\"-->\", at + 4);\n if (endOf === -1) {\n stop(\"a comment does not end\");\n break;\n }\n meter.steps += endOf + 3 - at;\n at = endOf + 3;\n continue;\n }\n if (text.startsWith(\"<![CDATA[\", at)) {\n const endOf = text.indexOf(\"]]>\", at + 9);\n if (endOf === -1) {\n stop(\"a CDATA section does not end\");\n break;\n }\n meter.steps += endOf + 3 - at;\n if (field !== undefined && !field.bad) {\n addText(endOf - at - 9 > SITEMAP_LIMITS.maxValue ? undefined : text.slice(at + 9, endOf));\n }\n at = endOf + 3;\n continue;\n }\n if (text.startsWith(\"<?\", at)) {\n const endOf = text.indexOf(\"?>\", at + 2);\n if (endOf === -1) {\n stop(\"a processing instruction does not end\");\n break;\n }\n meter.steps += endOf + 2 - at;\n at = endOf + 2;\n continue;\n }\n if (text.startsWith(\"<!\", at)) {\n const declaration = declarationEnd(text, at + 2, meter);\n if (declaration.end === -1) {\n stop(\"a declaration does not end\");\n break;\n }\n if (declaration.subset && /^<!DOCTYPE/i.test(text.slice(at, at + 9))) {\n notes.push(\"the file declares a DTD: the reader ignores it and expands no entity\");\n }\n at = declaration.end;\n continue;\n }\n if (text.startsWith(\"</\", at)) {\n const endOf = text.indexOf(\">\", at + 2);\n if (endOf === -1) {\n stop(\"an end tag does not end\");\n break;\n }\n meter.steps += endOf + 1 - at;\n const name = text.slice(at + 2, Math.min(endOf, at + 300)).trim();\n const top = stack.pop();\n if (top !== name) {\n stop(`the end tag </${short(name)}> does not match <${short(top)}>`);\n break;\n }\n close(stack.length + 1);\n at = endOf + 1;\n // the text after the root element is not part of the file\n if (stack.length === 0) break;\n continue;\n }\n const tag = readTag(text, at + 1, meter);\n if (tag === undefined) {\n stop(\"a start tag is not well formed\");\n break;\n }\n at = tag.end;\n const depth = stack.length + 1;\n if (depth > maxDepth) {\n stop(`the elements nest deeper than ${String(maxDepth)}`);\n break;\n }\n open(tag, depth);\n if (done) break;\n if (tag.selfClosing) close(depth);\n else stack.push(tag.name);\n if (depth === 1 && tag.selfClosing) break;\n }\n if (!done && stack.length > 0) stop(\"the file ends inside an element\");\n if (kind === \"unknown\" && notes.length === 0) {\n complete = false;\n notes.push(\"not a sitemap or a feed: the file has no root element\");\n }\n return { kind, entries, invalid, complete, notes };\n};\n\n/**\n * This function reads a sitemap, a sitemap index, an RSS or Atom feed, or a text sitemap. The\n * root element of the file tells its kind. The entries of a feed can have URLs relative to `base`\n * (the URL of the file). The entries of a sitemap must have absolute URLs.\n *\n * @beta\n */\nexport const readSitemap = (text: string, base: string, limits?: SitemapLimits): SitemapRead =>\n scanSitemap(text, base, limits);\n","import { createGunzip } from \"node:zlib\";\nimport type { Fetched } from \"./crawler.ts\";\nimport { DEADLINE_NOTE } from \"./crawler.ts\";\nimport { extractPage } from \"./html.ts\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport type { CrawlRuntime } from \"./runtime.ts\";\nimport { fetchAndIndex, isHeldBack } from \"./runtime.ts\";\nimport type { Changefreq, SitemapKind, SitemapRead } from \"./sitemaps.ts\";\nimport { readSitemap } from \"./sitemaps.ts\";\nimport { decodeBody } from \"./text.ts\";\nimport type { ScopeOptions } from \"./urls.ts\";\nimport { cacheKeyOf, scopeTest } from \"./urls.ts\";\n\n/**\n * Discovery finds the pages of a site from its sitemaps and its feeds. Each file is a fetch of the\n * crawler, so each polite rule applies to it. These rules are robots.txt, one request at a time for\n * each host, the pace, the stops, the egress policy and the capture log. These are the sources, in sequence:\n *\n * 1. The page at the URL of the site, for the RSS and Atom feeds that it links.\n * 2. The feeds that the caller gives.\n * 3. The sitemaps that the caller gives, and the `Sitemap:` lines of robots.txt. When there are\n * none, `/sitemap.xml`.\n * 4. The sitemaps of each sitemap index, the newest first, to a maximum depth.\n *\n * Discovery keeps a URL only when it is in scope (the same site by default) and robots.txt permits\n * it. It stops at a maximum of files, a maximum of URLs and a maximum time.\n */\n\n/**\n * These are the default limits of `discover`.\n *\n * @beta\n */\nexport const DISCOVER_LIMITS = {\n /** The most sitemap and feed files that one discovery reads. */\n maxFiles: 20,\n /** The most URLs that one discovery gives. */\n maxUrls: 10_000,\n /** The most bytes of one file, after decompression: 10 MiB. */\n maxBytes: 10_485_760,\n /** The most levels of sitemap index under a file that discovery found first. */\n maxNesting: 3,\n /** The longest time of one discovery: 1 minute. */\n budgetMs: 60_000,\n /** The most origins whose robots.txt one discovery reads for the URLs that it finds. */\n maxOrigins: 10,\n /** The most feeds that discovery reads from the links of the page. */\n maxLinkedFeeds: 5,\n} as const;\n\n/** These options control a discovery. Each limit has a default in `DISCOVER_LIMITS`. */\nexport interface DiscoverOptions extends ScopeOptions {\n /** This value is the most sitemap and feed files to read. The default is 20. */\n readonly maxFiles?: number | undefined;\n /** This value is the most URLs to give. The default is 10,000. */\n readonly maxUrls?: number | undefined;\n /** This value is the most bytes of one file, after decompression. The default is 10 MiB. */\n readonly maxBytes?: number | undefined;\n /** This value is the most levels of sitemap index to follow. The default is 3. */\n readonly maxNesting?: number | undefined;\n /** This value is the longest time of the discovery, in milliseconds. The default is 1 minute. */\n readonly budgetMs?: number | undefined;\n /** These sitemaps are read too, in addition to the `Sitemap:` lines of robots.txt. */\n readonly sitemaps?: readonly string[] | undefined;\n /** These RSS or Atom feeds are read too. */\n readonly feeds?: readonly string[] | undefined;\n /** With the default value true, discovery reads the page of the site for the feeds that it links. */\n readonly linkedFeeds?: boolean | undefined;\n /**\n * A stored copy of a file younger than this age answers, with no request. The default is `24h`.\n * The value `auto` uses the freshness that the server gave.\n */\n readonly maxAge?: Duration | undefined;\n /** These agents get their robots.txt rules applied to each fetch and to each URL. */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** When this signal aborts, the discovery stops, and it gives what it found. */\n readonly signal?: AbortSignal | undefined;\n /** This label goes into the capture records of the discovery. */\n readonly scope?: string | undefined;\n /** These labels go into the capture records of the discovery. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/**\n * These options control `discoverSite`: the options of `discover`, and a deadline.\n *\n * @beta\n */\nexport interface DiscoverRunOptions extends DiscoverOptions {\n /** This time (milliseconds since the epoch) ends the discovery when it comes before the end of `budgetMs`. */\n readonly deadline?: number | undefined;\n}\n\n/** One URL that a discovery found. */\nexport interface DiscoveredUrl {\n /** This URL is the canonical form of the URL of the page. */\n readonly url: string;\n /** This time (ISO 8601) is the last change that the file gave, or null. */\n readonly lastmod: string | null;\n readonly changefreq: Changefreq | null;\n /** This value tells if a sitemap or a feed listed the URL. */\n readonly via: \"sitemap\" | \"feed\";\n /** This URL is the file that listed the URL first. */\n readonly source: string;\n}\n\n/** One sitemap or feed file that a discovery read, or tried to read. */\nexport interface DiscoverSource {\n readonly url: string;\n readonly via: \"sitemap\" | \"feed\";\n /** This value tells how discovery found the file. */\n readonly from: \"robots.txt\" | \"default\" | \"given\" | \"page\" | \"index\";\n /** This value is the kind that the reader found, or null for a file that the crawler did not get. */\n readonly kind: SitemapKind | null;\n /** This value is the outcome of the fetch: `ok`, `not-modified`, `skipped`, `not-found` and others. */\n readonly outcome: string;\n readonly status: number;\n /** This id is the capture of the file. */\n readonly capture: string | null;\n readonly fetched_at: string | null;\n readonly from_cache: boolean;\n /** This value is the number of entries that the file listed. */\n readonly entries: number;\n /** This value is the number of new URLs that discovery kept from the file. */\n readonly urls: number;\n /** This value is true when the file was longer than `maxBytes`, so the reader read only its first part. */\n readonly truncated: boolean;\n readonly note: string | null;\n}\n\n/** The reason that a discovery stopped. */\nexport type DiscoverStop =\n \"done\" | \"max-files\" | \"max-urls\" | \"budget\" | \"cancelled\" | \"closed\" | \"robots\";\n\n/** The result of a discovery. */\nexport interface DiscoverResult {\n /** This URL is the origin of the site. */\n readonly site: string;\n /** These URLs are the pages that discovery found, the newest `lastmod` first. */\n readonly urls: readonly DiscoveredUrl[];\n /** These are the files that discovery read, in the sequence of the reads. */\n readonly sources: readonly DiscoverSource[];\n /** These values count the URLs that discovery did not keep, for each reason. */\n readonly dropped: {\n /** The URLs on a different site, and the files of an index on a different site. */\n readonly scope: number;\n /** The URLs that robots.txt does not permit. */\n readonly robots: number;\n /** The URLs that discovery found before. */\n readonly duplicate: number;\n /** The entries with no valid http(s) URL. */\n readonly invalid: number;\n /** The URLs and the files over a limit: the origins, the depth of the index, or the URLs. */\n readonly limit: number;\n };\n readonly stopped: DiscoverStop;\n /** This text tells why the discovery stopped early, or null. */\n readonly note: string | null;\n}\n\nconst PAGE_ACCEPT = \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.5\";\nconst SITEMAP_ACCEPT = \"application/xml,text/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5\";\nconst FEED_ACCEPT =\n \"application/rss+xml,application/atom+xml,application/xml;q=0.9,text/xml;q=0.8,*/*;q=0.5\";\nconst FEED_KINDS: ReadonlySet<SitemapKind> = new Set([\"rss\", \"atom\", \"rdf\"]);\n\ninterface Pending {\n readonly url: string;\n readonly via: \"sitemap\" | \"feed\";\n readonly from: DiscoverSource[\"from\"];\n readonly nesting: number;\n}\n\nconst whole = (name: string, value: number | undefined, fallback: number, min: number): number => {\n if (value === undefined) return fallback;\n if (!Number.isInteger(value) || value < min) {\n throw new RangeError(\n `${name} must be an integer of ${String(min)} or more, not ${String(value)}`,\n );\n }\n return value;\n};\n\n/**\n * This function reads a body that is gzip data (a `.xml.gz` sitemap). It keeps at most `maxBytes`\n * of the decompressed data, so a decompression bomb stops at the limit.\n */\nconst gunzipBounded = async (\n body: Uint8Array,\n maxBytes: number,\n): Promise<{\n readonly bytes: Uint8Array;\n readonly truncated: boolean;\n readonly error?: string;\n}> => {\n const gunzip = createGunzip();\n const chunks: Buffer[] = [];\n let size = 0;\n let truncated = false;\n let error: string | undefined;\n gunzip.end(body);\n try {\n for await (const chunk of gunzip as AsyncIterable<Buffer>) {\n const room = maxBytes - size;\n if (chunk.length > room) {\n chunks.push(chunk.subarray(0, room));\n size += room;\n truncated = true;\n break;\n }\n chunks.push(chunk);\n size += chunk.length;\n }\n } catch (caught) {\n // gzip data that the crawler cut at its limit ends early: the part before the end stays\n error = caught instanceof Error ? caught.message : String(caught);\n truncated = true;\n } finally {\n gunzip.destroy();\n }\n return {\n bytes: Buffer.concat(chunks, size),\n truncated,\n ...(error === undefined ? {} : { error }),\n };\n};\n\nconst isGzip = (body: Uint8Array): boolean => body[0] === 0x1f && body[1] === 0x8b;\n\nconst parsedUrl = (value: string, base?: string): URL | undefined => {\n try {\n const url = base === undefined ? new URL(value.trim()) : new URL(value.trim(), base);\n return url.protocol === \"http:\" || url.protocol === \"https:\" ? url : undefined;\n } catch {\n return undefined;\n }\n};\n\n/** This function gives the value of a promise, or rejects when the signal aborts first. */\nconst untilAbort = <T>(work: Promise<T>, signal: AbortSignal | undefined): Promise<T> => {\n if (signal === undefined) return work;\n return new Promise<T>((resolve, reject) => {\n const cancel = (): void => reject(signal.reason);\n if (signal.aborted) cancel();\n signal.addEventListener(\"abort\", cancel, { once: true });\n work.then(resolve, reject).finally(() => signal.removeEventListener(\"abort\", cancel));\n });\n};\n\n/** This function tells if the note of a fetch says that its deadline came first. */\nexport const pastDeadline = (note: string | null | undefined): boolean =>\n note?.startsWith(DEADLINE_NOTE) === true;\n\n/**\n * This function finds the pages of a site from its sitemaps and its feeds. `PoliteCrawler.discover`\n * uses it. Each file is a fetch of the crawler with the recipe `sitemap` or `feed`, so the\n * capture log records it, and robots.txt applies to it.\n *\n * @beta\n */\nexport const discoverSite = async (\n runtime: CrawlRuntime,\n site: string,\n options: DiscoverRunOptions = {},\n): Promise<DiscoverResult> => {\n const start = parsedUrl(site);\n if (start === undefined) throw new Error(`discover wants an http(s) URL, not \"${site}\"`);\n const maxFiles = whole(\"maxFiles\", options.maxFiles, DISCOVER_LIMITS.maxFiles, 1);\n const maxUrls = whole(\"maxUrls\", options.maxUrls, DISCOVER_LIMITS.maxUrls, 1);\n const maxBytes = whole(\"maxBytes\", options.maxBytes, DISCOVER_LIMITS.maxBytes, 1);\n const maxNesting = whole(\"maxNesting\", options.maxNesting, DISCOVER_LIMITS.maxNesting, 0);\n const maxAge = options.maxAge ?? \"24h\";\n const maxAgeMs = maxAge === \"auto\" ? \"auto\" : durationMs(maxAge);\n const budgetMs = options.budgetMs ?? DISCOVER_LIMITS.budgetMs;\n if (!(budgetMs >= 0)) throw new RangeError(`budgetMs must be 0 or more, not ${String(budgetMs)}`);\n const now = (): number => Date.parse(runtime.now());\n const deadline = Math.min(now() + budgetMs, options.deadline ?? Number.POSITIVE_INFINITY);\n const crawler = runtime.crawler;\n const signal = options.signal;\n // a function, so the check reads the signal again after each wait\n const cancelled = (): boolean => signal?.aborted === true;\n const marks = { scope: options.scope, labels: options.labels };\n const inScope = scopeTest([start], options);\n\n const urls = new Map<string, DiscoveredUrl>();\n const sources: DiscoverSource[] = [];\n const dropped = { scope: 0, robots: 0, duplicate: 0, invalid: 0, limit: 0 };\n const origins = new Set<string>();\n const queued = new Set<string>();\n const pending: Pending[] = [];\n let stopped: DiscoverStop | undefined;\n let note: string | null = null;\n let files = 0;\n let next = 0;\n const stopNote = (): string | null => {\n switch (stopped) {\n case \"max-files\":\n return `discovery read ${String(maxFiles)} files, its limit, and ${String(pending.length - next)} more files wait`;\n case \"max-urls\":\n return `discovery found ${String(maxUrls)} URLs, its limit`;\n case \"budget\":\n return `the budget of ${String(budgetMs)} ms ended`;\n case \"cancelled\":\n return \"the caller cancelled the discovery\";\n case \"closed\":\n return \"the crawler closed\";\n default:\n return null;\n }\n };\n const result = (): DiscoverResult => ({\n site: start.origin,\n urls: [...urls.values()]\n .map((url, order) => ({ url, order }))\n .sort(\n (a, b) =>\n (b.url.lastmod === null ? -Infinity : Date.parse(b.url.lastmod)) -\n (a.url.lastmod === null ? -Infinity : Date.parse(a.url.lastmod)) || a.order - b.order,\n )\n .map(({ url }) => url),\n sources,\n dropped,\n stopped: stopped ?? \"done\",\n note: note ?? stopNote(),\n });\n /** This function sets the reason of the stop from the note of a fetch, when the note gives one. */\n const stoppedBy = (fetched: { readonly note?: string | null | undefined }): boolean => {\n if (cancelled()) stopped = \"cancelled\";\n else if (pastDeadline(fetched.note)) stopped = \"budget\";\n else if (fetched.note === \"crawler closed\") stopped = \"closed\";\n else return false;\n return true;\n };\n const enqueue = (url: string, via: Pending[\"via\"], from: Pending[\"from\"], nesting = 0): void => {\n const parsed = parsedUrl(url, start.href);\n if (parsed === undefined) {\n dropped.invalid += 1;\n return;\n }\n const key = cacheKeyOf(parsed.href);\n if (queued.has(key)) return;\n queued.add(key);\n pending.push({ url: parsed.href, via, from, nesting });\n };\n\n if (cancelled()) {\n stopped = \"cancelled\";\n return result();\n }\n // robots.txt first: it names the sitemaps, and a site whose robots.txt does not read is closed\n let sitemapLines: readonly string[] = [];\n try {\n const robots = await untilAbort(crawler.readRobots(start.href, marks), signal);\n if (robots.robots.unreachable === \"crawler closed\") {\n stopped = \"closed\";\n return result();\n }\n if (robots.robots.unreachable !== undefined) {\n stopped = \"robots\";\n note = `robots.txt could not be read (${robots.robots.unreachable}), so nothing on the site is fetched`;\n return result();\n }\n sitemapLines = robots.robots.sitemaps;\n } catch {\n stopped = cancelled() ? \"cancelled\" : \"closed\";\n return result();\n }\n\n const fetchFile = (url: string, recipe: \"sitemap\" | \"feed\"): Promise<Fetched> =>\n crawler.fetch({\n url,\n mode: \"page\",\n recipe,\n accept: recipe === \"feed\" ? FEED_ACCEPT : SITEMAP_ACCEPT,\n maxBytes,\n maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n signal,\n scope: options.scope,\n labels: options.labels,\n deadline,\n });\n\n // the page of the site names its feeds\n if (options.linkedFeeds !== false) {\n const page = await fetchAndIndex(runtime, {\n url: start.href,\n mode: \"page\",\n recipe: \"page\",\n accept: PAGE_ACCEPT,\n maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n signal,\n scope: options.scope,\n labels: options.labels,\n deadline,\n });\n if (stoppedBy(page.fetched)) return result();\n const doc = page.docs[0];\n const body = page.fetched.body;\n const html = /html/i.test(page.fetched.contentType ?? \"text/html\");\n // the links of a page that asked not to be AI input are held back, its feeds too\n if (body !== undefined && html && doc !== undefined && !isHeldBack(doc)) {\n let linked: readonly string[] = [];\n try {\n linked =\n extractPage(decodeBody(body, page.fetched.contentType ?? null), page.fetched.url).feeds ??\n [];\n } catch {\n // a page that the reader cannot read names no feed\n }\n let taken = 0;\n for (const feed of linked) {\n const parsed = parsedUrl(feed);\n if (parsed === undefined || !inScope(parsed)) {\n dropped.scope += 1;\n continue;\n }\n if (taken >= DISCOVER_LIMITS.maxLinkedFeeds) {\n dropped.limit += 1;\n continue;\n }\n taken += 1;\n enqueue(parsed.href, \"feed\", \"page\");\n }\n }\n }\n for (const feed of options.feeds ?? []) enqueue(feed, \"feed\", \"given\");\n for (const sitemap of options.sitemaps ?? []) enqueue(sitemap, \"sitemap\", \"given\");\n for (const sitemap of sitemapLines) enqueue(sitemap, \"sitemap\", \"robots.txt\");\n if ((options.sitemaps ?? []).length === 0 && sitemapLines.length === 0)\n enqueue(new URL(\"/sitemap.xml\", start.origin).href, \"sitemap\", \"default\");\n\n /** This function keeps the page URLs of one file that are new, in scope and permitted. */\n const keep = async (\n read: SitemapRead,\n source: string,\n via: \"sitemap\" | \"feed\",\n ): Promise<number> => {\n let kept = 0;\n dropped.invalid += read.invalid;\n for (const entry of read.entries) {\n if (urls.size >= maxUrls) {\n stopped = \"max-urls\";\n return kept;\n }\n const known = urls.get(entry.url);\n if (known !== undefined) {\n dropped.duplicate += 1;\n // the same page in two files keeps the later change\n if (\n entry.lastmod !== null &&\n (known.lastmod === null || Date.parse(entry.lastmod) > Date.parse(known.lastmod))\n )\n urls.set(entry.url, { ...known, lastmod: entry.lastmod });\n continue;\n }\n const url = new URL(entry.url);\n if (!inScope(url)) {\n dropped.scope += 1;\n continue;\n }\n if (!origins.has(url.origin)) {\n if (origins.size >= DISCOVER_LIMITS.maxOrigins) {\n dropped.limit += 1;\n continue;\n }\n origins.add(url.origin);\n }\n let allowed: boolean;\n try {\n allowed = (\n await crawler.robotsAllow(entry.url, {\n onBehalfOf: options.onBehalfOf,\n signal,\n ...marks,\n })\n ).allowed;\n } catch {\n stopped = \"cancelled\";\n return kept;\n }\n if (!allowed) {\n dropped.robots += 1;\n continue;\n }\n urls.set(entry.url, {\n url: entry.url,\n lastmod: entry.lastmod,\n changefreq: entry.changefreq,\n via,\n source,\n });\n kept += 1;\n }\n return kept;\n };\n\n for (; next < pending.length && stopped === undefined; next += 1) {\n if (cancelled()) {\n stopped = \"cancelled\";\n break;\n }\n if (now() >= deadline) {\n stopped = \"budget\";\n break;\n }\n if (files >= maxFiles) {\n stopped = \"max-files\";\n break;\n }\n const file = pending[next]!;\n files += 1;\n const fetched = await fetchFile(file.url, file.via);\n const base = {\n url: fetched.url,\n via: file.via,\n from: file.from,\n outcome: fetched.outcome,\n status: fetched.status,\n capture: fetched.capture ?? null,\n fetched_at: fetched.fetchedAt ?? null,\n from_cache: fetched.fromCache,\n };\n if (stoppedBy(fetched)) {\n sources.push({\n ...base,\n kind: null,\n entries: 0,\n urls: 0,\n truncated: false,\n note: fetched.note ?? null,\n });\n break;\n }\n const body = fetched.body;\n if ((fetched.outcome !== \"ok\" && fetched.outcome !== \"not-modified\") || body === undefined) {\n sources.push({\n ...base,\n kind: null,\n entries: 0,\n urls: 0,\n truncated: false,\n note: fetched.note ?? `${fetched.outcome} ${String(fetched.status)}`,\n });\n continue;\n }\n let bytes = body;\n let truncated = fetched.truncated === true;\n const notes: string[] = [];\n if (isGzip(body)) {\n const unzipped = await gunzipBounded(body, maxBytes);\n bytes = unzipped.bytes;\n truncated ||= unzipped.truncated;\n if (unzipped.error !== undefined) notes.push(`the gzip data ended early (${unzipped.error})`);\n }\n const read = readSitemap(\n decodeBody(bytes, isGzip(body) ? null : (fetched.contentType ?? null)),\n fetched.url,\n );\n if (truncated)\n notes.push(\n `the file is longer than ${String(maxBytes)} bytes, so discovery read its first part`,\n );\n notes.push(...read.notes);\n let kept = 0;\n if (read.kind === \"sitemapindex\") {\n // the newest sitemaps first, so a limit leaves out the oldest\n const children = [...read.entries].sort(\n (a, b) =>\n (b.lastmod === null ? -Infinity : Date.parse(b.lastmod)) -\n (a.lastmod === null ? -Infinity : Date.parse(a.lastmod)),\n );\n for (const child of children) {\n const parsed = new URL(child.url);\n if (!inScope(parsed)) dropped.scope += 1;\n else if (file.nesting + 1 > maxNesting) dropped.limit += 1;\n else enqueue(child.url, \"sitemap\", \"index\", file.nesting + 1);\n }\n if (file.nesting + 1 > maxNesting && children.length > 0)\n notes.push(\n `the files of the index are deeper than maxNesting (${String(maxNesting)}), so discovery reads none of them`,\n );\n } else if (read.kind !== \"unknown\") {\n kept = await keep(read, fetched.url, FEED_KINDS.has(read.kind) ? \"feed\" : \"sitemap\");\n }\n sources.push({\n ...base,\n kind: read.kind,\n entries: read.entries.length,\n urls: kept,\n truncated,\n note: notes.length > 0 ? notes.join(\"; \") : null,\n });\n }\n if (stopped === undefined && urls.size >= maxUrls) stopped = \"max-urls\";\n return result();\n};\n","import { discoverSite, pastDeadline } from \"./discover.ts\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport { matchesPattern } from \"./robots.ts\";\nimport type { CrawlRuntime, PageResult } from \"./runtime.ts\";\nimport { fetchPage, isHeldBack } from \"./runtime.ts\";\nimport type { ScopeOptions } from \"./urls.ts\";\nimport { cacheKeyOf, scopeTest } from \"./urls.ts\";\n\n/**\n * The frontier is a small crawl of a site. It starts at the seeds. It follows the links of each page\n * that it fetched, to a maximum of pages, a maximum depth and a maximum time.\n *\n * Each page is a fetch of the crawler, so each polite rule applies. These rules are robots.txt, one\n * request at a time for each host, the pace, the stops, the egress policy and the capture log. The\n * frontier does not start a second request to a host with a request in progress. The crawler\n * limits the hosts in progress (`maxActive`).\n *\n * The state of the frontier is in memory, for one call. A later version can make it resumable.\n */\n\n/**\n * These are the default limits of `crawl`, and the limits of the memory of the frontier.\n *\n * @beta\n */\nexport const CRAWL_LIMITS = {\n /** The most pages that one crawl gives. */\n maxPages: 100,\n /** The most links from a seed to a page. A seed has the depth 0. */\n maxDepth: 3,\n /** The longest time of one crawl: 10 minutes. */\n budgetMs: 600_000,\n /** The most hosts that get a request of the crawl at one time. */\n hostsAtOnce: 16,\n /** The most URLs that wait in the frontier. */\n maxQueue: 10_000,\n /** The most URLs that the frontier remembers, so it fetches none of them two times. */\n maxSeen: 100_000,\n} as const;\n\n/**\n * A pattern for the URLs of a crawl. A text that starts with `/` matches the path and the query.\n * A different text matches the whole URL. In a text, `*` matches any characters and a `$` at the\n * end matches the end, as in robots.txt. A regular expression tests the whole URL.\n */\nexport type UrlPattern = string | RegExp;\n\n/** These options control a crawl. Each limit has a default in `CRAWL_LIMITS`. */\nexport interface CrawlOptions extends ScopeOptions {\n /** This value is the most pages that the crawl gives. The default is 100. */\n readonly maxPages?: number | undefined;\n /** This value is the most links from a seed to a page. The default is 3, and 0 fetches only the seeds. */\n readonly maxDepth?: number | undefined;\n /** This value is the longest time of the crawl in milliseconds. No request starts after it. The default is 10 minutes. */\n readonly budgetMs?: number | undefined;\n /** When this list is not empty, the crawl follows only a URL that matches a pattern of it. */\n readonly include?: readonly UrlPattern[] | undefined;\n /** The crawl follows no URL that matches a pattern of this list. */\n readonly exclude?: readonly UrlPattern[] | undefined;\n /** With the value true, the crawl adds the pages of the sitemaps of each site of the seeds (`discover`). */\n readonly useSitemaps?: boolean | undefined;\n /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */\n readonly render?: \"auto\" | \"never\" | \"always\" | undefined;\n /** A stored copy younger than this age answers, with no request. The default is `24h`. */\n readonly maxAge?: Duration | undefined;\n /** These agents get their robots.txt rules applied to each fetch. */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** When this signal aborts, the crawl stops the fetches in progress and ends. */\n readonly signal?: AbortSignal | undefined;\n /** This label goes into the capture records of the crawl. */\n readonly scope?: string | undefined;\n /** These labels go into the capture records of the crawl. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n}\n\n/**\n * These options control `crawlSite`: the options of `crawl`, and a step before the first fetch.\n *\n * @beta\n */\nexport interface CrawlRunOptions extends CrawlOptions {\n /** This function operates before the first fetch. `openCrawler` waits there for the open of the store. */\n readonly ready?: (() => Promise<void>) | undefined;\n}\n\n/** A page of a crawl: the result of its fetch, and the place of the page in the crawl. */\nexport interface CrawlPage extends PageResult {\n /** This value is the number of links from a seed to the page. */\n readonly depth: number;\n /** This value tells if the page is a seed, a page of a sitemap, or the target of a link. */\n readonly source: \"seed\" | \"sitemap\" | \"link\";\n /** This URL is the page whose link the crawl followed, or null. */\n readonly referrer: string | null;\n}\n\n/** The reason that a crawl stopped. */\nexport type CrawlStop = \"done\" | \"max-pages\" | \"budget\" | \"cancelled\" | \"closed\" | \"error\";\n\n/** The summary of a crawl. */\nexport interface CrawlSummary {\n /** This value is the number of pages that the crawl gave. */\n readonly pages: number;\n /** This value counts the pages with a document: `ok` or `not-modified`, from the store too. */\n readonly fetched: number;\n /** This value counts the pages that a stored copy answered, with no request. */\n readonly from_cache: number;\n /**\n * This value counts the refusals. These are the links that robots.txt does not permit, and the\n * pages that the crawler declined (`skipped`). The pages that the site refused (`blocked`,\n * `challenge`, `payment-required`) are refusals too.\n */\n readonly refused: number;\n /** This value counts the pages with a different outcome, for example `error` or `not-found`. */\n readonly failed: number;\n /** This value counts the links that the crawl did not follow. `skips` gives the reasons. */\n readonly skipped: number;\n readonly skips: {\n /** The links to a host that is not in scope. */\n readonly scope: number;\n /** The links that `include` or `exclude` left out. */\n readonly pattern: number;\n /** The links deeper than `maxDepth`. */\n readonly depth: number;\n /** The pages that asked not to be AI input: the crawl does not follow their links. */\n readonly held_back: number;\n /** The links over the limits of the memory of the frontier. */\n readonly limit: number;\n /** The URLs that a fetched page had as its final or its canonical URL. */\n readonly duplicate: number;\n };\n /** This value counts the URLs that waited in the frontier at the stop. */\n readonly queued: number;\n /** This value counts the URLs that the sitemaps gave, with `useSitemaps`. */\n readonly sitemap_urls: number;\n readonly stopped: CrawlStop;\n /** This text tells why the crawl stopped. */\n readonly reason: string;\n /** This value is the time of the crawl in milliseconds, on the clock of the crawler. */\n readonly elapsed_ms: number;\n}\n\n/**\n * A crawl in progress. A `for await` loop gets each page when its fetch ends. `summary` resolves\n * when the loop ends: at the last page, at a stop or at a `break`. The crawl starts at the first\n * step of the loop, and each crawl operates one time only.\n */\nexport interface CrawlRun extends AsyncIterable<CrawlPage> {\n readonly summary: Promise<CrawlSummary>;\n}\n\ninterface Item {\n readonly url: string;\n /** This key is the canonical form of the URL, with no fragment. */\n readonly key: string;\n readonly host: string;\n readonly depth: number;\n /** The seeds come first (0), then the URLs of the sitemaps (1), then the links (2). */\n readonly tier: number;\n readonly lastmod: number;\n readonly seq: number;\n readonly source: CrawlPage[\"source\"];\n readonly referrer: string | null;\n}\n\ntype Done =\n | { readonly kind: \"page\"; readonly item: Item; readonly page: PageResult }\n | { readonly kind: \"refused\"; readonly item: Item }\n | { readonly kind: \"aborted\"; readonly item: Item };\n\n/** The lower depth first, then the lower tier, then the newer `lastmod`, then the first found. */\nconst before = (a: Item, b: Item): number =>\n a.depth - b.depth || a.tier - b.tier || b.lastmod - a.lastmod || a.seq - b.seq;\n\n/** A binary heap of the URLs of one host, the first URL to fetch at the top. */\nclass Heap {\n private readonly items: Item[] = [];\n\n get size(): number {\n return this.items.length;\n }\n\n peek(): Item | undefined {\n return this.items[0];\n }\n\n push(item: Item): void {\n const items = this.items;\n items.push(item);\n let at = items.length - 1;\n while (at > 0) {\n const parent = (at - 1) >> 1;\n if (before(items[at]!, items[parent]!) >= 0) break;\n [items[at], items[parent]] = [items[parent]!, items[at]!];\n at = parent;\n }\n }\n\n pop(): Item | undefined {\n const items = this.items;\n const top = items[0];\n const last = items.pop();\n if (top === undefined || last === undefined || items.length === 0) return top;\n items[0] = last;\n let at = 0;\n for (;;) {\n const left = at * 2 + 1;\n const right = left + 1;\n let first = at;\n if (left < items.length && before(items[left]!, items[first]!) < 0) first = left;\n if (right < items.length && before(items[right]!, items[first]!) < 0) first = right;\n if (first === at) return top;\n [items[at], items[first]] = [items[first]!, items[at]!];\n at = first;\n }\n }\n}\n\nconst hostOf = (url: URL): string =>\n `${url.hostname.toLowerCase().replace(/\\.$/, \"\")}${url.port === \"\" ? \"\" : `:${url.port}`}`;\n\nconst matcher = (pattern: UrlPattern): ((url: URL) => boolean) => {\n if (pattern instanceof RegExp) {\n return (url) => {\n pattern.lastIndex = 0;\n return pattern.test(url.href);\n };\n }\n if (typeof pattern !== \"string\" || pattern.length === 0)\n throw new RangeError(\"a URL pattern is a regular expression or a text that is not empty\");\n return pattern.startsWith(\"/\")\n ? (url) => matchesPattern(pattern, `${url.pathname}${url.search}`)\n : (url) => matchesPattern(pattern, url.href);\n};\n\nconst whole = (name: string, value: number | undefined, fallback: number, min: number): number => {\n if (value === undefined) return fallback;\n if (!Number.isInteger(value) || value < min) {\n throw new RangeError(\n `${name} must be an integer of ${String(min)} or more, not ${String(value)}`,\n );\n }\n return value;\n};\n\nconst failedPage = (url: string, note: string): PageResult => ({\n url,\n doc: null,\n outcome: \"error\",\n status: 0,\n title: null,\n chars: 0,\n rendered: false,\n structured: [],\n links: 0,\n captures: [],\n capture: null,\n capture_count: 0,\n fetched_at: null,\n from_cache: false,\n held_back: false,\n note,\n});\n\nconst REFUSALS: ReadonlySet<string> = new Set([\n \"skipped\",\n \"blocked\",\n \"challenge\",\n \"payment-required\",\n]);\n\n/**\n * This function crawls a site from its seeds. `PoliteCrawler.crawl` uses it. It gives each page\n * when its fetch ends, and a summary at the end. The seeds come first, then the pages of the\n * sitemaps (the newest first), then the links, the lower depth first.\n *\n * @beta\n */\nexport const crawlSite = (\n runtime: CrawlRuntime,\n seeds: string | readonly string[],\n options: CrawlRunOptions = {},\n): CrawlRun => {\n let resolveSummary: (summary: CrawlSummary) => void = () => undefined;\n const summary = new Promise<CrawlSummary>((resolve) => {\n resolveSummary = resolve;\n });\n let used = false;\n\n async function* pages(): AsyncGenerator<CrawlPage, void, undefined> {\n const now = (): number => Date.parse(runtime.now());\n const startedAt = now();\n const abort = new AbortController();\n const outer = options.signal;\n const cancel = (): void => abort.abort(outer?.reason);\n if (outer?.aborted === true) cancel();\n outer?.addEventListener(\"abort\", cancel, { once: true });\n const inflight = new Map<string, Promise<Done>>();\n const heaps = new Map<string, Heap>();\n const seen = new Set<string>();\n const counts = { pages: 0, fetched: 0, from_cache: 0, refused: 0, failed: 0, sitemap_urls: 0 };\n const skips = { scope: 0, pattern: 0, depth: 0, held_back: 0, limit: 0, duplicate: 0 };\n // the final and the canonical URLs of the fetched pages: a URL that waits for one of them is not fetched\n const got = new Set<string>();\n let queued = 0;\n let returned = 0;\n let seq = 0;\n let started = 0;\n let stopping: CrawlStop | undefined;\n let reason: string | undefined;\n let finished = false;\n let maxPages: number = CRAWL_LIMITS.maxPages;\n let budgetMs: number = CRAWL_LIMITS.budgetMs;\n\n const crawler = runtime.crawler;\n const marks = { scope: options.scope, labels: options.labels };\n\n try {\n await options.ready?.();\n maxPages = whole(\"maxPages\", options.maxPages, CRAWL_LIMITS.maxPages, 1);\n const maxDepth = whole(\"maxDepth\", options.maxDepth, CRAWL_LIMITS.maxDepth, 0);\n budgetMs = options.budgetMs ?? CRAWL_LIMITS.budgetMs;\n if (!(budgetMs >= 0))\n throw new RangeError(`budgetMs must be 0 or more, not ${String(budgetMs)}`);\n const maxAge = options.maxAge ?? \"24h\";\n const maxAgeMs = maxAge === \"auto\" ? \"auto\" : durationMs(maxAge);\n const includes = (options.include ?? []).map(matcher);\n const excludes = (options.exclude ?? []).map(matcher);\n const deadline = startedAt + budgetMs;\n const list = typeof seeds === \"string\" ? [seeds] : [...seeds];\n if (list.length === 0) throw new RangeError(\"a crawl needs one seed URL or more\");\n const starts = list.flatMap((seed) => {\n try {\n const url = new URL(seed);\n return url.protocol === \"http:\" || url.protocol === \"https:\" ? [url] : [];\n } catch {\n return [];\n }\n });\n const inScope = scopeTest(starts, options);\n const followed = (url: URL): boolean =>\n (includes.length === 0 || includes.some((test) => test(url))) &&\n !excludes.some((test) => test(url));\n\n /** This function adds a URL to the frontier, when it is new, in scope and not too deep. */\n const offer = (\n raw: string,\n depth: number,\n tier: number,\n lastmod: number,\n source: Item[\"source\"],\n referrer: string | null,\n ): void => {\n let url: URL | undefined;\n try {\n url = new URL(raw);\n } catch {\n url = undefined;\n }\n const web = url !== undefined && (url.protocol === \"http:\" || url.protocol === \"https:\");\n if (!web && source !== \"seed\") return;\n // a fragment, a tracking parameter or a default port does not make a different page\n const key = cacheKeyOf(url?.href ?? raw);\n if (seen.has(key)) return;\n if (seen.size >= CRAWL_LIMITS.maxSeen) {\n skips.limit += 1;\n return;\n }\n seen.add(key);\n if (source !== \"seed\" && url !== undefined) {\n if (!inScope(url)) {\n skips.scope += 1;\n return;\n }\n if (!followed(url)) {\n skips.pattern += 1;\n return;\n }\n if (depth > maxDepth) {\n skips.depth += 1;\n return;\n }\n }\n if (queued >= CRAWL_LIMITS.maxQueue) {\n skips.limit += 1;\n return;\n }\n const host = url === undefined || !web ? `not-web:${raw}` : hostOf(url);\n let heap = heaps.get(host);\n if (heap === undefined) {\n heap = new Heap();\n heaps.set(host, heap);\n }\n seq += 1;\n heap.push({ url: raw, key, host, depth, tier, lastmod, seq, source, referrer });\n queued += 1;\n };\n\n /** This function gives the first URL of a host with no request of the crawl in progress. */\n const pick = (): Item | undefined => {\n for (;;) {\n let best: { readonly host: string; readonly heap: Heap } | undefined;\n for (const [host, heap] of heaps) {\n if (inflight.has(host)) continue;\n const top = heap.peek();\n if (top === undefined) continue;\n if (best === undefined || before(top, best.heap.peek()!) < 0) best = { host, heap };\n }\n if (best === undefined) return undefined;\n const item = best.heap.pop()!;\n queued -= 1;\n if (best.heap.size === 0) heaps.delete(best.host);\n if (!got.has(item.key)) return item;\n skips.duplicate += 1;\n }\n };\n\n const fetchItem = async (item: Item): Promise<Done> => {\n try {\n // a seed is the choice of the caller; a link that robots.txt refuses costs no fetch\n if (item.source !== \"seed\") {\n const verdict = await crawler.robotsAllow(item.url, {\n onBehalfOf: options.onBehalfOf,\n signal: abort.signal,\n ...marks,\n });\n if (!verdict.allowed) return { kind: \"refused\", item };\n }\n const page = await fetchPage(runtime, item.url, {\n render: options.render ?? \"auto\",\n maxAgeMs,\n onBehalfOf: options.onBehalfOf,\n signal: abort.signal,\n scope: options.scope,\n labels: options.labels,\n deadline,\n });\n return { kind: \"page\", item, page };\n } catch (error) {\n if (abort.signal.aborted) return { kind: \"aborted\", item };\n const message = error instanceof Error ? error.message : String(error);\n return { kind: \"page\", item, page: failedPage(item.url, `crawl error: ${message}`) };\n }\n };\n\n /** This function adds the links of a page to the frontier. The canonical URL of the page is known after its fetch. */\n const follow = (page: PageResult, item: Item): void => {\n if (page.doc === null) return;\n const doc = runtime.index.get(page.doc);\n if (doc === undefined) return;\n for (const same of [doc.url, doc.meta[\"canonical\"]]) {\n if (typeof same !== \"string\") continue;\n seen.add(cacheKeyOf(same));\n got.add(cacheKeyOf(same));\n }\n const links = doc.meta[\"links\"];\n if (!Array.isArray(links) || links.length === 0) return;\n // the links of a page that asked not to be AI input are held back with its text\n if (isHeldBack(doc)) {\n skips.held_back += 1;\n return;\n }\n for (const link of links as ReadonlyArray<{ readonly href?: unknown }>) {\n if (typeof link?.href === \"string\")\n offer(link.href, item.depth + 1, 2, Number.NEGATIVE_INFINITY, \"link\", page.url);\n }\n };\n\n for (const seed of list) offer(seed, 0, 0, Number.NEGATIVE_INFINITY, \"seed\", null);\n if (options.useSitemaps === true) {\n const sites = [...new Set(starts.map((start) => start.origin))];\n for (const site of sites) {\n if (abort.signal.aborted || now() >= deadline) break;\n try {\n const found = await discoverSite(runtime, site, {\n sameSite: options.sameSite,\n allowHosts: options.allowHosts,\n maxUrls: Math.min(10_000, Math.max(50, maxPages * 4)),\n // the crawl fetches the seed page itself, and never two times\n linkedFeeds: false,\n maxAge,\n onBehalfOf: options.onBehalfOf,\n signal: abort.signal,\n scope: options.scope,\n labels: options.labels,\n deadline,\n });\n counts.sitemap_urls += found.urls.length;\n for (const url of found.urls) {\n offer(\n url.url,\n 0,\n 1,\n url.lastmod === null ? Number.NEGATIVE_INFINITY : Date.parse(url.lastmod),\n \"sitemap\",\n null,\n );\n }\n } catch {\n // a site whose sitemaps do not read is crawled from its seeds\n }\n }\n }\n\n for (;;) {\n if (stopping === undefined && abort.signal.aborted) stopping = \"cancelled\";\n if (stopping === undefined && now() >= deadline) stopping = \"budget\";\n while (\n stopping === undefined &&\n inflight.size < CRAWL_LIMITS.hostsAtOnce &&\n started < maxPages\n ) {\n const item = pick();\n if (item === undefined) break;\n started += 1;\n inflight.set(item.host, fetchItem(item));\n }\n if (inflight.size === 0) break;\n const done = await Promise.race(inflight.values());\n inflight.delete(done.item.host);\n if (done.kind === \"aborted\") {\n started -= 1;\n stopping ??= \"cancelled\";\n continue;\n }\n if (done.kind === \"refused\") {\n started -= 1;\n counts.refused += 1;\n continue;\n }\n const page = done.page;\n if (pastDeadline(page.note)) {\n // the host was not free before the end of the budget: the page waits, and the crawl stops\n started -= 1;\n returned += 1;\n stopping ??= \"budget\";\n continue;\n }\n if (abort.signal.aborted) {\n started -= 1;\n stopping ??= \"cancelled\";\n continue;\n }\n if (page.outcome === \"skipped\" && page.note === \"crawler closed\") {\n started -= 1;\n stopping ??= \"closed\";\n continue;\n }\n counts.pages += 1;\n if (page.outcome === \"ok\" || page.outcome === \"not-modified\") {\n counts.fetched += 1;\n if (page.from_cache) counts.from_cache += 1;\n } else if (REFUSALS.has(page.outcome)) counts.refused += 1;\n else counts.failed += 1;\n follow(page, done.item);\n yield {\n ...page,\n depth: done.item.depth,\n source: done.item.source,\n referrer: done.item.referrer,\n };\n }\n stopping ??= queued > 0 && started >= maxPages ? \"max-pages\" : \"done\";\n finished = true;\n } catch (error) {\n stopping = \"error\";\n reason = error instanceof Error ? error.message : String(error);\n finished = true;\n throw error;\n } finally {\n // a loop that stopped early (a break) cancels the crawl\n if (!finished) {\n stopping ??= \"cancelled\";\n reason ??= \"the caller stopped the loop\";\n }\n outer?.removeEventListener(\"abort\", cancel);\n abort.abort(new Error(\"the crawl ended\"));\n // the fetches in progress end and write their records before the summary\n await Promise.allSettled(inflight.values());\n const stop = stopping ?? \"done\";\n reason ??=\n stop === \"done\"\n ? \"the crawl fetched each page in scope that it found\"\n : stop === \"max-pages\"\n ? `the crawl gave ${String(maxPages)} pages, its limit (maxPages)`\n : stop === \"budget\"\n ? `the budget of ${String(budgetMs)} ms ended (budgetMs)`\n : stop === \"cancelled\"\n ? \"the caller cancelled the crawl\"\n : \"the crawler closed\";\n resolveSummary({\n ...counts,\n skipped:\n skips.scope +\n skips.pattern +\n skips.depth +\n skips.held_back +\n skips.limit +\n skips.duplicate,\n skips,\n queued: queued + returned,\n stopped: stop,\n reason,\n elapsed_ms: Math.max(0, now() - startedAt),\n });\n }\n }\n\n return {\n summary,\n [Symbol.asyncIterator]() {\n if (used) throw new Error(\"a crawl runs one time: call crawl again for a new crawl\");\n used = true;\n return pages();\n },\n };\n};\n","import { randomUUID } from \"node:crypto\";\nimport { mkdirSync } from \"node:fs\";\nimport { dirname } from \"node:path\";\nimport type { StatementSync } from \"node:sqlite\";\nimport { DatabaseSync } from \"node:sqlite\";\nimport type {\n HostCounts,\n HostResult,\n HostStateStore,\n HostStop,\n ValidatorMemo,\n ValidatorReport,\n ValidatorVerdict,\n} from \"./politeness.ts\";\nimport { nextValidatorMemo } from \"./politeness.ts\";\n\n/** A stop with no end goes into the table as this value, which is past each date that JavaScript can hold. */\nconst NO_END = 9e15;\n/** A process that shares the store asks again after this wait, at most, during the lease of a different process. */\nconst SHARED_WAIT_MS = 1000;\n/** The store removes the rows of idle hosts after this many reservations. */\nconst SWEEP_EVERY = 1000;\n/**\n * This value is the version of the schema of the file (`PRAGMA user_version`). Version 0 (0.1.0)\n * has the table `hosts`. Version 1 adds the table `validators`, and it reads a file of version 0.\n */\nexport const HOST_STATE_SCHEMA = 1;\n\ninterface MemoRow {\n skip_etag: number;\n etag: string;\n last_modified: string;\n etag_misses: number;\n date_misses: number;\n since_probe: number;\n changed_at: number;\n}\n\nconst verdictOf = (value: string): ValidatorVerdict =>\n value === \"works\" || value === \"fails\" ? value : \"unknown\";\n\nconst memoOf = (row: MemoRow): ValidatorMemo => ({\n skipEtag: row.skip_etag === 1,\n etag: verdictOf(row.etag),\n lastModified: verdictOf(row.last_modified),\n etagMisses: row.etag_misses,\n dateMisses: row.date_misses,\n sinceProbe: row.since_probe,\n changedAt: row.changed_at,\n});\n\ninterface Row {\n next_at: number;\n leased_until: number;\n owner: string | null;\n failures: number;\n waits: number;\n stop_until: number | null;\n stop_reason: string | null;\n}\n\n/**\n * These options change a SQLite host state store.\n *\n * @beta\n */\nexport interface SqliteHostStateOptions {\n /** This value is the time in milliseconds that a step waits for the lock of a different process. The default is 10 seconds. */\n readonly busyTimeoutMs?: number | undefined;\n /** During the lease of a different process, a request asks again after this wait at most. The default is 1 second. */\n readonly pollMs?: number | undefined;\n}\n\n/**\n * This store keeps the state of each host in a SQLite file. Two or more processes on one store\n * share it, so they share the pace and the stops of each host. Thus a host gets one request at a\n * time from all of them. Each method is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock\n * of a different process (`busy_timeout`).\n *\n * The lease of a request belongs to the store that gave it. A different process that asks during\n * the lease gets a wait of one second or less, and asks again. A process that stops during a\n * request holds its lease until the lease ends (5 minutes by default).\n *\n * The table `validators` keeps the validator memo of each host (`ValidatorMemo`), so the processes\n * share it and it stays after a restart. A file of version 0.1.0 gets the table when it opens.\n *\n * The state is not a projection of the capture log. A deletion of the file loses only the pace,\n * the counts, the stops and the memos, and the crawler starts them again.\n *\n * @beta\n */\nexport class SqliteHostStateStore implements HostStateStore {\n private readonly db: DatabaseSync;\n private readonly statements = new Map<string, StatementSync>();\n /** This id names the leases of this store. */\n private readonly owner = randomUUID();\n private reservations = 0;\n private closed = false;\n private readonly pollMs: number;\n\n constructor(\n readonly path: string,\n options: SqliteHostStateOptions = {},\n ) {\n if (path !== \":memory:\") mkdirSync(dirname(path), { recursive: true, mode: 0o700 });\n this.pollMs = Math.max(1, Math.min(SHARED_WAIT_MS, options.pollMs ?? SHARED_WAIT_MS));\n this.db = new DatabaseSync(path);\n const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 10_000));\n this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);\n if (path !== \":memory:\") this.db.exec(\"PRAGMA journal_mode = WAL;\");\n this.db.exec(\"PRAGMA synchronous = NORMAL;\");\n this.migrate();\n }\n\n /**\n * This method makes the tables, or adds the tables of a later version to an older file. It is one\n * transaction, so two processes that open one old file at the same time do not collide.\n */\n private migrate(): void {\n this.step(() => {\n this.db.exec(`CREATE TABLE IF NOT EXISTS hosts (\n host TEXT PRIMARY KEY,\n next_at REAL NOT NULL DEFAULT 0,\n leased_until REAL NOT NULL DEFAULT 0,\n owner TEXT,\n failures INTEGER NOT NULL DEFAULT 0,\n waits INTEGER NOT NULL DEFAULT 0,\n stop_until REAL,\n stop_reason TEXT\n )`);\n this.db.exec(`CREATE TABLE IF NOT EXISTS validators (\n host TEXT PRIMARY KEY,\n skip_etag INTEGER NOT NULL DEFAULT 0,\n etag TEXT NOT NULL DEFAULT 'unknown',\n last_modified TEXT NOT NULL DEFAULT 'unknown',\n etag_misses INTEGER NOT NULL DEFAULT 0,\n date_misses INTEGER NOT NULL DEFAULT 0,\n since_probe INTEGER NOT NULL DEFAULT 0,\n changed_at REAL NOT NULL DEFAULT 0\n )`);\n const { user_version: version } = this.db.prepare(\"PRAGMA user_version\").get() as {\n user_version: number;\n };\n // a file of a later version keeps its number\n if (version < HOST_STATE_SCHEMA)\n this.db.exec(`PRAGMA user_version = ${String(HOST_STATE_SCHEMA)}`);\n });\n }\n\n private stmt(sql: string): StatementSync {\n let statement = this.statements.get(sql);\n if (statement === undefined) {\n statement = this.db.prepare(sql);\n this.statements.set(sql, statement);\n }\n return statement;\n }\n\n /** This method does one step in one transaction. */\n private step<T>(work: () => T): T {\n if (this.closed) throw new Error(\"the host state store is closed\");\n this.db.exec(\"BEGIN IMMEDIATE\");\n try {\n const result = work();\n this.db.exec(\"COMMIT\");\n return result;\n } catch (error) {\n if (this.db.isTransaction) this.db.exec(\"ROLLBACK\");\n throw error;\n }\n }\n\n private row(host: string): Row {\n this.stmt(\"INSERT OR IGNORE INTO hosts (host) VALUES (?)\").run(host);\n return this.stmt(\n \"SELECT next_at, leased_until, owner, failures, waits, stop_until, stop_reason FROM hosts WHERE host = ?\",\n ).get(host) as unknown as Row;\n }\n\n /** This method removes the hosts that have nothing to remember: no rest, no lease, no counts and no stop. */\n private sweep(now: number): void {\n this.stmt(\n `DELETE FROM hosts WHERE next_at <= ? AND leased_until <= ? AND failures = 0 AND waits = 0\n AND (stop_until IS NULL OR stop_until <= ?)`,\n ).run(now, now, now);\n }\n\n reserve(host: string, now: number, leaseMs: number): number {\n return this.step(() => {\n this.reservations += 1;\n if (this.reservations % SWEEP_EVERY === 0) this.sweep(now);\n const row = this.row(host);\n if (row.leased_until > now) {\n // the holder usually releases early, so a short wait and a new question\n return row.owner === this.owner\n ? row.leased_until - now\n : Math.min(this.pollMs, row.leased_until - now);\n }\n if (row.next_at > now) return row.next_at - now;\n this.stmt(\"UPDATE hosts SET leased_until = ?, owner = ? WHERE host = ?\").run(\n now + leaseMs,\n this.owner,\n host,\n );\n return 0;\n });\n }\n\n record(host: string, result: HostResult): HostCounts {\n return this.step(() => {\n const row = this.row(host);\n let { failures, waits } = row;\n if (result === \"success\") {\n failures = 0;\n waits = 0;\n } else if (result === \"failure\") failures += 1;\n else if (result === \"wait\") waits += 1;\n if (failures !== row.failures || waits !== row.waits) {\n this.stmt(\"UPDATE hosts SET failures = ?, waits = ? WHERE host = ?\").run(\n failures,\n waits,\n host,\n );\n }\n return { failures, waits };\n });\n }\n\n release(host: string, nextAt: number): void {\n this.step(() => {\n this.row(host);\n this.stmt(\n `UPDATE hosts SET next_at = max(next_at, ?),\n leased_until = CASE WHEN owner = ? THEN 0 ELSE leased_until END,\n owner = CASE WHEN owner = ? THEN NULL ELSE owner END\n WHERE host = ?`,\n ).run(nextAt, this.owner, this.owner, host);\n });\n }\n\n defer(host: string, until: number): void {\n this.step(() => {\n this.row(host);\n this.stmt(\"UPDATE hosts SET next_at = max(next_at, ?) WHERE host = ?\").run(until, host);\n });\n }\n\n stop(host: string, until: number, reason: string): void {\n const end = Number.isFinite(until) ? until : NO_END;\n this.step(() => {\n this.row(host);\n this.stmt(\n `UPDATE hosts SET stop_until = ?, stop_reason = ?\n WHERE host = ? AND (stop_until IS NULL OR stop_until < ?)`,\n ).run(end, reason, host, end);\n });\n }\n\n stopped(host: string, now: number): HostStop | undefined {\n if (this.closed) throw new Error(\"the host state store is closed\");\n const row = this.stmt(\"SELECT stop_until, stop_reason FROM hosts WHERE host = ?\").get(host) as\n { stop_until: number | null; stop_reason: string | null } | undefined;\n if (row === undefined || row.stop_until === null || row.stop_until <= now) return undefined;\n return {\n until: row.stop_until >= NO_END ? Number.POSITIVE_INFINITY : row.stop_until,\n reason: row.stop_reason ?? \"stopped\",\n };\n }\n\n reset(host?: string): void {\n this.step(() => {\n if (host === undefined) {\n this.stmt(\n \"UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL\",\n ).run();\n } else {\n this.stmt(\n \"UPDATE hosts SET failures = 0, waits = 0, stop_until = NULL, stop_reason = NULL WHERE host = ?\",\n ).run(host);\n }\n });\n }\n\n validators(host: string): ValidatorMemo | undefined {\n if (this.closed) throw new Error(\"the host state store is closed\");\n const row = this.stmt(\n `SELECT skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at\n FROM validators WHERE host = ?`,\n ).get(host) as MemoRow | undefined;\n return row === undefined ? undefined : memoOf(row);\n }\n\n noteValidators(host: string, report: ValidatorReport, now: number): ValidatorMemo {\n return this.step(() => {\n const memo = nextValidatorMemo(this.validators(host), report, now);\n this.stmt(\n `INSERT INTO validators (host, skip_etag, etag, last_modified, etag_misses, date_misses, since_probe, changed_at)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(host) DO UPDATE SET skip_etag = excluded.skip_etag, etag = excluded.etag,\n last_modified = excluded.last_modified, etag_misses = excluded.etag_misses,\n date_misses = excluded.date_misses, since_probe = excluded.since_probe,\n changed_at = excluded.changed_at`,\n ).run(\n host,\n memo.skipEtag ? 1 : 0,\n memo.etag,\n memo.lastModified,\n memo.etagMisses,\n memo.dateMisses,\n memo.sinceProbe,\n memo.changedAt,\n );\n return memo;\n });\n }\n\n /** This value is the number of hosts that the store holds. */\n get size(): number {\n return (this.stmt(\"SELECT count(*) AS n FROM hosts\").get() as { n: number }).n;\n }\n\n /** This method closes the file. A second close does nothing. */\n close(): void {\n if (this.closed) return;\n this.closed = true;\n this.statements.clear();\n this.db.close();\n }\n}\n","/**\n * The parts of hybrid search that need no store: the embedder interface, the chunks of a text, and\n * the fusion of ranked lists.\n *\n * An embedder changes texts into vectors. Two texts with a similar meaning get vectors that point\n * in a similar direction. Hybrid search ranks the stored documents two times: one time with the\n * full-text index (BM25), one time by the cosine of the vectors. Reciprocal rank fusion (RRF) then\n * makes one list of the two.\n *\n * `Embedder` is the extension point. `transformersEmbedder` and `ollamaEmbedder` are two adapters,\n * and an application can give an embedder of its own, as it can give a renderer of its own.\n */\n\n/** These options go to each call of `Embedder.embed`. */\nexport interface EmbedCallOptions {\n /** When this signal aborts, the embedder stops and rejects with the reason of the signal. */\n readonly signal?: AbortSignal | undefined;\n /**\n * This value tells what the texts are: `document` (parts of stored texts, the default) or `query`\n * (the words of a search). Some models want a different prefix for each.\n */\n readonly purpose?: \"document\" | \"query\" | undefined;\n}\n\n/**\n * An embedder changes texts into vectors. The vector store keeps each vector with the `id` of its\n * embedder, so the vectors of two models do not mix.\n */\nexport interface Embedder {\n /** This text names the model and its version. Change it when the vectors of the model change. */\n readonly id: string;\n /** This value is the length of each vector. The value 0 tells that the embedder learns it from its first answer. */\n readonly dimensions: number;\n /** This value is the largest input of the model, in tokens. The chunks of a text fit in it. The default is 256. */\n readonly maxInputTokens?: number | undefined;\n /** This method gives one vector for each text, in the same sequence. */\n embed(texts: readonly string[], options: EmbedCallOptions): Promise<Float32Array[]>;\n /** This optional method releases the model. A later call of `embed` loads it again. */\n close?(): Promise<void>;\n}\n\n// ── chunks ──────────────────────────────────────────────────────────────────\n\n/**\n * These are the limits of the chunks of a text.\n *\n * @beta\n */\nexport const CHUNK_LIMITS: {\n /** A chunk has this number of tokens at most, also for a model with a larger input. */\n readonly maxTokens: number;\n /** A chunk has this number of tokens when the embedder does not give `maxInputTokens`. */\n readonly defaultTokens: number;\n /** A chunk fills this share of its limit, because the count of the tokens is an estimate. */\n readonly fill: number;\n /** Two chunks next to each other share this share of the tokens of a chunk. */\n readonly overlap: number;\n /** A text gives this number of chunks at most. The rest of a longer text gets no vector. */\n readonly maxChunks: number;\n} = { maxTokens: 512, defaultTokens: 256, fill: 0.9, overlap: 0.1, maxChunks: 64 };\n\n/**\n * This is one chunk of a text: a window of paragraphs, with its position in the text.\n *\n * @beta\n */\nexport interface TextChunk {\n /** This value is the position of the chunk in the list, from 0. */\n readonly index: number;\n /** These values are the positions of the chunk in the text, in UTF-16 units. */\n readonly start: number;\n readonly end: number;\n readonly text: string;\n}\n\n/**\n * These options control the chunks of a text.\n *\n * @beta\n */\nexport interface ChunkOptions {\n /** This value is the largest number of tokens in a chunk. */\n readonly maxTokens: number;\n /** This value is the number of tokens that two chunks next to each other can share. The default is 10 percent of `maxTokens`. */\n readonly overlapTokens?: number | undefined;\n /** This value is the largest number of chunks. The default is `CHUNK_LIMITS.maxChunks`. */\n readonly maxChunks?: number | undefined;\n}\n\n/** These scripts write no space between words, so the estimate counts each character as a token. */\nconst SEGMENTED =\n /[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Thai}\\p{Script=Lao}\\p{Script=Khmer}\\p{Script=Myanmar}]/u;\nconst WORD = /[\\p{L}\\p{N}\\p{M}]/u;\nconst SPACE = /\\s/u;\n\n/** A word of this number of letters or fewer is one token in the estimate. A longer word is more tokens. */\nconst LETTERS_PER_TOKEN = 6;\n\nconst isAsciiWord = (code: number): boolean =>\n (code >= 0x30 && code <= 0x39) ||\n (code >= 0x41 && code <= 0x5a) ||\n (code >= 0x61 && code <= 0x7a);\nconst isAsciiSpace = (code: number): boolean => code === 0x20 || (code >= 0x09 && code <= 0x0d);\n\n/** This value is the kind of a code point for the estimate: a letter, a space, a segmented character or a different character. */\ntype Kind = \"word\" | \"space\" | \"segmented\" | \"other\";\n\nconst kindOf = (point: number): Kind => {\n if (point < 0x80) return isAsciiWord(point) ? \"word\" : isAsciiSpace(point) ? \"space\" : \"other\";\n const char = String.fromCodePoint(point);\n if (SEGMENTED.test(char)) return \"segmented\";\n if (WORD.test(char)) return \"word\";\n return SPACE.test(char) ? \"space\" : \"other\";\n};\n\n/**\n * This function estimates the number of tokens of a part of a text. A word is one token for each\n * six letters. A Chinese, Japanese or Thai character is one token. Each other character that is\n * not a space is one token. The estimate is a little high for English text, so a chunk fits.\n *\n * @beta\n */\nexport const estimateTokens = (\n text: string,\n start: number = 0,\n end: number = text.length,\n): number => {\n let tokens = 0;\n let run = 0;\n for (let at = start; at < end;) {\n const point = text.codePointAt(at)!;\n at += point > 0xffff ? 2 : 1;\n const kind = kindOf(point);\n if (kind === \"word\") {\n run += 1;\n continue;\n }\n if (run > 0) tokens += Math.ceil(run / LETTERS_PER_TOKEN);\n run = 0;\n if (kind !== \"space\") tokens += 1;\n }\n if (run > 0) tokens += Math.ceil(run / LETTERS_PER_TOKEN);\n return tokens;\n};\n\n/** A piece of a text: a sentence, or a part of a long sentence. A chunk is a run of pieces. */\ninterface Piece {\n readonly start: number;\n readonly end: number;\n readonly tokens: number;\n /** This value is true when the piece starts a paragraph. A chunk ends at a paragraph when it can. */\n readonly paragraph: boolean;\n}\n\n/** These characters end a sentence when a space follows them. */\nconst STOP = /[.!?…]+[\"'”’)\\]]*(?=\\s|$)|[。!?]+/gu;\n\n/** This function gives the ranges of the sentences of a part of a text, with no space at the ends. */\nconst sentencesOf = (\n text: string,\n start: number,\n end: number,\n): Array<readonly [number, number]> => {\n const part = text.slice(start, end);\n const out: Array<readonly [number, number]> = [];\n let from = 0;\n for (const match of part.matchAll(STOP)) {\n const to = match.index + match[0].length;\n if (to > from) out.push([from, to]);\n from = to;\n }\n if (from < part.length) out.push([from, part.length]);\n return out\n .map(([a, b]): readonly [number, number] => {\n let first = a;\n let last = b;\n while (first < last && SPACE.test(part[first]!)) first += 1;\n while (last > first && SPACE.test(part[last - 1]!)) last -= 1;\n return [start + first, start + last];\n })\n .filter(([a, b]) => b > a);\n};\n\n/** This function cuts a part of a text that is longer than the limit. It cuts at the spaces, and in a word that is too long. */\nconst cutLong = (text: string, start: number, end: number, size: number): Piece[] => {\n const out: Piece[] = [];\n let from = start;\n let tokens = 0;\n let run = 0;\n let lastSpace = -1;\n let at = start;\n const emit = (to: number): void => {\n let first = from;\n while (first < to && SPACE.test(text[first]!)) first += 1;\n let last = to;\n while (last > first && SPACE.test(text[last - 1]!)) last -= 1;\n if (last > first) {\n out.push({\n start: first,\n end: last,\n tokens: estimateTokens(text, first, last),\n paragraph: false,\n });\n }\n from = to;\n tokens = 0;\n run = 0;\n lastSpace = -1;\n };\n while (at < end) {\n const point = text.codePointAt(at)!;\n const width = point > 0xffff ? 2 : 1;\n const kind = kindOf(point);\n let cost = 0;\n if (kind === \"word\") {\n run += 1;\n if (run % LETTERS_PER_TOKEN === 1) cost = 1;\n } else {\n run = 0;\n if (kind !== \"space\") cost = 1;\n }\n if (tokens + cost > size && at > from) {\n // a cut at the last space keeps the words whole, else the cut is at this character\n if (lastSpace > from) {\n const to = lastSpace;\n emit(to);\n at = to;\n continue;\n }\n emit(at);\n }\n if (kind === \"space\") lastSpace = at;\n tokens += cost;\n at += width;\n }\n emit(end);\n return out;\n};\n\n/** This function gives the pieces of a text: its sentences, in paragraphs, and the parts of a sentence that is longer than the limit. */\nconst piecesOf = (text: string, size: number): Piece[] => {\n const pieces: Piece[] = [];\n for (const line of text.matchAll(/[^\\r\\n]+/g)) {\n let first = true;\n for (const [start, end] of sentencesOf(text, line.index, line.index + line[0].length)) {\n const tokens = estimateTokens(text, start, end);\n const parts =\n tokens <= size\n ? [{ start, end, tokens, paragraph: false }]\n : cutLong(text, start, end, size);\n for (const part of parts) {\n pieces.push({ ...part, paragraph: first });\n first = false;\n }\n }\n }\n return pieces;\n};\n\nconst positive = (name: string, value: number): number => {\n if (!Number.isFinite(value) || value < 1)\n throw new RangeError(`${name} must be a number of 1 or more, not ${String(value)}`);\n return Math.floor(value);\n};\n\n/**\n * This function cuts a text into chunks for an embedder. A chunk is a window of whole sentences.\n * It ends at the end of a paragraph when the paragraph fills half of the window or more. A\n * sentence that is longer than the window is cut at its spaces. Two chunks next to each other share\n * a few sentences (`overlapTokens`), so a passage at a cut is in one chunk. Each chunk keeps its\n * position in the text, so a hit points at a passage.\n *\n * @beta\n */\nexport const chunkText = (text: string, options: ChunkOptions): TextChunk[] => {\n const size = positive(\"maxTokens\", options.maxTokens);\n // an overlap of half a chunk or more gives each sentence to many chunks\n const overlap = Math.min(\n Math.floor(size / 2),\n Math.max(0, Math.floor(options.overlapTokens ?? size * CHUNK_LIMITS.overlap)),\n );\n const maxChunks = positive(\"maxChunks\", options.maxChunks ?? CHUNK_LIMITS.maxChunks);\n const pieces = piecesOf(text, size);\n const chunks: TextChunk[] = [];\n let first = 0;\n // the first piece that no earlier chunk holds\n let fresh = 0;\n while (first < pieces.length && chunks.length < maxChunks) {\n let last = first;\n let tokens = pieces[first]!.tokens;\n while (last + 1 < pieces.length && tokens + pieces[last + 1]!.tokens <= size) {\n last += 1;\n tokens += pieces[last]!.tokens;\n }\n // the chunk ends at the last paragraph that keeps it half full, when more text follows\n if (last + 1 < pieces.length && !pieces[last + 1]!.paragraph) {\n let filled = 0;\n let cut = -1;\n for (let at = first; at < last; at += 1) {\n filled += pieces[at]!.tokens;\n if (at >= fresh && pieces[at + 1]!.paragraph && filled * 2 >= size) cut = at;\n }\n if (cut !== -1) last = cut;\n }\n fresh = last + 1;\n const start = pieces[first]!.start;\n const end = pieces[last]!.end;\n chunks.push({ index: chunks.length, start, end, text: text.slice(start, end) });\n if (last + 1 >= pieces.length) break;\n // the next chunk starts with the last sentences of this chunk, to the overlap\n let next = last + 1;\n let shared = 0;\n while (next - 1 > first && shared + pieces[next - 1]!.tokens <= overlap) {\n next -= 1;\n shared += pieces[next]!.tokens;\n }\n if (shared + pieces[last + 1]!.tokens > size) next = last + 1;\n first = next;\n }\n return chunks;\n};\n\n/**\n * This function gives the number of tokens of a chunk for an embedder. It is the input limit with a\n * margin for the estimate, and `CHUNK_LIMITS.maxTokens` at most.\n *\n * @beta\n */\nexport const chunkTokensOf = (embedder: Pick<Embedder, \"maxInputTokens\">): number => {\n const limit = embedder.maxInputTokens;\n const tokens =\n limit === undefined || !Number.isFinite(limit) || limit < 1\n ? CHUNK_LIMITS.defaultTokens\n : Math.min(limit, CHUNK_LIMITS.maxTokens);\n return Math.max(8, Math.floor(tokens * CHUNK_LIMITS.fill));\n};\n\n// ── vectors ─────────────────────────────────────────────────────────────────\n\n/**\n * This function gives a copy of a vector with a length of 1. A vector of zeros stays zeros. A value that is not finite throws a `RangeError`.\n *\n * @beta\n */\nexport const unitVector = (vector: ArrayLike<number>): Float32Array => {\n const out = Float32Array.from(vector);\n let sum = 0;\n for (const value of out) {\n if (!Number.isFinite(value))\n throw new RangeError(\"the embedder gave a value that is not finite\");\n sum += value * value;\n }\n if (sum === 0) return out;\n const scale = 1 / Math.sqrt(sum);\n for (let at = 0; at < out.length; at += 1) out[at] = out[at]! * scale;\n return out;\n};\n\n// ── fusion ──────────────────────────────────────────────────────────────────\n\n/** These are the defaults of a hybrid search. */\nexport const HYBRID_DEFAULTS: {\n /** The full-text search gives this number of candidates. Each candidate gets vectors when it has none. */\n readonly candidates: number;\n /** The vector search gives this number of documents: the documents with the closest chunks. */\n readonly neighbors: number;\n /** This value is the constant k of reciprocal rank fusion. A larger k gives less weight to the top of each list. */\n readonly fusionK: number;\n} = { candidates: 50, neighbors: 50, fusionK: 60 };\n\n/**\n * This is one item of a fused list.\n *\n * @beta\n */\nexport interface FusedRank {\n readonly id: string;\n /** This value is the sum of 1 / (k + rank) over the lists that hold the item. */\n readonly score: number;\n /** These values are the rank of the item in each list, from 1, or null when a list does not hold it. */\n readonly ranks: ReadonlyArray<number | null>;\n}\n\n/**\n * This function fuses ranked lists with reciprocal rank fusion. Each item gets the sum of\n * 1 / (k + rank) over the lists that hold it, with the rank from 1. The best item is first. A tie goes to\n * the item with the best rank in a list, then to the earlier list, then to the smaller id. An item\n * that is two times in one list counts at its first place.\n *\n * @beta\n */\nexport const fuseRanks = (\n lists: ReadonlyArray<readonly string[]>,\n k: number = HYBRID_DEFAULTS.fusionK,\n): FusedRank[] => {\n if (!Number.isFinite(k) || k < 0)\n throw new RangeError(`the fusion constant k must be a number of 0 or more, not ${String(k)}`);\n const ranks = new Map<string, Array<number | null>>();\n lists.forEach((list, which) => {\n list.forEach((id, at) => {\n let entry = ranks.get(id);\n if (entry === undefined) {\n entry = lists.map(() => null);\n ranks.set(id, entry);\n }\n entry[which] ??= at + 1;\n });\n });\n // the keys of the sort come first, one time for each item: a long list sorts fast\n const keyed = [...ranks].map(([id, entry]) => {\n let score = 0;\n let best = Number.POSITIVE_INFINITY;\n for (const rank of entry) {\n if (rank === null) continue;\n score += 1 / (k + rank);\n best = Math.min(best, rank);\n }\n return { id, score, ranks: entry, best, first: entry.findIndex((rank) => rank !== null) };\n });\n keyed.sort(\n (a, b) =>\n b.score - a.score ||\n a.best - b.best ||\n a.first - b.first ||\n (a.id < b.id ? -1 : a.id > b.id ? 1 : 0),\n );\n return keyed.map(({ id, score, ranks: entry }) => ({ id, score, ranks: entry }));\n};\n","import { createHash } from \"node:crypto\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport type { Doc, DocDigest, Found, Passage } from \"./docs.ts\";\nimport { parseQuery } from \"./docs.ts\";\nimport type { Embedder } from \"./embed.ts\";\nimport {\n CHUNK_LIMITS,\n HYBRID_DEFAULTS,\n chunkText,\n chunkTokensOf,\n fuseRanks,\n unitVector,\n} from \"./embed.ts\";\nimport { archivedOf } from \"./fallback.ts\";\nimport { snippetOf } from \"./fold.ts\";\nimport type { CrawlRuntime } from \"./runtime.ts\";\nimport { isHeldBack } from \"./runtime.ts\";\nimport type { VectorHit, VectorStore } from \"./vectors.ts\";\n\n/**\n * Hybrid search ranks the stored documents two times and fuses the two lists:\n *\n * 1. The full-text index gives the BM25 candidates (`candidates`, 50 by default). Each candidate\n * whose text has no vectors gets them from the embedder first.\n * 2. The vector store gives the documents with the closest chunks (`neighbors`, 50 by default),\n * with a brute-force cosine over the vectors of the embedder.\n * 3. Reciprocal rank fusion (`fusionK`, 60 by default) makes one list of the two.\n *\n * The snippet and the `passage` of each hit come from its best chunk. A held-back document gets\n * no vectors, and no result gives it, exactly as for `find`. The text of a held-back version does not\n * goes to the embedder.\n */\n\n/** These options change a hybrid search. */\nexport interface HybridOptions {\n /** This value is the number of BM25 candidates. The default is `HYBRID_DEFAULTS.candidates`. */\n readonly candidates?: number | undefined;\n /** This value is the number of documents from the vector search. The default is `HYBRID_DEFAULTS.neighbors`. */\n readonly neighbors?: number | undefined;\n /** This value is the constant k of reciprocal rank fusion. The default is `HYBRID_DEFAULTS.fusionK`. */\n readonly fusionK?: number | undefined;\n /** When this signal aborts, the search stops the embedder and rejects. */\n readonly signal?: AbortSignal | undefined;\n}\n\n/** These options control a hybrid search of the store. */\nexport interface HybridFindOptions extends HybridOptions {\n /** This value is the maximum number of results. The default is 10, and the maximum is 1000. */\n readonly limit?: number | undefined;\n /** This value keeps only the documents of one kind, for example `page` or `pdf`. */\n readonly kind?: string | null | undefined;\n}\n\n/** This is one document of a ranking of a set of documents (`rankWithin`). */\nexport interface Ranked {\n readonly doc: string;\n /** This value is the score. A larger score is a better match. In the mode `hybrid`, it is the fused score. */\n readonly score: number;\n /** This value is the best chunk of the document, in the mode `hybrid`. */\n readonly passage?: Passage | undefined;\n}\n\n/** These options control `embed`, which gives vectors to the stored documents that have none. */\nexport interface EmbedOptions {\n /** This value is the longest time of the work, for example `\"10m\"` (the default). No embedding starts after it. */\n readonly budget?: Duration | undefined;\n /** This value is the largest number of texts that get new vectors. The default has no limit. */\n readonly maxDocs?: number | undefined;\n /** This value keeps only the documents of one kind, for example `page` or `pdf`. */\n readonly kind?: string | null | undefined;\n /** When this signal aborts, the work stops. The result then has `stopped: \"signal\"`. */\n readonly signal?: AbortSignal | undefined;\n}\n\n/** The reason of the end of `embed`. */\nexport type EmbedStop = \"done\" | \"budget\" | \"max-docs\" | \"signal\";\n\n/** The counts of `embed`. */\nexport interface EmbedResult {\n /** This value is the id of the embedder. */\n readonly embedder: string;\n /** This value is the number of documents that the work examined. */\n readonly docs: number;\n /** This value is the number of texts that got new vectors. */\n readonly embedded: number;\n /** This value is the number of documents whose text had vectors already. */\n readonly reused: number;\n /** This value is the number of documents that the work left out, because they are held back. */\n readonly held_back: number;\n /** This value is the number of new vectors: one for each chunk. */\n readonly chunks: number;\n readonly stopped: EmbedStop;\n /** This value is the time of the work in milliseconds. */\n readonly ms: number;\n}\n\n/** Hybrid search over a runtime and a vector store. `openCrawler` makes one when it has an embedder. */\nexport interface HybridSearch {\n readonly embedder: Embedder;\n /** This method searches the stored documents in the mode `hybrid`. It sends no request to a site. */\n find(q: string, options?: HybridFindOptions): Promise<Found[]>;\n /** This method ranks a set of documents in the mode `hybrid`, the best first. */\n rankWithin(q: string, docs: readonly string[], options?: HybridOptions): Promise<Ranked[]>;\n /** This method gives vectors to the stored documents that have none, within a budget. */\n embed(options?: EmbedOptions): Promise<EmbedResult>;\n}\n\n/**\n * These options make a hybrid search.\n *\n * @beta\n */\nexport interface HybridSearchOptions {\n readonly embedder: Embedder;\n /** This store keeps the vectors. The caller opens it and closes it. */\n readonly store: VectorStore;\n}\n\n/** The embedder gets this number of texts in one call at most. */\nconst BATCH = 32;\n/** The search keeps the vectors of this number of questions. */\nconst QUERY_CACHE = 32;\n/** The search checks this number of texts of the vector search against the index at one time. */\nconst PAGE = 64;\n\nconst sha256 = (text: string): string => createHash(\"sha256\").update(text, \"utf8\").digest(\"hex\");\n\nconst count = (name: string, value: number | undefined, fallback: number, max: number): number => {\n if (value === undefined) return fallback;\n if (!Number.isFinite(value) || value < 1)\n throw new RangeError(`${name} must be a number of 1 or more, not ${String(value)}`);\n return Math.min(max, Math.floor(value));\n};\n\n/** This function checks the constant k of reciprocal rank fusion. */\nconst fusionKOf = (value: number | undefined): number => {\n if (value === undefined) return HYBRID_DEFAULTS.fusionK;\n if (!Number.isFinite(value) || value < 0)\n throw new RangeError(`fusionK must be a number of 0 or more, not ${String(value)}`);\n return value;\n};\n\n/** This function gives a promise with its resolve and its reject. A rejection with no listener is not an error of the process. */\nconst deferred = <T>(): {\n readonly promise: Promise<T>;\n readonly resolve: (value: T) => void;\n readonly reject: (error: unknown) => void;\n} => {\n let resolve: (value: T) => void = () => undefined;\n let reject: (error: unknown) => void = () => undefined;\n const promise = new Promise<T>((yes, no) => {\n resolve = yes;\n reject = no;\n });\n promise.catch(() => undefined);\n return { promise, resolve, reject };\n};\n\nconst round = (value: number, digits: number): number => {\n const scale = 10 ** digits;\n return Math.round(value * scale) / scale;\n};\n\n/** The facts of a document that hybrid search checks: its kind, its AI input and the hash of its text. */\ntype Head = Pick<DocDigest, \"doc\" | \"kind\" | \"ai_input\" | \"text_sha\">;\n\n/**\n * This function makes a hybrid search over a runtime. The vectors go into the store, with the id of the embedder.\n *\n * @beta\n */\nexport const createHybridSearch = (\n runtime: CrawlRuntime,\n options: HybridSearchOptions,\n): HybridSearch => {\n const { embedder, store } = options;\n const tokens = chunkTokensOf(embedder);\n const plan = `chunks-1/${String(tokens)}/${String(Math.floor(tokens * CHUNK_LIMITS.overlap))}/${String(CHUNK_LIMITS.maxChunks)}`;\n const working = new Map<string, Promise<number>>();\n const queries = new Map<string, Float32Array>();\n let dimensions = embedder.dimensions > 0 ? embedder.dimensions : store.dimensions(embedder.id);\n\n /** This function gives the heads of documents, from `digests` when the index has it. */\n const headsOf = (docs: readonly string[]): Map<string, Head> => {\n const heads = new Map<string, Head>();\n if (docs.length === 0) return heads;\n const index = runtime.index;\n if (index.digests !== undefined) {\n for (const head of index.digests({ docs })) heads.set(head.doc, head);\n return heads;\n }\n for (const doc of docs) {\n const found = index.get(doc);\n if (found !== undefined) heads.set(doc, headOfDoc(found));\n }\n return heads;\n };\n const headOfDoc = (doc: Doc): Head => ({\n doc: doc.doc,\n kind: doc.kind,\n ai_input: doc.ai_input,\n text_sha: sha256(doc.text),\n });\n\n /** This function asks the embedder for the vectors of texts, in batches, and checks them. */\n const vectorsOf = async (\n texts: readonly string[],\n purpose: \"document\" | \"query\",\n signal: AbortSignal | undefined,\n ): Promise<Float32Array[]> => {\n const out: Float32Array[] = [];\n for (let at = 0; at < texts.length; at += BATCH) {\n signal?.throwIfAborted();\n const part = texts.slice(at, at + BATCH);\n const given = await embedder.embed(part, { signal, purpose });\n if (!Array.isArray(given) || given.length !== part.length) {\n throw new Error(\n `the embedder ${embedder.id} gave ${String(Array.isArray(given) ? given.length : 0)} vectors for ${String(part.length)} texts`,\n );\n }\n for (const vector of given) {\n const unit = unitVector(vector);\n dimensions ??= unit.length;\n if (unit.length !== dimensions || unit.length === 0) {\n throw new Error(\n `the embedder ${embedder.id} gave a vector of ${String(unit.length)} dimensions, not ${String(dimensions)}`,\n );\n }\n out.push(unit);\n }\n }\n return out;\n };\n\n const queryVector = async (q: string, signal: AbortSignal | undefined): Promise<Float32Array> => {\n const text = q.replace(/[\"*]/g, \" \").replace(/\\s+/g, \" \").trim();\n const known = queries.get(text);\n if (known !== undefined) {\n queries.delete(text);\n queries.set(text, known);\n return known;\n }\n const [vector] = await vectorsOf([text], \"query\", signal);\n queries.set(text, vector!);\n if (queries.size > QUERY_CACHE) queries.delete(queries.keys().next().value!);\n return vector!;\n };\n\n /**\n * This function gives vectors to the texts of documents that have none, and points each document\n * at its current text. It reads each text from the index, and it does not send the text\n * of a held-back document. Two calls for one text share the work. It gives the number of new vectors.\n */\n const ensure = async (\n docs: readonly string[],\n signal: AbortSignal | undefined,\n ): Promise<{ embedded: number; reused: number; held: number; chunks: number }> => {\n const tally = { embedded: 0, reused: 0, held: 0, chunks: 0 };\n const now = runtime.now();\n const heads = headsOf(docs);\n const waiting = new Map<string, { readonly text: string; readonly docs: string[] }>();\n const shared: Array<Promise<number>> = [];\n for (const id of docs) {\n const head = heads.get(id);\n if (head === undefined) continue;\n if (isHeldBack(head)) {\n tally.held += 1;\n if (store.pointed(embedder.id, id) !== undefined) store.forget(embedder.id, id);\n continue;\n }\n if (store.plan(embedder.id, head.text_sha) === plan) {\n tally.reused += 1;\n if (store.pointed(embedder.id, id) !== head.text_sha)\n store.point(embedder.id, id, head.text_sha, now);\n continue;\n }\n const doc = runtime.index.get(id);\n if (doc === undefined || isHeldBack(doc)) continue;\n const sha = sha256(doc.text);\n const entry = waiting.get(sha);\n if (entry !== undefined) {\n entry.docs.push(id);\n continue;\n }\n const busy = working.get(sha);\n if (busy !== undefined) {\n // a different call embeds this text now: this call points the document at it after\n const after = busy.then((made) => {\n store.point(embedder.id, id, sha, now);\n return made;\n });\n // This call can stop with an error before it waits for `shared`. Then a failure of the other\n // call is not an unhandled rejection, which stops the process.\n after.catch(() => undefined);\n shared.push(after);\n continue;\n }\n waiting.set(sha, { text: doc.text, docs: [id] });\n }\n // the chunks of the texts go to the embedder in batches, and each text is written when its vectors came\n const jobs = [...waiting].map(([sha, entry]) => {\n const job = {\n sha,\n entry,\n chunks: chunkText(entry.text, { maxTokens: tokens }),\n ...deferred<number>(),\n };\n working.set(sha, job.promise);\n return job;\n });\n try {\n for (let at = 0; at < jobs.length;) {\n const batch: typeof jobs = [];\n let size = 0;\n while (\n at < jobs.length &&\n (batch.length === 0 || size + jobs[at]!.chunks.length <= BATCH)\n ) {\n batch.push(jobs[at]!);\n size += jobs[at]!.chunks.length;\n at += 1;\n }\n const vectors = await vectorsOf(\n batch.flatMap((job) => job.chunks.map((chunk) => chunk.text)),\n \"document\",\n signal,\n );\n let next = 0;\n for (const job of batch) {\n const chunks = job.chunks.map((chunk) => ({\n start: chunk.start,\n end: chunk.end,\n vector: vectors[next++]!,\n }));\n store.put({\n embedder: embedder.id,\n dimensions: dimensions ?? chunks[0]?.vector.length ?? 0,\n sha: job.sha,\n plan,\n chars: job.entry.text.length,\n chunks,\n docs: job.entry.docs,\n at: now,\n });\n tally.embedded += 1;\n tally.chunks += chunks.length;\n working.delete(job.sha);\n job.resolve(chunks.length);\n }\n }\n } catch (error) {\n for (const job of jobs) {\n if (working.get(job.sha) === job.promise) working.delete(job.sha);\n job.reject(error);\n }\n throw error;\n }\n await Promise.all(shared);\n return tally;\n };\n\n /**\n * This function gives the documents of the vector search, the best first, to `limit`. It checks\n * each document against the index. Its current text must be the text of the hit, it must not be\n * held back, and it must have the kind. With `within`, it ranks only those documents.\n */\n const neighborsOf = (\n query: Float32Array,\n limit: number,\n kind: string | null | undefined,\n within: ReadonlySet<string> | undefined,\n ): Array<{ readonly doc: string; readonly hit: VectorHit }> => {\n const hits = store.search(embedder.id, query, within === undefined ? {} : { within });\n const out: Array<{ readonly doc: string; readonly hit: VectorHit }> = [];\n for (let at = 0; at < hits.length && out.length < limit; at += PAGE) {\n const page = hits.slice(at, at + PAGE);\n const docs = [\n ...new Set(\n page.flatMap((hit) =>\n within === undefined ? hit.docs : hit.docs.filter((doc) => within.has(doc)),\n ),\n ),\n ];\n const heads = headsOf(docs);\n for (const hit of page) {\n for (const doc of hit.docs) {\n if (out.length >= limit) break;\n if (within !== undefined && !within.has(doc)) continue;\n const head = heads.get(doc);\n if (head === undefined || head.text_sha !== hit.sha || isHeldBack(head)) continue;\n if (kind !== undefined && kind !== null && head.kind !== kind) continue;\n out.push({ doc, hit });\n }\n }\n }\n return out;\n };\n\n const passageOf = (hit: VectorHit): Passage => ({\n start: hit.start,\n end: hit.end,\n similarity: round(hit.similarity, 4),\n });\n\n return {\n embedder,\n async find(q, findOptions = {}) {\n const query = parseQuery(q);\n if (query === undefined) return [];\n // the limit reads as the limit of `find` in the other modes: 1 to 1000, and 10 for a value that is not a number\n const limit = Math.max(1, Math.min(1000, Math.floor(findOptions.limit ?? 10) || 10));\n const candidates = count(\n \"candidates\",\n findOptions.candidates,\n HYBRID_DEFAULTS.candidates,\n 1000,\n );\n const neighbors = count(\"neighbors\", findOptions.neighbors, HYBRID_DEFAULTS.neighbors, 1000);\n const k = fusionKOf(findOptions.fusionK);\n const signal = findOptions.signal;\n // the question first: an embedder that does not work fails before any other work\n const vector = await queryVector(q, signal);\n const kind = findOptions.kind ?? null;\n const lexical = runtime.index.find(q, { limit: candidates, kind, rank: \"exact\" });\n await ensure(\n lexical.map((hit) => hit.doc),\n signal,\n );\n const near = neighborsOf(vector, neighbors, kind, undefined);\n const fused = fuseRanks([lexical.map((hit) => hit.doc), near.map((one) => one.doc)], k).slice(\n 0,\n limit,\n );\n const byDoc = new Map(lexical.map((hit) => [hit.doc, hit]));\n const nearBy = new Map(near.map((one) => [one.doc, one.hit]));\n const results: Found[] = [];\n for (const one of fused) {\n const doc = runtime.index.get(one.id);\n if (doc === undefined || isHeldBack(doc)) continue;\n const best = nearBy.get(one.id) ?? store.best(embedder.id, sha256(doc.text), vector);\n const lexicalHit = byDoc.get(one.id);\n const archived = doc.kind === \"archived\" ? archivedOf(doc.meta) : undefined;\n results.push({\n doc: doc.doc,\n kind: doc.kind,\n title: doc.title,\n url: doc.url,\n snippet:\n best === undefined\n ? (lexicalHit?.snippet ?? snippetOf(doc.text, query.terms))\n : snippetOf(doc.text.slice(best.start, best.end), query.terms),\n score: round(one.score, 6),\n ai_input: doc.ai_input,\n fetched_at: doc.fetched_at,\n capture: doc.capture,\n ...(archived === undefined ? {} : { archived }),\n ...(best === undefined ? {} : { passage: passageOf(best) }),\n });\n }\n return results;\n },\n async rankWithin(q, docs, rankOptions = {}) {\n const query = parseQuery(q);\n if (query === undefined || docs.length === 0) return [];\n const candidates = count(\n \"candidates\",\n rankOptions.candidates,\n HYBRID_DEFAULTS.candidates,\n 1000,\n );\n const neighbors = count(\n \"neighbors\",\n rankOptions.neighbors,\n HYBRID_DEFAULTS.neighbors,\n 100_000,\n );\n const k = fusionKOf(rankOptions.fusionK);\n const signal = rankOptions.signal;\n const vector = await queryVector(q, signal);\n const lexical = runtime.index.rankWithin(q, docs, { rank: \"exact\" });\n // the BM25 matches first, then the other documents of the set, to the number of candidates\n const first = [...new Set([...lexical.map((hit) => hit.doc), ...docs])].slice(0, candidates);\n await ensure(first, signal);\n const within = new Set(docs);\n const near = neighborsOf(vector, neighbors, undefined, within);\n const nearBy = new Map(near.map((one) => [one.doc, one.hit]));\n const heads = headsOf(lexical.map((hit) => hit.doc).filter((doc) => !nearBy.has(doc)));\n return fuseRanks([lexical.map((hit) => hit.doc), near.map((one) => one.doc)], k).map(\n (one) => {\n const sha = heads.get(one.id)?.text_sha;\n const best =\n nearBy.get(one.id) ??\n (sha === undefined ? undefined : store.best(embedder.id, sha, vector));\n return {\n doc: one.id,\n score: round(one.score, 6),\n ...(best === undefined ? {} : { passage: passageOf(best) }),\n };\n },\n );\n },\n async embed(embedOptions = {}) {\n const started = Date.now();\n const budgetMs = durationMs(embedOptions.budget ?? \"10m\");\n const maxDocs =\n embedOptions.maxDocs === undefined\n ? Number.POSITIVE_INFINITY\n : count(\"maxDocs\", embedOptions.maxDocs, 1, Number.MAX_SAFE_INTEGER);\n const kind = embedOptions.kind ?? null;\n // the budget aborts the call in progress too, so no embedding runs past it\n const timer = new AbortController();\n // a timer holds 2^31 - 1 ms at most: a longer budget waits that long, and the check of the time ends it\n const clock = setTimeout(\n () => timer.abort(new Error(\"the budget of embed is over\")),\n Math.min(budgetMs, 2_147_483_647),\n );\n clock.unref();\n const signal =\n embedOptions.signal === undefined\n ? timer.signal\n : AbortSignal.any([embedOptions.signal, timer.signal]);\n const tally = { docs: 0, embedded: 0, reused: 0, held_back: 0, chunks: 0 };\n let stopped: EmbedStop = \"done\";\n const stopOf = (): EmbedStop | undefined =>\n embedOptions.signal?.aborted === true\n ? \"signal\"\n : timer.signal.aborted || Date.now() - started >= budgetMs\n ? \"budget\"\n : tally.embedded >= maxDocs\n ? \"max-docs\"\n : undefined;\n try {\n for await (const page of pagesOf(runtime)) {\n const docs = page.filter((head) => kind === null || head.kind === kind);\n for (let at = 0; at < docs.length;) {\n const stop = stopOf();\n if (stop !== undefined) {\n stopped = stop;\n break;\n }\n // a small group at a time, so the budget and maxDocs stop the work soon\n const room = Math.max(1, Math.min(BATCH, maxDocs - tally.embedded));\n const group = docs.slice(at, at + room).map((head) => head.doc);\n at += group.length;\n try {\n const done = await ensure(group, signal);\n tally.docs += group.length;\n tally.embedded += done.embedded;\n tally.reused += done.reused;\n tally.held_back += done.held;\n tally.chunks += done.chunks;\n } catch (error) {\n const stop = stopOf();\n if (stop === undefined) throw error;\n stopped = stop;\n break;\n }\n }\n if (stopped !== \"done\") break;\n }\n } finally {\n clearTimeout(clock);\n }\n return { embedder: embedder.id, ...tally, stopped, ms: Date.now() - started };\n },\n };\n};\n\n/**\n * This function gives the heads of each stored document, in pages. It uses `digests` when the\n * index has it. Else it reads the capture log and asks the index for the documents of each capture.\n */\nasync function* pagesOf(runtime: CrawlRuntime): AsyncGenerator<Head[]> {\n const index = runtime.index;\n if (index.digests !== undefined) {\n let after = \"\";\n for (;;) {\n const page = index.digests({ after, limit: 500 });\n if (page.length === 0) return;\n yield page;\n after = page.at(-1)!.doc;\n }\n }\n const seen = new Set<string>();\n let page: Head[] = [];\n for await (const envelope of runtime.captures.scan({ source: \"fetch\" })) {\n for (const doc of index.docsOfCapture(envelope.id)) {\n if (seen.has(doc.doc)) continue;\n seen.add(doc.doc);\n page.push({\n doc: doc.doc,\n kind: doc.kind,\n ai_input: doc.ai_input,\n text_sha: sha256(doc.text),\n });\n if (page.length >= 500) {\n yield page;\n page = [];\n }\n }\n }\n if (page.length > 0) yield page;\n}\n","import { mkdirSync } from \"node:fs\";\nimport { dirname } from \"node:path\";\nimport type { StatementSync } from \"node:sqlite\";\nimport { DatabaseSync } from \"node:sqlite\";\n\n/**\n * The vector store keeps the vectors of hybrid search in `vectors.sqlite`, beside the index. It is\n * a projection, as the index is. Delete the file, and the next hybrid search or `embed` makes the\n * vectors again from the stored texts.\n *\n * - A row is one chunk of one text for one embedder. Its key is the SHA-256 of the text, the\n * number of the chunk and the id of the embedder. Thus a reindex keeps each vector, and the same\n * text gets its vectors one time for each model, also when two documents hold it.\n * - A text is complete when the table `texts` has it, with the plan of its chunks. A text with a\n * different plan gets new chunks.\n * - The table `docs` points each document at the text that has its vectors. A search uses only\n * the texts that a document points at, and the caller checks each document against the index.\n * - A search compares the vector of the question with each stored vector (a brute-force cosine).\n * The store keeps the vectors of each embedder in memory after the first search. It reads them\n * again when a different process writes the file.\n *\n * Each connection waits for the lock of a different process (`busy_timeout`), and each write is\n * one transaction (`BEGIN IMMEDIATE`). The file has a schema version (`VECTORS_SCHEMA`). A file of\n * a different version opens empty.\n */\n\n/**\n * This value is the version of the schema of the vector file.\n *\n * @beta\n */\nexport const VECTORS_SCHEMA = 1;\n\n/**\n * This is one chunk of a text with its vector, as the store keeps it.\n *\n * @beta\n */\nexport interface StoredChunk {\n readonly chunk: number;\n /** These values are the positions of the chunk in the text, in UTF-16 units. */\n readonly start: number;\n readonly end: number;\n /** This vector has a length of 1, or it is zeros. */\n readonly vector: Float32Array;\n}\n\n/**\n * This is the best chunk of one text for a question.\n *\n * @beta\n */\nexport interface VectorHit {\n /** This value is the SHA-256 of the text, in hex. */\n readonly sha: string;\n readonly chunk: number;\n readonly start: number;\n readonly end: number;\n /** This value is the cosine of the chunk and the question, from -1 to 1. */\n readonly similarity: number;\n /** These documents point at the text. The caller checks each one against the index. */\n readonly docs: readonly string[];\n}\n\n/**\n * These are the counts of the store, for one embedder or for each embedder.\n *\n * @beta\n */\nexport interface VectorCounts {\n readonly texts: number;\n readonly chunks: number;\n readonly docs: number;\n /** This value is the size of the vectors in bytes, with no overhead of SQLite. */\n readonly bytes: number;\n}\n\n/**\n * These options open a vector store.\n *\n * @beta\n */\nexport interface VectorStoreOptions {\n /** This value is the time in milliseconds that a write waits for the lock of a different process. The default is 10 seconds. */\n readonly busyTimeoutMs?: number | undefined;\n}\n\n/**\n * These are the parts of one text in a write of the store.\n *\n * @beta\n */\nexport interface VectorWrite {\n readonly embedder: string;\n readonly dimensions: number;\n /** This value is the SHA-256 of the text, in hex. */\n readonly sha: string;\n /** This text names the rule of the chunks. A text with a different plan gets new chunks. */\n readonly plan: string;\n /** This value is the length of the text. */\n readonly chars: number;\n readonly chunks: ReadonlyArray<Omit<StoredChunk, \"chunk\">>;\n /** These documents hold the text as their current text. The store points each one at it. */\n readonly docs: readonly string[];\n /** This time (ISO 8601) is the time of the write. */\n readonly at: string;\n}\n\nconst DDL = `\nCREATE TABLE IF NOT EXISTS vectors_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);\nCREATE TABLE IF NOT EXISTS embedders (\n id TEXT PRIMARY KEY,\n dimensions INTEGER NOT NULL,\n first_at TEXT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS texts (\n sha TEXT NOT NULL,\n embedder TEXT NOT NULL,\n plan TEXT NOT NULL,\n chunks INTEGER NOT NULL,\n chars INTEGER NOT NULL,\n embedded_at TEXT NOT NULL,\n PRIMARY KEY (sha, embedder)\n);\nCREATE TABLE IF NOT EXISTS chunks (\n sha TEXT NOT NULL,\n chunk INTEGER NOT NULL,\n embedder TEXT NOT NULL,\n char_start INTEGER NOT NULL,\n char_end INTEGER NOT NULL,\n vector BLOB NOT NULL,\n PRIMARY KEY (sha, chunk, embedder)\n);\nCREATE INDEX IF NOT EXISTS chunks_embedder ON chunks (embedder, sha, chunk);\nCREATE TABLE IF NOT EXISTS docs (\n doc TEXT NOT NULL,\n embedder TEXT NOT NULL,\n sha TEXT NOT NULL,\n pointed_at TEXT NOT NULL,\n PRIMARY KEY (doc, embedder)\n);\nCREATE INDEX IF NOT EXISTS docs_sha ON docs (embedder, sha);\n`;\n\n/** The vectors of one embedder in memory: the rows of the chunks, and the texts and documents that they belong to. */\ninterface Matrix {\n readonly dimensions: number;\n rows: number;\n data: Float32Array;\n chunk: Int32Array;\n start: Int32Array;\n end: Int32Array;\n /** These are the rows of each text: from the first row to the row after the last row. */\n readonly texts: Map<string, { readonly from: number; readonly to: number }>;\n readonly docsOf: Map<string, Set<string>>;\n readonly shaOf: Map<string, string>;\n /** This value is the `data_version` of the file when the store read it. */\n readonly version: number;\n}\n\n/** This function gives the bytes of a vector for a BLOB. The bytes are in the order of the platform (little-endian on each common platform). */\nconst bytesOf = (vector: Float32Array): Uint8Array =>\n new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength);\n\n/** This function gives the vector of the bytes of a BLOB. It copies the bytes, so an odd offset of the BLOB is not a problem. */\nconst vectorOf = (bytes: Uint8Array): Float32Array => {\n const out = new Float32Array(Math.floor(bytes.byteLength / 4));\n new Uint8Array(out.buffer).set(bytes.subarray(0, out.length * 4));\n return out;\n};\n\nconst dot = (data: Float32Array, row: number, query: Float32Array, dims: number): number => {\n let sum = 0;\n const base = row * dims;\n for (let at = 0; at < dims; at += 1) sum += data[base + at]! * query[at]!;\n return sum;\n};\n\n/** @beta */\nexport class VectorStore {\n private readonly db: DatabaseSync;\n private readonly statements = new Map<string, StatementSync>();\n private readonly matrices = new Map<string, Matrix>();\n private closed = false;\n\n constructor(\n readonly path: string,\n options: VectorStoreOptions = {},\n ) {\n if (path !== \":memory:\") mkdirSync(dirname(path), { recursive: true, mode: 0o700 });\n this.db = new DatabaseSync(path);\n const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 10_000));\n this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);\n if (path !== \":memory:\") this.db.exec(\"PRAGMA journal_mode = WAL;\");\n // the store is a projection, so a power cut can cost its last transactions and nothing else\n this.db.exec(\"PRAGMA synchronous = NORMAL;\");\n this.write(() => {\n const version = (this.db.prepare(\"PRAGMA user_version\").get() as { user_version: number })\n .user_version;\n if (version !== VECTORS_SCHEMA) {\n // a store of a different schema is a projection: it opens empty\n const tables = this.db\n .prepare(\n \"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'\",\n )\n .all() as Array<{ name: string }>;\n for (const { name } of tables) this.db.exec(`DROP TABLE IF EXISTS \"${name}\"`);\n }\n this.db.exec(DDL);\n this.db.exec(`PRAGMA user_version = ${String(VECTORS_SCHEMA)}`);\n });\n }\n\n private stmt(sql: string): StatementSync {\n let statement = this.statements.get(sql);\n if (statement === undefined) {\n statement = this.db.prepare(sql);\n this.statements.set(sql, statement);\n }\n return statement;\n }\n\n private write<T>(work: () => T): T {\n if (this.closed) throw new Error(\"the vector store is closed\");\n this.db.exec(\"BEGIN IMMEDIATE\");\n try {\n const result = work();\n this.db.exec(\"COMMIT\");\n return result;\n } catch (error) {\n if (this.db.isTransaction) this.db.exec(\"ROLLBACK\");\n throw error;\n }\n }\n\n private live(): void {\n if (this.closed) throw new Error(\"the vector store is closed\");\n }\n\n /** This method gives the plan of the chunks of a text when the store has each of its vectors, or undefined. */\n plan(embedder: string, sha: string): string | undefined {\n this.live();\n return (\n this.stmt(\"SELECT plan FROM texts WHERE sha = ? AND embedder = ?\").get(sha, embedder) as\n { plan: string } | undefined\n )?.plan;\n }\n\n /** This method gives the text that a document points at, or undefined. */\n pointed(embedder: string, doc: string): string | undefined {\n this.live();\n return (\n this.stmt(\"SELECT sha FROM docs WHERE doc = ? AND embedder = ?\").get(doc, embedder) as\n { sha: string } | undefined\n )?.sha;\n }\n\n /** This method gives the dimensions that the store keeps for an embedder, or undefined. */\n dimensions(embedder: string): number | undefined {\n this.live();\n return (\n this.stmt(\"SELECT dimensions FROM embedders WHERE id = ?\").get(embedder) as\n { dimensions: number } | undefined\n )?.dimensions;\n }\n\n /**\n * This method writes the chunks of one text and points its documents at it, in one transaction.\n * A vector with a length that is different from the dimensions of the embedder throws an error.\n */\n put(input: VectorWrite): void {\n const dims = input.dimensions;\n for (const chunk of input.chunks) {\n if (chunk.vector.length !== dims) {\n throw new Error(\n `the embedder ${input.embedder} gave a vector of ${String(chunk.vector.length)} dimensions, not ${String(dims)}`,\n );\n }\n }\n let replaced = false;\n this.write(() => {\n const known = this.stmt(\"SELECT dimensions FROM embedders WHERE id = ?\").get(\n input.embedder,\n ) as { dimensions: number } | undefined;\n // a text with no chunk tells nothing of the dimensions\n if (known === undefined && input.chunks.length > 0) {\n this.stmt(\"INSERT INTO embedders (id, dimensions, first_at) VALUES (?, ?, ?)\").run(\n input.embedder,\n dims,\n input.at,\n );\n } else if (known !== undefined && input.chunks.length > 0 && known.dimensions !== dims) {\n throw new Error(\n `vectors.sqlite holds vectors of ${String(known.dimensions)} dimensions for the embedder ${input.embedder}, and it gave ${String(dims)}: give the embedder a new id, or delete vectors.sqlite`,\n );\n }\n const before = this.stmt(\"SELECT plan FROM texts WHERE sha = ? AND embedder = ?\").get(\n input.sha,\n input.embedder,\n ) as { plan: string } | undefined;\n replaced = before !== undefined && before.plan !== input.plan;\n this.stmt(\"DELETE FROM chunks WHERE sha = ? AND embedder = ?\").run(input.sha, input.embedder);\n const insert = this.stmt(\n \"INSERT INTO chunks (sha, chunk, embedder, char_start, char_end, vector) VALUES (?, ?, ?, ?, ?, ?)\",\n );\n input.chunks.forEach((chunk, at) => {\n insert.run(input.sha, at, input.embedder, chunk.start, chunk.end, bytesOf(chunk.vector));\n });\n this.stmt(\n `INSERT INTO texts (sha, embedder, plan, chunks, chars, embedded_at) VALUES (?, ?, ?, ?, ?, ?)\n ON CONFLICT(sha, embedder) DO UPDATE SET plan = excluded.plan, chunks = excluded.chunks,\n chars = excluded.chars, embedded_at = excluded.embedded_at`,\n ).run(input.sha, input.embedder, input.plan, input.chunks.length, input.chars, input.at);\n for (const doc of input.docs) this.pointRow(input.embedder, doc, input.sha, input.at);\n });\n const matrix = this.matrices.get(input.embedder);\n if (matrix === undefined) return;\n if (replaced || matrix.texts.has(input.sha)) {\n // a text with new chunks: the next search reads the vectors again\n this.matrices.delete(input.embedder);\n return;\n }\n this.append(\n matrix,\n input.sha,\n input.chunks.map((chunk, at) => ({ ...chunk, chunk: at })),\n );\n for (const doc of input.docs) this.pointMatrix(matrix, doc, input.sha);\n }\n\n private pointRow(embedder: string, doc: string, sha: string, at: string): void {\n this.stmt(\n `INSERT INTO docs (doc, embedder, sha, pointed_at) VALUES (?, ?, ?, ?)\n ON CONFLICT(doc, embedder) DO UPDATE SET sha = excluded.sha, pointed_at = excluded.pointed_at`,\n ).run(doc, embedder, sha, at);\n }\n\n /** This method points a document at a text that the store has. */\n point(embedder: string, doc: string, sha: string, at: string): void {\n this.write(() => this.pointRow(embedder, doc, sha, at));\n const matrix = this.matrices.get(embedder);\n if (matrix === undefined) return;\n if (!matrix.texts.has(sha)) this.append(matrix, sha, this.chunks(embedder, sha));\n this.pointMatrix(matrix, doc, sha);\n }\n\n /** This method removes the pointer of a document, for example when the document is held back. Its text keeps its vectors. */\n forget(embedder: string, doc: string): void {\n this.write(() =>\n this.stmt(\"DELETE FROM docs WHERE doc = ? AND embedder = ?\").run(doc, embedder),\n );\n const matrix = this.matrices.get(embedder);\n if (matrix === undefined) return;\n const sha = matrix.shaOf.get(doc);\n if (sha !== undefined) matrix.docsOf.get(sha)?.delete(doc);\n matrix.shaOf.delete(doc);\n }\n\n /** This method gives the chunks of a text, in their sequence. */\n chunks(embedder: string, sha: string): StoredChunk[] {\n this.live();\n return (\n this.stmt(\n \"SELECT chunk, char_start, char_end, vector FROM chunks WHERE sha = ? AND embedder = ? ORDER BY chunk\",\n ).all(sha, embedder) as Array<{\n chunk: number;\n char_start: number;\n char_end: number;\n vector: Uint8Array;\n }>\n ).map((row) => ({\n chunk: row.chunk,\n start: row.char_start,\n end: row.char_end,\n vector: vectorOf(row.vector),\n }));\n }\n\n private pointMatrix(matrix: Matrix, doc: string, sha: string): void {\n const before = matrix.shaOf.get(doc);\n if (before !== undefined && before !== sha) matrix.docsOf.get(before)?.delete(doc);\n matrix.shaOf.set(doc, sha);\n let docs = matrix.docsOf.get(sha);\n if (docs === undefined) {\n docs = new Set();\n matrix.docsOf.set(sha, docs);\n }\n docs.add(doc);\n }\n\n private append(matrix: Matrix, sha: string, chunks: readonly StoredChunk[]): void {\n const dims = matrix.dimensions;\n const need = matrix.rows + chunks.length;\n if (need > matrix.chunk.length) {\n const size = Math.max(need, matrix.chunk.length * 2, 64);\n const data = new Float32Array(size * dims);\n data.set(matrix.data.subarray(0, matrix.rows * dims));\n const grow = (old: Int32Array): Int32Array => {\n const next = new Int32Array(size);\n next.set(old.subarray(0, matrix.rows));\n return next;\n };\n matrix.data = data;\n matrix.chunk = grow(matrix.chunk);\n matrix.start = grow(matrix.start);\n matrix.end = grow(matrix.end);\n }\n const from = matrix.rows;\n for (const chunk of chunks) {\n const row = matrix.rows;\n if (chunk.vector.length === dims) matrix.data.set(chunk.vector, row * dims);\n matrix.chunk[row] = chunk.chunk;\n matrix.start[row] = chunk.start;\n matrix.end[row] = chunk.end;\n matrix.rows += 1;\n }\n matrix.texts.set(sha, { from, to: matrix.rows });\n }\n\n /** This method gives the vectors of an embedder in memory. It reads them again when a different process wrote the file. */\n private matrix(embedder: string): Matrix | undefined {\n this.live();\n const version = (this.db.prepare(\"PRAGMA data_version\").get() as { data_version: number })\n .data_version;\n const cached = this.matrices.get(embedder);\n if (cached !== undefined && cached.version === version) return cached;\n const dims = this.dimensions(embedder);\n if (dims === undefined) return undefined;\n // one read transaction: the count, the rows and the pointers come from one snapshot of the file\n this.db.exec(\"BEGIN\");\n try {\n const matrix = this.load(embedder, dims, version);\n this.matrices.set(embedder, matrix);\n return matrix;\n } finally {\n if (this.db.isTransaction) this.db.exec(\"COMMIT\");\n }\n }\n\n private load(embedder: string, dims: number, version: number): Matrix {\n const count = (\n this.db\n .prepare(\n \"SELECT count(*) AS n FROM chunks WHERE embedder = ? AND sha IN (SELECT sha FROM docs WHERE embedder = ?)\",\n )\n .get(embedder, embedder) as { n: number }\n ).n;\n const matrix: Matrix = {\n dimensions: dims,\n rows: 0,\n data: new Float32Array(Math.max(1, count) * dims),\n chunk: new Int32Array(Math.max(1, count)),\n start: new Int32Array(Math.max(1, count)),\n end: new Int32Array(Math.max(1, count)),\n texts: new Map(),\n docsOf: new Map(),\n shaOf: new Map(),\n version,\n };\n const rows = this.db\n .prepare(\n `SELECT sha, chunk, char_start, char_end, vector FROM chunks\n WHERE embedder = ? AND sha IN (SELECT sha FROM docs WHERE embedder = ?)\n ORDER BY sha, chunk`,\n )\n .iterate(embedder, embedder) as IterableIterator<{\n sha: string;\n chunk: number;\n char_start: number;\n char_end: number;\n vector: Uint8Array;\n }>;\n let sha: string | undefined;\n let from = 0;\n for (const row of rows) {\n if (row.sha !== sha) {\n if (sha !== undefined) matrix.texts.set(sha, { from, to: matrix.rows });\n sha = row.sha;\n from = matrix.rows;\n }\n if (matrix.rows >= matrix.chunk.length) break;\n const at = matrix.rows;\n if (row.vector.byteLength === dims * 4) {\n new Uint8Array(matrix.data.buffer, at * dims * 4, dims * 4).set(row.vector);\n }\n matrix.chunk[at] = row.chunk;\n matrix.start[at] = row.char_start;\n matrix.end[at] = row.char_end;\n matrix.rows += 1;\n }\n if (sha !== undefined) matrix.texts.set(sha, { from, to: matrix.rows });\n for (const row of this.db\n .prepare(\"SELECT doc, sha FROM docs WHERE embedder = ?\")\n .all(embedder) as Array<{ doc: string; sha: string }>) {\n this.pointMatrix(matrix, row.doc, row.sha);\n }\n return matrix;\n }\n\n /** This method gives the best chunk of one text for a question, or undefined when the store has no vector of the text. */\n best(embedder: string, sha: string, query: Float32Array): VectorHit | undefined {\n const matrix = this.matrix(embedder);\n const range = matrix?.texts.get(sha);\n if (matrix === undefined || range === undefined || range.to === range.from) return undefined;\n return this.hitOf(matrix, sha, range, query);\n }\n\n private hitOf(\n matrix: Matrix,\n sha: string,\n range: { readonly from: number; readonly to: number },\n query: Float32Array,\n ): VectorHit {\n let bestRow = range.from;\n let bestScore = Number.NEGATIVE_INFINITY;\n for (let row = range.from; row < range.to; row += 1) {\n const score = dot(matrix.data, row, query, matrix.dimensions);\n if (score > bestScore) {\n bestScore = score;\n bestRow = row;\n }\n }\n return {\n sha,\n chunk: matrix.chunk[bestRow]!,\n start: matrix.start[bestRow]!,\n end: matrix.end[bestRow]!,\n similarity: bestScore,\n docs: [...(matrix.docsOf.get(sha) ?? [])],\n };\n }\n\n /**\n * This method compares a question with each stored vector of an embedder (a brute-force cosine).\n * It gives the best chunk of each text that a document points at, the best first. With `within`,\n * it compares only the texts of those documents.\n */\n search(\n embedder: string,\n query: Float32Array,\n options: { readonly within?: Iterable<string> | undefined } = {},\n ): VectorHit[] {\n const matrix = this.matrix(embedder);\n if (matrix === undefined) return [];\n if (query.length !== matrix.dimensions) {\n throw new Error(\n `the question has a vector of ${String(query.length)} dimensions, and the store holds ${String(matrix.dimensions)} for the embedder ${embedder}`,\n );\n }\n const shas = new Set<string>();\n if (options.within === undefined) {\n for (const [sha, docs] of matrix.docsOf) if (docs.size > 0) shas.add(sha);\n } else {\n for (const doc of options.within) {\n const sha = matrix.shaOf.get(doc);\n if (sha !== undefined) shas.add(sha);\n }\n }\n const hits: VectorHit[] = [];\n for (const sha of shas) {\n const range = matrix.texts.get(sha);\n if (range === undefined || range.to === range.from) continue;\n hits.push(this.hitOf(matrix, sha, range, query));\n }\n return hits.sort((a, b) => b.similarity - a.similarity || (a.sha < b.sha ? -1 : 1));\n }\n\n /** This method gives the counts of the store, for one embedder or for each embedder. */\n counts(embedder?: string): VectorCounts {\n this.live();\n const where = embedder === undefined ? \"\" : \" WHERE embedder = ?\";\n const params = embedder === undefined ? [] : [embedder];\n const one = (sql: string): number =>\n (this.stmt(sql).get(...params) as { n: number | null }).n ?? 0;\n return {\n texts: one(`SELECT count(*) AS n FROM texts${where}`),\n chunks: one(`SELECT count(*) AS n FROM chunks${where}`),\n docs: one(`SELECT count(*) AS n FROM docs${where}`),\n bytes: one(`SELECT sum(length(vector)) AS n FROM chunks${where}`),\n };\n }\n\n /** This method closes the file. A second close does nothing. */\n close(): void {\n if (this.closed) return;\n this.closed = true;\n this.statements.clear();\n this.matrices.clear();\n this.db.close();\n }\n}\n","import { randomUUID } from \"node:crypto\";\nimport { mkdirSync } from \"node:fs\";\nimport { dirname } from \"node:path\";\nimport type { StatementSync } from \"node:sqlite\";\nimport { DatabaseSync } from \"node:sqlite\";\nimport type { DiffSummary } from \"./diff.ts\";\nimport { diffTexts } from \"./diff.ts\";\nimport { pastDeadline } from \"./discover.ts\";\nimport type { VersionText } from \"./docs.ts\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport { sha256, stableStringify } from \"./record.ts\";\nimport type { CrawlRuntime, PageResult } from \"./runtime.ts\";\nimport { fetchPage, isHeldBack } from \"./runtime.ts\";\nimport { cacheKeyOf } from \"./urls.ts\";\n\n/**\n * A watch list keeps the pages that the crawler checks again on a schedule. `watch.run` fetches the\n * pages that are due through the normal fetch, so each polite rule applies. It gives an event for\n * each page: new, changed, unchanged, gone, refused or error.\n *\n * These are the rules of a watch:\n *\n * - A watch compares the title, the text and the JSON-LD of the page with the last confirmed version.\n * - With `confirm: 2` (the default), a change is suspected when a check first sees it. The next\n * check confirms it when it sees the same new version. A check that sees the earlier version\n * again clears the suspicion, because one read can be a glitch. With `confirm: 1`, each check confirms.\n * - `every` has a floor of 1 hour by default. A check that a stop of the host refused is due again\n * at the end of the stop, or later.\n * - The list is in `watch.sqlite` in the store, so it stays after a restart. A run claims each entry\n * with a lease in that file, so two processes do not check one entry two times.\n *\n * A scheduler of the operator starts `watch.run`, or the daemon (`startDaemon`) starts it on a schedule.\n */\n\n/** This value is the version of the schema of `watch.sqlite`. */\nexport const WATCH_SCHEMA = 1;\n\n/** These are the defaults and the limits of the watch list. */\nexport const WATCH_LIMITS = {\n /** The shortest `every` of a watch by default: 1 hour, so a watch cannot hammer a site. */\n floorMs: 3_600_000,\n /** The default `every`: 24 hours. */\n everyMs: 86_400_000,\n /** The default time of one run: 10 minutes. No request starts after it. */\n budgetMs: 600_000,\n /** The default number of entries that one run checks. */\n maxPages: 100,\n /** The most hosts that get a check of one run at one time. */\n hostsAtOnce: 8,\n /** A run holds the lease of an entry until the end of its budget and this time more: 2 minutes. */\n leaseMs: 120_000,\n} as const;\n\n/** The kind of a check: the page is new, changed, unchanged or gone, or the check got no version. */\nexport type WatchKind = \"new\" | \"changed\" | \"unchanged\" | \"gone\" | \"refused\" | \"error\";\n\n/** These options add a page to the watch list, or change its watch. */\nexport interface WatchOptions {\n /** This value is the time between two checks. The default is `24h`, and the floor is `1h` (`watchFloor`). */\n readonly every?: Duration | undefined;\n /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */\n readonly render?: \"auto\" | \"never\" | \"always\" | undefined;\n /** These labels go into the capture records of each check. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n /** This label goes into the capture records of each check. */\n readonly scope?: string | undefined;\n /** These agents get their robots.txt rules applied to each check. */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** This value is the number of checks that must see a change before it is confirmed. The default is 2. */\n readonly confirm?: 1 | 2 | undefined;\n}\n\n/** A change that one check saw, and that the next check can confirm. */\nexport interface WatchPending {\n /** This value is `changed`, `gone` or `new` (a page that came back). */\n readonly kind: \"changed\" | \"gone\" | \"new\";\n /** This id is the capture of the suspected version, or null for a page that is gone. */\n readonly capture: string | null;\n /** This time (ISO 8601) is the time of the check that first saw the change. */\n readonly since: string;\n}\n\n/** This is one entry of the watch list. */\nexport interface WatchEntry {\n /** This URL is the page that the watch fetches. */\n readonly url: string;\n /** This value is the time between two checks in milliseconds. */\n readonly every_ms: number;\n readonly render: \"auto\" | \"never\" | \"always\";\n readonly confirm: 1 | 2;\n readonly scope: string | null;\n readonly labels: Readonly<Record<string, string>> | null;\n readonly on_behalf_of: readonly string[];\n /** This time (ISO 8601) is the time of the first add. */\n readonly added_at: string;\n /** This time (ISO 8601) is the time when the entry is due. */\n readonly next_at: string;\n /** This time (ISO 8601) is the time of the last check, or null before the first check. */\n readonly checked_at: string | null;\n readonly last_kind: WatchKind | null;\n readonly last_note: string | null;\n /** This value counts the checks. */\n readonly checks: number;\n /** This value counts the confirmed changes. */\n readonly changes: number;\n /** This id is the document of the page. */\n readonly doc: string | null;\n /** This id is the capture of the last confirmed version, or null. */\n readonly capture: string | null;\n /** This time (ISO 8601) is the fetch time of the last confirmed version, or null. */\n readonly fetched_at: string | null;\n /** This value is true when the last confirmed state is that the page is gone. */\n readonly gone: boolean;\n /** This value is the change that the last check suspected, or null. */\n readonly pending: WatchPending | null;\n /** This time (ISO 8601) is the end of the lease of a run that checks the entry, or null. */\n readonly leased_until: string | null;\n}\n\n/** These options control a run of the watch list. */\nexport interface WatchRunOptions {\n /** This value is the longest time of the run in milliseconds. No request starts after it. The default is 10 minutes. */\n readonly budgetMs?: number | undefined;\n /** This value is the most entries that the run checks. The default is 100. */\n readonly maxPages?: number | undefined;\n /** When this signal aborts, the run stops the checks in progress and ends. */\n readonly signal?: AbortSignal | undefined;\n /**\n * When this signal aborts, the run starts no new check. The checks in progress end and write\n * their state, and the run stops with `cancelled`. The daemon uses it for a clean stop.\n */\n readonly drain?: AbortSignal | undefined;\n}\n\n/**\n * These options control `runWatch`: the options of a run, and the parts of `openCrawler`.\n *\n * @beta\n */\nexport interface WatchRunnerOptions extends WatchRunOptions {\n /** These agents get their robots.txt rules applied to each check, before the agents of the entry. */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** This function operates before the first check. `openCrawler` waits there for the open of the store. */\n readonly ready?: (() => Promise<void>) | undefined;\n}\n\n/** This is the event of one check of a run. */\nexport interface WatchEvent {\n /** This URL is the page of the watch. */\n readonly url: string;\n readonly doc: string | null;\n readonly kind: WatchKind;\n /** This value is true when the check saw a change for the first time, and the next check must confirm it. */\n readonly suspected: boolean;\n /** This value is true when the check confirmed a change. Its version is then the last confirmed version. */\n readonly confirmed: boolean;\n /** This id is the capture that the check got, or null. */\n readonly capture: string | null;\n /** This time (ISO 8601) is the fetch time of that capture, or null. */\n readonly fetched_at: string | null;\n /** This id is the capture of the last confirmed version before the check, or null. */\n readonly previous: string | null;\n /** This time (ISO 8601) is the fetch time of that version, or null. */\n readonly previous_fetched_at: string | null;\n readonly outcome: string;\n readonly status: number;\n /** These are the counts of the diff from the last confirmed version, for a change. A held-back page gives null. */\n readonly diff: DiffSummary | null;\n /** This value is true when the page asked not to be AI input. */\n readonly held_back: boolean;\n /** This time (ISO 8601) is the time when the entry is due again. */\n readonly next_at: string;\n readonly note: string | null;\n}\n\n/** The reason that a run stopped. */\nexport type WatchStop = \"done\" | \"max-pages\" | \"budget\" | \"cancelled\" | \"closed\" | \"error\";\n\n/** The summary of a run. */\nexport interface WatchSummary {\n /** This value counts the checks that gave an event. */\n readonly checked: number;\n readonly new: number;\n readonly changed: number;\n readonly unchanged: number;\n readonly gone: number;\n readonly refused: number;\n readonly error: number;\n /** This value counts the changes that wait for the next check. */\n readonly suspected: number;\n /** This value counts the changes that the run confirmed. */\n readonly confirmed: number;\n /** This value counts the entries that are due at the end of the run. */\n readonly due: number;\n readonly stopped: WatchStop;\n /** This text tells why the run stopped. */\n readonly reason: string;\n /** This value is the time of the run in milliseconds, on the clock of the crawler. */\n readonly elapsed_ms: number;\n}\n\n/** A run in progress. A `for await` loop gets each event. `summary` resolves when the loop ends. */\nexport interface WatchRun extends AsyncIterable<WatchEvent> {\n readonly summary: Promise<WatchSummary>;\n}\n\n/** This is the watch list of `openCrawler`. Each method gives a promise, except `run`. */\nexport interface WatchList {\n /** This method adds a page, or changes the options of its watch. A new entry is due at once. */\n add(url: string, options?: WatchOptions): Promise<WatchEntry>;\n /** This method removes a page from the list. It gives false when the list does not have the page. */\n remove(url: string): Promise<boolean>;\n /** This method gives the entries, the first due first. */\n list(): Promise<WatchEntry[]>;\n /** This method checks the entries that are due, to a budget. A `for await` loop gets each event. */\n run(options?: WatchRunOptions): WatchRun;\n}\n\n/**\n * These options open a watch store.\n *\n * @beta\n */\nexport interface WatchStoreOptions {\n /** This value is the time in milliseconds that a step waits for the lock of a different process. The default is 10 seconds. */\n readonly busyTimeoutMs?: number | undefined;\n /** This value is the shortest `every` in milliseconds. The default is 1 hour. */\n readonly floorMs?: number | undefined;\n}\n\n/**\n * This is a row of the table of the watch list, as `WatchStore` reads it.\n *\n * @beta\n */\nexport interface WatchRow {\n url: string;\n target: string;\n host: string;\n every_ms: number;\n render: string;\n confirm: number;\n scope: string | null;\n labels: string | null;\n on_behalf_of: string | null;\n added_at: number;\n next_at: number;\n lease_until: number;\n lease_owner: string | null;\n checks: number;\n changes: number;\n last_at: number | null;\n last_kind: string | null;\n last_note: string | null;\n doc: string | null;\n base_state: string | null;\n base_sha: string | null;\n base_capture: string | null;\n base_fetched_at: string | null;\n pend_state: string | null;\n pend_sha: string | null;\n pend_capture: string | null;\n pend_fetched_at: string | null;\n pend_since: number | null;\n}\n\n/**\n * The state of a page after a check: what a later check compares.\n *\n * @beta\n */\nexport interface WatchState {\n readonly state: \"present\" | \"gone\";\n readonly sha: string | null;\n readonly capture: string | null;\n readonly fetched_at: string | null;\n}\n\n/**\n * This is the change of an entry after a check.\n *\n * @beta\n */\nexport interface WatchUpdate {\n readonly next_at: number;\n readonly last_at: number;\n readonly last_kind: WatchKind;\n readonly last_note: string | null;\n readonly changed: boolean;\n readonly doc: string | null;\n /** The new confirmed state, or undefined when it stays. */\n readonly base?: WatchState | undefined;\n /** The new suspected state, null to clear it, or undefined when it stays. */\n readonly pending?: (WatchState & { readonly since: number }) | null | undefined;\n}\n\nconst iso = (ms: number): string => new Date(ms).toISOString();\n\nconst parseJson = <T>(text: string | null, fallback: T): T => {\n if (text === null) return fallback;\n try {\n return JSON.parse(text) as T;\n } catch {\n return fallback;\n }\n};\n\nconst RENDERS: ReadonlySet<string> = new Set([\"auto\", \"never\", \"always\"]);\n\nconst entryOf = (row: WatchRow, now: number): WatchEntry => ({\n url: row.target,\n every_ms: row.every_ms,\n render: row.render as WatchEntry[\"render\"],\n confirm: row.confirm === 1 ? 1 : 2,\n scope: row.scope,\n labels: parseJson<Record<string, string> | null>(row.labels, null),\n on_behalf_of: parseJson<string[]>(row.on_behalf_of, []),\n added_at: iso(row.added_at),\n next_at: iso(row.next_at),\n checked_at: row.last_at === null ? null : iso(row.last_at),\n last_kind: row.last_kind as WatchKind | null,\n last_note: row.last_note,\n checks: row.checks,\n changes: row.changes,\n doc: row.doc,\n capture: row.base_capture,\n fetched_at: row.base_fetched_at,\n gone: row.base_state === \"gone\",\n pending:\n row.pend_state === null\n ? null\n : {\n kind: row.pend_state === \"gone\" ? \"gone\" : row.base_state === \"gone\" ? \"new\" : \"changed\",\n capture: row.pend_capture,\n since: iso(row.pend_since ?? row.last_at ?? row.added_at),\n },\n leased_until: row.lease_until > now ? iso(row.lease_until) : null,\n});\n\n/** This function gives the canonical key, the URL and the host of a page to watch. A URL that is not http(s) throws a `RangeError`. */\nconst targetOf = (\n url: string,\n): { readonly key: string; readonly target: string; readonly host: string } => {\n let parsed: URL;\n try {\n parsed = new URL(url);\n } catch {\n throw new RangeError(`not a valid URL: ${url}`);\n }\n if (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\")\n throw new RangeError(`a watch needs an http or https URL, not ${url}`);\n parsed.hash = \"\";\n return { key: cacheKeyOf(parsed.href), target: parsed.href, host: parsed.host.toLowerCase() };\n};\n\n/**\n * This store keeps the watch list in a SQLite file (`watch.sqlite`). Two or more processes on one\n * store share it. Each step is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock of a\n * different process (`busy_timeout`).\n *\n * The list is the decision of the operator, not a projection of the capture log. Thus it has its\n * own file: a deletion of `hosts.sqlite` or of the index does not lose it.\n *\n * @beta\n */\nexport class WatchStore {\n private readonly db: DatabaseSync;\n private readonly statements = new Map<string, StatementSync>();\n private closed = false;\n /** This value is the shortest `every` in milliseconds. */\n readonly floorMs: number;\n\n constructor(\n readonly path: string,\n options: WatchStoreOptions = {},\n ) {\n if (path !== \":memory:\") mkdirSync(dirname(path), { recursive: true, mode: 0o700 });\n const floor = options.floorMs ?? WATCH_LIMITS.floorMs;\n if (!Number.isFinite(floor) || floor < 0)\n throw new RangeError(\n `the floor of every must be 0 or more milliseconds, not ${String(floor)}`,\n );\n this.floorMs = floor;\n this.db = new DatabaseSync(path);\n const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 10_000));\n this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);\n if (path !== \":memory:\") this.db.exec(\"PRAGMA journal_mode = WAL;\");\n this.db.exec(\"PRAGMA synchronous = NORMAL;\");\n this.step(() => {\n this.db.exec(`CREATE TABLE IF NOT EXISTS watches (\n url TEXT PRIMARY KEY,\n target TEXT NOT NULL,\n host TEXT NOT NULL,\n every_ms REAL NOT NULL,\n render TEXT NOT NULL,\n confirm INTEGER NOT NULL,\n scope TEXT,\n labels TEXT,\n on_behalf_of TEXT,\n added_at REAL NOT NULL,\n next_at REAL NOT NULL,\n lease_until REAL NOT NULL DEFAULT 0,\n lease_owner TEXT,\n checks INTEGER NOT NULL DEFAULT 0,\n changes INTEGER NOT NULL DEFAULT 0,\n last_at REAL,\n last_kind TEXT,\n last_note TEXT,\n doc TEXT,\n base_state TEXT,\n base_sha TEXT,\n base_capture TEXT,\n base_fetched_at TEXT,\n pend_state TEXT,\n pend_sha TEXT,\n pend_capture TEXT,\n pend_fetched_at TEXT,\n pend_since REAL\n )`);\n this.db.exec(\"CREATE INDEX IF NOT EXISTS watches_due ON watches (next_at)\");\n const { user_version: version } = this.db.prepare(\"PRAGMA user_version\").get() as {\n user_version: number;\n };\n if (version < WATCH_SCHEMA) this.db.exec(`PRAGMA user_version = ${String(WATCH_SCHEMA)}`);\n });\n }\n\n private stmt(sql: string): StatementSync {\n let statement = this.statements.get(sql);\n if (statement === undefined) {\n statement = this.db.prepare(sql);\n this.statements.set(sql, statement);\n }\n return statement;\n }\n\n /** This method does one step in one transaction. */\n private step<T>(work: () => T): T {\n if (this.closed) throw new Error(\"the watch store is closed\");\n this.db.exec(\"BEGIN IMMEDIATE\");\n try {\n const result = work();\n this.db.exec(\"COMMIT\");\n return result;\n } catch (error) {\n if (this.db.isTransaction) this.db.exec(\"ROLLBACK\");\n throw error;\n }\n }\n\n private row(key: string): WatchRow | undefined {\n return this.stmt(\"SELECT * FROM watches WHERE url = ?\").get(key) as WatchRow | undefined;\n }\n\n /**\n * This method adds a page, or changes the options of its watch and keeps its state. A new entry\n * is due at once. A changed entry is due one `every` after its last check. A bad option throws\n * a `RangeError`.\n */\n add(url: string, options: WatchOptions, now: number): WatchEntry {\n const { key, target, host } = targetOf(url);\n const every = options.every === undefined ? WATCH_LIMITS.everyMs : durationMs(options.every);\n if (every < this.floorMs) {\n throw new RangeError(\n `every must be ${String(this.floorMs / 60_000)} minutes or more, so a watch cannot hammer a site, not ${String(every / 60_000)} minutes`,\n );\n }\n const render = options.render ?? \"auto\";\n if (!RENDERS.has(render))\n throw new RangeError(`render is auto, never or always, not ${JSON.stringify(render)}`);\n const confirm = options.confirm ?? 2;\n if (confirm !== 1 && confirm !== 2)\n throw new RangeError(`confirm is 1 or 2, not ${JSON.stringify(confirm)}`);\n const labels =\n options.labels === undefined || Object.keys(options.labels).length === 0\n ? null\n : JSON.stringify(options.labels);\n const agents =\n options.onBehalfOf === undefined || options.onBehalfOf.length === 0\n ? null\n : JSON.stringify([...options.onBehalfOf]);\n return this.step(() => {\n this.stmt(\n `INSERT INTO watches (url, target, host, every_ms, render, confirm, scope, labels, on_behalf_of, added_at, next_at)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(url) DO UPDATE SET target = excluded.target, host = excluded.host,\n every_ms = excluded.every_ms, render = excluded.render, confirm = excluded.confirm,\n scope = excluded.scope, labels = excluded.labels, on_behalf_of = excluded.on_behalf_of,\n next_at = CASE WHEN watches.last_at IS NULL THEN watches.next_at\n ELSE watches.last_at + excluded.every_ms END`,\n ).run(\n key,\n target,\n host,\n every,\n render,\n confirm,\n options.scope ?? null,\n labels,\n agents,\n now,\n now,\n );\n return entryOf(this.row(key)!, now);\n });\n }\n\n /** This method removes a page. It gives false when the list does not have the page. */\n remove(url: string): boolean {\n const { key } = targetOf(url);\n return this.step(\n () => Number(this.stmt(\"DELETE FROM watches WHERE url = ?\").run(key).changes) > 0,\n );\n }\n\n /** This method gives the entry of a page, or undefined. */\n get(url: string, now: number): WatchEntry | undefined {\n if (this.closed) throw new Error(\"the watch store is closed\");\n const row = this.row(targetOf(url).key);\n return row === undefined ? undefined : entryOf(row, now);\n }\n\n /** This method gives the entries, the first due first. */\n list(now: number): WatchEntry[] {\n if (this.closed) throw new Error(\"the watch store is closed\");\n return (\n this.stmt(\"SELECT * FROM watches ORDER BY next_at, url\").all() as unknown as WatchRow[]\n ).map((row) => entryOf(row, now));\n }\n\n /** This method counts the entries that are due at a time. */\n due(now: number): number {\n if (this.closed) throw new Error(\"the watch store is closed\");\n return (\n this.stmt(\"SELECT count(*) AS n FROM watches WHERE next_at <= ?\").get(now) as { n: number }\n ).n;\n }\n\n /**\n * This method claims the first due entry with no lease, on a host that is not in `busy`. The\n * lease of `owner` holds until `until`. Thus two processes do not get one entry at one time.\n */\n claim(now: number, owner: string, until: number, busy: readonly string[]): WatchRow | undefined {\n return this.step(() => {\n const row = this.stmt(\n `SELECT * FROM watches WHERE next_at <= ? AND lease_until <= ?\n AND host NOT IN (SELECT value FROM json_each(?)) ORDER BY next_at, url LIMIT 1`,\n ).get(now, now, JSON.stringify(busy)) as WatchRow | undefined;\n if (row === undefined) return undefined;\n this.stmt(\"UPDATE watches SET lease_until = ?, lease_owner = ? WHERE url = ?\").run(\n until,\n owner,\n row.url,\n );\n return { ...row, lease_until: until, lease_owner: owner };\n });\n }\n\n /** This method ends the lease of `owner` and changes nothing else, so the entry stays due. A closed store does nothing. */\n release(key: string, owner: string): void {\n if (this.closed) return;\n this.step(() => {\n this.stmt(\n \"UPDATE watches SET lease_until = 0, lease_owner = NULL WHERE url = ? AND lease_owner = ?\",\n ).run(key, owner);\n });\n }\n\n /**\n * This method writes the result of a check and ends the lease. It gives false when `owner` does\n * not hold the lease, or when the store is closed.\n */\n settle(key: string, owner: string, update: WatchUpdate): boolean {\n if (this.closed) return false;\n return this.step(() => {\n const row = this.row(key);\n if (row === undefined || row.lease_owner !== owner) return false;\n const base = update.base;\n const pending = update.pending;\n this.stmt(\n `UPDATE watches SET lease_until = 0, lease_owner = NULL, next_at = ?, last_at = ?, last_kind = ?,\n last_note = ?, checks = checks + 1, changes = changes + ?, doc = ?,\n base_state = ?, base_sha = ?, base_capture = ?, base_fetched_at = ?,\n pend_state = ?, pend_sha = ?, pend_capture = ?, pend_fetched_at = ?, pend_since = ?\n WHERE url = ?`,\n ).run(\n update.next_at,\n update.last_at,\n update.last_kind,\n update.last_note,\n update.changed ? 1 : 0,\n update.doc ?? row.doc,\n base === undefined ? row.base_state : base.state,\n base === undefined ? row.base_sha : base.sha,\n base === undefined ? row.base_capture : base.capture,\n base === undefined ? row.base_fetched_at : base.fetched_at,\n pending === undefined ? row.pend_state : (pending?.state ?? null),\n pending === undefined ? row.pend_sha : (pending?.sha ?? null),\n pending === undefined ? row.pend_capture : (pending?.capture ?? null),\n pending === undefined ? row.pend_fetched_at : (pending?.fetched_at ?? null),\n pending === undefined ? row.pend_since : (pending?.since ?? null),\n key,\n );\n return true;\n });\n }\n\n /** This method closes the file. A second close does nothing. */\n close(): void {\n if (this.closed) return;\n this.closed = true;\n this.statements.clear();\n this.db.close();\n }\n}\n\n/** The outcomes of a check that the site or the crawler refused. */\nconst REFUSALS: ReadonlySet<string> = new Set([\n \"skipped\",\n \"blocked\",\n \"challenge\",\n \"payment-required\",\n]);\n\n/** This function gives the fingerprint of a version: a hash of its title, its text and its JSON-LD. */\nexport const watchFingerprint = (version: VersionText): string =>\n sha256(stableStringify([version.title, version.text, version.meta[\"jsonld\"] ?? null]));\n\nconst sameState = (a: WatchState | undefined, b: WatchState): boolean =>\n a !== undefined && a.state === b.state && (a.state === \"gone\" || a.sha === b.sha);\n\n/** One check that ended: an event, or an entry that stays due because the run stopped first. */\ntype Done =\n | { readonly kind: \"event\"; readonly host: string; readonly event: WatchEvent }\n | { readonly kind: \"deferred\"; readonly host: string; readonly stop: WatchStop | undefined };\n\n/**\n * This function checks the entries of a watch store that are due. `PoliteCrawler.watch.run` uses\n * it. Each check is a fetch of the page with `fetchPage`, with no stored copy (`maxAgeMs: 0`), so\n * each polite rule applies. A check that the budget stops before its request stays due. The store\n * can be a function that opens it: the run uses it before the first check.\n *\n * @beta\n */\nexport const runWatch = (\n runtime: CrawlRuntime,\n source: WatchStore | (() => Promise<WatchStore>),\n options: WatchRunnerOptions = {},\n): WatchRun => {\n let resolveSummary: (summary: WatchSummary) => void = () => undefined;\n const summary = new Promise<WatchSummary>((resolve) => {\n resolveSummary = resolve;\n });\n let used = false;\n\n async function* events(): AsyncGenerator<WatchEvent, void, undefined> {\n const now = (): number => Date.parse(runtime.now());\n const startedAt = now();\n const owner = randomUUID();\n const abort = new AbortController();\n const outer = options.signal;\n const cancel = (): void => abort.abort(outer?.reason);\n if (outer?.aborted === true) cancel();\n outer?.addEventListener(\"abort\", cancel, { once: true });\n const inflight = new Map<string, Promise<Done>>();\n const counts = {\n checked: 0,\n new: 0,\n changed: 0,\n unchanged: 0,\n gone: 0,\n refused: 0,\n error: 0,\n suspected: 0,\n confirmed: 0,\n };\n let started = 0;\n let stopping: WatchStop | undefined;\n let reason: string | undefined;\n let finished = false;\n let budgetMs: number = WATCH_LIMITS.budgetMs;\n let maxPages: number = WATCH_LIMITS.maxPages;\n let opened: WatchStore | undefined;\n\n /** This function fetches one entry, compares it with its state, and writes the new state. */\n const check = async (store: WatchStore, row: WatchRow, deadline: number): Promise<Done> => {\n const agents = [...(options.onBehalfOf ?? []), ...parseJson<string[]>(row.on_behalf_of, [])];\n const labels = parseJson<Record<string, string> | undefined>(row.labels, undefined);\n let page: PageResult | undefined;\n let failure: string | undefined;\n try {\n page = await fetchPage(runtime, row.target, {\n render: row.render as \"auto\" | \"never\" | \"always\",\n maxAgeMs: 0,\n onBehalfOf: agents,\n signal: abort.signal,\n scope: row.scope ?? undefined,\n labels,\n deadline,\n });\n } catch (error) {\n failure = `watch error: ${error instanceof Error ? error.message : String(error)}`;\n }\n const deferred = (stop: WatchStop | undefined): Done => {\n store.release(row.url, owner);\n return { kind: \"deferred\", host: row.host, stop };\n };\n if (abort.signal.aborted) return deferred(\"cancelled\");\n if (page !== undefined && pastDeadline(page.note)) return deferred(\"budget\");\n if (page?.outcome === \"skipped\" && page.note === \"crawler closed\") return deferred(\"closed\");\n const at = now();\n const base: WatchState | undefined =\n row.base_state === null\n ? undefined\n : {\n state: row.base_state as WatchState[\"state\"],\n sha: row.base_sha,\n capture: row.base_capture,\n fetched_at: row.base_fetched_at,\n };\n const pending: WatchState | undefined =\n row.pend_state === null\n ? undefined\n : {\n state: row.pend_state as WatchState[\"state\"],\n sha: row.pend_sha,\n capture: row.pend_capture,\n fetched_at: row.pend_fetched_at,\n };\n let seen: WatchState | undefined;\n let version: VersionText | undefined;\n let held = false;\n let kind: WatchKind;\n let note: string | null = page?.note ?? failure ?? null;\n const ok = page !== undefined && (page.outcome === \"ok\" || page.outcome === \"not-modified\");\n if (page !== undefined && ok && page.doc !== null && page.capture !== null) {\n version = runtime.index.versionText(page.doc, page.capture);\n const doc = runtime.index.get(page.doc);\n held =\n (doc !== undefined && isHeldBack(doc)) || (version !== undefined && isHeldBack(version));\n if (version !== undefined) {\n seen = {\n state: \"present\",\n sha: watchFingerprint(version),\n capture: version.capture,\n fetched_at: version.fetched_at,\n };\n }\n } else if (page?.outcome === \"not-found\") {\n seen = { state: \"gone\", sha: null, capture: null, fetched_at: page.fetched_at };\n }\n let update: Omit<WatchUpdate, \"next_at\" | \"last_at\" | \"last_kind\" | \"last_note\" | \"doc\">;\n let suspected = false;\n let confirmed = false;\n if (seen === undefined) {\n kind = page !== undefined && REFUSALS.has(page.outcome) ? \"refused\" : \"error\";\n if (ok) note = note ?? \"the page gave no document, so the watch cannot compare it\";\n update = { changed: false };\n } else if (base === undefined) {\n // the first version that a watch sees is not a change: it is the start\n kind = seen.state === \"gone\" ? \"gone\" : \"new\";\n update = { changed: false, base: seen, pending: null };\n } else if (sameState(base, seen)) {\n kind = \"unchanged\";\n if (pending !== undefined)\n note =\n \"the change that the last check saw did not repeat, so the watch takes it for a glitch\";\n update = { changed: false, pending: null };\n } else {\n kind = seen.state === \"gone\" ? \"gone\" : base.state === \"gone\" ? \"new\" : \"changed\";\n // a second check of the same new version confirms it, and a stored copy is not a second check\n const repeated =\n sameState(pending, seen) && (seen.state === \"gone\" || pending?.capture !== seen.capture);\n if (row.confirm === 1 || repeated) {\n confirmed = true;\n update = { changed: true, base: seen, pending: null };\n } else {\n suspected = true;\n note =\n pending === undefined\n ? \"the watch saw this change one time: the next check confirms it when it sees the same version\"\n : \"the watch saw a different change than the last check: the next check confirms it when it sees the same version\";\n update = { changed: false, pending: { ...seen, since: at } };\n }\n }\n let diff: DiffSummary | null = null;\n if (\n kind === \"changed\" &&\n base?.state === \"present\" &&\n base.capture !== null &&\n version !== undefined &&\n !held\n ) {\n const docId = row.doc ?? version.doc;\n const before = runtime.index.versionText(docId, base.capture);\n if (before !== undefined && !isHeldBack(before)) {\n diff = diffTexts(\n { text: before.text, fields: { title: before.title, meta: before.meta } },\n { text: version.text, fields: { title: version.title, meta: version.meta } },\n ).summary;\n }\n }\n // a stopped host is due at the end of its stop, or later\n let next = at + row.every_ms;\n const stop = runtime.crawler.hostStop(row.target);\n if (stop !== undefined && Number.isFinite(stop.until) && stop.until > next) next = stop.until;\n store.settle(row.url, owner, {\n ...update,\n next_at: next,\n last_at: at,\n last_kind: kind,\n last_note: note,\n doc: page?.doc ?? null,\n });\n const event: WatchEvent = {\n url: row.target,\n doc: page?.doc ?? row.doc,\n kind,\n suspected,\n confirmed,\n capture: seen?.capture ?? null,\n fetched_at: seen?.fetched_at ?? null,\n previous: base?.capture ?? null,\n previous_fetched_at: base?.fetched_at ?? null,\n outcome: page?.outcome ?? \"error\",\n status: page?.status ?? 0,\n diff,\n held_back: held,\n next_at: iso(next),\n note,\n };\n return { kind: \"event\", host: row.host, event };\n };\n\n try {\n const store = typeof source === \"function\" ? await source() : source;\n opened = store;\n await options.ready?.();\n budgetMs = options.budgetMs ?? WATCH_LIMITS.budgetMs;\n if (!(budgetMs >= 0))\n throw new RangeError(`budgetMs must be 0 or more, not ${String(budgetMs)}`);\n maxPages = options.maxPages ?? WATCH_LIMITS.maxPages;\n if (!Number.isInteger(maxPages) || maxPages < 1)\n throw new RangeError(`maxPages must be an integer of 1 or more, not ${String(maxPages)}`);\n const deadline = startedAt + budgetMs;\n for (;;) {\n if (stopping === undefined && abort.signal.aborted) stopping = \"cancelled\";\n if (stopping === undefined && options.drain?.aborted === true) {\n // a drain starts no new check, and the checks in progress end with their state\n stopping = \"cancelled\";\n reason = \"the run stopped for a shutdown: the checks in progress ended first\";\n }\n if (stopping === undefined && now() >= deadline) stopping = \"budget\";\n while (\n stopping === undefined &&\n inflight.size < WATCH_LIMITS.hostsAtOnce &&\n started < maxPages\n ) {\n const claimed = store.claim(\n now(),\n owner,\n Math.max(now(), deadline) + WATCH_LIMITS.leaseMs,\n [...inflight.keys()],\n );\n if (claimed === undefined) break;\n started += 1;\n inflight.set(claimed.host, check(store, claimed, deadline));\n }\n if (inflight.size === 0) break;\n const done = await Promise.race(inflight.values());\n inflight.delete(done.host);\n if (done.kind === \"deferred\") {\n started -= 1;\n stopping ??= done.stop;\n continue;\n }\n const event = done.event;\n counts.checked += 1;\n counts[event.kind] += 1;\n if (event.suspected) counts.suspected += 1;\n if (event.confirmed) counts.confirmed += 1;\n yield event;\n }\n stopping ??= started >= maxPages && store.due(now()) > 0 ? \"max-pages\" : \"done\";\n finished = true;\n } catch (error) {\n stopping = \"error\";\n reason = error instanceof Error ? error.message : String(error);\n finished = true;\n throw error;\n } finally {\n // a loop that stopped early (a break) cancels the run\n if (!finished) {\n stopping ??= \"cancelled\";\n reason ??= \"the caller stopped the loop\";\n }\n outer?.removeEventListener(\"abort\", cancel);\n abort.abort(new Error(\"the watch run ended\"));\n // the checks in progress end, write their state or release their lease, before the summary\n await Promise.allSettled(inflight.values());\n const stop = stopping ?? \"done\";\n reason ??=\n stop === \"done\"\n ? \"the run checked each entry that was due\"\n : stop === \"max-pages\"\n ? `the run checked ${String(maxPages)} entries, its limit (maxPages)`\n : stop === \"budget\"\n ? `the budget of ${String(budgetMs)} ms ended (budgetMs)`\n : stop === \"cancelled\"\n ? \"the caller cancelled the run\"\n : \"the crawler closed\";\n let due = 0;\n try {\n due = opened?.due(now()) ?? 0;\n } catch {\n // a closed store has no count\n }\n resolveSummary({\n ...counts,\n due,\n stopped: stop,\n reason,\n elapsed_ms: Math.max(0, now() - startedAt),\n });\n }\n }\n\n return {\n summary,\n [Symbol.asyncIterator]() {\n if (used) throw new Error(\"a watch run operates one time: call run again for a new run\");\n used = true;\n return events();\n },\n };\n};\n","import { existsSync, lstatSync, mkdirSync, realpathSync, statSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport type { Dispatcher } from \"undici\";\nimport type { BlobStore } from \"./blobs.ts\";\nimport { checkedRenderer } from \"./browser.ts\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport { FileBlobStore } from \"./blobs.ts\";\nimport type { CrawlEvent } from \"./crawler.ts\";\nimport { Crawler } from \"./crawler.ts\";\nimport type { DiffOptions, DiffResult } from \"./diff.ts\";\nimport { diffDocument } from \"./diff.ts\";\nimport type { DiscoverOptions, DiscoverResult } from \"./discover.ts\";\nimport { discoverSite } from \"./discover.ts\";\nimport type { DocVersion, Found, RankMode, ReadingIndex, SearchRank } from \"./docs.ts\";\nimport { DocIndex, NO_EMBEDDER } from \"./docs.ts\";\nimport type { Embedder } from \"./embed.ts\";\nimport type { EgressOptions } from \"./egress.ts\";\nimport type { ExtractOptions, ExtractResult, FieldClaim } from \"./extract.ts\";\nimport { extractDocument } from \"./extract.ts\";\nimport type { CrawlOptions, CrawlRun } from \"./frontier.ts\";\nimport { crawlSite } from \"./frontier.ts\";\nimport { SqliteHostStateStore } from \"./hoststate.ts\";\nimport type {\n EmbedOptions,\n EmbedResult,\n HybridFindOptions,\n HybridOptions,\n HybridSearch,\n Ranked,\n} from \"./hybrid.ts\";\nimport { createHybridSearch } from \"./hybrid.ts\";\nimport { CaptureLog } from \"./log.ts\";\nimport type { Clock, HostStateStore, PolicyOptions } from \"./politeness.ts\";\nimport { MemoryHostStateStore, realClock } from \"./politeness.ts\";\nimport type { Renderer } from \"./render.ts\";\nimport { playwrightRenderer } from \"./render.ts\";\nimport type {\n CrawlRuntime,\n Extractor,\n PageResult,\n ReadOptions,\n ReadResult,\n ReindexResult,\n VerifyOptions,\n VerifyResult,\n} from \"./runtime.ts\";\nimport {\n Extractors,\n catchUpIndex,\n fetchPage,\n findDocuments,\n readDocument,\n reindex,\n verifyQuote,\n} from \"./runtime.ts\";\nimport type { Recipe } from \"./recipe.ts\";\nimport { registerRecipe } from \"./recipe.ts\";\nimport { VectorStore } from \"./vectors.ts\";\nimport { BOT_PAGE, DEFAULT_TOKEN, VERSION } from \"./version.ts\";\nimport type { WatchList } from \"./watch.ts\";\nimport { WATCH_LIMITS, WatchStore, runWatch } from \"./watch.ts\";\nimport type { RequestSigner, WebBotAuthOptions } from \"./webbotauth.ts\";\n\nexport type { ReadOptions, ReadResult, VerifyOptions, VerifyResult } from \"./runtime.ts\";\n\nexport { BOT_PAGE, DEFAULT_TOKEN } from \"./version.ts\";\n\nexport type { Duration } from \"./duration.ts\";\nexport { durationMs } from \"./duration.ts\";\n\n/**\n * This function gives the default `User-Agent` header. The header names the crawler, its version\n * and the page about it, so a site owner can find out what made each request.\n */\nexport const defaultUserAgent = (contact?: string): string =>\n `${DEFAULT_TOKEN}/${VERSION} (+${BOT_PAGE}${contact === undefined ? \"\" : `; ${contact}`})`;\n\n/** A mailbox is `name@host`, or a name and `<name@host>`. A URL is not a mailbox, also when it holds an `@`. */\nconst MAILBOX = /^(?:[^\\s<>@:/]+@[^\\s<>@/]+|[^<>]*<[^\\s<>@:/]+@[^\\s<>@/]+>)$/;\n\n/**\n * This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,\n * section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in\n * the `User-Agent` only.\n *\n * @beta\n */\nexport const isMailbox = (contact: string): boolean => MAILBOX.test(contact.trim());\n\n/**\n * These options make a runtime. A store needs a log, a blob store and an index. Give `dir`, and\n * the runtime makes the three parts in it, or give the parts. A part that you give replaces the\n * part in `dir`.\n */\nexport interface RuntimeOptions {\n /** This folder keeps the capture log (`captures/`), the bodies (`blobs/`), the index (`index.sqlite`) and the state of each host (`hosts.sqlite`). */\n readonly dir?: string | undefined;\n /**\n * This log replaces `<dir>/captures`.\n *\n * @beta\n */\n readonly log?: CaptureLog | undefined;\n /**\n * This blob store replaces `<dir>/blobs`.\n *\n * @beta\n */\n readonly blobs?: BlobStore | undefined;\n /**\n * This index replaces `<dir>/index.sqlite`. A runtime takes any `ReadingIndex`.\n *\n * @beta\n */\n readonly index?: ReadingIndex | undefined;\n /**\n * This store keeps the pace and the stops of each host. The default with `dir` is a SQLite file\n * (`hosts.sqlite`), so the processes on one store share the pace. With no `dir`, it is in memory.\n */\n readonly hostState?: HostStateStore | undefined;\n /** This token is the name that robots.txt rules match. The default is `PoliteCrawler`. */\n readonly token?: string | undefined;\n /** This text is the full `User-Agent` header. The default names the crawler and its page. */\n readonly userAgent?: string | undefined;\n /**\n * This address goes into the `User-Agent` header. A mailbox (`ops@example.com`) also goes into the\n * `From` header, and a URL does not. The crawler sends no contact without it.\n */\n readonly contact?: string | undefined;\n /**\n * This browser renders the pages that need JavaScript. The value `false` turns rendering off. The\n * default checks one time that `playwright-core` and its browser are installed. With no browser,\n * each page that needs JavaScript gets a note at once, with no wait.\n *\n * @beta\n */\n readonly renderer?: Renderer | false | undefined;\n /** This value is the number of sites that the crawler fetches from at one time. The default is 16. */\n readonly maxActive?: number | undefined;\n /** This value is the number of pages that the browser renders at one time. The default is 2: each render holds a browser context. */\n readonly maxRenders?: number | undefined;\n /** This `undici` dispatcher sends the requests. Tests give a mock agent, and a proxy is possible. */\n readonly dispatcher?: Dispatcher | undefined;\n /** This policy controls the addresses that the crawler fetches. By default, it fetches only public addresses. */\n readonly egress?: EgressOptions | undefined;\n /** These values change the limits of the politeness policy, for each host or for some hosts. */\n readonly politeness?: PolicyOptions | undefined;\n /** This clock gives the time to the crawler and to the index. Tests give a clock that they control. */\n readonly clock?: Clock | undefined;\n /** This function gives the random numbers of the jitter. The default is `Math.random`. */\n readonly random?: (() => number) | undefined;\n /** This label goes into each capture record. */\n readonly scope?: string | undefined;\n /** With the value false, the crawler does not also obey the group of `PoliteCrawler` when its `User-Agent` links the bot page (`CrawlerOptions.obeyDefaultToken`). */\n readonly obeyDefaultToken?: boolean | undefined;\n /** This function ranks the scopes, the most private highest. A shared fetch of a robots.txt records the scope with the highest rank (`CrawlerOptions.scopeRank`). */\n readonly scopeRank?: ((scope: string) => number) | undefined;\n /** This function gets an event for each request, response, wait, stop and skip. */\n readonly onEvent?: ((event: CrawlEvent) => void) | undefined;\n /** These extractors read the captures of the recipes of an application. The recipes `page` and `archive-copy` are built in. */\n readonly extractors?: Readonly<Record<string, Extractor>> | undefined;\n /**\n * These recipes of `defineRecipe` get their extractors at open. Thus `reindex` reads their\n * captures before the first fetch. A recipe with the name of an entry of `extractors` throws an error.\n */\n readonly recipes?: ReadonlyArray<Recipe<never>> | undefined;\n /**\n * This signer signs each request with Web Bot Auth (`CrawlerOptions.signer`). Give the options of\n * `createWebBotAuthSigner`, or a `RequestSigner`. Without it, the crawler signs nothing.\n */\n readonly signer?: RequestSigner | WebBotAuthOptions | undefined;\n}\n\n/**\n * A runtime that a store made. It owns the parts that it made, and it closes them.\n *\n * @beta\n */\nexport interface StoreRuntime extends CrawlRuntime {\n readonly hostState: HostStateStore;\n /** This method closes the crawler, the browser, the log, the index and the host state store that the runtime made. A second close does nothing. */\n close(): Promise<void>;\n}\n\n/** The parts of a store that can be a symbolic link to a different place only by mistake or by attack. */\nconst STORE_PARTS = [\n \"captures\",\n \"blobs\",\n \"index.sqlite\",\n \"hosts.sqlite\",\n \"watch.sqlite\",\n \"vectors.sqlite\",\n];\n\n/**\n * This function makes the folder of a store for the owner only, where the system has modes. It\n * refuses a store folder of a different user, and a part of the store that is a symbolic link.\n * The store folder itself can be a link, for example to a larger disk.\n */\nconst guardStore = (dir: string): void => {\n mkdirSync(dir, { recursive: true, mode: 0o700 });\n const real = lstatSync(dir).isSymbolicLink() ? realpathSync(dir) : dir;\n const info = statSync(real);\n if (!info.isDirectory()) throw new Error(`the store ${dir} is not a folder`);\n if (\n process.platform !== \"win32\" &&\n typeof process.getuid === \"function\" &&\n info.uid !== process.getuid()\n ) {\n throw new Error(`the store ${dir} belongs to a different user: use a folder of your own`);\n }\n for (const part of STORE_PARTS) {\n let link = false;\n try {\n link = lstatSync(join(dir, part)).isSymbolicLink();\n } catch {\n // a part that is not there yet\n }\n if (link)\n throw new Error(\n `${part} in the store ${dir} is a symbolic link: the crawler does not follow it`,\n );\n }\n};\n\n/**\n * This function makes a runtime. It is the one place that puts the log, the blob store, the index,\n * the host state, the crawler and the extractors together. The function `openCrawler` uses it.\n * An application that needs the parts uses it too. The runtime and the crawler share one clock.\n *\n * @beta\n */\nexport const createRuntime = (options: RuntimeOptions): StoreRuntime => {\n const dir = options.dir;\n if (\n dir === undefined &&\n (options.log === undefined || options.blobs === undefined || options.index === undefined)\n ) {\n throw new Error(\"a runtime needs dir, or a log, a blob store and an index\");\n }\n const contact = options.contact;\n if (contact !== undefined && !/^[\\t -~]*$/.test(contact)) {\n throw new Error(\n `contact \"${contact}\" has a character that an HTTP header cannot hold: use printable ASCII`,\n );\n }\n // the extractors first: a recipe that is not valid throws before a file of the store opens\n const extractors = new Extractors();\n for (const [recipe, extractor] of Object.entries(options.extractors ?? {})) {\n extractors.register(recipe, extractor);\n }\n for (const recipe of options.recipes ?? []) registerRecipe(extractors, recipe);\n if (dir !== undefined) guardStore(dir);\n const owned: Array<() => Promise<void> | void> = [];\n const log = options.log ?? new CaptureLog(join(dir!, \"captures\"));\n const blobs = options.blobs ?? new FileBlobStore(join(dir!, \"blobs\"));\n let index = options.index;\n if (index === undefined) {\n const made = new DocIndex(join(dir!, \"index.sqlite\"));\n owned.push(() => made.close());\n index = made;\n }\n let hostState = options.hostState;\n if (hostState === undefined) {\n if (dir === undefined) hostState = new MemoryHostStateStore();\n else {\n const made = new SqliteHostStateStore(join(dir, \"hosts.sqlite\"));\n owned.push(() => made.close());\n hostState = made;\n }\n }\n const clock = options.clock ?? realClock;\n // the default browser is checked one time: with none, each page gets its note at once\n const renderer =\n options.renderer === false\n ? undefined\n : (options.renderer ?? checkedRenderer(playwrightRenderer()));\n const crawler = new Crawler({\n token: options.token ?? DEFAULT_TOKEN,\n userAgent: options.userAgent ?? defaultUserAgent(contact),\n from: contact !== undefined && isMailbox(contact) ? contact.trim() : undefined,\n log,\n blobs,\n index,\n renderer,\n maxActive: options.maxActive,\n maxRenders: options.maxRenders,\n dispatcher: options.dispatcher,\n egress: options.egress,\n politeness: options.politeness,\n hostState,\n clock,\n random: options.random,\n scope: options.scope,\n scopeRank: options.scopeRank,\n obeyDefaultToken: options.obeyDefaultToken,\n onEvent: options.onEvent,\n signer: options.signer,\n });\n let closing: Promise<void> | undefined;\n return {\n crawler,\n index,\n blobs,\n captures: log,\n log,\n extractors,\n hostState,\n now: () => new Date(clock.now()).toISOString(),\n close() {\n closing ??= (async () => {\n // the crawler closes the browser and the log\n await crawler.close();\n for (const close of owned) await close();\n })();\n return closing;\n },\n };\n};\n\n/** These options open a crawler on a folder. Only `dir` is necessary. */\nexport interface OpenOptions extends Omit<RuntimeOptions, \"dir\" | \"log\" | \"blobs\" | \"index\"> {\n /** This folder keeps the capture log, the response bodies, the search index and the state of each host. */\n readonly dir: string;\n /** These agents get their robots.txt rules applied to each fetch, for example `Claude-User`. */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** This value is the shortest `every` of a watch. The default is `1h`, so a watch cannot hammer a site. */\n readonly watchFloor?: Duration | undefined;\n /**\n * This embedder makes the vectors of hybrid search (`rank: \"hybrid\"`): `transformersEmbedder`,\n * `ollamaEmbedder` or an embedder of your own. The vectors go into `vectors.sqlite` in the store.\n * Without it, `rank: \"hybrid\"` throws an error. `close` also closes the embedder.\n */\n readonly embedder?: Embedder | undefined;\n}\n\n/** These options control one fetch. */\nexport interface FetchOptions {\n /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */\n readonly render?: \"auto\" | \"never\" | \"always\" | undefined;\n /**\n * A stored copy younger than this age answers, and the crawler sends no request. The default is\n * `24h`. The value `auto` uses the freshness that the server gave. The value `0s` asks again.\n */\n readonly maxAge?: Duration | undefined;\n /** These agents get their robots.txt rules applied to this fetch, in addition to the defaults. */\n readonly onBehalfOf?: readonly string[] | undefined;\n /** When this signal aborts, the crawler stops the fetch and the render. */\n readonly signal?: AbortSignal | undefined;\n /** This label goes into the capture records of this fetch. */\n readonly scope?: string | undefined;\n /** These labels go into the capture records of this fetch. They are not part of the capture ids. */\n readonly labels?: Readonly<Record<string, string>> | undefined;\n /**\n * The value `archive` gives a copy from a public web archive (the Wayback Machine, then Common\n * Crawl) when the site refuses the page or is down. The result then has `archived`. The default is `none`.\n */\n readonly fallback?: \"archive\" | \"none\" | undefined;\n /**\n * The value true also gives an archived copy of a page that the current robots.txt of its site\n * disallows for this agent. The default is false: only a stop, an error or a dead host gets a copy.\n */\n readonly archiveDespiteRobots?: boolean | undefined;\n}\n\n/** These options control a search. */\nexport interface FindOptions {\n /** This value is the maximum number of results. The default is 10. */\n readonly limit?: number | undefined;\n /** This value keeps only the documents of one kind, for example `page` or `pdf`. */\n readonly kind?: string | null | undefined;\n /**\n * This value chooses the ranking. The default `exact` ranks each match with BM25. `fast` leaves\n * the common words out of the second part of the search, so a long question is faster (`RankMode`).\n */\n readonly rank?: RankMode | undefined;\n}\n\n/**\n * These options control a search of `openCrawler`: the options of `find`, and `rank: \"hybrid\"`\n * with the options of a hybrid search. The mode `hybrid` needs `OpenOptions.embedder`.\n */\nexport interface SearchOptions extends Omit<FindOptions, \"rank\">, HybridFindOptions {\n /** This value chooses the ranking. `hybrid` fuses BM25 with the ranking of the vectors (`SearchRank`). */\n readonly rank?: SearchRank | undefined;\n}\n\n/** These options control `rankWithin` of a crawler. */\nexport interface RankOptions extends HybridOptions {\n /** This value chooses the ranking. The default is `exact`. */\n readonly rank?: SearchRank | undefined;\n}\n\n/** A polite crawler fetches each source one time, keeps it, and answers the next questions from the disk. */\nexport interface PoliteCrawler {\n /**\n * The runtime gives the parts to an application that needs more control.\n *\n * @beta\n */\n readonly runtime: StoreRuntime;\n /** This method fetches a page, reads it and adds it to the index. */\n fetch(url: string, options?: FetchOptions): Promise<PageResult>;\n /**\n * This method searches the stored documents. It sends no request. A held-back document is not\n * in the results. The syntax of the query is the syntax of `parseQuery`.\n */\n find(q: string, options?: FindOptions): Promise<Found[]>;\n /** This method reads a range of characters of one stored document, the current version or the version of one capture. */\n read(doc: string, options?: ReadOptions): Promise<ReadResult>;\n /**\n * This method finds a quote in a stored document. It ignores case, spaces, typographic\n * punctuation and Unicode forms. It tells which capture holds the quote.\n */\n verify(doc: string, quote: string, options?: VerifyOptions): Promise<VerifyResult>;\n /** This method gives the versions of a stored document, the newest first. A held-back version shows no title. */\n versions(doc: string, options?: { readonly limit?: number }): Promise<DocVersion[]>;\n /**\n * This method builds the index again from the capture log and the stored bodies, with no request.\n * When the crawler rebuilt a stale index at open, the first reindex gives the counts of that\n * rebuild, unless a fetch came before it.\n */\n reindex(): Promise<ReindexResult>;\n /** This method closes the browser, the log files and the index. A second close does nothing. */\n close(): Promise<void>;\n // The members after 0.1.1 are optional, so an object of your own that implements 0.1.1 stays correct.\n /**\n * This optional method finds the pages of a site from its sitemaps and its feeds. Each file is a\n * polite fetch with its capture. It keeps the URLs on the site that robots.txt permits, to its\n * limits. `openCrawler` gives it. Over a runtime of your own, use `discoverSite`.\n */\n discover?(site: string, options?: DiscoverOptions): Promise<DiscoverResult>;\n /**\n * This optional method crawls a site from its seeds. It follows the links of each page to a\n * maximum of pages, a maximum depth and a maximum time. A `for await` loop gets each page.\n * `summary` tells why the crawl stopped. `openCrawler` gives it. Over a runtime of your own, use\n * `crawlSite`.\n */\n crawl?(seeds: string | readonly string[], options?: CrawlOptions): CrawlRun;\n /**\n * This optional method compares two versions of a stored document: the lines and the words of\n * the text, and the metadata at its paths. It sends no request. `openCrawler` gives it.\n */\n diff?(doc: string, options?: DiffOptions): Promise<DiffResult>;\n /** This optional list keeps the pages that the crawler checks again on a schedule. `openCrawler` gives it. */\n readonly watch?: WatchList;\n // ── extract ──\n /**\n * This optional method checks the proposed values of fields against a stored document, and it\n * locates each value. It does not make values. It sends no request. `openCrawler` gives it.\n */\n extract?(\n doc: string,\n fields: Readonly<Record<string, FieldClaim>>,\n options?: ExtractOptions,\n ): Promise<ExtractResult>;\n // ── end of extract ──\n // ── hybrid search ──\n /** This optional search ranks with the vectors of an embedder. `openCrawler` gives it when it has an embedder. */\n readonly hybrid?: HybridSearch | undefined;\n /** This optional method gives vectors to the stored documents that have none, within a budget. */\n embed?(options?: EmbedOptions): Promise<EmbedResult>;\n /** This optional method ranks a set of stored documents against a query, the best first. */\n rankWithin?(q: string, docs: readonly string[], options?: RankOptions): Promise<Ranked[]>;\n // ── end of hybrid search ──\n}\n\n/** This crawler is the result of `openCrawler` in version 0.1.2: a `PoliteCrawler` with `discover`, `crawl`, `diff`, `watch` and `extract`. */\nexport interface OpenedCrawler extends PoliteCrawler {\n /** This method finds the pages of a site from its sitemaps and its feeds. */\n discover(site: string, options?: DiscoverOptions): Promise<DiscoverResult>;\n /** This method crawls a site from its seeds, within its limits. A `for await` loop gets each page. */\n crawl(seeds: string | readonly string[], options?: CrawlOptions): CrawlRun;\n /** This method compares two versions of a stored document. It sends no request. */\n diff(doc: string, options?: DiffOptions): Promise<DiffResult>;\n /** This list keeps the pages that the crawler checks again on a schedule, in `watch.sqlite`. */\n readonly watch: WatchList;\n /**\n * This method checks the proposed values of fields against a stored document. Each field gets\n * `verified`, `normalized`, `ambiguous`, `not-found`, `held-back` or `invalid`.\n */\n extract(\n doc: string,\n fields: Readonly<Record<string, FieldClaim>>,\n options?: ExtractOptions,\n ): Promise<ExtractResult>;\n}\n\n/**\n * This crawler is the result of `openCrawler`: an `OpenedCrawler` with hybrid search. Its `find`\n * takes `rank: \"hybrid\"`, and it has `hybrid`, `embed` and `rankWithin`. `OpenedCrawler` does not\n * change, so an object of your own that implements the `OpenedCrawler` of 0.1.2 stays correct.\n */\nexport interface HybridCrawler extends OpenedCrawler {\n /** This method searches the stored documents. It also takes `rank: \"hybrid\"`, when the crawler has an embedder. */\n find(q: string, options?: SearchOptions): Promise<Found[]>;\n /** This search ranks with the vectors of the embedder, or it is undefined when the crawler has no embedder. */\n readonly hybrid: HybridSearch | undefined;\n /** This method gives vectors to the stored documents that have none, within a budget. It rejects when the crawler has no embedder. */\n embed(options?: EmbedOptions): Promise<EmbedResult>;\n /**\n * This method ranks a set of stored documents against a query, the best first. The modes\n * `exact` and `fast` are `DocIndex.rankWithin`. The mode `hybrid` needs an embedder.\n */\n rankWithin(q: string, docs: readonly string[], options?: RankOptions): Promise<Ranked[]>;\n}\n\n/**\n * This function opens a crawler on a folder. The folder keeps the capture log (`captures/`), the\n * response bodies (`blobs/`), the search index (`index.sqlite`) and the state of each host\n * (`hosts.sqlite`). The index is a projection: `reindex` makes it again from the log and the bodies.\n * At open, `catchUpIndex` rebuilds an index of an earlier version, and gives an index that is\n * behind the log the captures that it missed.\n *\n * Each method gives a promise, and none throws before its promise. After `close`, each method\n * rejects.\n */\nexport const openCrawler = (options: OpenOptions): HybridCrawler => {\n const floorMs =\n options.watchFloor === undefined ? WATCH_LIMITS.floorMs : durationMs(options.watchFloor);\n const runtime = createRuntime(options);\n let closed = false;\n // the rebuild at open, which the first reindex() gives when no fetch came before it\n let rebuilt: ReindexResult | undefined;\n const ready: Promise<unknown> = existsSync(join(options.dir, \"captures\"))\n ? catchUpIndex(runtime).then(\n (done) => {\n if (done?.mode === \"rebuild\") {\n const { mode, ...counts } = done;\n rebuilt = mode === \"rebuild\" ? counts : undefined;\n }\n },\n () => undefined,\n )\n : Promise.resolve();\n const live = async (): Promise<void> => {\n if (closed) throw new Error(\"the crawler is closed\");\n await ready;\n if (closed) throw new Error(\"the crawler is closed\");\n };\n // the watch list opens at its first use, so a store with no watch has no watch.sqlite\n let watches: WatchStore | undefined;\n const watchStore = async (): Promise<WatchStore> => {\n await live();\n watches ??= new WatchStore(join(options.dir, \"watch.sqlite\"), { floorMs });\n return watches;\n };\n const clockNow = (): number => Date.parse(runtime.now());\n // ── hybrid search: vectors.sqlite opens at the first hybrid search, as watch.sqlite does ──\n const embedder = options.embedder;\n let vectors: VectorStore | undefined;\n let search: HybridSearch | undefined;\n const hybridSearch = async (): Promise<HybridSearch> => {\n await live();\n if (embedder === undefined) throw new Error(NO_EMBEDDER);\n vectors ??= new VectorStore(join(options.dir, \"vectors.sqlite\"));\n search ??= createHybridSearch(runtime, { embedder, store: vectors });\n return search;\n };\n const hybrid: HybridSearch | undefined =\n embedder === undefined\n ? undefined\n : {\n embedder,\n find: async (q, findOptions) => (await hybridSearch()).find(q, findOptions),\n rankWithin: async (q, docs, rankOptions) =>\n (await hybridSearch()).rankWithin(q, docs, rankOptions),\n embed: async (embedOptions) => (await hybridSearch()).embed(embedOptions),\n };\n // ── end of hybrid search ──\n let closing: Promise<void> | undefined;\n const watch: WatchList = {\n async add(url, watchOptions = {}) {\n return (await watchStore()).add(url, watchOptions, clockNow());\n },\n async remove(url) {\n return (await watchStore()).remove(url);\n },\n async list() {\n return (await watchStore()).list(clockNow());\n },\n run(runOptions = {}) {\n return runWatch(runtime, watchStore, {\n ...runOptions,\n onBehalfOf: options.onBehalfOf,\n ready: () => {\n rebuilt = undefined;\n return Promise.resolve();\n },\n });\n },\n };\n return {\n runtime,\n async fetch(url, fetchOptions = {}) {\n await live();\n rebuilt = undefined;\n const maxAge = fetchOptions.maxAge ?? \"24h\";\n return fetchPage(runtime, url, {\n render: fetchOptions.render ?? \"auto\",\n maxAgeMs: maxAge === \"auto\" ? \"auto\" : durationMs(maxAge),\n onBehalfOf: [...(options.onBehalfOf ?? []), ...(fetchOptions.onBehalfOf ?? [])],\n signal: fetchOptions.signal,\n scope: fetchOptions.scope,\n labels: fetchOptions.labels,\n fallback: fetchOptions.fallback,\n archiveDespiteRobots: fetchOptions.archiveDespiteRobots,\n });\n },\n async find(q, findOptions = {}) {\n await live();\n const { rank, ...rest } = findOptions;\n if (rank === \"hybrid\") return (await hybridSearch()).find(q, rest);\n return findDocuments(runtime, q, { ...rest, ...(rank === undefined ? {} : { rank }) });\n },\n hybrid,\n async embed(embedOptions = {}) {\n return (await hybridSearch()).embed(embedOptions);\n },\n async rankWithin(q, docs, rankOptions = {}) {\n await live();\n const { rank, ...rest } = rankOptions;\n if (rank === \"hybrid\") return (await hybridSearch()).rankWithin(q, docs, rest);\n return runtime.index.rankWithin(q, docs, rank === undefined ? {} : { rank });\n },\n async read(id, readOptions = {}) {\n await live();\n return readDocument(runtime, id, readOptions);\n },\n async verify(id, quote, verifyOptions = {}) {\n await live();\n return verifyQuote(runtime, id, quote, verifyOptions);\n },\n // ── extract ──\n async extract(id, fields, extractOptions = {}) {\n await live();\n return extractDocument(runtime, id, fields, extractOptions);\n },\n // ── end of extract ──\n async versions(id, versionOptions = {}) {\n await live();\n // a held-back version shows no title, as a read of its capture does\n const held = runtime.index.get(id)?.ai_input === \"no\";\n return runtime.index\n .versions(id, versionOptions)\n .map((version) =>\n held || version.ai_input === \"no\" ? { ...version, title: \"\" } : version,\n );\n },\n async discover(site, discoverOptions = {}) {\n await live();\n rebuilt = undefined;\n return discoverSite(runtime, site, {\n ...discoverOptions,\n onBehalfOf: [...(options.onBehalfOf ?? []), ...(discoverOptions.onBehalfOf ?? [])],\n });\n },\n crawl(seeds, crawlOptions = {}) {\n return crawlSite(runtime, seeds, {\n ...crawlOptions,\n onBehalfOf: [...(options.onBehalfOf ?? []), ...(crawlOptions.onBehalfOf ?? [])],\n ready: async () => {\n await live();\n rebuilt = undefined;\n },\n });\n },\n async reindex() {\n await live();\n // the index that the open rebuilt is the index that a second replay of the log makes\n const done = rebuilt;\n rebuilt = undefined;\n return done ?? reindex(runtime);\n },\n async diff(id, diffOptions = {}) {\n await live();\n return diffDocument(runtime, id, diffOptions);\n },\n watch,\n close() {\n closed = true;\n // a second close gives the first one: the embedder closes one time\n closing ??= (async () => {\n await ready;\n await runtime.close();\n watches?.close();\n vectors?.close();\n await embedder?.close?.();\n })();\n return closing;\n },\n };\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;AA+CA,MAAM,kBAAkB;;;;;;AAmIxB,MAAa,kBAAkB;;AAU/B,MAAa,cACX;;AAGF,MAAM,cAAc,UAA6B;CAC/C,IAAI,UAAU,KAAA,KAAa,UAAU,SAAS,OAAO;CACrD,IAAI,UAAU,QAAQ,OAAO;CAE7B,IAAI,UAAU,UACZ,MAAM,IAAI,WACR,qKACF;CACF,MAAM,IAAI,WACR,uCAAuC,KAAK,UAAU,KAAK,KAAK,aAClE;AACF;AAoJA,MAAM,WAAW,gBAA+B;CAC9C,MAAM,QAAQ,OAAO,UAAU;CAC/B,KAAK,OAAO,OAAO,UAAU;CAC7B,UAAU,YAAY,OAAO,UAAU;CACvC,OAAO,SAAS,OAAO,UAAU;CACjC,QAAQ,UAAU,OAAO,UAAU;AACrC;AAEA,MAAM,mBAAmB;;AAGzB,MAAM,eAAe;;AAGrB,MAAM,YAAY;AAElB,MAAM,SAAS,MAAqB;6BACP,EAAE,MAAM;;;;6BAIR,EAAE,SAAS;;;;;;;;;;;;;;;;;;;;6BAoBX,EAAE,SAAS,WAAW,EAAE,SAAS;6BACjC,EAAE,SAAS,cAAc,EAAE,SAAS;6BACpC,EAAE,SAAS,cAAc,EAAE,SAAS;6BACpC,EAAE,SAAS,aAAa,EAAE,SAAS;6BACnC,EAAE,KAAK;;;;;;;;;;;;;;;;;;;;qCAoBC,EAAE,IAAI;uDACY,UAAU;;6BAEpC,EAAE,OAAO;;;;;;;;;;;;;;;;;6BAiBT,EAAE,OAAO,cAAc,EAAE,OAAO;;AA8E7D,MAAM,gBAAgB;;AAGtB,MAAM,SACJ,KACA,YACwD;CACxD,IAAI,IAAI,SAAS,aAAa,GAC5B,OAAO;EAAE,KAAK,WAAW,IAAI,MAAM,GAAG,EAAqB,CAAC;EAAG,SAAS;CAAW;CAErF,OAAO;EAAE,KAAK,WAAW,GAAG;EAAG,SAAS,WAAW;CAAQ;AAC7D;AAEA,MAAM,OAAO,SAAyB,WAAW,QAAQ,CAAC,CAAC,OAAO,MAAM,MAAM,CAAC,CAAC,OAAO,KAAK;AAE5F,MAAMA,eAAgB,MAAiC,aAAmB;CACxE,IAAI,SAAS,QAAQ,SAAS,KAAA,GAAW,OAAO;CAChD,IAAI;EACF,OAAO,KAAK,MAAM,IAAI;CACxB,QAAQ;EACN,OAAO;CACT;AACF;AAEA,MAAM,YAAY,SAA4B;CAC5C,KAAK,IAAI;CACT,SAAS,IAAI;CACb,MAAM,IAAI;CACV,MAAM,IAAI;CACV,eAAe,IAAI;CACnB,cAAc,IAAI;CAClB,YAAY,IAAI;CAChB,QAAQ,IAAI;CACZ,UAAU,IAAI;CACd,UAAU,IAAI;CACd,WAAW,IAAI,cAAc,IAAI,OAAO,KAAA;CACxC,aAAa,IAAI;CACjB,UAAU,IAAI;CACd,SAAS,IAAI;AACf;AAEA,MAAM,aAAa,SAAiC;CAClD,KAAK,IAAI;CACT,SAAS,IAAI;CACb,QAAQ,IAAI;CACZ,UAAU,IAAI,aAAa;CAC3B,YAAY,IAAI;CAChB,MAAM,IAAI;CACV,MAAM,IAAI;CACV,OAAO,IAAI;CACX,KAAK,IAAI;CACT,UAAU,IAAI;CACd,OAAO,IAAI;CACX,KAAK,IAAI,aAAa;AACxB;AAEA,MAAM,aAAa,UACjB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;;AAyDvD,IAAa,WAAb,MAA8C;CAgBjC;CAfX;CACA,6BAA8B,IAAI,IAA2B;CAC7D,QAAgB;CAChB,SAAiB;CACjB;;;;;CAMA,IAAI,QAAiB;EACnB,OAAO,KAAK;CACd;CAEA,YACE,MACA,UAA2B,CAAC,GAC5B;EAFS,KAAA,OAAA;EAGT,IAAI,SAAS,YAAY,UAAU,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAClF,KAAK,KAAK,IAAI,aAAa,IAAI;EAE/B,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,iBAAiB,GAAM,CAAC;EACpE,KAAK,GAAG,KAAK,yBAAyB,OAAO,IAAI,EAAE,EAAE;EACrD,IAAI;GACF,KAAK,GAAG,KACN,qIACF;EACF,SAAS,OAAO;GACd,KAAK,GAAG,MAAM;GACd,MAAM,IAAI,MACR,uFAAuF,QAAQ,QAAQ,gDACvG,EAAE,MAAM,CACV;EACF;EACA,IAAI,SAAS,YAAY,KAAK,GAAG,KAAK,4BAA4B;EAElE,KAAK,GAAG,KAAK,8BAA8B;EAC3C,KAAK,QAAQ,KAAK,YAAY,KAAK,cAAc,CAAC;CACpD;;CAGA,gBAAiC;EAC/B,MAAM,UAAW,KAAK,GAAG,QAAQ,qBAAqB,CAAC,CAAC,IAAI,CAAC,CAC1D;EACH,MAAM,SAAS,KAAK,GACjB,QAAQ,kFAAkF,CAAC,CAC3F,IAAI;EACP,IAAI,YAAA,KAA4B,OAAO,MAAM,UAAU,MAAM,SAAS,YAAY,GAAG;GACnF,KAAK,cAAc;GACnB,OAAO;EACT;EAEA,MAAM,UAAU,KAAK,GAClB,QACC,0FACF,CAAC,CACA,IAAI;EACP,KAAK,MAAM,EAAE,UAAU,SAAS,KAAK,GAAG,KAAK,yBAAyB,KAAK,EAAE;EAC7E,MAAM,OAAO,KAAK,GACf,QAAQ,kFAAkF,CAAC,CAC3F,IAAI;EACP,KAAK,MAAM,EAAE,UAAU,MAAM,KAAK,GAAG,KAAK,yBAAyB,KAAK,EAAE;EAC1E,KAAK,GAAG,KAAK,qEAAqE;EAClF,KAAK,GAAG,KAAK,MAAM,QAAQ,CAAC,CAAC,CAAC;EAC9B,KAAK,WAAW,MAAM;EACtB,KAAK,QAAQ,cAAc,GAAG;EAC9B,KAAK,QAAQ,mBAAmB,GAAG;EACnC,KAAK,GAAG,KAAK,yBAAyB,OAAA,CAAmB,GAAG;EAC5D,OAAO;CACT;CAEA,QAAgB,KAAa,OAAqB;EAChD,KAAK,KACH,yGACF,CAAC,CAAC,IAAI,KAAK,KAAK;CAClB;CAEA,QAAgB,KAAiC;EAC/C,OACE,KAAK,KAAK,4CAA4C,CAAC,CAAC,IAAI,GAAG,CAAC,EAE/D;CACL;CAEA,KAAa,KAA4B;EACvC,IAAI,YAAY,KAAK,WAAW,IAAI,GAAG;EACvC,IAAI,cAAc,KAAA,GAAW;GAC3B,YAAY,KAAK,GAAG,QAAQ,GAAG;GAC/B,KAAK,WAAW,IAAI,KAAK,SAAS;EACpC;EACA,OAAO;CACT;;;;;CAMA,MAAiB,MAAkB;EACjC,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,qBAAqB;EACtD,IAAI,KAAK,QAAQ,KAAK,KAAK,GAAG,eAAe;GAC3C,KAAK,SAAS;GACd,IAAI;IACF,OAAO,KAAK;GACd,UAAU;IACR,KAAK,SAAS;GAChB;EACF;EACA,KAAK,GAAG,KAAK,iBAAiB;EAC9B,KAAK,QAAQ;EACb,IAAI;GACF,MAAM,SAAS,KAAK;GACpB,KAAK,GAAG,KAAK,QAAQ;GACrB,OAAO;EACT,SAAS,OAAO;GACd,IAAI,KAAK,GAAG,eAAe,KAAK,GAAG,KAAK,UAAU;GAClD,MAAM;EACR,UAAU;GACR,KAAK,QAAQ;EACf;CACF;CAEA,aAA6B;EAC3B,OAAO,OAAO,KAAK,QAAQ,YAAY,KAAK,GAAG;CACjD;;;;;;CAOA,KAAgB,MAA8B;EAC5C,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,qBAAqB;EACtD,IAAI;GACF,OAAO,KAAK,QAAQ,KAAK,WAAW,CAAC,CAAC;EACxC,SAAS,OAAO;GACd,IAAI,CAAC,iBAAiB,KAAK,UAAU,KAAK,CAAC,KAAK,KAAK,QAAQ,GAAG,MAAM;GACtE,KAAK,WAAW,MAAM;GACtB,OAAO,KAAK,QAAQ,KAAK,WAAW,CAAC,CAAC;EACxC;CACF;;CAGA,eAAuB,YAA0B;EAC/C,MAAM,IAAI,QAAQ,UAAU;EAC5B,KAAK,MAAM,SAAS;GAAC,EAAE;GAAK,EAAE;GAAM,EAAE;GAAU,EAAE;GAAO,EAAE;EAAM,GAC/D,KAAK,GAAG,KAAK,wBAAwB,OAAO;EAE9C,KAAK,GAAG,QAAQ,sCAAsC,CAAC,CAAC,IAAI,YAAY,OAAO,UAAU,GAAG;EAC5F,KAAK,WAAW,MAAM;CACxB;;CAGA,gBAA8B;EAC5B,MAAM,UAAU,KAAK,WAAW;EAChC,MAAM,8BAAc,IAAI,IAAY;EACpC,KAAK,MAAM,EAAE,UAAU,KAAK,GACzB,QAAQ,qDAAqD,CAAC,CAC9D,IAAI,GAA8B;GACnC,MAAM,QAAQ,iBAAiB,KAAK,IAAI;GACxC,IAAI,UAAU,MAAM,YAAY,IAAI,OAAO,MAAM,EAAE,CAAC;EACtD;EACA,KAAK,MAAM,cAAc,aAAa;GACpC,IAAI,eAAe,SAAS;GAC5B,MAAM,UAAU,KAAK,QAAQ,YAAY,OAAO,UAAU,GAAG;GAG7D,IAAI,GADF,YAAY,KAAA,IAAY,OAAO,oBAAoB,KAAK,IAAI,IAAI,KAAK,MAAM,OAAO,KACxE,eAAe,KAAK,eAAe,UAAU;EAC3D;CACF;CAIA,QAAgB,GAAU,MAAsB;EAC9C,MAAM,MAAM,IAAI,IAAI;EACpB,KAAK,KAAK,yBAAyB,EAAE,MAAM,2BAA2B,CAAC,CAAC,IAAI,KAAK,IAAI;EACrF,OAAO;CACT;CAEA,OAAe,GAAU,KAAwC;EAC/D,IAAI,QAAQ,MAAM,OAAO,KAAA;EACzB,OACE,KAAK,KAAK,oBAAoB,EAAE,MAAM,eAAe,CAAC,CAAC,IAAI,GAAG,CAAC,EAE9D;CACL;CAEA,WAAmB,GAAU,OAAiB,KAAmB;EAC/D,MAAM,OACJ,MAAM,cAAc,KAAA,KAAa,MAAM,cAAc,MAAM,OACvD,KAAK,QAAQ,GAAG,MAAM,SAAS,IAC/B;EACN,KAAK,KACH,eAAe,EAAE,SAAS;;;;;;;kFAQ5B,CAAC,CAAC,IACA,MAAM,KACN,MAAM,WAAW,MAAM,SAAS,MAAM,IACtC,MAAM,UAAU,IAChB,MAAM,aAAa,OAAO,IAAI,GAC9B,MAAM,cAAc,KACpB,MAAM,QAAQ,MACd,MAAM,MACN,MAAM,OACN,MAAM,KACN,KAAK,QAAQ,GAAG,KAAK,UAAU,MAAM,QAAQ,CAAC,CAAC,CAAC,GAChD,KAAK,QAAQ,GAAG,MAAM,IAAI,GAC1B,MAAM,KAAK,QACX,MACA,MAAM,YAAY,SAClB,MAAM,SAAS,WACf,MAAM,SAAS,MACf,MAAM,WAAW,KAAA,IAAY,OAAO,KAAK,UAAU,MAAM,MAAM,CACjE;CACF;;CAGA,OACE,GACA,KACA,QACA,UACA,OACwB;EACxB,IAAI,UAAU,KAAA,GACZ,OAAO,KAAK,KACV,iBAAiB,EAAE,SAAS;yDAE9B,CAAC,CAAC,IAAI,KAAK,QAAQ,QAAQ;EAE7B,OAAO,KAAK,KACV,iBAAiB,EAAE,SAAS;uDAE9B,CAAC,CAAC,IAAI,KAAK,QAAQ,UAAU,KAAK;CACpC;;;;;;CAOA,OAAe,GAAU,KAAa,OAAoC;EACxE,MAAM,SACJ,UAAU,KAAA,IACN,KAAK,KAAK,yCAAyC,EAAE,SAAS,eAAe,CAAC,CAAC,IAAI,GAAG,IACtF,KAAK,KACH,yCAAyC,EAAE,SAAS,mCACtD,CAAC,CAAC,IAAI,KAAK,KAAK;EAEtB,IAAI,OAAO,WAAW,GAAG,OAAO,KAAA;EAChC,MAAM,2BAAW,IAAI,IAA2D;EAChF,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,MAAM,KAAK,OAAO,GAAG,KAAK,MAAM,QAAQ,MAAM,UAAU,KAAK;GACnE,IAAI,QAAQ,KAAA,GAAW;GACvB,MAAM,QAAQ,SAAS,IAAI,MAAM,MAAM,KAAK,CAAC;GAC7C,IAAI,MAAM,aAAa,GAAG,MAAM,WAAW;QACtC,MAAM,QAAQ;GACnB,SAAS,IAAI,MAAM,QAAQ,KAAK;EAClC;EACA,MAAM,aAA0B,CAAC;EACjC,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,eAAe,UAAU;GACpD,IAAI;GACJ,IAAI,aAAa,KAAA,GAAW,OAAO;QAC9B,IAAI,UAAU,KAAA,KAAa,SAAS,cAAc,MAAM,YAAY,OAAO;QAC3E;IAEH,MAAM,OAAO,KAAK,KAChB,oBAAoB,EAAE,SAAS;2DAEjC,CAAC,CAAC,IAAI,KAAK,QAAQ,SAAS,UAAU;IACtC,OACE,SAAS,KAAA,KAAa,KAAK,SAAS,QAAQ,KAAK,SAAS,MAAM,OAAO,WAAW;GACtF;GACA,MAAM,SACJ,UAAU,KAAA,IACN,WACA,aAAa,KAAA,KAAa,MAAM,aAAa,SAAS,aACpD,QACA;GACR,MAAM,SAAS,CAAC,OAAO,QAAQ,CAAC,CAAC,QAC9B,QAA2B,QAAQ,KAAA,KAAa,QAAQ,IAC3D;GACA,WAAW,KAAK;IAAE,KAAK;IAAM,IAAI,OAAO;IAAqB;GAAO,CAAC;EACvE;EACA,IAAI,SAAS,WAAW;EACxB,IAAI,WAAW,SAAS,GAAG;GACzB,MAAM,SAAS,WAAW,OAAO,cAAc,UAAU,IAAI,UAAU,QAAQ;GAC/E,KAAK,MAAM,aAAa,WAAW,MAAM,CAAC,GAAG;IAC3C,MAAM,IAAI,UAAU;IACpB,MAAM,IAAI,OAAO;IAKjB,IAJe,SACX,EAAE,aAAa,EAAE,aACjB,EAAE,aAAa,EAAE,cAChB,EAAE,eAAe,EAAE,cAAc,EAAE,aAAa,EAAE,YAC3C,SAAS;GACvB;EACF;EAEA,MAAM,UACJ,OAAO,IAAI,aAAa,QAAQ,WAAW,MAAM,cAAc,UAAU,OAAO,IAAI,IAChF,OACA,OAAO,OAAO,UACZ,OAAO,KACP,WAAW,MAAM,cAAc,UAAU,OAAO,KAAK,IACnD,QACA;EACV,OAAO;GAAE;GAAQ;GAAY;EAAQ;CACvC;;CAGA,UAAkB,GAAU,KAAa,KAA8B;EACrE,MAAM,SAAS,KAAK,KAAK,sCAAsC,EAAE,KAAK,eAAe,CAAC,CAAC,IACrF,GACF;EACA,MAAM,SAAS,KAAK,OAAO,GAAG,GAAG;EACjC,IAAI,WAAW,KAAA,GAAW;GACxB,IAAI,WAAW,KAAA,GAAW;IACxB,KAAK,KAAK,eAAe,EAAE,IAAI,iBAAiB,CAAC,CAAC,IAAI,OAAO,KAAK;IAClE,KAAK,KAAK,eAAe,EAAE,KAAK,eAAe,CAAC,CAAC,IAAI,GAAG;GAC1D;GACA;EACF;EACA,MAAM,EAAE,QAAQ,YAAY,YAAY;EACxC,MAAM,MAAM,OAAO;EACnB,IAAI,OAAOA,YAAmC,KAAK,OAAO,GAAG,IAAI,QAAQ,GAAG,CAAC,CAAC;EAC9E,KAAK,MAAM,aAAa,YAEtB,IAAI,cAAc,QAChB,OAAO;GACL,GAAGA,YAAmC,KAAK,OAAO,GAAG,UAAU,IAAI,QAAQ,GAAG,CAAC,CAAC;GAChF,GAAG;EACL;EAGJ,MAAM,OAAO,KAAK,KAChB,uBAAuB,EAAE,SAAS;uDAEpC,CAAC,CAAC,IAAI,KAAK,IAAI,UAAA,EAAsB;EACrC,MAAM,WAAW,CACf,mBAAG,IAAI,IAAI;GACT,IAAI;GACJ,GAAG,KAAK,KAAK,UAAU,MAAM,OAAO;GACpC,GAAG,OAAO,OAAO,KAAK,UAAU,MAAM,OAAO;GAC7C,GAAG,WACA,QAAQ,cAAc,cAAc,MAAM,CAAC,CAC3C,KAAK,cAAc,UAAU,IAAI,OAAO;EAC7C,CAAC,CACH,CAAC,CAAC,MAAM,GAAA,EAAe;EACvB,MAAM,QACJ,KAAK,KAAK,6BAA6B,EAAE,SAAS,eAAe,CAAC,CAAC,IAAI,GAAG,CAAC,CAC3E;EACF,MAAM,MACJ,IAAI,OACJ,WAAW,KAAK,cAAc,UAAU,IAAI,GAAG,CAAC,CAAC,MAAM,UAAU,UAAU,IAAI,KAC/E;EACF,MAAM,QAAQ,KAAK,KACjB,eAAe,EAAE,KAAK;;;;;;;;;uBAUxB,CAAC,CAAC,IACA,KACA,IAAI,MACJ,IAAI,OACJ,KACA,KAAK,UAAU,IAAI,GACnB,IAAI,UACJ,IAAI,UACJ,KAAK,UAAU,QAAQ,GACvB,OACA,IAAI,SACJ,IAAI,MACJ,IAAI,YACJ,IAAI,QACJ,IAAI,UACJ,SACA,IAAI,OACJ,IAAI,QACJ,GACF;EACA,MAAM,OAAO,KAAK,OAAO,GAAG,IAAI,QAAQ,KAAK;EAC7C,IAAI,WAAW,KAAA,KAAa,OAAO,aAAa,IAAI,YAAY,OAAO,UAAU,IAAI,OAAO;GAC1F,IAAI,WAAW,KAAA,GAAW,KAAK,KAAK,eAAe,EAAE,IAAI,iBAAiB,CAAC,CAAC,IAAI,OAAO,KAAK;GAC5F,KAAK,KAAK,eAAe,EAAE,IAAI,uCAAuC,CAAC,CAAC,IACtE,MAAM,OACN,KAAK,IAAI,OAAO,QAAQ,GACxB,KAAK,KAAK,SAAS,kBAAkB,KAAK,MAAM,GAAG,eAAe,IAAI,MAAM,QAAQ,CACtF;EACF;EACA,OAAO,KAAK,MAAM,GAAG,KAAK,IAAI;CAChC;CAEA,MAAc,GAAU,KAAa,WAAqC;EACxE,MAAM,MAAM,KAAK,KAAK,wBAAwB,EAAE,KAAK,eAAe,CAAC,CAAC,IAAI,GAAG;EAE7E,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;EAC9B,OAAO,KAAK,MAAM,GAAG,KAAK,SAAS;CACrC;CAEA,MAAc,GAAU,KAAa,WAAyB;EAC5D,MAAM,SAASA,YAA8C,IAAI,QAAQ,KAAA,CAAS;EAClF,OAAO;GACL,KAAK,IAAI;GACT,MAAM,IAAI;GACV,OAAO,IAAI;GACX,MAAM,aAAa,KAAK,OAAO,GAAG,IAAI,QAAQ,KAAK;GACnD,KAAK,IAAI;GACT,MAAMA,YAAmC,IAAI,MAAM,CAAC,CAAC;GACrD,UAAUA,YAAoB,IAAI,UAAU,CAAC,CAAC;GAC9C,eAAe,IAAI;GACnB,SAAS,IAAI;GACb,MAAM,IAAI;GACV,YAAY,IAAI;GAChB,QAAQ,IAAI;GACZ,UAAU,IAAI,aAAa;GAC3B,UAAU,IAAI;GACd,KAAK,IAAI,aAAa;GACtB,GAAI,IAAI,UAAU,OAAO,EAAE,OAAO,IAAI,MAAM,IAAI,CAAC;GACjD,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;GACzC,YAAY,IAAI;EAClB;CACF;CAEA,cAAsB,GAAU,KAA8B;EAC5D,OAAO;GACL,GAAG,UAAU,GAAG;GAChB,MAAM,KAAK,OAAO,GAAG,IAAI,QAAQ,KAAK;GACtC,WAAW,KAAK,OAAO,GAAG,IAAI,QAAQ;GACtC,MAAMA,YAAmC,KAAK,OAAO,GAAG,IAAI,QAAQ,GAAG,CAAC,CAAC;EAC3E;CACF;CAIA,IAAI,KAA8B;EAChC,OAAO,KAAK,MAAM,MAAM,KAAK,MAAM,GAAG,GAAG,CAAC;CAC5C;;;;;CAMA,OAAO,OAAiB,KAAkB;EACxC,OAAO,KAAK,MAAM,MAChB,KAAK,YAAY;GACf,KAAK,WAAW,GAAG,OAAO,GAAG;GAC7B,OAAO,KAAK,UAAU,GAAG,MAAM,KAAK,GAAG;EACzC,CAAC,CACH;CACF;;CAGA,SAAS,KAAa,UAAuC,CAAC,GAAiB;EAC7E,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,GAAG,CAAC;EAC1D,OAAO,KAAK,MAAM,MAEd,KAAK,KACH,iBAAiB,EAAE,SAAS,8DAC9B,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CACjB,IAAI,SAAS,CACjB;CACF;;;;;;CAOA,YAAY,KAAa,SAA2C;EAClE,OAAO,KAAK,MAAM,MAAM;GACtB,MAAM,KAAK,WAAW,KAAK,MAAM,GAAG,GAAG,CAAC,EAAE;GAC1C,IAAI,OAAO,KAAA,GAAW,OAAO,KAAA;GAC7B,MAAM,MAAM,KAAK,KAAK,iBAAiB,EAAE,SAAS,+BAA+B,CAAC,CAAC,IACjF,KACA,EACF;GACA,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,KAAK,cAAc,GAAG,GAAG;EAClE,CAAC;CACH;;;;;;;CAQA,UAAU,KAAa,IAAuB;EAC5C,OAAO,KAAK,MAAM,MAAM;GACtB,MAAM,QAEF,KAAK,KAAK,wCAAwC,EAAE,SAAS,eAAe,CAAC,CAAC,IAAI,GAAG,CAAC,EAErF,SAAS,KAAA;GACd,MAAM,SAAS,KAAK,OAAO,GAAG,KAAK,EAAE;GACrC,IAAI,WAAW,KAAA,GAAW,OAAO;IAAE,SAAS,KAAA;IAAW;GAAM;GAC7D,MAAM,UAAU,KAAK,cAAc,GAAG,OAAO,OAAO,GAAG;GACvD,OAAO;IACL,SAAS,OAAO,YAAY,OAAO;KAAE,GAAG;KAAS,UAAU;IAAK,IAAI;IACpE;GACF;EACF,CAAC;CACH;;;;;;CAOA,aAAa,KAAa,UAAuC,CAAC,GAAkB;EAClF,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,EAAE,CAAC;EACzD,OAAO,KAAK,MAAM,MAAM;GACtB,MAAM,OAAO,KAAK,KAAK,wBAAwB,EAAE,KAAK,eAAe,CAAC,CAAC,IAAI,GAAG;GAE9E,IAAI,SAAS,KAAA,GAAW,OAAO,CAAC;GAChC,MAAM,OAAO,KAAK,KAChB,iBAAiB,EAAE,SAAS,wEAC9B,CAAC,CAAC,IAAI,KAAK,KAAK,QAAQ;GACxB,MAAM,uBAAO,IAAI,IAAY;GAC7B,MAAM,MAAqB,CAAC;GAC5B,KAAK,MAAM,OAAO,MAAM;IACtB,IAAI,KAAK,IAAI,IAAI,QAAQ,GAAG;IAC5B,KAAK,IAAI,IAAI,QAAQ;IACrB,IAAI,KAAK,KAAK,cAAc,GAAG,GAAG,CAAC;IACnC,IAAI,IAAI,UAAU,OAAO;GAC3B;GACA,OAAO;EACT,CAAC;CACH;;CAGA,WAAW,SAA0B;EACnC,OAAO,KAAK,MACT,MACC,KAAK,KAAK,0BAA0B,EAAE,SAAS,2BAA2B,CAAC,CAAC,IAAI,OAAO,MACvF,KAAA,CACJ;CACF;;;;;CAMA,QAAQ,UAAyB,CAAC,GAAgB;EAChD,OAAO,KAAK,MAAM,MAAM;GACtB,IAAI,QAAQ,SAAS,KAAA,GAAW;IAC9B,IAAI,QAAQ,KAAK,WAAW,GAAG,OAAO,CAAC;IACvC,OAAO,KAAK,KACV,6CAA6C,EAAE,KAAK,+CACtD,CAAC,CAAC,IAAI,KAAK,UAAU,QAAQ,IAAI,CAAC;GACpC;GACA,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,GAAG,CAAC;GAC1D,OAAO,KAAK,KACV,6CAA6C,EAAE,KAAK,oCACtD,CAAC,CAAC,IAAI,QAAQ,SAAS,IAAI,KAAK;EAClC,CAAC;CACH;;CAGA,cAAkC;EAChC,OAAO,KAAK,MACT,MAEG,KAAK,KAAK,yCAAyC,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,EAEnE,UAAU,KAAA,CACjB;CACF;;CAGA,cAAc,SAAwB;EACpC,OAAO,KAAK,MAAM,MAEd,KAAK,KAAK,4BAA4B,EAAE,SAAS,mBAAmB,CAAC,CAAC,IACpE,OACF,CAAC,CAEA,KAAK,QAAQ,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CACpC,QAAQ,QAAoB,QAAQ,KAAA,CAAS,CAClD;CACF;;;;;;;CAQA,KAAK,GAAW,SAAoC;EAClD,MAAM,OAAO,WAAW,QAAQ,IAAI;EACpC,MAAM,QAAQ,WAAW,CAAC;EAC1B,IAAI,UAAU,KAAA,GAAW,OAAO,CAAC;EACjC,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,KAAM,KAAK,MAAM,QAAQ,KAAK,KAAK,EAAE,CAAC;EACzE,OAAO,KAAK,MAAM,MAAM;GACtB,MAAM,aACJ,QAAQ,SAAS,KAAA,KAAa,QAAQ,SAAS,OAAO,KAAK;GAC7D,MAAM,MAAM,sCAAsC,EAAE,IAAI;iBAC7C,EAAE,IAAI,QAAQ,EAAE,KAAK,kBAAkB,EAAE,IAAI;iBAC7C,EAAE,IAAI,kCAAkC,WAAW;;GAE9D,MAAM,OAAO,UAAoD;IAC/D,MAAM,SAAiC,CAAC,KAAK;IAC7C,IAAI,WAAW,SAAS,GAAG,OAAO,KAAK,QAAQ,IAAc;IAC7D,OAAO,KAAK,KAAK;IACjB,OAAO,KAAK,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,MAAM;GACrC;GACA,MAAM,OAAO,IAAI,MAAM,GAAG;GAC1B,IAAI,KAAK,SAAS,SAAS,MAAM,QAAQ,GAAG;IAC1C,MAAM,OAAO,IAAI,IAAI,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC;IAC/C,KAAK,MAAM,OAAO,IAAI,SAAS,SAAS,KAAK,UAAU,GAAG,MAAM,QAAQ,IAAI,MAAM,GAAG,GAAG;KACtF,IAAI,KAAK,UAAU,OAAO;KAC1B,IAAI,CAAC,KAAK,IAAI,IAAI,GAAG,GAAG,KAAK,KAAK,GAAG;IACvC;GACF;GACA,OAAO,KAAK,KAAK,SAAS;IACxB,KAAK,IAAI;IACT,MAAM,IAAI;IACV,OAAO,IAAI;IACX,KAAK,IAAI;IACT,SAAS,UAAU,KAAK,OAAO,GAAG,IAAI,QAAQ,KAAK,IAAI,MAAM,KAAK;IAElE,OAAO,KAAK,MAAM,CAAC,IAAI,OAAO,GAAI,IAAI;IACtC,UAAU,IAAI;IACd,YAAY,IAAI;IAChB,SAAS,IAAI;GACf,EAAE;EACJ,CAAC;CACH;;;;;;;CAQA,WACE,GACA,MACA,UAAoD,CAAC,GACI;EACzD,MAAM,OAAO,WAAW,QAAQ,IAAI;EACpC,MAAM,QAAQ,WAAW,CAAC;EAC1B,IAAI,UAAU,KAAA,KAAa,KAAK,WAAW,GAAG,OAAO,CAAC;EACtD,OAAO,KAAK,MAAM,MAEd,KAAK,KACH,6BAA6B,EAAE,IAAI;oBACzB,EAAE,IAAI,QAAQ,EAAE,KAAK,kBAAkB,EAAE,IAAI;oBAC7C,EAAE,IAAI;0BAElB,CAAC,CAAC,IACA,SAAS,SAAS,KAAK,UAAU,GAAG,MAAM,QAAQ,IAAI,MAAM,KAC5D,KAAK,UAAU,IAAI,CACrB,CAAC,CACD,KAAK,SAAS;GAAE,KAAK,IAAI;GAAK,OAAO,KAAK,MAAM,CAAC,IAAI,OAAO,GAAI,IAAI;EAAK,EAAE,CAC/E;CACF;;;;;;CAOA,UAAkB,GAAU,OAAkC;EAC5D,IAAI,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,MAAM;EAC9C,MAAM,QAAS,KAAK,KAAK,6BAA6B,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAmB;EACxF,MAAM,UAAU,MAAM,KAAK,UAAU;GACnC;GACA,OACE,KAAK,KAAK,6BAA6B,EAAE,IAAI,SAAS,EAAE,IAAI,SAAS,CAAC,CAAC,IAAI,IAAI,CAAC,CAGhF;EACJ,EAAE;EACF,MAAM,OAAO,QAAQ,QAAQ,QAAQ,IAAI,SAAS,kBAAkB,KAAK;EACzE,IAAI,KAAK,SAAS,GAAG,OAAO,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,KAAK,MAAM;EACnE,OAAO,QAAQ,QAAQ,MAAM,QAAS,IAAI,QAAQ,KAAK,QAAQ,MAAM,IAAK,CAAC,CAAC;CAC9E;;CAGA,OAAO,KAAa,SAAuC;EACzD,MAAM,MAAM,MAAM,KAAK,OAAO;EAC9B,OAAO,KAAK,MAAM,MAAM;GACtB,MAAM,MAAM,KAAK,KAAK,iBAAiB,EAAE,OAAO,+BAA+B,CAAC,CAAC,IAC/E,IAAI,KACJ,IAAI,OACN;GACA,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,SAAS,GAAG;EACrD,CAAC;CACH;CAEA,UAAkB,GAAU,OAAqB;EAC/C,MAAM,MAAM,MAAM,MAAM,KAAK,MAAM,OAAO;EAE1C,KAAK,KACH,eAAe,EAAE,OAAO;;;;;;;;sCAQQ,EAAE,OAAO,YAC3C,CAAC,CAAC,IACA,IAAI,KACJ,IAAI,SACJ,MAAM,SACN,MAAM,MACN,MAAM,MACN,MAAM,eACN,MAAM,cACN,MAAM,YACN,MAAM,QACN,MAAM,UACN,MAAM,YAAY,MAClB,MAAM,cAAc,OAAO,IAAI,MAC/B,MAAM,eAAe,MACrB,MAAM,YAAY,IACpB;CACF;;CAGA,UAAU,OAAqB;EAC7B,KAAK,MAAM,MAAM,KAAK,YAAY,KAAK,UAAU,GAAG,KAAK,CAAC,CAAC;CAC7D;CAEA,SAAsB;EACpB,OAAO,KAAK,MAAM,OAAO;GACvB,MAAO,KAAK,KAAK,6BAA6B,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAmB;GAChF,MAAO,KAAK,KAAK,6BAA6B,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAmB;EACpF,EAAE;CACJ;;CAGA,QAAc;EACZ,KAAK,MAAM,MACT,KAAK,YAAY;GACf,KAAK,MAAM,SAAS;IAAC,EAAE;IAAK,EAAE;IAAM,EAAE;IAAU,EAAE;IAAO,EAAE;GAAM,GAC/D,KAAK,GAAG,KAAK,eAAe,OAAO;EAEvC,CAAC,CACH;CACF;;;;;;CAOA,aAAa,UAAuC,CAAC,GAAe;EAClE,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,SAAS,GAAG,CAAC;EAC9D,MAAM,aAAa,KAAK,YAAY;GAClC,MAAM,OAAO,OAAO,KAAK,QAAQ,iBAAiB,KAAK,GAAG;GAC1D,KAAK,QAAQ,mBAAmB,OAAO,OAAO,CAAC,CAAC;GAChD,KAAK,QAAQ,YAAY,OAAO,IAAI,sBAAK,IAAI,KAAK,EAAA,CAAE,YAAY,CAAC;GACjE,KAAK,GAAG,KAAK,MAAM,QAAQ,IAAI,CAAC,CAAC;GACjC,OAAO;EACT,CAAC;EACD,MAAM,IAAI,QAAQ,UAAU;EAC5B,MAAM,WAAsE,CAAC;EAC7E,MAAM,yBAAS,IAAI,IAAoB;EACvC,IAAI,OAAO;EACX,MAAM,WAAW,KAAa,YAAyC;GACrE,MAAM,MAAM,MAAM,KAAK,OAAO;GAC9B,OAAO,GAAG,IAAI,QAAQ,GAAG,IAAI;EAC/B;EACA,MAAM,cAAoB;GACxB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qBAAqB;GAChD,IAAI,SAAS,WAAW,KAAK,OAAO,SAAS,GAAG;GAChD,KAAK,YAAY;IACf,MAAM,0BAAU,IAAI,IAAoB;IACxC,KAAK,MAAM,EAAE,OAAO,SAAS,UAAU;KACrC,KAAK,WAAW,GAAG,OAAO,GAAG;KAC7B,QAAQ,IAAI,MAAM,KAAK,GAAG;IAC5B;IACA,KAAK,MAAM,SAAS,OAAO,OAAO,GAAG,KAAK,UAAU,GAAG,KAAK;IAC5D,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS,KAAK,UAAU,GAAG,KAAK,GAAG;GAC9D,CAAC;GACD,SAAS,SAAS;GAClB,OAAO,MAAM;EACf;;EAEA,MAAM,aAAa,SAAgB,OAAe,WAA2B;GAC3E,KAAK,MAAM,UAAU;IAAC;IAAY;IAAY;GAAU,GACtD,KAAK,GACF,QACC,yBAAyB,EAAE,MAAM;yCACJ,QAAQ,MAAM;uCAChB,OAAO,QAAQ,QAAQ,SAAS,SAAS,MAAM,EAC5E,CAAC,CACA,IAAI,GAAG,MAAM;EAEpB;EACA,OAAO;GACL,SAAS,OAAO,QAAQ;IACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qBAAqB;IAChD,SAAS,KAAK;KAAE;KAAO;IAAI,CAAC;IAC5B,IAAI,SAAS,UAAU,WAAW,MAAM;GAC1C;GACA,YAAY,UAAU;IACpB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qBAAqB;IAChD,MAAM,MAAM,QAAQ,MAAM,KAAK,MAAM,OAAO;IAC5C,MAAM,SAAS,OAAO,IAAI,GAAG;IAC7B,IAAI,WAAW,KAAA,KAAa,MAAM,cAAc,OAAO,YAAY,OAAO,IAAI,KAAK,KAAK;IACxF,IAAI,OAAO,QAAQ,WAAW,MAAM;GACtC;GACA,SAAS,KAAK,YAAY;IACxB,MAAM,UAAU,OAAO,IAAI,QAAQ,KAAK,OAAO,CAAC;IAChD,IAAI,YAAY,KAAA,GAAW,OAAO;IAClC,MAAM,MAAM,MAAM,KAAK,OAAO;IAC9B,MAAM,MAAM,KAAK,KAAK,iBAAiB,EAAE,OAAO,+BAA+B,CAAC,CAAC,IAC/E,IAAI,KACJ,IAAI,OACN;IACA,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,SAAS,GAAG;GACrD;GACA;GACA,QAAQ,YAAY;IAClB,MAAM;IACN,MAAM,QAAQ,KAAK,UAAU,CAAC,GAAG,OAAO,CAAC;IACzC,MAAM,UAAU,QAAQ,KAAK,WAAW,CAAC;IACzC,MAAM,OAAO,KAAK,GACf,QACC,4BAA4B,QAAQ,SAAS,sDAC/C,CAAC,CACA,IAAI,KAAK;IACZ,MAAM,QACJ;IACF,KAAK,IAAI,KAAK,GAAG,KAAK,KAAK,QAAQ,MAAM,WAAW;KAClD,MAAM,OAAO,KAAK,UAAU,KAAK,MAAM,IAAI,KAAK,SAAS,CAAC,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC;KAChF,KAAK,YAAY;MACf,UAAU,SAAS,OAAO,CAAC,OAAO,IAAI,CAAC;MACvC,KAAK,GACF,QACC,yBAAyB,EAAE,SAAS,iBAAiB,QAAQ,SAAS,SAAS,OACjF,CAAC,CACA,IAAI,OAAO,IAAI;MAClB,MAAM,uBAAM,IAAI,KAAK,EAAA,CAAE,YAAY;MACnC,KAAK,MAAM,OAAO,KAAK,MAAM,IAAI,GAAe,KAAK,UAAU,GAAG,KAAK,GAAG;KAC5E,CAAC;IACH;IACA,OAAO,KAAK;GACd;GACA,SAAS,UAAU;IACjB,MAAM;IACN,OAAO;IACP,OAAO,KAAK,YAAY;KACtB,MAAM,UAAU,KAAK,WAAW;KAChC,MAAM,MAAM,QAAQ,OAAO;KAE3B,UAAU,KAAK,mBAAmB,CAAC,KAAK,CAAC;KACzC,MAAM,QAAQ,KAAK,GAChB,QAAQ,4BAA4B,IAAI,SAAS,uBAAuB,CAAC,CACzE,IAAI,KAAK;KACZ,KAAK,GACF,QACC,yBAAyB,EAAE,SAAS,iBAAiB,IAAI,SAAS,uBACpE,CAAC,CACA,IAAI,KAAK;KACZ,MAAM,uBAAM,IAAI,KAAK,EAAA,CAAE,YAAY;KACnC,KAAK,MAAM,EAAE,SAAS,OAAO,KAAK,UAAU,GAAG,KAAK,GAAG;KACvD,KAAK,MAAM,OAAO,KAAK,GACpB,QAAQ,iBAAiB,IAAI,OAAO,uBAAuB,CAAC,CAC5D,IAAI,KAAK,GACV,KAAK,UAAU,GAAG,SAAS,GAAG,CAAC;KAEjC,KAAK,QAAQ,cAAc,OAAO,UAAU,CAAC;KAC7C,KAAK,GACF,QAAQ,sCAAsC,CAAC,CAC/C,IAAI,YAAY,OAAO,UAAU,GAAG;KACvC,KAAK,eAAe,OAAO;KAC3B,KAAK,QAAQ;KACb,OAAO;MACL,MAAO,KAAK,GAAG,QAAQ,6BAA6B,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAmB;MACtF,MAAO,KAAK,GAAG,QAAQ,6BAA6B,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,CAClE;KACL;IACF,CAAC;GACH;GACA,aAAa;IACX,OAAO;IACP,SAAS,SAAS;IAClB,OAAO,MAAM;IACb,IAAI,KAAK,QAAQ;IACjB,KAAK,YAAY,KAAK,eAAe,UAAU,CAAC;GAClD;EACF;CACF;;CAGA,QAAc;EACZ,IAAI,KAAK,QAAQ;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,MAAM;EACtB,KAAK,GAAG,MAAM;CAChB;AACF;;;;;;;;ACz7CA,MAAa,iBAAiB;CAC5B,YAAY;CACZ,UAAU;;CAEV,UAAU;AACZ;AAOA,MAAM,6BAAkC,IAAI,IAAI;CAC9C;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AACD,MAAM,aAAa;CACjB;CACA;CACA;AACF;AACA,MAAM,UAAU,CAAC,6BAA6B;AAC9C,MAAM,UAAU,CAAC,0BAA0B;AAC3C,MAAM,QAAQ,CAAC,kCAAkC;AACjD,MAAM,SAAS,CAAC,6CAA6C;AAC7D,MAAM,eAAiD;CACrD,KAAK;CACL,IAAI;CACJ,IAAI;CACJ,MAAM;CACN,MAAM;AACR;;AAEA,MAAM,iBAAiB;AACvB,MAAM,OAAO;AACb,MAAM,WACJ;AACF,MAAM,YACJ;AAEF,MAAM,SAAS,SACb,SAAS,KAAA,IAAY,YAAY,KAAK,SAAS,KAAK,GAAG,KAAK,MAAM,GAAG,EAAE,EAAE,OAAO;;;;;;;;AASlF,MAAa,YAAY,UAAiC;CACxD,MAAM,OAAO,MAAM,KAAK;CACxB,IAAI,KAAK,WAAW,KAAK,KAAK,SAAS,IAAI,OAAO;CAClD,IAAI;CACJ,MAAM,MAAM,SAAS,KAAK,IAAI;CAC9B,IAAI,QAAQ,MAAM;EAChB,MAAM,MAAM,KAAK,QAAQ,KAAK,GAAG;EAEjC,QAAQ,IAAI,SAAS,GAAG,KAAK,IAAI,OAAO,KAAA,IAAY,GAAG,IAAI,KAAK;CAClE,OAAO,IAAI,UAAU,KAAK,IAAI,GAAG,QAAQ;CACzC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,KAAK,KAAK,MAAM,KAAK;CAC3B,OAAO,OAAO,SAAS,EAAE,KAAK,KAAK,IAAI,EAAE,IAAI,SAAU,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY,IAAI;AACtF;;AAGA,MAAM,UAAU,OAAe,SAAiD;CAC9E,MAAM,OAAO,MAAM,KAAK;CACxB,IAAI,KAAK,WAAW,KAAK,KAAK,SAAS,eAAe,UAAU,OAAO,KAAA;CACvE,IAAI;EACF,MAAM,MAAM,SAAS,KAAA,IAAY,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,IAAI;EACnE,IAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAAU,OAAO,KAAA;EAClE,OAAO,aAAa,IAAI,IAAI;CAC9B,QAAQ;EACN;CACF;AACF;;AAGA,MAAM,cAAc,KAAa,UAAqC;CACpE,MAAM,SAAS,IAAI;CACnB,IAAI,CAAC,IAAI,SAAS,GAAG,GAAG,OAAO;CAC/B,IAAI,MAAM;CACV,IAAI,KAAK;CACT,SAAS;EACP,MAAM,MAAM,IAAI,QAAQ,KAAK,EAAE;EAC/B,IAAI,QAAQ,IAAI,OAAO,MAAM,IAAI,MAAM,EAAE;EACzC,OAAO,IAAI,MAAM,IAAI,GAAG;EACxB,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,CAAC;EACrC,IAAI,SAAS,MAAM,OAAO,MAAM,IAAI,OAAO,KAAA;EAC3C,MAAM,OAAO,IAAI,MAAM,MAAM,GAAG,IAAI;EACpC,IAAI,OAAO,aAAa;EACxB,IAAI,SAAS,KAAA,KAAa,mCAAmC,KAAK,IAAI,GAAG;GACvE,MAAM,OAAO,KAAK,OAAO,MAAM,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,OAAO,KAAK,MAAM,CAAC,CAAC;GACxF,IAAI,OAAO,KAAK,QAAQ,YAAa,OAAO,SAAU,OAAO,QAC3D,OAAO,OAAO,cAAc,IAAI;EACpC;EAEA,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;EAC/B,OAAO;EACP,KAAK,OAAO;CACd;AACF;AAUA,MAAM,WAAW,SAA0B,SAAS,MAAM,SAAS,KAAK,SAAS,MAAM,SAAS;;AAGhG,MAAM,WAAW,MAAc,MAAc,UAAkC;CAC7E,MAAM,MAAM,MAAM,MAAM,IAAI;CAE5B,MAAM,UAAU,KAAK,OAAO,KAAK,UAAU;CAC3C,OAAO;AACT;AAEA,MAAM,SAAS,MAAc,SAAkC;CAC7D,MAAM,IAAI,KAAK;CACf,IAAI,KAAK;CACT,OAAO,KAAK,GAAG;EACb,MAAM,OAAO,KAAK,WAAW,EAAE;EAC/B,IAAI,QAAQ,IAAI,KAAK,SAAS,MAAM,SAAS,IAAI;EACjD,MAAM;CACR;CACA,MAAM,OAAO,KAAK,MAAM,MAAM,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC;CACtD,IAAI,KAAK,OAAO,OAAO,CAAC,KAAK,KAAK,IAAI,GAAG,OAAO,KAAA;CAChD,MAAM,aAA+C,CAAC;CACtD,SAAS;EACP,OAAO,KAAK,KAAK,QAAQ,KAAK,WAAW,EAAE,CAAC,GAAG,MAAM;EACrD,IAAI,MAAM,GAAG,OAAO,KAAA;EACpB,MAAM,OAAO,KAAK,WAAW,EAAE;EAC/B,IAAI,SAAS,IAAI,OAAO;GAAE;GAAM;GAAY,aAAa;GAAO,KAAK,KAAK;EAAE;EAC5E,IAAI,SAAS,IACX,OAAO,KAAK,WAAW,KAAK,CAAC,MAAM,KAC/B;GAAE;GAAM;GAAY,aAAa;GAAM,KAAK,KAAK;EAAE,IACnD,KAAA;EAEN,MAAM,QAAQ;EACd,OAAO,KAAK,GAAG;GACb,MAAM,OAAO,KAAK,WAAW,EAAE;GAC/B,IAAI,QAAQ,IAAI,KAAK,SAAS,MAAM,SAAS,MAAM,SAAS,IAAI;GAChE,MAAM;EACR;EACA,MAAM,MAAM,KAAK,MAAM,OAAO,KAAK,IAAI,IAAI,QAAQ,GAAG,CAAC;EACvD,IAAI,KAAK,QAAQ,OAAO,CAAC,KAAK,KAAK,GAAG,GAAG,OAAO,KAAA;EAChD,OAAO,KAAK,KAAK,QAAQ,KAAK,WAAW,EAAE,CAAC,GAAG,MAAM;EACrD,IAAI,KAAK,WAAW,EAAE,MAAM,IAAI,OAAO,KAAA;EACvC,MAAM;EACN,OAAO,KAAK,KAAK,QAAQ,KAAK,WAAW,EAAE,CAAC,GAAG,MAAM;EACrD,MAAM,QAAQ,KAAK;EACnB,IAAI,UAAU,QAAO,UAAU,KAAK,OAAO,KAAA;EAC3C,MAAM,QAAQ,KAAK,QAAQ,OAAO,KAAK,CAAC;EACxC,IAAI,UAAU,IAAI,OAAO,KAAA;EACzB,IAAI,WAAW,SAAS,kBAAkB,QAAQ,MAAM,eAAe,UACrE,WAAW,KAAK,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC;EAClD,KAAK,QAAQ;CACf;AACF;;;;;AAMA,MAAM,kBACJ,MACA,MACA,UACuD;CACvD,IAAI,QAAQ;CACZ,IAAI,QAAQ;CACZ,IAAI,SAAS;CACb,KAAK,IAAI,KAAK,MAAM,KAAK,KAAK,QAAQ,MAAM,GAAG;EAC7C,MAAM,SAAS;EACf,MAAM,OAAO,KAAK,WAAW,EAAE;EAC/B,IAAI,UAAU,GACR;OAAA,SAAS,OAAO,QAAQ;EAAA,OACvB,IAAI,SAAS,MAAM,SAAS,IAAI,QAAQ;OAC1C,IAAI,SAAS,IAAI;GACpB,SAAS;GACT,SAAS;EACX,OAAO,IAAI,SAAS,IAAI,QAAQ,KAAK,IAAI,GAAG,QAAQ,CAAC;OAChD,IAAI,SAAS,MAAM,UAAU,GAAG,OAAO;GAAE,KAAK,KAAK;GAAG;EAAO;CACpE;CACA,OAAO;EAAE,KAAK;EAAI;CAAO;AAC3B;;AA2BA,MAAM,aAAa,MAAc,YAAoB,UAA8B;CACjF,MAAM,UAA0B,CAAC;CACjC,IAAI,UAAU;CACd,IAAI,KAAK;CACT,OAAO,KAAK,KAAK,QAAQ;EACvB,IAAI,MAAM,KAAK,QAAQ,MAAM,EAAE;EAC/B,IAAI,QAAQ,IAAI,MAAM,KAAK;EAC3B,MAAM,SAAS,MAAM,KAAK;EAC1B,MAAM,OAAO,MAAM,KAAK,eAAe,WAAW,IAAI,MAAM,KAAK,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK;EACrF,KAAK,MAAM;EACX,IAAI,KAAK,WAAW,GAAG;EACvB,MAAM,MAAM,OAAO,MAAM,KAAA,CAAS;EAClC,IAAI,QAAQ,KAAA,GAAW,WAAW;OAC7B,QAAQ,KAAK;GAAE;GAAK,SAAS;GAAM,YAAY;EAAK,CAAC;EAC1D,IAAI,QAAQ,UAAU,YAAY;GAChC,MAAM,WAAW,MAAM,KAAK;GAC5B,OAAO;IACL,MAAM;IACN;IACA;IACA;IACA,OAAO,WAAW,CAAC,IAAI,CAAC,yBAAyB,OAAO,UAAU,EAAE,SAAS;GAC/E;EACF;CACF;CACA,OAAO;EAAE,MAAM;EAAQ;EAAS;EAAS,UAAU;EAAM,OAAO,CAAC;CAAE;AACrE;;;;;AAMA,MAAa,eACX,MACA,MACA,SAAwB,CAAC,GACzB,QAAe,EAAE,OAAO,EAAE,MACV;CAChB,MAAM,aAAa,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,cAAc,eAAe,UAAU,CAAC;CACzF,MAAM,WAAW,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,YAAY,eAAe,QAAQ,CAAC;CACnF,IAAI,KAAK,KAAK,WAAW,CAAC,MAAM,QAAS,IAAI;CAC7C,OAAO,KAAK,KAAK,UAAU,QAAQ,KAAK,WAAW,EAAE,CAAC,GAAG,MAAM;CAC/D,IAAI,KAAK,KAAK,UAAU,KAAK,QAAQ,KAAK,OAAO,UAAU,KAAK,MAAM,EAAE,GAAG,YAAY,KAAK;CAE5F,MAAM,IAAI,KAAK;CACf,MAAM,QAAkB,CAAC;CACzB,MAAM,UAA0B,CAAC;CACjC,MAAM,QAAkB,CAAC;CACzB,MAAM,2BAAW,IAAI,IAAoB;CACzC,IAAI,OAAoB;CACxB,IAAI,UAAU;CACd,IAAI,WAAW;CACf,IAAI,OAAO;CACX,IAAI,WAAW;CACf,IAAI;CACJ,IAAI;CACJ,MAAM,QAAQ,SAAuB;EACnC,WAAW;EACX,OAAO;EACP,MAAM,KAAK,IAAI;CACjB;;CAGA,MAAM,SAAS,OAAe,OAAe,WAAuC;EAClF,MAAM,QAAQ,MAAM,QAAQ,GAAG;EAC/B,IAAI,UAAU,IAAI,OAAO,UAAU;EACnC,OACE,MAAM,MAAM,QAAQ,CAAC,MAAM,SAAS,OAAO,SAAS,SAAS,IAAI,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE;CAEjG;CACA,MAAM,aAAa,KAAU,OAAe,WAAmD;EAC7F,KAAK,MAAM,CAAC,KAAK,QAAQ,IAAI,YAC3B,IAAI,WAAW,KAAA,IAAY,QAAQ,QAAQ,MAAM,KAAK,OAAO,MAAM,KAAK,IAAI,SAAS,GAAG,GACtF,OAAO,WAAW,KAAK,KAAK;CAGlC;CAEA,MAAM,WAAW,UAAqC;EACpD,QAAQ,MAAR;GACE,KAAK;GACL,KAAK;IACH,IAAI,MAAM,OAAO,OAAO,UAAU,GAAG,OAAO;IAC5C,IAAI,MAAM,OAAO,WAAW,UAAU,GAAG,OAAO;IAChD,OAAO,SAAS,YAAY,MAAM,OAAO,cAAc,UAAU,IAC7D,eACA,KAAA;GACN,KAAK;IACH,IAAI,UAAU,QAAQ,OAAO;IAC7B,IAAI,UAAU,QAAQ,OAAO;IAC7B,OAAO,UAAU,YAAY,SAAS,KAAA;GACxC,KAAK,QACH,OAAO,MAAM,OAAO,WAAW,OAAO,KAAK,MAAM,OAAO,aAAa,OAAO,IACxE,SACA,KAAA;GACN,KAAK;IACH,IAAI,MAAM,OAAO,QAAQ,OAAO,GAAG,OAAO;IAC1C,OAAO,MAAM,OAAO,QAAQ,KAAK,IAAI,SAAS,KAAA;GAChD,SACE;EACJ;CACF;CAEA,MAAM,WAAW,OAAe,UAA2B;EACzD,QAAQ,MAAR;GACE,KAAK,UACH,OAAO,UAAU,KAAK,MAAM,OAAO,OAAO,UAAU;GACtD,KAAK,gBACH,OAAO,UAAU,KAAK,MAAM,OAAO,WAAW,UAAU;GAC1D,KAAK,OACH,OAAO,UAAU,KAAK,UAAU,UAAU,MAAM,OAAO;GACzD,KAAK,QACH,OAAO,UAAU,KAAK,MAAM,OAAO,SAAS,OAAO;GACrD,KAAK,OACH,OAAO,UAAU,KAAK,MAAM,OAAO,QAAQ,OAAO;GACpD,SACE,OAAO;EACX;CACF;CAEA,MAAM,eAAe,KAAU,YAA4B;EACzD,MAAM,WAAW,UAAU,KAAK,UAAU;EAC1C,IAAI,aAAa,KAAA,GAAW,OAAO;EACnC,IAAI;GACF,OAAO,IAAI,IAAI,SAAS,KAAK,GAAG,OAAO,CAAC,CAAC;EAC3C,QAAQ;GACN,OAAO;EACT;CACF;CAEA,MAAM,QAAQ,KAAU,UAAwB;EAC9C,KAAK,MAAM,CAAC,KAAK,QAAQ,IAAI,YAC3B,IAAI,IAAI,WAAW,QAAQ,GAAG;GAC5B,MAAM,MAAM,WAAW,KAAK,KAAK;GACjC,IAAI,QAAQ,KAAA,GAAW,SAAS,IAAI,IAAI,MAAM,CAAC,GAAG,IAAI,KAAK,CAAC;EAC9D;EAEF,IAAI,UAAU,GAAG;GACf,MAAM,QAAQ,IAAI,KAAK,MAAM,IAAI,KAAK,QAAQ,GAAG,IAAI,CAAC;GACtD,IAAI,UAAU,YAAY,UAAU,kBAAkB,UAAU,OAAO,OAAO;QACzE,IAAI,UAAU,QAAQ,OAAO;QAC7B,IAAI,UAAU,OAAO,OAAO;QAC5B;IACH,KAAK,iDAAiD,MAAM,IAAI,IAAI,EAAE,EAAE;IACxE;GACF;GACA,WAAW,YAAY,KAAK,IAAI;GAChC;EACF;EACA,IAAI,UAAU,KAAA,GAAW;GACvB,IAAI,QAAQ,IAAI,MAAM,KAAK,GACzB,QAAQ;IACN;IACA,MAAM,SAAS,SAAS,YAAY,KAAK,QAAQ,IAAI;IACrD,KAAK,KAAA;IACL,MAAM,KAAA;IACN,MAAM,KAAA;IACN,SAAS,KAAA;IACT,YAAY,KAAA;IACZ,OAAO,SAAS,QAAQ,UAAU,KAAK,SAAS,MAAM,IAAI,KAAA;IAC1D,UAAU;GACZ;GAEF;EACF;EACA,IAAI,UAAU,KAAA,GAAW;GAEvB,MAAM,MAAM;GACZ;EACF;EACA,IAAI,UAAU,MAAM,QAAQ,GAAG;EAC/B,IAAI,SAAS,UAAU,MAAM,IAAI,MAAM,QAAQ,OAAO,GAAG;GACvD,MAAM,OAAO,UAAU,KAAK,KAAK,KAAK,YAAA,CAAa,KAAK,CAAC,CAAC,YAAY;GACtE,MAAM,OAAO,UAAU,KAAK,MAAM;GAClC,MAAM,QAAQ,UAAU,KAAK,MAAM,KAAK,YAAA,CAAa,KAAK,CAAC,CAAC,YAAY;GAExE,MAAM,OAAO,KAAK,WAAW,WAAW,KAAK,KAAK,WAAW,mBAAmB;GAChF,IACE,QAAQ,eACR,SAAS,KAAA,MACR,MAAM,SAAS,KAAA,KAAc,QAAQ,CAAC,MAAM,WAC7C;IACA,MAAM,OAAO;IACb,MAAM,WAAW;GACnB;GACA;EACF;EACA,MAAM,QAAQ,QAAQ,IAAI,IAAI;EAC9B,IAAI,UAAU,KAAA,GAAW;EACzB,IAAI,UAAU,QAEO;QAAA,UAAU,KAAK,aAAa,KAAK,OAAA,CAAQ,KAAK,CAAC,CAAC,YACvD,MAAM,SAAS;EAAA;EAE7B,QAAQ;GAAE,OAAO;GAAO;GAAO,OAAO,CAAC;GAAG,QAAQ;GAAG,KAAK;EAAM;CAClE;CAEA,MAAM,eAAe,YAA6B;EAChD,MAAM,MACJ,SAAS,YAAY,SAAS,iBAC1B,QAAQ,MACP,QAAQ,SAAS,SAAS,QAAQ,QAAQ,QAAQ,QAAQ;EACjE,MAAM,MACJ,QAAQ,KAAA,IACJ,KAAA,IACA,OAAO,KAAK,SAAS,YAAY,SAAS,iBAAiB,KAAA,IAAY,QAAQ,IAAI;EACzF,IAAI,QAAQ,KAAA,GAAW;GACrB,WAAW;GACX;EACF;EACA,MAAM,OAAO,QAAQ,YAAY,KAAK,CAAC,CAAC,YAAY;EACpD,QAAQ,KAAK;GACX;GACA,SAAS,QAAQ,YAAY,KAAA,IAAY,OAAO,SAAS,QAAQ,OAAO;GACxE,YACE,SAAS,KAAA,KAAa,WAAW,IAAI,IAAI,KAAK,SAAS,WAClD,OACD;EACR,CAAC;EACD,IAAI,QAAQ,UAAU,YAAY,KAAK,yBAAyB,OAAO,UAAU,EAAE,SAAS;CAC9F;CAEA,MAAM,SAAS,UAAwB;EACrC,IAAI,UAAU,KAAA,KAAa,UAAU,MAAM,OAAO;GAChD,MAAM,WAAW;GACjB,QAAQ,KAAA;GACR,IAAI,SAAS,OAAO,UAAU,KAAA,GAAW;GACzC,MAAM,QAAQ,SAAS,MAAM,KAAK,EAAE,CAAC,CAAC,KAAK;GAE3C,IAAI,MAAM,WAAW,GAAG;GACxB,IAAI,SAAS,UAAU,OAAO,MAAM,QAAQ;QACvC,IAAI,SAAS,UAAU,QAAQ,MAAM,SAAS;QAC9C,IAAI,SAAS,UAAU,QAAQ,MAAM,SAAS;QAC9C,IAAI,SAAS,UAAU,aAAa,SAAS,UAAU,QAE1D,MAAM,YAAY;QACb,MAAM,eAAe;GAC5B;EACF;EACA,IAAI,UAAU,KAAA,KAAa,UAAU,MAAM,OAAO;GAChD,MAAM,WAAW;GACjB,QAAQ,KAAA;GACR,YAAY,QAAQ;EACtB;CACF;CAEA,MAAM,WAAW,UAAoC;EACnD,IAAI,UAAU,KAAA,KAAa,MAAM,KAAK;EACtC,IAAI,UAAU,KAAA,KAAa,MAAM,SAAS,MAAM,SAAS,eAAe,UAAU;GAChF,MAAM,MAAM;GACZ;EACF;EACA,MAAM,MAAM,KAAK,KAAK;EACtB,MAAM,UAAU,MAAM;CACxB;CAEA,OAAO,KAAK,KAAK,CAAC,MAAM;EACtB,MAAM,SAAS;EACf,MAAM,KAAK,KAAK,QAAQ,KAAK,EAAE;EAC/B,MAAM,MAAM,OAAO,KAAK,IAAI;EAC5B,IAAI,MAAM,IAAI;GACZ,MAAM,SAAS,MAAM;GACrB,IAAI,UAAU,KAAA,KAAa,CAAC,MAAM,KAChC,QACE,MAAM,KAAK,eAAe,WAAW,KAAA,IAAY,WAAW,KAAK,MAAM,IAAI,GAAG,GAAG,KAAK,CACxF;EAEJ;EACA,IAAI,OAAO,IAAI;EACf,KAAK;EACL,IAAI,KAAK,WAAW,QAAQ,EAAE,GAAG;GAC/B,MAAM,QAAQ,KAAK,QAAQ,OAAO,KAAK,CAAC;GACxC,IAAI,UAAU,IAAI;IAChB,KAAK,wBAAwB;IAC7B;GACF;GACA,MAAM,SAAS,QAAQ,IAAI;GAC3B,KAAK,QAAQ;GACb;EACF;EACA,IAAI,KAAK,WAAW,aAAa,EAAE,GAAG;GACpC,MAAM,QAAQ,KAAK,QAAQ,OAAO,KAAK,CAAC;GACxC,IAAI,UAAU,IAAI;IAChB,KAAK,8BAA8B;IACnC;GACF;GACA,MAAM,SAAS,QAAQ,IAAI;GAC3B,IAAI,UAAU,KAAA,KAAa,CAAC,MAAM,KAChC,QAAQ,QAAQ,KAAK,IAAI,eAAe,WAAW,KAAA,IAAY,KAAK,MAAM,KAAK,GAAG,KAAK,CAAC;GAE1F,KAAK,QAAQ;GACb;EACF;EACA,IAAI,KAAK,WAAW,MAAM,EAAE,GAAG;GAC7B,MAAM,QAAQ,KAAK,QAAQ,MAAM,KAAK,CAAC;GACvC,IAAI,UAAU,IAAI;IAChB,KAAK,uCAAuC;IAC5C;GACF;GACA,MAAM,SAAS,QAAQ,IAAI;GAC3B,KAAK,QAAQ;GACb;EACF;EACA,IAAI,KAAK,WAAW,MAAM,EAAE,GAAG;GAC7B,MAAM,cAAc,eAAe,MAAM,KAAK,GAAG,KAAK;GACtD,IAAI,YAAY,QAAQ,IAAI;IAC1B,KAAK,4BAA4B;IACjC;GACF;GACA,IAAI,YAAY,UAAU,cAAc,KAAK,KAAK,MAAM,IAAI,KAAK,CAAC,CAAC,GACjE,MAAM,KAAK,sEAAsE;GAEnF,KAAK,YAAY;GACjB;EACF;EACA,IAAI,KAAK,WAAW,MAAM,EAAE,GAAG;GAC7B,MAAM,QAAQ,KAAK,QAAQ,KAAK,KAAK,CAAC;GACtC,IAAI,UAAU,IAAI;IAChB,KAAK,yBAAyB;IAC9B;GACF;GACA,MAAM,SAAS,QAAQ,IAAI;GAC3B,MAAM,OAAO,KAAK,MAAM,KAAK,GAAG,KAAK,IAAI,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK;GAChE,MAAM,MAAM,MAAM,IAAI;GACtB,IAAI,QAAQ,MAAM;IAChB,KAAK,iBAAiB,MAAM,IAAI,EAAE,oBAAoB,MAAM,GAAG,EAAE,EAAE;IACnE;GACF;GACA,MAAM,MAAM,SAAS,CAAC;GACtB,KAAK,QAAQ;GAEb,IAAI,MAAM,WAAW,GAAG;GACxB;EACF;EACA,MAAM,MAAM,QAAQ,MAAM,KAAK,GAAG,KAAK;EACvC,IAAI,QAAQ,KAAA,GAAW;GACrB,KAAK,gCAAgC;GACrC;EACF;EACA,KAAK,IAAI;EACT,MAAM,QAAQ,MAAM,SAAS;EAC7B,IAAI,QAAQ,UAAU;GACpB,KAAK,iCAAiC,OAAO,QAAQ,GAAG;GACxD;EACF;EACA,KAAK,KAAK,KAAK;EACf,IAAI,MAAM;EACV,IAAI,IAAI,aAAa,MAAM,KAAK;OAC3B,MAAM,KAAK,IAAI,IAAI;EACxB,IAAI,UAAU,KAAK,IAAI,aAAa;CACtC;CACA,IAAI,CAAC,QAAQ,MAAM,SAAS,GAAG,KAAK,iCAAiC;CACrE,IAAI,SAAS,aAAa,MAAM,WAAW,GAAG;EAC5C,WAAW;EACX,MAAM,KAAK,uDAAuD;CACpE;CACA,OAAO;EAAE;EAAM;EAAS;EAAS;EAAU;CAAM;AACnD;;;;;;;;AASA,MAAa,eAAe,MAAc,MAAc,WACtD,YAAY,MAAM,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;AClnBhC,MAAa,kBAAkB;;CAE7B,UAAU;;CAEV,SAAS;;CAET,UAAU;;CAEV,YAAY;;CAEZ,UAAU;;CAEV,YAAY;;CAEZ,gBAAgB;AAClB;AAgHA,MAAM,cAAc;AACpB,MAAM,iBAAiB;AACvB,MAAM,cACJ;AACF,MAAM,6BAAuC,IAAI,IAAI;CAAC;CAAO;CAAQ;AAAK,CAAC;AAS3E,MAAMC,WAAS,MAAc,OAA2B,UAAkB,QAAwB;CAChG,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,KACtC,MAAM,IAAI,WACR,GAAG,KAAK,yBAAyB,OAAO,GAAG,EAAE,gBAAgB,OAAO,KAAK,GAC3E;CAEF,OAAO;AACT;;;;;AAMA,MAAM,gBAAgB,OACpB,MACA,aAKI;CACJ,MAAM,SAAS,aAAa;CAC5B,MAAM,SAAmB,CAAC;CAC1B,IAAI,OAAO;CACX,IAAI,YAAY;CAChB,IAAI;CACJ,OAAO,IAAI,IAAI;CACf,IAAI;EACF,WAAW,MAAM,SAAS,QAAiC;GACzD,MAAM,OAAO,WAAW;GACxB,IAAI,MAAM,SAAS,MAAM;IACvB,OAAO,KAAK,MAAM,SAAS,GAAG,IAAI,CAAC;IACnC,QAAQ;IACR,YAAY;IACZ;GACF;GACA,OAAO,KAAK,KAAK;GACjB,QAAQ,MAAM;EAChB;CACF,SAAS,QAAQ;EAEf,QAAQ,kBAAkB,QAAQ,OAAO,UAAU,OAAO,MAAM;EAChE,YAAY;CACd,UAAU;EACR,OAAO,QAAQ;CACjB;CACA,OAAO;EACL,OAAO,OAAO,OAAO,QAAQ,IAAI;EACjC;EACA,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM;CACzC;AACF;AAEA,MAAM,UAAU,SAA8B,KAAK,OAAO,MAAQ,KAAK,OAAO;AAE9E,MAAM,aAAa,OAAe,SAAmC;CACnE,IAAI;EACF,MAAM,MAAM,SAAS,KAAA,IAAY,IAAI,IAAI,MAAM,KAAK,CAAC,IAAI,IAAI,IAAI,MAAM,KAAK,GAAG,IAAI;EACnF,OAAO,IAAI,aAAa,WAAW,IAAI,aAAa,WAAW,MAAM,KAAA;CACvE,QAAQ;EACN;CACF;AACF;;AAGA,MAAM,cAAiB,MAAkB,WAAgD;CACvF,IAAI,WAAW,KAAA,GAAW,OAAO;CACjC,OAAO,IAAI,SAAY,SAAS,WAAW;EACzC,MAAM,eAAqB,OAAO,OAAO,MAAM;EAC/C,IAAI,OAAO,SAAS,OAAO;EAC3B,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;EACvD,KAAK,KAAK,SAAS,MAAM,CAAC,CAAC,cAAc,OAAO,oBAAoB,SAAS,MAAM,CAAC;CACtF,CAAC;AACH;;AAGA,MAAa,gBAAgB,SAC3B,MAAM,WAAW,aAAa,MAAM;;;;;;;;AAStC,MAAa,eAAe,OAC1B,SACA,MACA,UAA8B,CAAC,MACH;CAC5B,MAAM,QAAQ,UAAU,IAAI;CAC5B,IAAI,UAAU,KAAA,GAAW,MAAM,IAAI,MAAM,uCAAuC,KAAK,EAAE;CACvF,MAAM,WAAWA,QAAM,YAAY,QAAQ,UAAU,gBAAgB,UAAU,CAAC;CAChF,MAAM,UAAUA,QAAM,WAAW,QAAQ,SAAS,gBAAgB,SAAS,CAAC;CAC5E,MAAM,WAAWA,QAAM,YAAY,QAAQ,UAAU,gBAAgB,UAAU,CAAC;CAChF,MAAM,aAAaA,QAAM,cAAc,QAAQ,YAAY,gBAAgB,YAAY,CAAC;CACxF,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,WAAW,WAAW,SAAS,SAAS,WAAW,MAAM;CAC/D,MAAM,WAAW,QAAQ,YAAY,gBAAgB;CACrD,IAAI,EAAE,YAAY,IAAI,MAAM,IAAI,WAAW,mCAAmC,OAAO,QAAQ,GAAG;CAChG,MAAM,YAAoB,KAAK,MAAM,QAAQ,IAAI,CAAC;CAClD,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,YAAY,OAAO,iBAAiB;CACxF,MAAM,UAAU,QAAQ;CACxB,MAAM,SAAS,QAAQ;CAEvB,MAAM,kBAA2B,QAAQ,YAAY;CACrD,MAAM,QAAQ;EAAE,OAAO,QAAQ;EAAO,QAAQ,QAAQ;CAAO;CAC7D,MAAM,UAAU,UAAU,CAAC,KAAK,GAAG,OAAO;CAE1C,MAAM,uBAAO,IAAI,IAA2B;CAC5C,MAAM,UAA4B,CAAC;CACnC,MAAM,UAAU;EAAE,OAAO;EAAG,QAAQ;EAAG,WAAW;EAAG,SAAS;EAAG,OAAO;CAAE;CAC1E,MAAM,0BAAU,IAAI,IAAY;CAChC,MAAM,yBAAS,IAAI,IAAY;CAC/B,MAAM,UAAqB,CAAC;CAC5B,IAAI;CACJ,IAAI,OAAsB;CAC1B,IAAI,QAAQ;CACZ,IAAI,OAAO;CACX,MAAM,iBAAgC;EACpC,QAAQ,SAAR;GACE,KAAK,aACH,OAAO,kBAAkB,OAAO,QAAQ,EAAE,yBAAyB,OAAO,QAAQ,SAAS,IAAI,EAAE;GACnG,KAAK,YACH,OAAO,mBAAmB,OAAO,OAAO,EAAE;GAC5C,KAAK,UACH,OAAO,iBAAiB,OAAO,QAAQ,EAAE;GAC3C,KAAK,aACH,OAAO;GACT,KAAK,UACH,OAAO;GACT,SACE,OAAO;EACX;CACF;CACA,MAAM,gBAAgC;EACpC,MAAM,MAAM;EACZ,MAAM,CAAC,GAAG,KAAK,OAAO,CAAC,CAAC,CACrB,KAAK,KAAK,WAAW;GAAE;GAAK;EAAM,EAAE,CAAC,CACrC,MACE,GAAG,OACD,EAAE,IAAI,YAAY,OAAO,YAAY,KAAK,MAAM,EAAE,IAAI,OAAO,MAC3D,EAAE,IAAI,YAAY,OAAO,YAAY,KAAK,MAAM,EAAE,IAAI,OAAO,MAAM,EAAE,QAAQ,EAAE,KACtF,CAAC,CACA,KAAK,EAAE,UAAU,GAAG;EACvB;EACA;EACA,SAAS,WAAW;EACpB,MAAM,QAAQ,SAAS;CACzB;;CAEA,MAAM,aAAa,YAAoE;EACrF,IAAI,UAAU,GAAG,UAAU;OACtB,IAAI,aAAa,QAAQ,IAAI,GAAG,UAAU;OAC1C,IAAI,QAAQ,SAAS,kBAAkB,UAAU;OACjD,OAAO;EACZ,OAAO;CACT;CACA,MAAM,WAAW,KAAa,KAAqB,MAAuB,UAAU,MAAY;EAC9F,MAAM,SAAS,UAAU,KAAK,MAAM,IAAI;EACxC,IAAI,WAAW,KAAA,GAAW;GACxB,QAAQ,WAAW;GACnB;EACF;EACA,MAAM,MAAM,WAAW,OAAO,IAAI;EAClC,IAAI,OAAO,IAAI,GAAG,GAAG;EACrB,OAAO,IAAI,GAAG;EACd,QAAQ,KAAK;GAAE,KAAK,OAAO;GAAM;GAAK;GAAM;EAAQ,CAAC;CACvD;CAEA,IAAI,UAAU,GAAG;EACf,UAAU;EACV,OAAO,OAAO;CAChB;CAEA,IAAI,eAAkC,CAAC;CACvC,IAAI;EACF,MAAM,SAAS,MAAM,WAAW,QAAQ,WAAW,MAAM,MAAM,KAAK,GAAG,MAAM;EAC7E,IAAI,OAAO,OAAO,gBAAgB,kBAAkB;GAClD,UAAU;GACV,OAAO,OAAO;EAChB;EACA,IAAI,OAAO,OAAO,gBAAgB,KAAA,GAAW;GAC3C,UAAU;GACV,OAAO,iCAAiC,OAAO,OAAO,YAAY;GAClE,OAAO,OAAO;EAChB;EACA,eAAe,OAAO,OAAO;CAC/B,QAAQ;EACN,UAAU,UAAU,IAAI,cAAc;EACtC,OAAO,OAAO;CAChB;CAEA,MAAM,aAAa,KAAa,WAC9B,QAAQ,MAAM;EACZ;EACA,MAAM;EACN;EACA,QAAQ,WAAW,SAAS,cAAc;EAC1C;EACA;EACA,YAAY,QAAQ;EACpB;EACA,OAAO,QAAQ;EACf,QAAQ,QAAQ;EAChB;CACF,CAAC;CAGH,IAAI,QAAQ,gBAAgB,OAAO;EACjC,MAAM,OAAO,MAAM,cAAc,SAAS;GACxC,KAAK,MAAM;GACX,MAAM;GACN,QAAQ;GACR,QAAQ;GACR;GACA,YAAY,QAAQ;GACpB;GACA,OAAO,QAAQ;GACf,QAAQ,QAAQ;GAChB;EACF,CAAC;EACD,IAAI,UAAU,KAAK,OAAO,GAAG,OAAO,OAAO;EAC3C,MAAM,MAAM,KAAK,KAAK;EACtB,MAAM,OAAO,KAAK,QAAQ;EAC1B,MAAM,OAAO,QAAQ,KAAK,KAAK,QAAQ,eAAe,WAAW;EAEjE,IAAI,SAAS,KAAA,KAAa,QAAQ,QAAQ,KAAA,KAAa,CAAC,WAAW,GAAG,GAAG;GACvE,IAAI,SAA4B,CAAC;GACjC,IAAI;IACF,SACE,YAAY,WAAW,MAAM,KAAK,QAAQ,eAAe,IAAI,GAAG,KAAK,QAAQ,GAAG,CAAC,CAAC,SAClF,CAAC;GACL,QAAQ,CAER;GACA,IAAI,QAAQ;GACZ,KAAK,MAAM,QAAQ,QAAQ;IACzB,MAAM,SAAS,UAAU,IAAI;IAC7B,IAAI,WAAW,KAAA,KAAa,CAAC,QAAQ,MAAM,GAAG;KAC5C,QAAQ,SAAS;KACjB;IACF;IACA,IAAI,SAAS,gBAAgB,gBAAgB;KAC3C,QAAQ,SAAS;KACjB;IACF;IACA,SAAS;IACT,QAAQ,OAAO,MAAM,QAAQ,MAAM;GACrC;EACF;CACF;CACA,KAAK,MAAM,QAAQ,QAAQ,SAAS,CAAC,GAAG,QAAQ,MAAM,QAAQ,OAAO;CACrE,KAAK,MAAM,WAAW,QAAQ,YAAY,CAAC,GAAG,QAAQ,SAAS,WAAW,OAAO;CACjF,KAAK,MAAM,WAAW,cAAc,QAAQ,SAAS,WAAW,YAAY;CAC5E,KAAK,QAAQ,YAAY,CAAC,EAAA,CAAG,WAAW,KAAK,aAAa,WAAW,GACnE,QAAQ,IAAI,IAAI,gBAAgB,MAAM,MAAM,CAAC,CAAC,MAAM,WAAW,SAAS;;CAG1E,MAAM,OAAO,OACX,MACA,QACA,QACoB;EACpB,IAAI,OAAO;EACX,QAAQ,WAAW,KAAK;EACxB,KAAK,MAAM,SAAS,KAAK,SAAS;GAChC,IAAI,KAAK,QAAQ,SAAS;IACxB,UAAU;IACV,OAAO;GACT;GACA,MAAM,QAAQ,KAAK,IAAI,MAAM,GAAG;GAChC,IAAI,UAAU,KAAA,GAAW;IACvB,QAAQ,aAAa;IAErB,IACE,MAAM,YAAY,SACjB,MAAM,YAAY,QAAQ,KAAK,MAAM,MAAM,OAAO,IAAI,KAAK,MAAM,MAAM,OAAO,IAE/E,KAAK,IAAI,MAAM,KAAK;KAAE,GAAG;KAAO,SAAS,MAAM;IAAQ,CAAC;IAC1D;GACF;GACA,MAAM,MAAM,IAAI,IAAI,MAAM,GAAG;GAC7B,IAAI,CAAC,QAAQ,GAAG,GAAG;IACjB,QAAQ,SAAS;IACjB;GACF;GACA,IAAI,CAAC,QAAQ,IAAI,IAAI,MAAM,GAAG;IAC5B,IAAI,QAAQ,QAAQ,gBAAgB,YAAY;KAC9C,QAAQ,SAAS;KACjB;IACF;IACA,QAAQ,IAAI,IAAI,MAAM;GACxB;GACA,IAAI;GACJ,IAAI;IACF,WACE,MAAM,QAAQ,YAAY,MAAM,KAAK;KACnC,YAAY,QAAQ;KACpB;KACA,GAAG;IACL,CAAC,EAAA,CACD;GACJ,QAAQ;IACN,UAAU;IACV,OAAO;GACT;GACA,IAAI,CAAC,SAAS;IACZ,QAAQ,UAAU;IAClB;GACF;GACA,KAAK,IAAI,MAAM,KAAK;IAClB,KAAK,MAAM;IACX,SAAS,MAAM;IACf,YAAY,MAAM;IAClB;IACA;GACF,CAAC;GACD,QAAQ;EACV;EACA,OAAO;CACT;CAEA,OAAO,OAAO,QAAQ,UAAU,YAAY,KAAA,GAAW,QAAQ,GAAG;EAChE,IAAI,UAAU,GAAG;GACf,UAAU;GACV;EACF;EACA,IAAI,IAAI,KAAK,UAAU;GACrB,UAAU;GACV;EACF;EACA,IAAI,SAAS,UAAU;GACrB,UAAU;GACV;EACF;EACA,MAAM,OAAO,QAAQ;EACrB,SAAS;EACT,MAAM,UAAU,MAAM,UAAU,KAAK,KAAK,KAAK,GAAG;EAClD,MAAM,OAAO;GACX,KAAK,QAAQ;GACb,KAAK,KAAK;GACV,MAAM,KAAK;GACX,SAAS,QAAQ;GACjB,QAAQ,QAAQ;GAChB,SAAS,QAAQ,WAAW;GAC5B,YAAY,QAAQ,aAAa;GACjC,YAAY,QAAQ;EACtB;EACA,IAAI,UAAU,OAAO,GAAG;GACtB,QAAQ,KAAK;IACX,GAAG;IACH,MAAM;IACN,SAAS;IACT,MAAM;IACN,WAAW;IACX,MAAM,QAAQ,QAAQ;GACxB,CAAC;GACD;EACF;EACA,MAAM,OAAO,QAAQ;EACrB,IAAK,QAAQ,YAAY,QAAQ,QAAQ,YAAY,kBAAmB,SAAS,KAAA,GAAW;GAC1F,QAAQ,KAAK;IACX,GAAG;IACH,MAAM;IACN,SAAS;IACT,MAAM;IACN,WAAW;IACX,MAAM,QAAQ,QAAQ,GAAG,QAAQ,QAAQ,GAAG,OAAO,QAAQ,MAAM;GACnE,CAAC;GACD;EACF;EACA,IAAI,QAAQ;EACZ,IAAI,YAAY,QAAQ,cAAc;EACtC,MAAM,QAAkB,CAAC;EACzB,IAAI,OAAO,IAAI,GAAG;GAChB,MAAM,WAAW,MAAM,cAAc,MAAM,QAAQ;GACnD,QAAQ,SAAS;GACjB,cAAc,SAAS;GACvB,IAAI,SAAS,UAAU,KAAA,GAAW,MAAM,KAAK,8BAA8B,SAAS,MAAM,EAAE;EAC9F;EACA,MAAM,OAAO,YACX,WAAW,OAAO,OAAO,IAAI,IAAI,OAAQ,QAAQ,eAAe,IAAK,GACrE,QAAQ,GACV;EACA,IAAI,WACF,MAAM,KACJ,2BAA2B,OAAO,QAAQ,EAAE,yCAC9C;EACF,MAAM,KAAK,GAAG,KAAK,KAAK;EACxB,IAAI,OAAO;EACX,IAAI,KAAK,SAAS,gBAAgB;GAEhC,MAAM,WAAW,CAAC,GAAG,KAAK,OAAO,CAAC,CAAC,MAChC,GAAG,OACD,EAAE,YAAY,OAAO,YAAY,KAAK,MAAM,EAAE,OAAO,MACrD,EAAE,YAAY,OAAO,YAAY,KAAK,MAAM,EAAE,OAAO,EAC1D;GACA,KAAK,MAAM,SAAS,UAElB,IAAI,CAAC,QAAQ,IADM,IAAI,MAAM,GACX,CAAC,GAAG,QAAQ,SAAS;QAClC,IAAI,KAAK,UAAU,IAAI,YAAY,QAAQ,SAAS;QACpD,QAAQ,MAAM,KAAK,WAAW,SAAS,KAAK,UAAU,CAAC;GAE9D,IAAI,KAAK,UAAU,IAAI,cAAc,SAAS,SAAS,GACrD,MAAM,KACJ,sDAAsD,OAAO,UAAU,EAAE,mCAC3E;EACJ,OAAO,IAAI,KAAK,SAAS,WACvB,OAAO,MAAM,KAAK,MAAM,QAAQ,KAAK,WAAW,IAAI,KAAK,IAAI,IAAI,SAAS,SAAS;EAErF,QAAQ,KAAK;GACX,GAAG;GACH,MAAM,KAAK;GACX,SAAS,KAAK,QAAQ;GACtB,MAAM;GACN;GACA,MAAM,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI;EAC9C,CAAC;CACH;CACA,IAAI,YAAY,KAAA,KAAa,KAAK,QAAQ,SAAS,UAAU;CAC7D,OAAO,OAAO;AAChB;;;;;;;;;;;;;;;;;;;AC5jBA,MAAa,eAAe;;CAE1B,UAAU;;CAEV,UAAU;;CAEV,UAAU;;CAEV,aAAa;;CAEb,UAAU;;CAEV,SAAS;AACX;;AAoIA,MAAM,UAAU,GAAS,MACvB,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE;;AAG7E,IAAM,OAAN,MAAW;CACT,QAAiC,CAAC;CAElC,IAAI,OAAe;EACjB,OAAO,KAAK,MAAM;CACpB;CAEA,OAAyB;EACvB,OAAO,KAAK,MAAM;CACpB;CAEA,KAAK,MAAkB;EACrB,MAAM,QAAQ,KAAK;EACnB,MAAM,KAAK,IAAI;EACf,IAAI,KAAK,MAAM,SAAS;EACxB,OAAO,KAAK,GAAG;GACb,MAAM,SAAU,KAAK,KAAM;GAC3B,IAAI,OAAO,MAAM,KAAM,MAAM,OAAQ,KAAK,GAAG;GAC7C,CAAC,MAAM,KAAK,MAAM,WAAW,CAAC,MAAM,SAAU,MAAM,GAAI;GACxD,KAAK;EACP;CACF;CAEA,MAAwB;EACtB,MAAM,QAAQ,KAAK;EACnB,MAAM,MAAM,MAAM;EAClB,MAAM,OAAO,MAAM,IAAI;EACvB,IAAI,QAAQ,KAAA,KAAa,SAAS,KAAA,KAAa,MAAM,WAAW,GAAG,OAAO;EAC1E,MAAM,KAAK;EACX,IAAI,KAAK;EACT,SAAS;GACP,MAAM,OAAO,KAAK,IAAI;GACtB,MAAM,QAAQ,OAAO;GACrB,IAAI,QAAQ;GACZ,IAAI,OAAO,MAAM,UAAU,OAAO,MAAM,OAAQ,MAAM,MAAO,IAAI,GAAG,QAAQ;GAC5E,IAAI,QAAQ,MAAM,UAAU,OAAO,MAAM,QAAS,MAAM,MAAO,IAAI,GAAG,QAAQ;GAC9E,IAAI,UAAU,IAAI,OAAO;GACzB,CAAC,MAAM,KAAK,MAAM,UAAU,CAAC,MAAM,QAAS,MAAM,GAAI;GACtD,KAAK;EACP;CACF;AACF;AAEA,MAAM,UAAU,QACd,GAAG,IAAI,SAAS,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE,IAAI,IAAI,SAAS,KAAK,KAAK,IAAI,IAAI;AAEpF,MAAM,WAAW,YAAiD;CAChE,IAAI,mBAAmB,QACrB,QAAQ,QAAQ;EACd,QAAQ,YAAY;EACpB,OAAO,QAAQ,KAAK,IAAI,IAAI;CAC9B;CAEF,IAAI,OAAO,YAAY,YAAY,QAAQ,WAAW,GACpD,MAAM,IAAI,WAAW,mEAAmE;CAC1F,OAAO,QAAQ,WAAW,GAAG,KACxB,QAAQ,eAAe,SAAS,GAAG,IAAI,WAAW,IAAI,QAAQ,KAC9D,QAAQ,eAAe,SAAS,IAAI,IAAI;AAC/C;AAEA,MAAM,SAAS,MAAc,OAA2B,UAAkB,QAAwB;CAChG,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,KACtC,MAAM,IAAI,WACR,GAAG,KAAK,yBAAyB,OAAO,GAAG,EAAE,gBAAgB,OAAO,KAAK,GAC3E;CAEF,OAAO;AACT;AAEA,MAAM,cAAc,KAAa,UAA8B;CAC7D;CACA,KAAK;CACL,SAAS;CACT,QAAQ;CACR,OAAO;CACP,OAAO;CACP,UAAU;CACV,YAAY,CAAC;CACb,OAAO;CACP,UAAU,CAAC;CACX,SAAS;CACT,eAAe;CACf,YAAY;CACZ,YAAY;CACZ,WAAW;CACX;AACF;AAEA,MAAMC,6BAAgC,IAAI,IAAI;CAC5C;CACA;CACA;CACA;AACF,CAAC;;;;;;;;AASD,MAAa,aACX,SACA,OACA,UAA2B,CAAC,MACf;CACb,IAAI,uBAAwD,KAAA;CAC5D,MAAM,UAAU,IAAI,SAAuB,YAAY;EACrD,iBAAiB;CACnB,CAAC;CACD,IAAI,OAAO;CAEX,gBAAgB,QAAoD;EAClE,MAAM,YAAoB,KAAK,MAAM,QAAQ,IAAI,CAAC;EAClD,MAAM,YAAY,IAAI;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,MAAM,QAAQ,QAAQ;EACtB,MAAM,eAAqB,MAAM,MAAM,OAAO,MAAM;EACpD,IAAI,OAAO,YAAY,MAAM,OAAO;EACpC,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;EACvD,MAAM,2BAAW,IAAI,IAA2B;EAChD,MAAM,wBAAQ,IAAI,IAAkB;EACpC,MAAM,uBAAO,IAAI,IAAY;EAC7B,MAAM,SAAS;GAAE,OAAO;GAAG,SAAS;GAAG,YAAY;GAAG,SAAS;GAAG,QAAQ;GAAG,cAAc;EAAE;EAC7F,MAAM,QAAQ;GAAE,OAAO;GAAG,SAAS;GAAG,OAAO;GAAG,WAAW;GAAG,OAAO;GAAG,WAAW;EAAE;EAErF,MAAM,sBAAM,IAAI,IAAY;EAC5B,IAAI,SAAS;EACb,IAAI,WAAW;EACf,IAAI,MAAM;EACV,IAAI,UAAU;EACd,IAAI;EACJ,IAAI;EACJ,IAAI,WAAW;EACf,IAAI,WAAmB,aAAa;EACpC,IAAI,WAAmB,aAAa;EAEpC,MAAM,UAAU,QAAQ;EACxB,MAAM,QAAQ;GAAE,OAAO,QAAQ;GAAO,QAAQ,QAAQ;EAAO;EAE7D,IAAI;GACF,MAAM,QAAQ,QAAQ;GACtB,WAAW,MAAM,YAAY,QAAQ,UAAU,aAAa,UAAU,CAAC;GACvE,MAAM,WAAW,MAAM,YAAY,QAAQ,UAAU,aAAa,UAAU,CAAC;GAC7E,WAAW,QAAQ,YAAY,aAAa;GAC5C,IAAI,EAAE,YAAY,IAChB,MAAM,IAAI,WAAW,mCAAmC,OAAO,QAAQ,GAAG;GAC5E,MAAM,SAAS,QAAQ,UAAU;GACjC,MAAM,WAAW,WAAW,SAAS,SAAS,WAAW,MAAM;GAC/D,MAAM,YAAY,QAAQ,WAAW,CAAC,EAAA,CAAG,IAAI,OAAO;GACpD,MAAM,YAAY,QAAQ,WAAW,CAAC,EAAA,CAAG,IAAI,OAAO;GACpD,MAAM,WAAW,YAAY;GAC7B,MAAM,OAAO,OAAO,UAAU,WAAW,CAAC,KAAK,IAAI,CAAC,GAAG,KAAK;GAC5D,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,WAAW,oCAAoC;GAChF,MAAM,SAAS,KAAK,SAAS,SAAS;IACpC,IAAI;KACF,MAAM,MAAM,IAAI,IAAI,IAAI;KACxB,OAAO,IAAI,aAAa,WAAW,IAAI,aAAa,WAAW,CAAC,GAAG,IAAI,CAAC;IAC1E,QAAQ;KACN,OAAO,CAAC;IACV;GACF,CAAC;GACD,MAAM,UAAU,UAAU,QAAQ,OAAO;GACzC,MAAM,YAAY,SACf,SAAS,WAAW,KAAK,SAAS,MAAM,SAAS,KAAK,GAAG,CAAC,MAC3D,CAAC,SAAS,MAAM,SAAS,KAAK,GAAG,CAAC;;GAGpC,MAAM,SACJ,KACA,OACA,MACA,SACA,QACA,aACS;IACT,IAAI;IACJ,IAAI;KACF,MAAM,IAAI,IAAI,GAAG;IACnB,QAAQ;KACN,MAAM,KAAA;IACR;IACA,MAAM,MAAM,QAAQ,KAAA,MAAc,IAAI,aAAa,WAAW,IAAI,aAAa;IAC/E,IAAI,CAAC,OAAO,WAAW,QAAQ;IAE/B,MAAM,MAAM,WAAW,KAAK,QAAQ,GAAG;IACvC,IAAI,KAAK,IAAI,GAAG,GAAG;IACnB,IAAI,KAAK,QAAQ,aAAa,SAAS;KACrC,MAAM,SAAS;KACf;IACF;IACA,KAAK,IAAI,GAAG;IACZ,IAAI,WAAW,UAAU,QAAQ,KAAA,GAAW;KAC1C,IAAI,CAAC,QAAQ,GAAG,GAAG;MACjB,MAAM,SAAS;MACf;KACF;KACA,IAAI,CAAC,SAAS,GAAG,GAAG;MAClB,MAAM,WAAW;MACjB;KACF;KACA,IAAI,QAAQ,UAAU;MACpB,MAAM,SAAS;MACf;KACF;IACF;IACA,IAAI,UAAU,aAAa,UAAU;KACnC,MAAM,SAAS;KACf;IACF;IACA,MAAM,OAAO,QAAQ,KAAA,KAAa,CAAC,MAAM,WAAW,QAAQ,OAAO,GAAG;IACtE,IAAI,OAAO,MAAM,IAAI,IAAI;IACzB,IAAI,SAAS,KAAA,GAAW;KACtB,OAAO,IAAI,KAAK;KAChB,MAAM,IAAI,MAAM,IAAI;IACtB;IACA,OAAO;IACP,KAAK,KAAK;KAAE,KAAK;KAAK;KAAK;KAAM;KAAO;KAAM;KAAS;KAAK;KAAQ;IAAS,CAAC;IAC9E,UAAU;GACZ;;GAGA,MAAM,aAA+B;IACnC,SAAS;KACP,IAAI;KACJ,KAAK,MAAM,CAAC,MAAM,SAAS,OAAO;MAChC,IAAI,SAAS,IAAI,IAAI,GAAG;MACxB,MAAM,MAAM,KAAK,KAAK;MACtB,IAAI,QAAQ,KAAA,GAAW;MACvB,IAAI,SAAS,KAAA,KAAa,OAAO,KAAK,KAAK,KAAK,KAAK,CAAE,IAAI,GAAG,OAAO;OAAE;OAAM;MAAK;KACpF;KACA,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;KAC/B,MAAM,OAAO,KAAK,KAAK,IAAI;KAC3B,UAAU;KACV,IAAI,KAAK,KAAK,SAAS,GAAG,MAAM,OAAO,KAAK,IAAI;KAChD,IAAI,CAAC,IAAI,IAAI,KAAK,GAAG,GAAG,OAAO;KAC/B,MAAM,aAAa;IACrB;GACF;GAEA,MAAM,YAAY,OAAO,SAA8B;IACrD,IAAI;KAEF,IAAI,KAAK,WAAW,QAMd;UAAA,EAAC,MALiB,QAAQ,YAAY,KAAK,KAAK;OAClD,YAAY,QAAQ;OACpB,QAAQ,MAAM;OACd,GAAG;MACL,CAAC,EAAA,CACY,SAAS,OAAO;OAAE,MAAM;OAAW;MAAK;KAAA;KAWvD,OAAO;MAAE,MAAM;MAAQ;MAAM,MAAA,MATV,UAAU,SAAS,KAAK,KAAK;OAC9C,QAAQ,QAAQ,UAAU;OAC1B;OACA,YAAY,QAAQ;OACpB,QAAQ,MAAM;OACd,OAAO,QAAQ;OACf,QAAQ,QAAQ;OAChB;MACF,CAAC;KACiC;IACpC,SAAS,OAAO;KACd,IAAI,MAAM,OAAO,SAAS,OAAO;MAAE,MAAM;MAAW;KAAK;KACzD,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;KACrE,OAAO;MAAE,MAAM;MAAQ;MAAM,MAAM,WAAW,KAAK,KAAK,gBAAgB,SAAS;KAAE;IACrF;GACF;;GAGA,MAAM,UAAU,MAAkB,SAAqB;IACrD,IAAI,KAAK,QAAQ,MAAM;IACvB,MAAM,MAAM,QAAQ,MAAM,IAAI,KAAK,GAAG;IACtC,IAAI,QAAQ,KAAA,GAAW;IACvB,KAAK,MAAM,QAAQ,CAAC,IAAI,KAAK,IAAI,KAAK,YAAY,GAAG;KACnD,IAAI,OAAO,SAAS,UAAU;KAC9B,KAAK,IAAI,WAAW,IAAI,CAAC;KACzB,IAAI,IAAI,WAAW,IAAI,CAAC;IAC1B;IACA,MAAM,QAAQ,IAAI,KAAK;IACvB,IAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,WAAW,GAAG;IAEjD,IAAI,WAAW,GAAG,GAAG;KACnB,MAAM,aAAa;KACnB;IACF;IACA,KAAK,MAAM,QAAQ,OACjB,IAAI,OAAO,MAAM,SAAS,UACxB,MAAM,KAAK,MAAM,KAAK,QAAQ,GAAG,GAAG,OAAO,mBAAmB,QAAQ,KAAK,GAAG;GAEpF;GAEA,KAAK,MAAM,QAAQ,MAAM,MAAM,MAAM,GAAG,GAAG,OAAO,mBAAmB,QAAQ,IAAI;GACjF,IAAI,QAAQ,gBAAgB,MAAM;IAChC,MAAM,QAAQ,CAAC,GAAG,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;IAC9D,KAAK,MAAM,QAAQ,OAAO;KACxB,IAAI,MAAM,OAAO,WAAW,IAAI,KAAK,UAAU;KAC/C,IAAI;MACF,MAAM,QAAQ,MAAM,aAAa,SAAS,MAAM;OAC9C,UAAU,QAAQ;OAClB,YAAY,QAAQ;OACpB,SAAS,KAAK,IAAI,KAAQ,KAAK,IAAI,IAAI,WAAW,CAAC,CAAC;OAEpD,aAAa;OACb;OACA,YAAY,QAAQ;OACpB,QAAQ,MAAM;OACd,OAAO,QAAQ;OACf,QAAQ,QAAQ;OAChB;MACF,CAAC;MACD,OAAO,gBAAgB,MAAM,KAAK;MAClC,KAAK,MAAM,OAAO,MAAM,MACtB,MACE,IAAI,KACJ,GACA,GACA,IAAI,YAAY,OAAO,OAAO,oBAAoB,KAAK,MAAM,IAAI,OAAO,GACxE,WACA,IACF;KAEJ,QAAQ,CAER;IACF;GACF;GAEA,SAAS;IACP,IAAI,aAAa,KAAA,KAAa,MAAM,OAAO,SAAS,WAAW;IAC/D,IAAI,aAAa,KAAA,KAAa,IAAI,KAAK,UAAU,WAAW;IAC5D,OACE,aAAa,KAAA,KACb,SAAS,OAAO,aAAa,eAC7B,UAAU,UACV;KACA,MAAM,OAAO,KAAK;KAClB,IAAI,SAAS,KAAA,GAAW;KACxB,WAAW;KACX,SAAS,IAAI,KAAK,MAAM,UAAU,IAAI,CAAC;IACzC;IACA,IAAI,SAAS,SAAS,GAAG;IACzB,MAAM,OAAO,MAAM,QAAQ,KAAK,SAAS,OAAO,CAAC;IACjD,SAAS,OAAO,KAAK,KAAK,IAAI;IAC9B,IAAI,KAAK,SAAS,WAAW;KAC3B,WAAW;KACX,aAAa;KACb;IACF;IACA,IAAI,KAAK,SAAS,WAAW;KAC3B,WAAW;KACX,OAAO,WAAW;KAClB;IACF;IACA,MAAM,OAAO,KAAK;IAClB,IAAI,aAAa,KAAK,IAAI,GAAG;KAE3B,WAAW;KACX,YAAY;KACZ,aAAa;KACb;IACF;IACA,IAAI,MAAM,OAAO,SAAS;KACxB,WAAW;KACX,aAAa;KACb;IACF;IACA,IAAI,KAAK,YAAY,aAAa,KAAK,SAAS,kBAAkB;KAChE,WAAW;KACX,aAAa;KACb;IACF;IACA,OAAO,SAAS;IAChB,IAAI,KAAK,YAAY,QAAQ,KAAK,YAAY,gBAAgB;KAC5D,OAAO,WAAW;KAClB,IAAI,KAAK,YAAY,OAAO,cAAc;IAC5C,OAAO,IAAIA,WAAS,IAAI,KAAK,OAAO,GAAG,OAAO,WAAW;SACpD,OAAO,UAAU;IACtB,OAAO,MAAM,KAAK,IAAI;IACtB,MAAM;KACJ,GAAG;KACH,OAAO,KAAK,KAAK;KACjB,QAAQ,KAAK,KAAK;KAClB,UAAU,KAAK,KAAK;IACtB;GACF;GACA,aAAa,SAAS,KAAK,WAAW,WAAW,cAAc;GAC/D,WAAW;EACb,SAAS,OAAO;GACd,WAAW;GACX,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D,WAAW;GACX,MAAM;EACR,UAAU;GAER,IAAI,CAAC,UAAU;IACb,aAAa;IACb,WAAW;GACb;GACA,OAAO,oBAAoB,SAAS,MAAM;GAC1C,MAAM,sBAAM,IAAI,MAAM,iBAAiB,CAAC;GAExC,MAAM,QAAQ,WAAW,SAAS,OAAO,CAAC;GAC1C,MAAM,OAAO,YAAY;GACzB,WACE,SAAS,SACL,uDACA,SAAS,cACP,kBAAkB,OAAO,QAAQ,EAAE,gCACnC,SAAS,WACP,iBAAiB,OAAO,QAAQ,EAAE,wBAClC,SAAS,cACP,mCACA;GACZ,eAAe;IACb,GAAG;IACH,SACE,MAAM,QACN,MAAM,UACN,MAAM,QACN,MAAM,YACN,MAAM,QACN,MAAM;IACR;IACA,QAAQ,SAAS;IACjB,SAAS;IACT;IACA,YAAY,KAAK,IAAI,GAAG,IAAI,IAAI,SAAS;GAC3C,CAAC;EACH;CACF;CAEA,OAAO;EACL;EACA,CAAC,OAAO,iBAAiB;GACvB,IAAI,MAAM,MAAM,IAAI,MAAM,yDAAyD;GACnF,OAAO;GACP,OAAO,MAAM;EACf;CACF;AACF;;;;ACvlBA,MAAM,SAAS;;AAEf,MAAM,iBAAiB;;AAEvB,MAAM,cAAc;AAiBpB,MAAM,aAAa,UACjB,UAAU,WAAW,UAAU,UAAU,QAAQ;AAEnD,MAAM,UAAU,SAAiC;CAC/C,UAAU,IAAI,cAAc;CAC5B,MAAM,UAAU,IAAI,IAAI;CACxB,cAAc,UAAU,IAAI,aAAa;CACzC,YAAY,IAAI;CAChB,YAAY,IAAI;CAChB,YAAY,IAAI;CAChB,WAAW,IAAI;AACjB;;;;;;;;;;;;;;;;;;;AA0CA,IAAa,uBAAb,MAA4D;CAU/C;CATX;CACA,6BAA8B,IAAI,IAA2B;;CAE7D,QAAyB,WAAW;CACpC,eAAuB;CACvB,SAAiB;CACjB;CAEA,YACE,MACA,UAAkC,CAAC,GACnC;EAFS,KAAA,OAAA;EAGT,IAAI,SAAS,YAAY,UAAU,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAClF,KAAK,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,QAAQ,UAAU,cAAc,CAAC;EACpF,KAAK,KAAK,IAAI,aAAa,IAAI;EAC/B,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,iBAAiB,GAAM,CAAC;EACpE,KAAK,GAAG,KAAK,yBAAyB,OAAO,IAAI,EAAE,EAAE;EACrD,IAAI,SAAS,YAAY,KAAK,GAAG,KAAK,4BAA4B;EAClE,KAAK,GAAG,KAAK,8BAA8B;EAC3C,KAAK,QAAQ;CACf;;;;;CAMA,UAAwB;EACtB,KAAK,WAAW;GACd,KAAK,GAAG,KAAK;;;;;;;;;QASX;GACF,KAAK,GAAG,KAAK;;;;;;;;;QASX;GACF,MAAM,EAAE,cAAc,YAAY,KAAK,GAAG,QAAQ,qBAAqB,CAAC,CAAC,IAAI;GAI7E,IAAI,UAAA,GACF,KAAK,GAAG,KAAK,yBAAyB,OAAA,CAAwB,GAAG;EACrE,CAAC;CACH;CAEA,KAAa,KAA4B;EACvC,IAAI,YAAY,KAAK,WAAW,IAAI,GAAG;EACvC,IAAI,cAAc,KAAA,GAAW;GAC3B,YAAY,KAAK,GAAG,QAAQ,GAAG;GAC/B,KAAK,WAAW,IAAI,KAAK,SAAS;EACpC;EACA,OAAO;CACT;;CAGA,KAAgB,MAAkB;EAChC,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,gCAAgC;EACjE,KAAK,GAAG,KAAK,iBAAiB;EAC9B,IAAI;GACF,MAAM,SAAS,KAAK;GACpB,KAAK,GAAG,KAAK,QAAQ;GACrB,OAAO;EACT,SAAS,OAAO;GACd,IAAI,KAAK,GAAG,eAAe,KAAK,GAAG,KAAK,UAAU;GAClD,MAAM;EACR;CACF;CAEA,IAAY,MAAmB;EAC7B,KAAK,KAAK,+CAA+C,CAAC,CAAC,IAAI,IAAI;EACnE,OAAO,KAAK,KACV,yGACF,CAAC,CAAC,IAAI,IAAI;CACZ;;CAGA,MAAc,KAAmB;EAC/B,KAAK,KACH;qDAEF,CAAC,CAAC,IAAI,KAAK,KAAK,GAAG;CACrB;CAEA,QAAQ,MAAc,KAAa,SAAyB;EAC1D,OAAO,KAAK,WAAW;GACrB,KAAK,gBAAgB;GACrB,IAAI,KAAK,eAAe,gBAAgB,GAAG,KAAK,MAAM,GAAG;GACzD,MAAM,MAAM,KAAK,IAAI,IAAI;GACzB,IAAI,IAAI,eAAe,KAErB,OAAO,IAAI,UAAU,KAAK,QACtB,IAAI,eAAe,MACnB,KAAK,IAAI,KAAK,QAAQ,IAAI,eAAe,GAAG;GAElD,IAAI,IAAI,UAAU,KAAK,OAAO,IAAI,UAAU;GAC5C,KAAK,KAAK,6DAA6D,CAAC,CAAC,IACvE,MAAM,SACN,KAAK,OACL,IACF;GACA,OAAO;EACT,CAAC;CACH;CAEA,OAAO,MAAc,QAAgC;EACnD,OAAO,KAAK,WAAW;GACrB,MAAM,MAAM,KAAK,IAAI,IAAI;GACzB,IAAI,EAAE,UAAU,UAAU;GAC1B,IAAI,WAAW,WAAW;IACxB,WAAW;IACX,QAAQ;GACV,OAAO,IAAI,WAAW,WAAW,YAAY;QACxC,IAAI,WAAW,QAAQ,SAAS;GACrC,IAAI,aAAa,IAAI,YAAY,UAAU,IAAI,OAC7C,KAAK,KAAK,yDAAyD,CAAC,CAAC,IACnE,UACA,OACA,IACF;GAEF,OAAO;IAAE;IAAU;GAAM;EAC3B,CAAC;CACH;CAEA,QAAQ,MAAc,QAAsB;EAC1C,KAAK,WAAW;GACd,KAAK,IAAI,IAAI;GACb,KAAK,KACH;;;wBAIF,CAAC,CAAC,IAAI,QAAQ,KAAK,OAAO,KAAK,OAAO,IAAI;EAC5C,CAAC;CACH;CAEA,MAAM,MAAc,OAAqB;EACvC,KAAK,WAAW;GACd,KAAK,IAAI,IAAI;GACb,KAAK,KAAK,2DAA2D,CAAC,CAAC,IAAI,OAAO,IAAI;EACxF,CAAC;CACH;CAEA,KAAK,MAAc,OAAe,QAAsB;EACtD,MAAM,MAAM,OAAO,SAAS,KAAK,IAAI,QAAQ;EAC7C,KAAK,WAAW;GACd,KAAK,IAAI,IAAI;GACb,KAAK,KACH;mEAEF,CAAC,CAAC,IAAI,KAAK,QAAQ,MAAM,GAAG;EAC9B,CAAC;CACH;CAEA,QAAQ,MAAc,KAAmC;EACvD,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,gCAAgC;EACjE,MAAM,MAAM,KAAK,KAAK,0DAA0D,CAAC,CAAC,IAAI,IAAI;EAE1F,IAAI,QAAQ,KAAA,KAAa,IAAI,eAAe,QAAQ,IAAI,cAAc,KAAK,OAAO,KAAA;EAClF,OAAO;GACL,OAAO,IAAI,cAAc,SAAS,OAAO,oBAAoB,IAAI;GACjE,QAAQ,IAAI,eAAe;EAC7B;CACF;CAEA,MAAM,MAAqB;EACzB,KAAK,WAAW;GACd,IAAI,SAAS,KAAA,GACX,KAAK,KACH,iFACF,CAAC,CAAC,IAAI;QAEN,KAAK,KACH,gGACF,CAAC,CAAC,IAAI,IAAI;EAEd,CAAC;CACH;CAEA,WAAW,MAAyC;EAClD,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,gCAAgC;EACjE,MAAM,MAAM,KAAK,KACf;wCAEF,CAAC,CAAC,IAAI,IAAI;EACV,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,OAAO,GAAG;CACnD;CAEA,eAAe,MAAc,QAAyB,KAA4B;EAChF,OAAO,KAAK,WAAW;GACrB,MAAM,OAAO,kBAAkB,KAAK,WAAW,IAAI,GAAG,QAAQ,GAAG;GACjE,KAAK,KACH;;;;;4CAMF,CAAC,CAAC,IACA,MACA,KAAK,WAAW,IAAI,GACpB,KAAK,MACL,KAAK,cACL,KAAK,YACL,KAAK,YACL,KAAK,YACL,KAAK,SACP;GACA,OAAO;EACT,CAAC;CACH;;CAGA,IAAI,OAAe;EACjB,OAAQ,KAAK,KAAK,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAmB;CAC/E;;CAGA,QAAc;EACZ,IAAI,KAAK,QAAQ;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,MAAM;EACtB,KAAK,GAAG,MAAM;CAChB;AACF;;;;;;;;ACxRA,MAAa,eAWT;CAAE,WAAW;CAAK,eAAe;CAAK,MAAM;CAAK,SAAS;CAAK,WAAW;AAAG;;AA+BjF,MAAM,YACJ;AACF,MAAM,OAAO;AACb,MAAM,QAAQ;;AAGd,MAAM,oBAAoB;AAE1B,MAAM,eAAe,SAClB,QAAQ,MAAQ,QAAQ,MACxB,QAAQ,MAAQ,QAAQ,MACxB,QAAQ,MAAQ,QAAQ;AAC3B,MAAM,gBAAgB,SAA0B,SAAS,MAAS,QAAQ,KAAQ,QAAQ;AAK1F,MAAM,UAAU,UAAwB;CACtC,IAAI,QAAQ,KAAM,OAAO,YAAY,KAAK,IAAI,SAAS,aAAa,KAAK,IAAI,UAAU;CACvF,MAAM,OAAO,OAAO,cAAc,KAAK;CACvC,IAAI,UAAU,KAAK,IAAI,GAAG,OAAO;CACjC,IAAI,KAAK,KAAK,IAAI,GAAG,OAAO;CAC5B,OAAO,MAAM,KAAK,IAAI,IAAI,UAAU;AACtC;;;;;;;;AASA,MAAa,kBACX,MACA,QAAgB,GAChB,MAAc,KAAK,WACR;CACX,IAAI,SAAS;CACb,IAAI,MAAM;CACV,KAAK,IAAI,KAAK,OAAO,KAAK,MAAM;EAC9B,MAAM,QAAQ,KAAK,YAAY,EAAE;EACjC,MAAM,QAAQ,QAAS,IAAI;EAC3B,MAAM,OAAO,OAAO,KAAK;EACzB,IAAI,SAAS,QAAQ;GACnB,OAAO;GACP;EACF;EACA,IAAI,MAAM,GAAG,UAAU,KAAK,KAAK,MAAM,iBAAiB;EACxD,MAAM;EACN,IAAI,SAAS,SAAS,UAAU;CAClC;CACA,IAAI,MAAM,GAAG,UAAU,KAAK,KAAK,MAAM,iBAAiB;CACxD,OAAO;AACT;;AAYA,MAAM,OAAO;;AAGb,MAAM,eACJ,MACA,OACA,QACqC;CACrC,MAAM,OAAO,KAAK,MAAM,OAAO,GAAG;CAClC,MAAM,MAAwC,CAAC;CAC/C,IAAI,OAAO;CACX,KAAK,MAAM,SAAS,KAAK,SAAS,IAAI,GAAG;EACvC,MAAM,KAAK,MAAM,QAAQ,MAAM,EAAE,CAAC;EAClC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;EAClC,OAAO;CACT;CACA,IAAI,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;CACpD,OAAO,IACJ,KAAK,CAAC,GAAG,OAAkC;EAC1C,IAAI,QAAQ;EACZ,IAAI,OAAO;EACX,OAAO,QAAQ,QAAQ,MAAM,KAAK,KAAK,MAAO,GAAG,SAAS;EAC1D,OAAO,OAAO,SAAS,MAAM,KAAK,KAAK,OAAO,EAAG,GAAG,QAAQ;EAC5D,OAAO,CAAC,QAAQ,OAAO,QAAQ,IAAI;CACrC,CAAC,CAAC,CACD,QAAQ,CAAC,GAAG,OAAO,IAAI,CAAC;AAC7B;;AAGA,MAAM,WAAW,MAAc,OAAe,KAAa,SAA0B;CACnF,MAAM,MAAe,CAAC;CACtB,IAAI,OAAO;CACX,IAAI,SAAS;CACb,IAAI,MAAM;CACV,IAAI,YAAY;CAChB,IAAI,KAAK;CACT,MAAM,QAAQ,OAAqB;EACjC,IAAI,QAAQ;EACZ,OAAO,QAAQ,MAAM,MAAM,KAAK,KAAK,MAAO,GAAG,SAAS;EACxD,IAAI,OAAO;EACX,OAAO,OAAO,SAAS,MAAM,KAAK,KAAK,OAAO,EAAG,GAAG,QAAQ;EAC5D,IAAI,OAAO,OACT,IAAI,KAAK;GACP,OAAO;GACP,KAAK;GACL,QAAQ,eAAe,MAAM,OAAO,IAAI;GACxC,WAAW;EACb,CAAC;EAEH,OAAO;EACP,SAAS;EACT,MAAM;EACN,YAAY;CACd;CACA,OAAO,KAAK,KAAK;EACf,MAAM,QAAQ,KAAK,YAAY,EAAE;EACjC,MAAM,QAAQ,QAAQ,QAAS,IAAI;EACnC,MAAM,OAAO,OAAO,KAAK;EACzB,IAAI,OAAO;EACX,IAAI,SAAS,QAAQ;GACnB,OAAO;GACP,IAAI,MAAM,sBAAsB,GAAG,OAAO;EAC5C,OAAO;GACL,MAAM;GACN,IAAI,SAAS,SAAS,OAAO;EAC/B;EACA,IAAI,SAAS,OAAO,QAAQ,KAAK,MAAM;GAErC,IAAI,YAAY,MAAM;IACpB,MAAM,KAAK;IACX,KAAK,EAAE;IACP,KAAK;IACL;GACF;GACA,KAAK,EAAE;EACT;EACA,IAAI,SAAS,SAAS,YAAY;EAClC,UAAU;EACV,MAAM;CACR;CACA,KAAK,GAAG;CACR,OAAO;AACT;;AAGA,MAAM,YAAY,MAAc,SAA0B;CACxD,MAAM,SAAkB,CAAC;CACzB,KAAK,MAAM,QAAQ,KAAK,SAAS,WAAW,GAAG;EAC7C,IAAI,QAAQ;EACZ,KAAK,MAAM,CAAC,OAAO,QAAQ,YAAY,MAAM,KAAK,OAAO,KAAK,QAAQ,KAAK,EAAE,CAAC,MAAM,GAAG;GACrF,MAAM,SAAS,eAAe,MAAM,OAAO,GAAG;GAC9C,MAAM,QACJ,UAAU,OACN,CAAC;IAAE;IAAO;IAAK;IAAQ,WAAW;GAAM,CAAC,IACzC,QAAQ,MAAM,OAAO,KAAK,IAAI;GACpC,KAAK,MAAM,QAAQ,OAAO;IACxB,OAAO,KAAK;KAAE,GAAG;KAAM,WAAW;IAAM,CAAC;IACzC,QAAQ;GACV;EACF;CACF;CACA,OAAO;AACT;AAEA,MAAM,YAAY,MAAc,UAA0B;CACxD,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GACrC,MAAM,IAAI,WAAW,GAAG,KAAK,sCAAsC,OAAO,KAAK,GAAG;CACpF,OAAO,KAAK,MAAM,KAAK;AACzB;;;;;;;;;;AAWA,MAAa,aAAa,MAAc,YAAuC;CAC7E,MAAM,OAAO,SAAS,aAAa,QAAQ,SAAS;CAEpD,MAAM,UAAU,KAAK,IACnB,KAAK,MAAM,OAAO,CAAC,GACnB,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,iBAAiB,OAAO,aAAa,OAAO,CAAC,CAC9E;CACA,MAAM,YAAY,SAAS,aAAa,QAAQ,aAAa,aAAa,SAAS;CACnF,MAAM,SAAS,SAAS,MAAM,IAAI;CAClC,MAAM,SAAsB,CAAC;CAC7B,IAAI,QAAQ;CAEZ,IAAI,QAAQ;CACZ,OAAO,QAAQ,OAAO,UAAU,OAAO,SAAS,WAAW;EACzD,IAAI,OAAO;EACX,IAAI,SAAS,OAAO,MAAM,CAAE;EAC5B,OAAO,OAAO,IAAI,OAAO,UAAU,SAAS,OAAO,OAAO,EAAE,CAAE,UAAU,MAAM;GAC5E,QAAQ;GACR,UAAU,OAAO,KAAK,CAAE;EAC1B;EAEA,IAAI,OAAO,IAAI,OAAO,UAAU,CAAC,OAAO,OAAO,EAAE,CAAE,WAAW;GAC5D,IAAI,SAAS;GACb,IAAI,MAAM;GACV,KAAK,IAAI,KAAK,OAAO,KAAK,MAAM,MAAM,GAAG;IACvC,UAAU,OAAO,GAAG,CAAE;IACtB,IAAI,MAAM,SAAS,OAAO,KAAK,EAAE,CAAE,aAAa,SAAS,KAAK,MAAM,MAAM;GAC5E;GACA,IAAI,QAAQ,IAAI,OAAO;EACzB;EACA,QAAQ,OAAO;EACf,MAAM,QAAQ,OAAO,MAAM,CAAE;EAC7B,MAAM,MAAM,OAAO,KAAK,CAAE;EAC1B,OAAO,KAAK;GAAE,OAAO,OAAO;GAAQ;GAAO;GAAK,MAAM,KAAK,MAAM,OAAO,GAAG;EAAE,CAAC;EAC9E,IAAI,OAAO,KAAK,OAAO,QAAQ;EAE/B,IAAI,OAAO,OAAO;EAClB,IAAI,SAAS;EACb,OAAO,OAAO,IAAI,SAAS,SAAS,OAAO,OAAO,EAAE,CAAE,UAAU,SAAS;GACvE,QAAQ;GACR,UAAU,OAAO,KAAK,CAAE;EAC1B;EACA,IAAI,SAAS,OAAO,OAAO,EAAE,CAAE,SAAS,MAAM,OAAO,OAAO;EAC5D,QAAQ;CACV;CACA,OAAO;AACT;;;;;;;AAQA,MAAa,iBAAiB,aAAuD;CACnF,MAAM,QAAQ,SAAS;CACvB,MAAM,SACJ,UAAU,KAAA,KAAa,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,IACtD,aAAa,gBACb,KAAK,IAAI,OAAO,aAAa,SAAS;CAC5C,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,SAAS,aAAa,IAAI,CAAC;AAC3D;;;;;;AASA,MAAa,cAAc,WAA4C;CACrE,MAAM,MAAM,aAAa,KAAK,MAAM;CACpC,IAAI,MAAM;CACV,KAAK,MAAM,SAAS,KAAK;EACvB,IAAI,CAAC,OAAO,SAAS,KAAK,GACxB,MAAM,IAAI,WAAW,8CAA8C;EACrE,OAAO,QAAQ;CACjB;CACA,IAAI,QAAQ,GAAG,OAAO;CACtB,MAAM,QAAQ,IAAI,KAAK,KAAK,GAAG;CAC/B,KAAK,IAAI,KAAK,GAAG,KAAK,IAAI,QAAQ,MAAM,GAAG,IAAI,MAAM,IAAI,MAAO;CAChE,OAAO;AACT;;AAKA,MAAa,kBAOT;CAAE,YAAY;CAAI,WAAW;CAAI,SAAS;AAAG;;;;;;;;;AAuBjD,MAAa,aACX,OACA,IAAY,gBAAgB,YACZ;CAChB,IAAI,CAAC,OAAO,SAAS,CAAC,KAAK,IAAI,GAC7B,MAAM,IAAI,WAAW,4DAA4D,OAAO,CAAC,GAAG;CAC9F,MAAM,wBAAQ,IAAI,IAAkC;CACpD,MAAM,SAAS,MAAM,UAAU;EAC7B,KAAK,SAAS,IAAI,OAAO;GACvB,IAAI,QAAQ,MAAM,IAAI,EAAE;GACxB,IAAI,UAAU,KAAA,GAAW;IACvB,QAAQ,MAAM,UAAU,IAAI;IAC5B,MAAM,IAAI,IAAI,KAAK;GACrB;GACA,MAAM,WAAW,KAAK;EACxB,CAAC;CACH,CAAC;CAED,MAAM,QAAQ,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,WAAW;EAC5C,IAAI,QAAQ;EACZ,IAAI,OAAO,OAAO;EAClB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,SAAS,MAAM;GACnB,SAAS,KAAK,IAAI;GAClB,OAAO,KAAK,IAAI,MAAM,IAAI;EAC5B;EACA,OAAO;GAAE;GAAI;GAAO,OAAO;GAAO;GAAM,OAAO,MAAM,WAAW,SAAS,SAAS,IAAI;EAAE;CAC1F,CAAC;CACD,MAAM,MACH,GAAG,MACF,EAAE,QAAQ,EAAE,SACZ,EAAE,OAAO,EAAE,QACX,EAAE,QAAQ,EAAE,UACX,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI,EAC1C;CACA,OAAO,MAAM,KAAK,EAAE,IAAI,OAAO,OAAO,aAAa;EAAE;EAAI;EAAO,OAAO;CAAM,EAAE;AACjF;;;;ACnTA,MAAM,QAAQ;;AAEd,MAAM,cAAc;;AAEpB,MAAM,OAAO;AAEb,MAAM,UAAU,SAAyB,WAAW,QAAQ,CAAC,CAAC,OAAO,MAAM,MAAM,CAAC,CAAC,OAAO,KAAK;AAE/F,MAAM,SAAS,MAAc,OAA2B,UAAkB,QAAwB;CAChG,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GACrC,MAAM,IAAI,WAAW,GAAG,KAAK,sCAAsC,OAAO,KAAK,GAAG;CACpF,OAAO,KAAK,IAAI,KAAK,KAAK,MAAM,KAAK,CAAC;AACxC;;AAGA,MAAM,aAAa,UAAsC;CACvD,IAAI,UAAU,KAAA,GAAW,OAAO,gBAAgB;CAChD,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GACrC,MAAM,IAAI,WAAW,8CAA8C,OAAO,KAAK,GAAG;CACpF,OAAO;AACT;;AAGA,MAAM,iBAID;CACH,IAAI,gBAAoC,KAAA;CACxC,IAAI,eAAyC,KAAA;CAC7C,MAAM,UAAU,IAAI,SAAY,KAAK,OAAO;EAC1C,UAAU;EACV,SAAS;CACX,CAAC;CACD,QAAQ,YAAY,KAAA,CAAS;CAC7B,OAAO;EAAE;EAAS;EAAS;CAAO;AACpC;AAEA,MAAM,SAAS,OAAe,WAA2B;CACvD,MAAM,QAAQ,MAAM;CACpB,OAAO,KAAK,MAAM,QAAQ,KAAK,IAAI;AACrC;;;;;;AAUA,MAAa,sBACX,SACA,YACiB;CACjB,MAAM,EAAE,UAAU,UAAU;CAC5B,MAAM,SAAS,cAAc,QAAQ;CACrC,MAAM,OAAO,YAAY,OAAO,MAAM,EAAE,GAAG,OAAO,KAAK,MAAM,SAAS,aAAa,OAAO,CAAC,EAAE,GAAG,OAAO,aAAa,SAAS;CAC7H,MAAM,0BAAU,IAAI,IAA6B;CACjD,MAAM,0BAAU,IAAI,IAA0B;CAC9C,IAAI,aAAa,SAAS,aAAa,IAAI,SAAS,aAAa,MAAM,WAAW,SAAS,EAAE;;CAG7F,MAAM,WAAW,SAA+C;EAC9D,MAAM,wBAAQ,IAAI,IAAkB;EACpC,IAAI,KAAK,WAAW,GAAG,OAAO;EAC9B,MAAM,QAAQ,QAAQ;EACtB,IAAI,MAAM,YAAY,KAAA,GAAW;GAC/B,KAAK,MAAM,QAAQ,MAAM,QAAQ,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,KAAK,KAAK,IAAI;GACpE,OAAO;EACT;EACA,KAAK,MAAM,OAAO,MAAM;GACtB,MAAM,QAAQ,MAAM,IAAI,GAAG;GAC3B,IAAI,UAAU,KAAA,GAAW,MAAM,IAAI,KAAK,UAAU,KAAK,CAAC;EAC1D;EACA,OAAO;CACT;CACA,MAAM,aAAa,SAAoB;EACrC,KAAK,IAAI;EACT,MAAM,IAAI;EACV,UAAU,IAAI;EACd,UAAU,OAAO,IAAI,IAAI;CAC3B;;CAGA,MAAM,YAAY,OAChB,OACA,SACA,WAC4B;EAC5B,MAAM,MAAsB,CAAC;EAC7B,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,QAAQ,MAAM,OAAO;GAC/C,QAAQ,eAAe;GACvB,MAAM,OAAO,MAAM,MAAM,IAAI,KAAK,KAAK;GACvC,MAAM,QAAQ,MAAM,SAAS,MAAM,MAAM;IAAE;IAAQ;GAAQ,CAAC;GAC5D,IAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,WAAW,KAAK,QACjD,MAAM,IAAI,MACR,gBAAgB,SAAS,GAAG,QAAQ,OAAO,MAAM,QAAQ,KAAK,IAAI,MAAM,SAAS,CAAC,EAAE,eAAe,OAAO,KAAK,MAAM,EAAE,OACzH;GAEF,KAAK,MAAM,UAAU,OAAO;IAC1B,MAAM,OAAO,WAAW,MAAM;IAC9B,eAAe,KAAK;IACpB,IAAI,KAAK,WAAW,cAAc,KAAK,WAAW,GAChD,MAAM,IAAI,MACR,gBAAgB,SAAS,GAAG,oBAAoB,OAAO,KAAK,MAAM,EAAE,mBAAmB,OAAO,UAAU,GAC1G;IAEF,IAAI,KAAK,IAAI;GACf;EACF;EACA,OAAO;CACT;CAEA,MAAM,cAAc,OAAO,GAAW,WAA2D;EAC/F,MAAM,OAAO,EAAE,QAAQ,SAAS,GAAG,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;EAC/D,MAAM,QAAQ,QAAQ,IAAI,IAAI;EAC9B,IAAI,UAAU,KAAA,GAAW;GACvB,QAAQ,OAAO,IAAI;GACnB,QAAQ,IAAI,MAAM,KAAK;GACvB,OAAO;EACT;EACA,MAAM,CAAC,UAAU,MAAM,UAAU,CAAC,IAAI,GAAG,SAAS,MAAM;EACxD,QAAQ,IAAI,MAAM,MAAO;EACzB,IAAI,QAAQ,OAAO,aAAa,QAAQ,OAAO,QAAQ,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAM;EAC3E,OAAO;CACT;;;;;;CAOA,MAAM,SAAS,OACb,MACA,WACgF;EAChF,MAAM,QAAQ;GAAE,UAAU;GAAG,QAAQ;GAAG,MAAM;GAAG,QAAQ;EAAE;EAC3D,MAAM,MAAM,QAAQ,IAAI;EACxB,MAAM,QAAQ,QAAQ,IAAI;EAC1B,MAAM,0BAAU,IAAI,IAAgE;EACpF,MAAM,SAAiC,CAAC;EACxC,KAAK,MAAM,MAAM,MAAM;GACrB,MAAM,OAAO,MAAM,IAAI,EAAE;GACzB,IAAI,SAAS,KAAA,GAAW;GACxB,IAAI,WAAW,IAAI,GAAG;IACpB,MAAM,QAAQ;IACd,IAAI,MAAM,QAAQ,SAAS,IAAI,EAAE,MAAM,KAAA,GAAW,MAAM,OAAO,SAAS,IAAI,EAAE;IAC9E;GACF;GACA,IAAI,MAAM,KAAK,SAAS,IAAI,KAAK,QAAQ,MAAM,MAAM;IACnD,MAAM,UAAU;IAChB,IAAI,MAAM,QAAQ,SAAS,IAAI,EAAE,MAAM,KAAK,UAC1C,MAAM,MAAM,SAAS,IAAI,IAAI,KAAK,UAAU,GAAG;IACjD;GACF;GACA,MAAM,MAAM,QAAQ,MAAM,IAAI,EAAE;GAChC,IAAI,QAAQ,KAAA,KAAa,WAAW,GAAG,GAAG;GAC1C,MAAM,MAAM,OAAO,IAAI,IAAI;GAC3B,MAAM,QAAQ,QAAQ,IAAI,GAAG;GAC7B,IAAI,UAAU,KAAA,GAAW;IACvB,MAAM,KAAK,KAAK,EAAE;IAClB;GACF;GACA,MAAM,OAAO,QAAQ,IAAI,GAAG;GAC5B,IAAI,SAAS,KAAA,GAAW;IAEtB,MAAM,QAAQ,KAAK,MAAM,SAAS;KAChC,MAAM,MAAM,SAAS,IAAI,IAAI,KAAK,GAAG;KACrC,OAAO;IACT,CAAC;IAGD,MAAM,YAAY,KAAA,CAAS;IAC3B,OAAO,KAAK,KAAK;IACjB;GACF;GACA,QAAQ,IAAI,KAAK;IAAE,MAAM,IAAI;IAAM,MAAM,CAAC,EAAE;GAAE,CAAC;EACjD;EAEA,MAAM,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW;GAC9C,MAAM,MAAM;IACV;IACA;IACA,QAAQ,UAAU,MAAM,MAAM,EAAE,WAAW,OAAO,CAAC;IACnD,GAAG,SAAiB;GACtB;GACA,QAAQ,IAAI,KAAK,IAAI,OAAO;GAC5B,OAAO;EACT,CAAC;EACD,IAAI;GACF,KAAK,IAAI,KAAK,GAAG,KAAK,KAAK,SAAS;IAClC,MAAM,QAAqB,CAAC;IAC5B,IAAI,OAAO;IACX,OACE,KAAK,KAAK,WACT,MAAM,WAAW,KAAK,OAAO,KAAK,GAAG,CAAE,OAAO,UAAU,QACzD;KACA,MAAM,KAAK,KAAK,GAAI;KACpB,QAAQ,KAAK,GAAG,CAAE,OAAO;KACzB,MAAM;IACR;IACA,MAAM,UAAU,MAAM,UACpB,MAAM,SAAS,QAAQ,IAAI,OAAO,KAAK,UAAU,MAAM,IAAI,CAAC,GAC5D,YACA,MACF;IACA,IAAI,OAAO;IACX,KAAK,MAAM,OAAO,OAAO;KACvB,MAAM,SAAS,IAAI,OAAO,KAAK,WAAW;MACxC,OAAO,MAAM;MACb,KAAK,MAAM;MACX,QAAQ,QAAQ;KAClB,EAAE;KACF,MAAM,IAAI;MACR,UAAU,SAAS;MACnB,YAAY,cAAc,OAAO,EAAE,EAAE,OAAO,UAAU;MACtD,KAAK,IAAI;MACT;MACA,OAAO,IAAI,MAAM,KAAK;MACtB;MACA,MAAM,IAAI,MAAM;MAChB,IAAI;KACN,CAAC;KACD,MAAM,YAAY;KAClB,MAAM,UAAU,OAAO;KACvB,QAAQ,OAAO,IAAI,GAAG;KACtB,IAAI,QAAQ,OAAO,MAAM;IAC3B;GACF;EACF,SAAS,OAAO;GACd,KAAK,MAAM,OAAO,MAAM;IACtB,IAAI,QAAQ,IAAI,IAAI,GAAG,MAAM,IAAI,SAAS,QAAQ,OAAO,IAAI,GAAG;IAChE,IAAI,OAAO,KAAK;GAClB;GACA,MAAM;EACR;EACA,MAAM,QAAQ,IAAI,MAAM;EACxB,OAAO;CACT;;;;;;CAOA,MAAM,eACJ,OACA,OACA,MACA,WAC6D;EAC7D,MAAM,OAAO,MAAM,OAAO,SAAS,IAAI,OAAO,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,CAAC;EACpF,MAAM,MAAgE,CAAC;EACvE,KAAK,IAAI,KAAK,GAAG,KAAK,KAAK,UAAU,IAAI,SAAS,OAAO,MAAM,MAAM;GACnE,MAAM,OAAO,KAAK,MAAM,IAAI,KAAK,IAAI;GACrC,MAAM,OAAO,CACX,GAAG,IAAI,IACL,KAAK,SAAS,QACZ,WAAW,KAAA,IAAY,IAAI,OAAO,IAAI,KAAK,QAAQ,QAAQ,OAAO,IAAI,GAAG,CAAC,CAC5E,CACF,CACF;GACA,MAAM,QAAQ,QAAQ,IAAI;GAC1B,KAAK,MAAM,OAAO,MAChB,KAAK,MAAM,OAAO,IAAI,MAAM;IAC1B,IAAI,IAAI,UAAU,OAAO;IACzB,IAAI,WAAW,KAAA,KAAa,CAAC,OAAO,IAAI,GAAG,GAAG;IAC9C,MAAM,OAAO,MAAM,IAAI,GAAG;IAC1B,IAAI,SAAS,KAAA,KAAa,KAAK,aAAa,IAAI,OAAO,WAAW,IAAI,GAAG;IACzE,IAAI,SAAS,KAAA,KAAa,SAAS,QAAQ,KAAK,SAAS,MAAM;IAC/D,IAAI,KAAK;KAAE;KAAK;IAAI,CAAC;GACvB;EAEJ;EACA,OAAO;CACT;CAEA,MAAM,aAAa,SAA6B;EAC9C,OAAO,IAAI;EACX,KAAK,IAAI;EACT,YAAY,MAAM,IAAI,YAAY,CAAC;CACrC;CAEA,OAAO;EACL;EACA,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG;GAC9B,MAAM,QAAQ,WAAW,CAAC;GAC1B,IAAI,UAAU,KAAA,GAAW,OAAO,CAAC;GAEjC,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,KAAM,KAAK,MAAM,YAAY,SAAS,EAAE,KAAK,EAAE,CAAC;GACnF,MAAM,aAAa,MACjB,cACA,YAAY,YACZ,gBAAgB,YAChB,GACF;GACA,MAAM,YAAY,MAAM,aAAa,YAAY,WAAW,gBAAgB,WAAW,GAAI;GAC3F,MAAM,IAAI,UAAU,YAAY,OAAO;GACvC,MAAM,SAAS,YAAY;GAE3B,MAAM,SAAS,MAAM,YAAY,GAAG,MAAM;GAC1C,MAAM,OAAO,YAAY,QAAQ;GACjC,MAAM,UAAU,QAAQ,MAAM,KAAK,GAAG;IAAE,OAAO;IAAY;IAAM,MAAM;GAAQ,CAAC;GAChF,MAAM,OACJ,QAAQ,KAAK,QAAQ,IAAI,GAAG,GAC5B,MACF;GACA,MAAM,OAAO,YAAY,QAAQ,WAAW,MAAM,KAAA,CAAS;GAC3D,MAAM,QAAQ,UAAU,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,GAAG,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MACtF,GACA,KACF;GACA,MAAM,QAAQ,IAAI,IAAI,QAAQ,KAAK,QAAQ,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;GAC1D,MAAM,SAAS,IAAI,IAAI,KAAK,KAAK,QAAQ,CAAC,IAAI,KAAK,IAAI,GAAG,CAAC,CAAC;GAC5D,MAAM,UAAmB,CAAC;GAC1B,KAAK,MAAM,OAAO,OAAO;IACvB,MAAM,MAAM,QAAQ,MAAM,IAAI,IAAI,EAAE;IACpC,IAAI,QAAQ,KAAA,KAAa,WAAW,GAAG,GAAG;IAC1C,MAAM,OAAO,OAAO,IAAI,IAAI,EAAE,KAAK,MAAM,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,GAAG,MAAM;IACnF,MAAM,aAAa,MAAM,IAAI,IAAI,EAAE;IACnC,MAAM,WAAW,IAAI,SAAS,aAAa,WAAW,IAAI,IAAI,IAAI,KAAA;IAClE,QAAQ,KAAK;KACX,KAAK,IAAI;KACT,MAAM,IAAI;KACV,OAAO,IAAI;KACX,KAAK,IAAI;KACT,SACE,SAAS,KAAA,IACJ,YAAY,WAAW,UAAU,IAAI,MAAM,MAAM,KAAK,IACvD,UAAU,IAAI,KAAK,MAAM,KAAK,OAAO,KAAK,GAAG,GAAG,MAAM,KAAK;KACjE,OAAO,MAAM,IAAI,OAAO,CAAC;KACzB,UAAU,IAAI;KACd,YAAY,IAAI;KAChB,SAAS,IAAI;KACb,GAAI,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS;KAC7C,GAAI,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,UAAU,IAAI,EAAE;IAC3D,CAAC;GACH;GACA,OAAO;EACT;EACA,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,GAAG;GAE1C,IADc,WAAW,CACjB,MAAM,KAAA,KAAa,KAAK,WAAW,GAAG,OAAO,CAAC;GACtD,MAAM,aAAa,MACjB,cACA,YAAY,YACZ,gBAAgB,YAChB,GACF;GACA,MAAM,YAAY,MAChB,aACA,YAAY,WACZ,gBAAgB,WAChB,GACF;GACA,MAAM,IAAI,UAAU,YAAY,OAAO;GACvC,MAAM,SAAS,YAAY;GAC3B,MAAM,SAAS,MAAM,YAAY,GAAG,MAAM;GAC1C,MAAM,UAAU,QAAQ,MAAM,WAAW,GAAG,MAAM,EAAE,MAAM,QAAQ,CAAC;GAEnE,MAAM,QAAQ,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,QAAQ,KAAK,QAAQ,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,UAAU;GAC3F,MAAM,OAAO,OAAO,MAAM;GAC1B,MAAM,SAAS,IAAI,IAAI,IAAI;GAC3B,MAAM,OAAO,YAAY,QAAQ,WAAW,KAAA,GAAW,MAAM;GAC7D,MAAM,SAAS,IAAI,IAAI,KAAK,KAAK,QAAQ,CAAC,IAAI,KAAK,IAAI,GAAG,CAAC,CAAC;GAC5D,MAAM,QAAQ,QAAQ,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,CAAC,QAAQ,QAAQ,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC;GACrF,OAAO,UAAU,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,GAAG,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,KAC9E,QAAQ;IACP,MAAM,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC,EAAE;IAC/B,MAAM,OACJ,OAAO,IAAI,IAAI,EAAE,MAChB,QAAQ,KAAA,IAAY,KAAA,IAAY,MAAM,KAAK,SAAS,IAAI,KAAK,MAAM;IACtE,OAAO;KACL,KAAK,IAAI;KACT,OAAO,MAAM,IAAI,OAAO,CAAC;KACzB,GAAI,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,UAAU,IAAI,EAAE;IAC3D;GACF,CACF;EACF;EACA,MAAM,MAAM,eAAe,CAAC,GAAG;GAC7B,MAAM,UAAU,KAAK,IAAI;GACzB,MAAM,WAAW,WAAW,aAAa,UAAU,KAAK;GACxD,MAAM,UACJ,aAAa,YAAY,KAAA,IACrB,OAAO,oBACP,MAAM,WAAW,aAAa,SAAS,GAAG,OAAO,gBAAgB;GACvE,MAAM,OAAO,aAAa,QAAQ;GAElC,MAAM,QAAQ,IAAI,gBAAgB;GAElC,MAAM,QAAQ,iBACN,MAAM,sBAAM,IAAI,MAAM,6BAA6B,CAAC,GAC1D,KAAK,IAAI,UAAU,UAAa,CAClC;GACA,MAAM,MAAM;GACZ,MAAM,SACJ,aAAa,WAAW,KAAA,IACpB,MAAM,SACN,YAAY,IAAI,CAAC,aAAa,QAAQ,MAAM,MAAM,CAAC;GACzD,MAAM,QAAQ;IAAE,MAAM;IAAG,UAAU;IAAG,QAAQ;IAAG,WAAW;IAAG,QAAQ;GAAE;GACzE,IAAI,UAAqB;GACzB,MAAM,eACJ,aAAa,QAAQ,YAAY,OAC7B,WACA,MAAM,OAAO,WAAW,KAAK,IAAI,IAAI,WAAW,WAC9C,WACA,MAAM,YAAY,UAChB,aACA,KAAA;GACV,IAAI;IACF,WAAW,MAAM,QAAQ,QAAQ,OAAO,GAAG;KACzC,MAAM,OAAO,KAAK,QAAQ,SAAS,SAAS,QAAQ,KAAK,SAAS,IAAI;KACtE,KAAK,IAAI,KAAK,GAAG,KAAK,KAAK,SAAS;MAClC,MAAM,OAAO,OAAO;MACpB,IAAI,SAAS,KAAA,GAAW;OACtB,UAAU;OACV;MACF;MAEA,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,OAAO,UAAU,MAAM,QAAQ,CAAC;MAClE,MAAM,QAAQ,KAAK,MAAM,IAAI,KAAK,IAAI,CAAC,CAAC,KAAK,SAAS,KAAK,GAAG;MAC9D,MAAM,MAAM;MACZ,IAAI;OACF,MAAM,OAAO,MAAM,OAAO,OAAO,MAAM;OACvC,MAAM,QAAQ,MAAM;OACpB,MAAM,YAAY,KAAK;OACvB,MAAM,UAAU,KAAK;OACrB,MAAM,aAAa,KAAK;OACxB,MAAM,UAAU,KAAK;MACvB,SAAS,OAAO;OACd,MAAM,OAAO,OAAO;OACpB,IAAI,SAAS,KAAA,GAAW,MAAM;OAC9B,UAAU;OACV;MACF;KACF;KACA,IAAI,YAAY,QAAQ;IAC1B;GACF,UAAU;IACR,aAAa,KAAK;GACpB;GACA,OAAO;IAAE,UAAU,SAAS;IAAI,GAAG;IAAO;IAAS,IAAI,KAAK,IAAI,IAAI;GAAQ;EAC9E;CACF;AACF;;;;;AAMA,gBAAgB,QAAQ,SAA+C;CACrE,MAAM,QAAQ,QAAQ;CACtB,IAAI,MAAM,YAAY,KAAA,GAAW;EAC/B,IAAI,QAAQ;EACZ,SAAS;GACP,MAAM,OAAO,MAAM,QAAQ;IAAE;IAAO,OAAO;GAAI,CAAC;GAChD,IAAI,KAAK,WAAW,GAAG;GACvB,MAAM;GACN,QAAQ,KAAK,GAAG,EAAE,CAAC,CAAE;EACvB;CACF;CACA,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI,OAAe,CAAC;CACpB,WAAW,MAAM,YAAY,QAAQ,SAAS,KAAK,EAAE,QAAQ,QAAQ,CAAC,GACpE,KAAK,MAAM,OAAO,MAAM,cAAc,SAAS,EAAE,GAAG;EAClD,IAAI,KAAK,IAAI,IAAI,GAAG,GAAG;EACvB,KAAK,IAAI,IAAI,GAAG;EAChB,KAAK,KAAK;GACR,KAAK,IAAI;GACT,MAAM,IAAI;GACV,UAAU,IAAI;GACd,UAAU,OAAO,IAAI,IAAI;EAC3B,CAAC;EACD,IAAI,KAAK,UAAU,KAAK;GACtB,MAAM;GACN,OAAO,CAAC;EACV;CACF;CAEF,IAAI,KAAK,SAAS,GAAG,MAAM;AAC7B;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC5jBA,MAAa,iBAAiB;AA6E9B,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDZ,MAAM,WAAW,WACf,IAAI,WAAW,OAAO,QAAQ,OAAO,YAAY,OAAO,UAAU;;AAGpE,MAAM,YAAY,UAAoC;CACpD,MAAM,MAAM,IAAI,aAAa,KAAK,MAAM,MAAM,aAAa,CAAC,CAAC;CAC7D,IAAI,WAAW,IAAI,MAAM,CAAC,CAAC,IAAI,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,CAAC;CAChE,OAAO;AACT;AAEA,MAAM,OAAO,MAAoB,KAAa,OAAqB,SAAyB;CAC1F,IAAI,MAAM;CACV,MAAM,OAAO,MAAM;CACnB,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,MAAM,GAAG,OAAO,KAAK,OAAO,MAAO,MAAM;CACrE,OAAO;AACT;;AAGA,IAAa,cAAb,MAAyB;CAOZ;CANX;CACA,6BAA8B,IAAI,IAA2B;CAC7D,2BAA4B,IAAI,IAAoB;CACpD,SAAiB;CAEjB,YACE,MACA,UAA8B,CAAC,GAC/B;EAFS,KAAA,OAAA;EAGT,IAAI,SAAS,YAAY,UAAU,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAClF,KAAK,KAAK,IAAI,aAAa,IAAI;EAC/B,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,iBAAiB,GAAM,CAAC;EACpE,KAAK,GAAG,KAAK,yBAAyB,OAAO,IAAI,EAAE,EAAE;EACrD,IAAI,SAAS,YAAY,KAAK,GAAG,KAAK,4BAA4B;EAElE,KAAK,GAAG,KAAK,8BAA8B;EAC3C,KAAK,YAAY;GAGf,IAFiB,KAAK,GAAG,QAAQ,qBAAqB,CAAC,CAAC,IAAI,CAAC,CAC1D,iBAAA,GAC6B;IAE9B,MAAM,SAAS,KAAK,GACjB,QACC,kFACF,CAAC,CACA,IAAI;IACP,KAAK,MAAM,EAAE,UAAU,QAAQ,KAAK,GAAG,KAAK,yBAAyB,KAAK,EAAE;GAC9E;GACA,KAAK,GAAG,KAAK,GAAG;GAChB,KAAK,GAAG,KAAK,yBAAyB,OAAA,CAAqB,GAAG;EAChE,CAAC;CACH;CAEA,KAAa,KAA4B;EACvC,IAAI,YAAY,KAAK,WAAW,IAAI,GAAG;EACvC,IAAI,cAAc,KAAA,GAAW;GAC3B,YAAY,KAAK,GAAG,QAAQ,GAAG;GAC/B,KAAK,WAAW,IAAI,KAAK,SAAS;EACpC;EACA,OAAO;CACT;CAEA,MAAiB,MAAkB;EACjC,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,4BAA4B;EAC7D,KAAK,GAAG,KAAK,iBAAiB;EAC9B,IAAI;GACF,MAAM,SAAS,KAAK;GACpB,KAAK,GAAG,KAAK,QAAQ;GACrB,OAAO;EACT,SAAS,OAAO;GACd,IAAI,KAAK,GAAG,eAAe,KAAK,GAAG,KAAK,UAAU;GAClD,MAAM;EACR;CACF;CAEA,OAAqB;EACnB,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,4BAA4B;CAC/D;;CAGA,KAAK,UAAkB,KAAiC;EACtD,KAAK,KAAK;EACV,OACE,KAAK,KAAK,uDAAuD,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,EAEpF;CACL;;CAGA,QAAQ,UAAkB,KAAiC;EACzD,KAAK,KAAK;EACV,OACE,KAAK,KAAK,qDAAqD,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,EAElF;CACL;;CAGA,WAAW,UAAsC;EAC/C,KAAK,KAAK;EACV,OACE,KAAK,KAAK,+CAA+C,CAAC,CAAC,IAAI,QAAQ,CAAC,EAEvE;CACL;;;;;CAMA,IAAI,OAA0B;EAC5B,MAAM,OAAO,MAAM;EACnB,KAAK,MAAM,SAAS,MAAM,QACxB,IAAI,MAAM,OAAO,WAAW,MAC1B,MAAM,IAAI,MACR,gBAAgB,MAAM,SAAS,oBAAoB,OAAO,MAAM,OAAO,MAAM,EAAE,mBAAmB,OAAO,IAAI,GAC/G;EAGJ,IAAI,WAAW;EACf,KAAK,YAAY;GACf,MAAM,QAAQ,KAAK,KAAK,+CAA+C,CAAC,CAAC,IACvE,MAAM,QACR;GAEA,IAAI,UAAU,KAAA,KAAa,MAAM,OAAO,SAAS,GAC/C,KAAK,KAAK,mEAAmE,CAAC,CAAC,IAC7E,MAAM,UACN,MACA,MAAM,EACR;QACK,IAAI,UAAU,KAAA,KAAa,MAAM,OAAO,SAAS,KAAK,MAAM,eAAe,MAChF,MAAM,IAAI,MACR,mCAAmC,OAAO,MAAM,UAAU,EAAE,+BAA+B,MAAM,SAAS,gBAAgB,OAAO,IAAI,EAAE,uDACzI;GAEF,MAAM,SAAS,KAAK,KAAK,uDAAuD,CAAC,CAAC,IAChF,MAAM,KACN,MAAM,QACR;GACA,WAAW,WAAW,KAAA,KAAa,OAAO,SAAS,MAAM;GACzD,KAAK,KAAK,mDAAmD,CAAC,CAAC,IAAI,MAAM,KAAK,MAAM,QAAQ;GAC5F,MAAM,SAAS,KAAK,KAClB,mGACF;GACA,MAAM,OAAO,SAAS,OAAO,OAAO;IAClC,OAAO,IAAI,MAAM,KAAK,IAAI,MAAM,UAAU,MAAM,OAAO,MAAM,KAAK,QAAQ,MAAM,MAAM,CAAC;GACzF,CAAC;GACD,KAAK,KACH;;sEAGF,CAAC,CAAC,IAAI,MAAM,KAAK,MAAM,UAAU,MAAM,MAAM,MAAM,OAAO,QAAQ,MAAM,OAAO,MAAM,EAAE;GACvF,KAAK,MAAM,OAAO,MAAM,MAAM,KAAK,SAAS,MAAM,UAAU,KAAK,MAAM,KAAK,MAAM,EAAE;EACtF,CAAC;EACD,MAAM,SAAS,KAAK,SAAS,IAAI,MAAM,QAAQ;EAC/C,IAAI,WAAW,KAAA,GAAW;EAC1B,IAAI,YAAY,OAAO,MAAM,IAAI,MAAM,GAAG,GAAG;GAE3C,KAAK,SAAS,OAAO,MAAM,QAAQ;GACnC;EACF;EACA,KAAK,OACH,QACA,MAAM,KACN,MAAM,OAAO,KAAK,OAAO,QAAQ;GAAE,GAAG;GAAO,OAAO;EAAG,EAAE,CAC3D;EACA,KAAK,MAAM,OAAO,MAAM,MAAM,KAAK,YAAY,QAAQ,KAAK,MAAM,GAAG;CACvE;CAEA,SAAiB,UAAkB,KAAa,KAAa,IAAkB;EAC7E,KAAK,KACH;qGAEF,CAAC,CAAC,IAAI,KAAK,UAAU,KAAK,EAAE;CAC9B;;CAGA,MAAM,UAAkB,KAAa,KAAa,IAAkB;EAClE,KAAK,YAAY,KAAK,SAAS,UAAU,KAAK,KAAK,EAAE,CAAC;EACtD,MAAM,SAAS,KAAK,SAAS,IAAI,QAAQ;EACzC,IAAI,WAAW,KAAA,GAAW;EAC1B,IAAI,CAAC,OAAO,MAAM,IAAI,GAAG,GAAG,KAAK,OAAO,QAAQ,KAAK,KAAK,OAAO,UAAU,GAAG,CAAC;EAC/E,KAAK,YAAY,QAAQ,KAAK,GAAG;CACnC;;CAGA,OAAO,UAAkB,KAAmB;EAC1C,KAAK,YACH,KAAK,KAAK,iDAAiD,CAAC,CAAC,IAAI,KAAK,QAAQ,CAChF;EACA,MAAM,SAAS,KAAK,SAAS,IAAI,QAAQ;EACzC,IAAI,WAAW,KAAA,GAAW;EAC1B,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG;EAChC,IAAI,QAAQ,KAAA,GAAW,OAAO,OAAO,IAAI,GAAG,CAAC,EAAE,OAAO,GAAG;EACzD,OAAO,MAAM,OAAO,GAAG;CACzB;;CAGA,OAAO,UAAkB,KAA4B;EACnD,KAAK,KAAK;EACV,OACE,KAAK,KACH,sGACF,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAMpB,KAAK,SAAS;GACd,OAAO,IAAI;GACX,OAAO,IAAI;GACX,KAAK,IAAI;GACT,QAAQ,SAAS,IAAI,MAAM;EAC7B,EAAE;CACJ;CAEA,YAAoB,QAAgB,KAAa,KAAmB;EAClE,MAAM,SAAS,OAAO,MAAM,IAAI,GAAG;EACnC,IAAI,WAAW,KAAA,KAAa,WAAW,KAAK,OAAO,OAAO,IAAI,MAAM,CAAC,EAAE,OAAO,GAAG;EACjF,OAAO,MAAM,IAAI,KAAK,GAAG;EACzB,IAAI,OAAO,OAAO,OAAO,IAAI,GAAG;EAChC,IAAI,SAAS,KAAA,GAAW;GACtB,uBAAO,IAAI,IAAI;GACf,OAAO,OAAO,IAAI,KAAK,IAAI;EAC7B;EACA,KAAK,IAAI,GAAG;CACd;CAEA,OAAe,QAAgB,KAAa,QAAsC;EAChF,MAAM,OAAO,OAAO;EACpB,MAAM,OAAO,OAAO,OAAO,OAAO;EAClC,IAAI,OAAO,OAAO,MAAM,QAAQ;GAC9B,MAAM,OAAO,KAAK,IAAI,MAAM,OAAO,MAAM,SAAS,GAAG,EAAE;GACvD,MAAM,OAAO,IAAI,aAAa,OAAO,IAAI;GACzC,KAAK,IAAI,OAAO,KAAK,SAAS,GAAG,OAAO,OAAO,IAAI,CAAC;GACpD,MAAM,QAAQ,QAAgC;IAC5C,MAAM,OAAO,IAAI,WAAW,IAAI;IAChC,KAAK,IAAI,IAAI,SAAS,GAAG,OAAO,IAAI,CAAC;IACrC,OAAO;GACT;GACA,OAAO,OAAO;GACd,OAAO,QAAQ,KAAK,OAAO,KAAK;GAChC,OAAO,QAAQ,KAAK,OAAO,KAAK;GAChC,OAAO,MAAM,KAAK,OAAO,GAAG;EAC9B;EACA,MAAM,OAAO,OAAO;EACpB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,MAAM,OAAO;GACnB,IAAI,MAAM,OAAO,WAAW,MAAM,OAAO,KAAK,IAAI,MAAM,QAAQ,MAAM,IAAI;GAC1E,OAAO,MAAM,OAAO,MAAM;GAC1B,OAAO,MAAM,OAAO,MAAM;GAC1B,OAAO,IAAI,OAAO,MAAM;GACxB,OAAO,QAAQ;EACjB;EACA,OAAO,MAAM,IAAI,KAAK;GAAE;GAAM,IAAI,OAAO;EAAK,CAAC;CACjD;;CAGA,OAAe,UAAsC;EACnD,KAAK,KAAK;EACV,MAAM,UAAW,KAAK,GAAG,QAAQ,qBAAqB,CAAC,CAAC,IAAI,CAAC,CAC1D;EACH,MAAM,SAAS,KAAK,SAAS,IAAI,QAAQ;EACzC,IAAI,WAAW,KAAA,KAAa,OAAO,YAAY,SAAS,OAAO;EAC/D,MAAM,OAAO,KAAK,WAAW,QAAQ;EACrC,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;EAE/B,KAAK,GAAG,KAAK,OAAO;EACpB,IAAI;GACF,MAAM,SAAS,KAAK,KAAK,UAAU,MAAM,OAAO;GAChD,KAAK,SAAS,IAAI,UAAU,MAAM;GAClC,OAAO;EACT,UAAU;GACR,IAAI,KAAK,GAAG,eAAe,KAAK,GAAG,KAAK,QAAQ;EAClD;CACF;CAEA,KAAa,UAAkB,MAAc,SAAyB;EACpE,MAAM,QACJ,KAAK,GACF,QACC,0GACF,CAAC,CACA,IAAI,UAAU,QAAQ,CAAC,CAC1B;EACF,MAAM,SAAiB;GACrB,YAAY;GACZ,MAAM;GACN,MAAM,IAAI,aAAa,KAAK,IAAI,GAAG,KAAK,IAAI,IAAI;GAChD,OAAO,IAAI,WAAW,KAAK,IAAI,GAAG,KAAK,CAAC;GACxC,OAAO,IAAI,WAAW,KAAK,IAAI,GAAG,KAAK,CAAC;GACxC,KAAK,IAAI,WAAW,KAAK,IAAI,GAAG,KAAK,CAAC;GACtC,uBAAO,IAAI,IAAI;GACf,wBAAQ,IAAI,IAAI;GAChB,uBAAO,IAAI,IAAI;GACf;EACF;EACA,MAAM,OAAO,KAAK,GACf,QACC;;8BAGF,CAAC,CACA,QAAQ,UAAU,QAAQ;EAO7B,IAAI;EACJ,IAAI,OAAO;EACX,KAAK,MAAM,OAAO,MAAM;GACtB,IAAI,IAAI,QAAQ,KAAK;IACnB,IAAI,QAAQ,KAAA,GAAW,OAAO,MAAM,IAAI,KAAK;KAAE;KAAM,IAAI,OAAO;IAAK,CAAC;IACtE,MAAM,IAAI;IACV,OAAO,OAAO;GAChB;GACA,IAAI,OAAO,QAAQ,OAAO,MAAM,QAAQ;GACxC,MAAM,KAAK,OAAO;GAClB,IAAI,IAAI,OAAO,eAAe,OAAO,GACnC,IAAI,WAAW,OAAO,KAAK,QAAQ,KAAK,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,MAAM;GAE5E,OAAO,MAAM,MAAM,IAAI;GACvB,OAAO,MAAM,MAAM,IAAI;GACvB,OAAO,IAAI,MAAM,IAAI;GACrB,OAAO,QAAQ;EACjB;EACA,IAAI,QAAQ,KAAA,GAAW,OAAO,MAAM,IAAI,KAAK;GAAE;GAAM,IAAI,OAAO;EAAK,CAAC;EACtE,KAAK,MAAM,OAAO,KAAK,GACpB,QAAQ,8CAA8C,CAAC,CACvD,IAAI,QAAQ,GACb,KAAK,YAAY,QAAQ,IAAI,KAAK,IAAI,GAAG;EAE3C,OAAO;CACT;;CAGA,KAAK,UAAkB,KAAa,OAA4C;EAC9E,MAAM,SAAS,KAAK,OAAO,QAAQ;EACnC,MAAM,QAAQ,QAAQ,MAAM,IAAI,GAAG;EACnC,IAAI,WAAW,KAAA,KAAa,UAAU,KAAA,KAAa,MAAM,OAAO,MAAM,MAAM,OAAO,KAAA;EACnF,OAAO,KAAK,MAAM,QAAQ,KAAK,OAAO,KAAK;CAC7C;CAEA,MACE,QACA,KACA,OACA,OACW;EACX,IAAI,UAAU,MAAM;EACpB,IAAI,YAAY,OAAO;EACvB,KAAK,IAAI,MAAM,MAAM,MAAM,MAAM,MAAM,IAAI,OAAO,GAAG;GACnD,MAAM,QAAQ,IAAI,OAAO,MAAM,KAAK,OAAO,OAAO,UAAU;GAC5D,IAAI,QAAQ,WAAW;IACrB,YAAY;IACZ,UAAU;GACZ;EACF;EACA,OAAO;GACL;GACA,OAAO,OAAO,MAAM;GACpB,OAAO,OAAO,MAAM;GACpB,KAAK,OAAO,IAAI;GAChB,YAAY;GACZ,MAAM,CAAC,GAAI,OAAO,OAAO,IAAI,GAAG,KAAK,CAAC,CAAE;EAC1C;CACF;;;;;;CAOA,OACE,UACA,OACA,UAA8D,CAAC,GAClD;EACb,MAAM,SAAS,KAAK,OAAO,QAAQ;EACnC,IAAI,WAAW,KAAA,GAAW,OAAO,CAAC;EAClC,IAAI,MAAM,WAAW,OAAO,YAC1B,MAAM,IAAI,MACR,gCAAgC,OAAO,MAAM,MAAM,EAAE,mCAAmC,OAAO,OAAO,UAAU,EAAE,oBAAoB,UACxI;EAEF,MAAM,uBAAO,IAAI,IAAY;EAC7B,IAAI,QAAQ,WAAW,KAAA,GAChB;QAAA,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,IAAI,KAAK,OAAO,GAAG,KAAK,IAAI,GAAG;EAAA,OAExE,KAAK,MAAM,OAAO,QAAQ,QAAQ;GAChC,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG;GAChC,IAAI,QAAQ,KAAA,GAAW,KAAK,IAAI,GAAG;EACrC;EAEF,MAAM,OAAoB,CAAC;EAC3B,KAAK,MAAM,OAAO,MAAM;GACtB,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;GAClC,IAAI,UAAU,KAAA,KAAa,MAAM,OAAO,MAAM,MAAM;GACpD,KAAK,KAAK,KAAK,MAAM,QAAQ,KAAK,OAAO,KAAK,CAAC;EACjD;EACA,OAAO,KAAK,MAAM,GAAG,MAAM,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,KAAK,EAAE;CACpF;;CAGA,OAAO,UAAiC;EACtC,KAAK,KAAK;EACV,MAAM,QAAQ,aAAa,KAAA,IAAY,KAAK;EAC5C,MAAM,SAAS,aAAa,KAAA,IAAY,CAAC,IAAI,CAAC,QAAQ;EACtD,MAAM,OAAO,QACV,KAAK,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAA0B,KAAK;EAC/D,OAAO;GACL,OAAO,IAAI,kCAAkC,OAAO;GACpD,QAAQ,IAAI,mCAAmC,OAAO;GACtD,MAAM,IAAI,iCAAiC,OAAO;GAClD,OAAO,IAAI,8CAA8C,OAAO;EAClE;CACF;;CAGA,QAAc;EACZ,IAAI,KAAK,QAAQ;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,MAAM;EACtB,KAAK,SAAS,MAAM;EACpB,KAAK,GAAG,MAAM;CAChB;AACF;;;;;;;;;;;;;;;;;;;;;;AC1iBA,MAAa,eAAe;;AAG5B,MAAa,eAAe;;CAE1B,SAAS;;CAET,SAAS;;CAET,UAAU;;CAEV,UAAU;;CAEV,aAAa;;CAEb,SAAS;AACX;AAqPA,MAAM,OAAO,OAAuB,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY;AAE7D,MAAM,aAAgB,MAAqB,aAAmB;CAC5D,IAAI,SAAS,MAAM,OAAO;CAC1B,IAAI;EACF,OAAO,KAAK,MAAM,IAAI;CACxB,QAAQ;EACN,OAAO;CACT;AACF;AAEA,MAAM,0BAA+B,IAAI,IAAI;CAAC;CAAQ;CAAS;AAAQ,CAAC;AAExE,MAAM,WAAW,KAAe,SAA6B;CAC3D,KAAK,IAAI;CACT,UAAU,IAAI;CACd,QAAQ,IAAI;CACZ,SAAS,IAAI,YAAY,IAAI,IAAI;CACjC,OAAO,IAAI;CACX,QAAQ,UAAyC,IAAI,QAAQ,IAAI;CACjE,cAAc,UAAoB,IAAI,cAAc,CAAC,CAAC;CACtD,UAAU,IAAI,IAAI,QAAQ;CAC1B,SAAS,IAAI,IAAI,OAAO;CACxB,YAAY,IAAI,YAAY,OAAO,OAAO,IAAI,IAAI,OAAO;CACzD,WAAW,IAAI;CACf,WAAW,IAAI;CACf,QAAQ,IAAI;CACZ,SAAS,IAAI;CACb,KAAK,IAAI;CACT,SAAS,IAAI;CACb,YAAY,IAAI;CAChB,MAAM,IAAI,eAAe;CACzB,SACE,IAAI,eAAe,OACf,OACA;EACE,MAAM,IAAI,eAAe,SAAS,SAAS,IAAI,eAAe,SAAS,QAAQ;EAC/E,SAAS,IAAI;EACb,OAAO,IAAI,IAAI,cAAc,IAAI,WAAW,IAAI,QAAQ;CAC1D;CACN,cAAc,IAAI,cAAc,MAAM,IAAI,IAAI,WAAW,IAAI;AAC/D;;AAGA,MAAM,YACJ,QAC6E;CAC7E,IAAI;CACJ,IAAI;EACF,SAAS,IAAI,IAAI,GAAG;CACtB,QAAQ;EACN,MAAM,IAAI,WAAW,oBAAoB,KAAK;CAChD;CACA,IAAI,OAAO,aAAa,WAAW,OAAO,aAAa,UACrD,MAAM,IAAI,WAAW,2CAA2C,KAAK;CACvE,OAAO,OAAO;CACd,OAAO;EAAE,KAAK,WAAW,OAAO,IAAI;EAAG,QAAQ,OAAO;EAAM,MAAM,OAAO,KAAK,YAAY;CAAE;AAC9F;;;;;;;;;;;AAYA,IAAa,aAAb,MAAwB;CAQX;CAPX;CACA,6BAA8B,IAAI,IAA2B;CAC7D,SAAiB;;CAEjB;CAEA,YACE,MACA,UAA6B,CAAC,GAC9B;EAFS,KAAA,OAAA;EAGT,IAAI,SAAS,YAAY,UAAU,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAClF,MAAM,QAAQ,QAAQ,WAAW,aAAa;EAC9C,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GACrC,MAAM,IAAI,WACR,0DAA0D,OAAO,KAAK,GACxE;EACF,KAAK,UAAU;EACf,KAAK,KAAK,IAAI,aAAa,IAAI;EAC/B,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,iBAAiB,GAAM,CAAC;EACpE,KAAK,GAAG,KAAK,yBAAyB,OAAO,IAAI,EAAE,EAAE;EACrD,IAAI,SAAS,YAAY,KAAK,GAAG,KAAK,4BAA4B;EAClE,KAAK,GAAG,KAAK,8BAA8B;EAC3C,KAAK,WAAW;GACd,KAAK,GAAG,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QA6BX;GACF,KAAK,GAAG,KAAK,6DAA6D;GAC1E,MAAM,EAAE,cAAc,YAAY,KAAK,GAAG,QAAQ,qBAAqB,CAAC,CAAC,IAAI;GAG7E,IAAI,UAAA,GAAwB,KAAK,GAAG,KAAK,yBAAyB,OAAA,CAAmB,GAAG;EAC1F,CAAC;CACH;CAEA,KAAa,KAA4B;EACvC,IAAI,YAAY,KAAK,WAAW,IAAI,GAAG;EACvC,IAAI,cAAc,KAAA,GAAW;GAC3B,YAAY,KAAK,GAAG,QAAQ,GAAG;GAC/B,KAAK,WAAW,IAAI,KAAK,SAAS;EACpC;EACA,OAAO;CACT;;CAGA,KAAgB,MAAkB;EAChC,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,2BAA2B;EAC5D,KAAK,GAAG,KAAK,iBAAiB;EAC9B,IAAI;GACF,MAAM,SAAS,KAAK;GACpB,KAAK,GAAG,KAAK,QAAQ;GACrB,OAAO;EACT,SAAS,OAAO;GACd,IAAI,KAAK,GAAG,eAAe,KAAK,GAAG,KAAK,UAAU;GAClD,MAAM;EACR;CACF;CAEA,IAAY,KAAmC;EAC7C,OAAO,KAAK,KAAK,qCAAqC,CAAC,CAAC,IAAI,GAAG;CACjE;;;;;;CAOA,IAAI,KAAa,SAAuB,KAAyB;EAC/D,MAAM,EAAE,KAAK,QAAQ,SAAS,SAAS,GAAG;EAC1C,MAAM,QAAQ,QAAQ,UAAU,KAAA,IAAY,aAAa,UAAU,WAAW,QAAQ,KAAK;EAC3F,IAAI,QAAQ,KAAK,SACf,MAAM,IAAI,WACR,iBAAiB,OAAO,KAAK,UAAU,GAAM,EAAE,yDAAyD,OAAO,QAAQ,GAAM,EAAE,SACjI;EAEF,MAAM,SAAS,QAAQ,UAAU;EACjC,IAAI,CAAC,QAAQ,IAAI,MAAM,GACrB,MAAM,IAAI,WAAW,wCAAwC,KAAK,UAAU,MAAM,GAAG;EACvF,MAAM,UAAU,QAAQ,WAAW;EACnC,IAAI,YAAY,KAAK,YAAY,GAC/B,MAAM,IAAI,WAAW,0BAA0B,KAAK,UAAU,OAAO,GAAG;EAC1E,MAAM,SACJ,QAAQ,WAAW,KAAA,KAAa,OAAO,KAAK,QAAQ,MAAM,CAAC,CAAC,WAAW,IACnE,OACA,KAAK,UAAU,QAAQ,MAAM;EACnC,MAAM,SACJ,QAAQ,eAAe,KAAA,KAAa,QAAQ,WAAW,WAAW,IAC9D,OACA,KAAK,UAAU,CAAC,GAAG,QAAQ,UAAU,CAAC;EAC5C,OAAO,KAAK,WAAW;GACrB,KAAK,KACH;;;;;;uEAOF,CAAC,CAAC,IACA,KACA,QACA,MACA,OACA,QACA,SACA,QAAQ,SAAS,MACjB,QACA,QACA,KACA,GACF;GACA,OAAO,QAAQ,KAAK,IAAI,GAAG,GAAI,GAAG;EACpC,CAAC;CACH;;CAGA,OAAO,KAAsB;EAC3B,MAAM,EAAE,QAAQ,SAAS,GAAG;EAC5B,OAAO,KAAK,WACJ,OAAO,KAAK,KAAK,mCAAmC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,OAAO,IAAI,CAClF;CACF;;CAGA,IAAI,KAAa,KAAqC;EACpD,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,2BAA2B;EAC5D,MAAM,MAAM,KAAK,IAAI,SAAS,GAAG,CAAC,CAAC,GAAG;EACtC,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,QAAQ,KAAK,GAAG;CACzD;;CAGA,KAAK,KAA2B;EAC9B,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,2BAA2B;EAC5D,OACE,KAAK,KAAK,6CAA6C,CAAC,CAAC,IAAI,CAAC,CAC9D,KAAK,QAAQ,QAAQ,KAAK,GAAG,CAAC;CAClC;;CAGA,IAAI,KAAqB;EACvB,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,2BAA2B;EAC5D,OACE,KAAK,KAAK,sDAAsD,CAAC,CAAC,IAAI,GAAG,CAAC,CAC1E;CACJ;;;;;CAMA,MAAM,KAAa,OAAe,OAAe,MAA+C;EAC9F,OAAO,KAAK,WAAW;GACrB,MAAM,MAAM,KAAK,KACf;0FAEF,CAAC,CAAC,IAAI,KAAK,KAAK,KAAK,UAAU,IAAI,CAAC;GACpC,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;GAC9B,KAAK,KAAK,mEAAmE,CAAC,CAAC,IAC7E,OACA,OACA,IAAI,GACN;GACA,OAAO;IAAE,GAAG;IAAK,aAAa;IAAO,aAAa;GAAM;EAC1D,CAAC;CACH;;CAGA,QAAQ,KAAa,OAAqB;EACxC,IAAI,KAAK,QAAQ;EACjB,KAAK,WAAW;GACd,KAAK,KACH,0FACF,CAAC,CAAC,IAAI,KAAK,KAAK;EAClB,CAAC;CACH;;;;;CAMA,OAAO,KAAa,OAAe,QAA8B;EAC/D,IAAI,KAAK,QAAQ,OAAO;EACxB,OAAO,KAAK,WAAW;GACrB,MAAM,MAAM,KAAK,IAAI,GAAG;GACxB,IAAI,QAAQ,KAAA,KAAa,IAAI,gBAAgB,OAAO,OAAO;GAC3D,MAAM,OAAO,OAAO;GACpB,MAAM,UAAU,OAAO;GACvB,KAAK,KACH;;;;uBAKF,CAAC,CAAC,IACA,OAAO,SACP,OAAO,SACP,OAAO,WACP,OAAO,WACP,OAAO,UAAU,IAAI,GACrB,OAAO,OAAO,IAAI,KAClB,SAAS,KAAA,IAAY,IAAI,aAAa,KAAK,OAC3C,SAAS,KAAA,IAAY,IAAI,WAAW,KAAK,KACzC,SAAS,KAAA,IAAY,IAAI,eAAe,KAAK,SAC7C,SAAS,KAAA,IAAY,IAAI,kBAAkB,KAAK,YAChD,YAAY,KAAA,IAAY,IAAI,aAAc,SAAS,SAAS,MAC5D,YAAY,KAAA,IAAY,IAAI,WAAY,SAAS,OAAO,MACxD,YAAY,KAAA,IAAY,IAAI,eAAgB,SAAS,WAAW,MAChE,YAAY,KAAA,IAAY,IAAI,kBAAmB,SAAS,cAAc,MACtE,YAAY,KAAA,IAAY,IAAI,aAAc,SAAS,SAAS,MAC5D,GACF;GACA,OAAO;EACT,CAAC;CACH;;CAGA,QAAc;EACZ,IAAI,KAAK,QAAQ;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,MAAM;EACtB,KAAK,GAAG,MAAM;CAChB;AACF;;AAGA,MAAM,2BAAgC,IAAI,IAAI;CAC5C;CACA;CACA;CACA;AACF,CAAC;;AAGD,MAAa,oBAAoB,YAC/BC,SAAO,gBAAgB;CAAC,QAAQ;CAAO,QAAQ;CAAM,QAAQ,KAAK,aAAa;AAAI,CAAC,CAAC;AAEvF,MAAM,aAAa,GAA2B,MAC5C,MAAM,KAAA,KAAa,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,UAAU,EAAE,QAAQ,EAAE;;;;;;;;;AAe/E,MAAa,YACX,SACA,QACA,UAA8B,CAAC,MAClB;CACb,IAAI,uBAAwD,KAAA;CAC5D,MAAM,UAAU,IAAI,SAAuB,YAAY;EACrD,iBAAiB;CACnB,CAAC;CACD,IAAI,OAAO;CAEX,gBAAgB,SAAsD;EACpE,MAAM,YAAoB,KAAK,MAAM,QAAQ,IAAI,CAAC;EAClD,MAAM,YAAY,IAAI;EACtB,MAAM,QAAQ,WAAW;EACzB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,MAAM,QAAQ,QAAQ;EACtB,MAAM,eAAqB,MAAM,MAAM,OAAO,MAAM;EACpD,IAAI,OAAO,YAAY,MAAM,OAAO;EACpC,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;EACvD,MAAM,2BAAW,IAAI,IAA2B;EAChD,MAAM,SAAS;GACb,SAAS;GACT,KAAK;GACL,SAAS;GACT,WAAW;GACX,MAAM;GACN,SAAS;GACT,OAAO;GACP,WAAW;GACX,WAAW;EACb;EACA,IAAI,UAAU;EACd,IAAI;EACJ,IAAI;EACJ,IAAI,WAAW;EACf,IAAI,WAAmB,aAAa;EACpC,IAAI,WAAmB,aAAa;EACpC,IAAI;;EAGJ,MAAM,QAAQ,OAAO,OAAmB,KAAe,aAAoC;GACzF,MAAM,SAAS,CAAC,GAAI,QAAQ,cAAc,CAAC,GAAI,GAAG,UAAoB,IAAI,cAAc,CAAC,CAAC,CAAC;GAC3F,MAAM,SAAS,UAA8C,IAAI,QAAQ,KAAA,CAAS;GAClF,IAAI;GACJ,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,UAAU,SAAS,IAAI,QAAQ;KAC1C,QAAQ,IAAI;KACZ,UAAU;KACV,YAAY;KACZ,QAAQ,MAAM;KACd,OAAO,IAAI,SAAS,KAAA;KACpB;KACA;IACF,CAAC;GACH,SAAS,OAAO;IACd,UAAU,gBAAgB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACjF;GACA,MAAM,YAAY,SAAsC;IACtD,MAAM,QAAQ,IAAI,KAAK,KAAK;IAC5B,OAAO;KAAE,MAAM;KAAY,MAAM,IAAI;KAAM;IAAK;GAClD;GACA,IAAI,MAAM,OAAO,SAAS,OAAO,SAAS,WAAW;GACrD,IAAI,SAAS,KAAA,KAAa,aAAa,KAAK,IAAI,GAAG,OAAO,SAAS,QAAQ;GAC3E,IAAI,MAAM,YAAY,aAAa,KAAK,SAAS,kBAAkB,OAAO,SAAS,QAAQ;GAC3F,MAAM,KAAK,IAAI;GACf,MAAM,OACJ,IAAI,eAAe,OACf,KAAA,IACA;IACE,OAAO,IAAI;IACX,KAAK,IAAI;IACT,SAAS,IAAI;IACb,YAAY,IAAI;GAClB;GACN,MAAM,UACJ,IAAI,eAAe,OACf,KAAA,IACA;IACE,OAAO,IAAI;IACX,KAAK,IAAI;IACT,SAAS,IAAI;IACb,YAAY,IAAI;GAClB;GACN,IAAI;GACJ,IAAI;GACJ,IAAI,OAAO;GACX,IAAI;GACJ,IAAI,OAAsB,MAAM,QAAQ,WAAW;GACnD,MAAM,KAAK,SAAS,KAAA,MAAc,KAAK,YAAY,QAAQ,KAAK,YAAY;GAC5E,IAAI,SAAS,KAAA,KAAa,MAAM,KAAK,QAAQ,QAAQ,KAAK,YAAY,MAAM;IAC1E,UAAU,QAAQ,MAAM,YAAY,KAAK,KAAK,KAAK,OAAO;IAC1D,MAAM,MAAM,QAAQ,MAAM,IAAI,KAAK,GAAG;IACtC,OACG,QAAQ,KAAA,KAAa,WAAW,GAAG,KAAO,YAAY,KAAA,KAAa,WAAW,OAAO;IACxF,IAAI,YAAY,KAAA,GACd,OAAO;KACL,OAAO;KACP,KAAK,iBAAiB,OAAO;KAC7B,SAAS,QAAQ;KACjB,YAAY,QAAQ;IACtB;GAEJ,OAAO,IAAI,MAAM,YAAY,aAC3B,OAAO;IAAE,OAAO;IAAQ,KAAK;IAAM,SAAS;IAAM,YAAY,KAAK;GAAW;GAEhF,IAAI;GACJ,IAAI,YAAY;GAChB,IAAI,YAAY;GAChB,IAAI,SAAS,KAAA,GAAW;IACtB,OAAO,SAAS,KAAA,KAAa,SAAS,IAAI,KAAK,OAAO,IAAI,YAAY;IACtE,IAAI,IAAI,OAAO,QAAQ;IACvB,SAAS,EAAE,SAAS,MAAM;GAC5B,OAAO,IAAI,SAAS,KAAA,GAAW;IAE7B,OAAO,KAAK,UAAU,SAAS,SAAS;IACxC,SAAS;KAAE,SAAS;KAAO,MAAM;KAAM,SAAS;IAAK;GACvD,OAAO,IAAI,UAAU,MAAM,IAAI,GAAG;IAChC,OAAO;IACP,IAAI,YAAY,KAAA,GACd,OACE;IACJ,SAAS;KAAE,SAAS;KAAO,SAAS;IAAK;GAC3C,OAAO;IACL,OAAO,KAAK,UAAU,SAAS,SAAS,KAAK,UAAU,SAAS,QAAQ;IAExE,MAAM,WACJ,UAAU,SAAS,IAAI,MAAM,KAAK,UAAU,UAAU,SAAS,YAAY,KAAK;IAClF,IAAI,IAAI,YAAY,KAAK,UAAU;KACjC,YAAY;KACZ,SAAS;MAAE,SAAS;MAAM,MAAM;MAAM,SAAS;KAAK;IACtD,OAAO;KACL,YAAY;KACZ,OACE,YAAY,KAAA,IACR,iGACA;KACN,SAAS;MAAE,SAAS;MAAO,SAAS;OAAE,GAAG;OAAM,OAAO;MAAG;KAAE;IAC7D;GACF;GACA,IAAI,OAA2B;GAC/B,IACE,SAAS,aACT,MAAM,UAAU,aAChB,KAAK,YAAY,QACjB,YAAY,KAAA,KACZ,CAAC,MACD;IACA,MAAM,QAAQ,IAAI,OAAO,QAAQ;IACjC,MAAM,SAAS,QAAQ,MAAM,YAAY,OAAO,KAAK,OAAO;IAC5D,IAAI,WAAW,KAAA,KAAa,CAAC,WAAW,MAAM,GAC5C,OAAO,UACL;KAAE,MAAM,OAAO;KAAM,QAAQ;MAAE,OAAO,OAAO;MAAO,MAAM,OAAO;KAAK;IAAE,GACxE;KAAE,MAAM,QAAQ;KAAM,QAAQ;MAAE,OAAO,QAAQ;MAAO,MAAM,QAAQ;KAAK;IAAE,CAC7E,CAAC,CAAC;GAEN;GAEA,IAAI,OAAO,KAAK,IAAI;GACpB,MAAM,OAAO,QAAQ,QAAQ,SAAS,IAAI,MAAM;GAChD,IAAI,SAAS,KAAA,KAAa,OAAO,SAAS,KAAK,KAAK,KAAK,KAAK,QAAQ,MAAM,OAAO,KAAK;GACxF,MAAM,OAAO,IAAI,KAAK,OAAO;IAC3B,GAAG;IACH,SAAS;IACT,SAAS;IACT,WAAW;IACX,WAAW;IACX,KAAK,MAAM,OAAO;GACpB,CAAC;GACD,MAAM,QAAoB;IACxB,KAAK,IAAI;IACT,KAAK,MAAM,OAAO,IAAI;IACtB;IACA;IACA;IACA,SAAS,MAAM,WAAW;IAC1B,YAAY,MAAM,cAAc;IAChC,UAAU,MAAM,WAAW;IAC3B,qBAAqB,MAAM,cAAc;IACzC,SAAS,MAAM,WAAW;IAC1B,QAAQ,MAAM,UAAU;IACxB;IACA,WAAW;IACX,SAAS,IAAI,IAAI;IACjB;GACF;GACA,OAAO;IAAE,MAAM;IAAS,MAAM,IAAI;IAAM;GAAM;EAChD;EAEA,IAAI;GACF,MAAM,QAAQ,OAAO,WAAW,aAAa,MAAM,OAAO,IAAI;GAC9D,SAAS;GACT,MAAM,QAAQ,QAAQ;GACtB,WAAW,QAAQ,YAAY,aAAa;GAC5C,IAAI,EAAE,YAAY,IAChB,MAAM,IAAI,WAAW,mCAAmC,OAAO,QAAQ,GAAG;GAC5E,WAAW,QAAQ,YAAY,aAAa;GAC5C,IAAI,CAAC,OAAO,UAAU,QAAQ,KAAK,WAAW,GAC5C,MAAM,IAAI,WAAW,iDAAiD,OAAO,QAAQ,GAAG;GAC1F,MAAM,WAAW,YAAY;GAC7B,SAAS;IACP,IAAI,aAAa,KAAA,KAAa,MAAM,OAAO,SAAS,WAAW;IAC/D,IAAI,aAAa,KAAA,KAAa,QAAQ,OAAO,YAAY,MAAM;KAE7D,WAAW;KACX,SAAS;IACX;IACA,IAAI,aAAa,KAAA,KAAa,IAAI,KAAK,UAAU,WAAW;IAC5D,OACE,aAAa,KAAA,KACb,SAAS,OAAO,aAAa,eAC7B,UAAU,UACV;KACA,MAAM,UAAU,MAAM,MACpB,IAAI,GACJ,OACA,KAAK,IAAI,IAAI,GAAG,QAAQ,IAAI,aAAa,SACzC,CAAC,GAAG,SAAS,KAAK,CAAC,CACrB;KACA,IAAI,YAAY,KAAA,GAAW;KAC3B,WAAW;KACX,SAAS,IAAI,QAAQ,MAAM,MAAM,OAAO,SAAS,QAAQ,CAAC;IAC5D;IACA,IAAI,SAAS,SAAS,GAAG;IACzB,MAAM,OAAO,MAAM,QAAQ,KAAK,SAAS,OAAO,CAAC;IACjD,SAAS,OAAO,KAAK,IAAI;IACzB,IAAI,KAAK,SAAS,YAAY;KAC5B,WAAW;KACX,aAAa,KAAK;KAClB;IACF;IACA,MAAM,QAAQ,KAAK;IACnB,OAAO,WAAW;IAClB,OAAO,MAAM,SAAS;IACtB,IAAI,MAAM,WAAW,OAAO,aAAa;IACzC,IAAI,MAAM,WAAW,OAAO,aAAa;IACzC,MAAM;GACR;GACA,aAAa,WAAW,YAAY,MAAM,IAAI,IAAI,CAAC,IAAI,IAAI,cAAc;GACzE,WAAW;EACb,SAAS,OAAO;GACd,WAAW;GACX,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D,WAAW;GACX,MAAM;EACR,UAAU;GAER,IAAI,CAAC,UAAU;IACb,aAAa;IACb,WAAW;GACb;GACA,OAAO,oBAAoB,SAAS,MAAM;GAC1C,MAAM,sBAAM,IAAI,MAAM,qBAAqB,CAAC;GAE5C,MAAM,QAAQ,WAAW,SAAS,OAAO,CAAC;GAC1C,MAAM,OAAO,YAAY;GACzB,WACE,SAAS,SACL,4CACA,SAAS,cACP,mBAAmB,OAAO,QAAQ,EAAE,kCACpC,SAAS,WACP,iBAAiB,OAAO,QAAQ,EAAE,wBAClC,SAAS,cACP,iCACA;GACZ,IAAI,MAAM;GACV,IAAI;IACF,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK;GAC9B,QAAQ,CAER;GACA,eAAe;IACb,GAAG;IACH;IACA,SAAS;IACT;IACA,YAAY,KAAK,IAAI,GAAG,IAAI,IAAI,SAAS;GAC3C,CAAC;EACH;CACF;CAEA,OAAO;EACL;EACA,CAAC,OAAO,iBAAiB;GACvB,IAAI,MAAM,MAAM,IAAI,MAAM,6DAA6D;GACvF,OAAO;GACP,OAAO,OAAO;EAChB;CACF;AACF;;;;;;;;;;;;;;;AC71BA,MAAa,oBAAoB,YAC/B,GAAG,cAAc,GAAG,QAAQ,KAAK,WAAW,YAAY,KAAA,IAAY,KAAK,KAAK,UAAU;;AAG1F,MAAM,UAAU;;;;;;;;AAShB,MAAa,aAAa,YAA6B,QAAQ,KAAK,QAAQ,KAAK,CAAC;;AAkGlF,MAAM,cAAc;CAClB;CACA;CACA;CACA;CACA;CACA;AACF;;;;;;AAOA,MAAM,cAAc,QAAsB;CACxC,UAAU,KAAK;EAAE,WAAW;EAAM,MAAM;CAAM,CAAC;CAC/C,MAAM,OAAO,UAAU,GAAG,CAAC,CAAC,eAAe,IAAI,aAAa,GAAG,IAAI;CACnE,MAAM,OAAO,SAAS,IAAI;CAC1B,IAAI,CAAC,KAAK,YAAY,GAAG,MAAM,IAAI,MAAM,aAAa,IAAI,iBAAiB;CAC3E,IACE,QAAQ,aAAa,WACrB,OAAO,QAAQ,WAAW,cAC1B,KAAK,QAAQ,QAAQ,OAAO,GAE5B,MAAM,IAAI,MAAM,aAAa,IAAI,uDAAuD;CAE1F,KAAK,MAAM,QAAQ,aAAa;EAC9B,IAAI,OAAO;EACX,IAAI;GACF,OAAO,UAAU,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe;EACnD,QAAQ,CAER;EACA,IAAI,MACF,MAAM,IAAI,MACR,GAAG,KAAK,gBAAgB,IAAI,oDAC9B;CACJ;AACF;;;;;;;;AASA,MAAa,iBAAiB,YAA0C;CACtE,MAAM,MAAM,QAAQ;CACpB,IACE,QAAQ,KAAA,MACP,QAAQ,QAAQ,KAAA,KAAa,QAAQ,UAAU,KAAA,KAAa,QAAQ,UAAU,KAAA,IAE/E,MAAM,IAAI,MAAM,0DAA0D;CAE5E,MAAM,UAAU,QAAQ;CACxB,IAAI,YAAY,KAAA,KAAa,CAAC,aAAa,KAAK,OAAO,GACrD,MAAM,IAAI,MACR,YAAY,QAAQ,uEACtB;CAGF,MAAM,aAAa,IAAI,WAAW;CAClC,KAAK,MAAM,CAAC,QAAQ,cAAc,OAAO,QAAQ,QAAQ,cAAc,CAAC,CAAC,GACvE,WAAW,SAAS,QAAQ,SAAS;CAEvC,KAAK,MAAM,UAAU,QAAQ,WAAW,CAAC,GAAG,eAAe,YAAY,MAAM;CAC7E,IAAI,QAAQ,KAAA,GAAW,WAAW,GAAG;CACrC,MAAM,QAA2C,CAAC;CAClD,MAAM,MAAM,QAAQ,OAAO,IAAI,WAAW,KAAK,KAAM,UAAU,CAAC;CAChE,MAAM,QAAQ,QAAQ,SAAS,IAAI,cAAc,KAAK,KAAM,OAAO,CAAC;CACpE,IAAI,QAAQ,QAAQ;CACpB,IAAI,UAAU,KAAA,GAAW;EACvB,MAAM,OAAO,IAAI,SAAS,KAAK,KAAM,cAAc,CAAC;EACpD,MAAM,WAAW,KAAK,MAAM,CAAC;EAC7B,QAAQ;CACV;CACA,IAAI,YAAY,QAAQ;CACxB,IAAI,cAAc,KAAA,GAAW;EAC3B,IAAI,QAAQ,KAAA,GAAW,YAAY,IAAI,qBAAqB;OACvD;GACH,MAAM,OAAO,IAAI,qBAAqB,KAAK,KAAK,cAAc,CAAC;GAC/D,MAAM,WAAW,KAAK,MAAM,CAAC;GAC7B,YAAY;EACd;CACF;CACA,MAAM,QAAQ,QAAQ,SAAS;CAE/B,MAAM,WACJ,QAAQ,aAAa,QACjB,KAAA,IACC,QAAQ,YAAY,gBAAgB,mBAAmB,CAAC;CAC/D,MAAM,UAAU,IAAI,QAAQ;EAC1B,OAAO,QAAQ,SAAA;EACf,WAAW,QAAQ,aAAa,iBAAiB,OAAO;EACxD,MAAM,YAAY,KAAA,KAAa,UAAU,OAAO,IAAI,QAAQ,KAAK,IAAI,KAAA;EACrE;EACA;EACA;EACA;EACA,WAAW,QAAQ;EACnB,YAAY,QAAQ;EACpB,YAAY,QAAQ;EACpB,QAAQ,QAAQ;EAChB,YAAY,QAAQ;EACpB;EACA;EACA,QAAQ,QAAQ;EAChB,OAAO,QAAQ;EACf,WAAW,QAAQ;EACnB,kBAAkB,QAAQ;EAC1B,SAAS,QAAQ;EACjB,QAAQ,QAAQ;CAClB,CAAC;CACD,IAAI;CACJ,OAAO;EACL;EACA;EACA;EACA,UAAU;EACV;EACA;EACA;EACA,WAAW,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,YAAY;EAC7C,QAAQ;GACN,aAAa,YAAY;IAEvB,MAAM,QAAQ,MAAM;IACpB,KAAK,MAAM,SAAS,OAAO,MAAM,MAAM;GACzC,EAAA,CAAG;GACH,OAAO;EACT;CACF;AACF;;;;;;;;;;;AAuMA,MAAa,eAAe,YAAwC;CAClE,MAAM,UACJ,QAAQ,eAAe,KAAA,IAAY,aAAa,UAAU,WAAW,QAAQ,UAAU;CACzF,MAAM,UAAU,cAAc,OAAO;CACrC,IAAI,SAAS;CAEb,IAAI;CACJ,MAAM,QAA0B,WAAW,KAAK,QAAQ,KAAK,UAAU,CAAC,IACpE,aAAa,OAAO,CAAC,CAAC,MACnB,SAAS;EACR,IAAI,MAAM,SAAS,WAAW;GAC5B,MAAM,EAAE,MAAM,GAAG,WAAW;GAC5B,UAAU,SAAS,YAAY,SAAS,KAAA;EAC1C;CACF,SACM,KAAA,CACR,IACA,QAAQ,QAAQ;CACpB,MAAM,OAAO,YAA2B;EACtC,IAAI,QAAQ,MAAM,IAAI,MAAM,uBAAuB;EACnD,MAAM;EACN,IAAI,QAAQ,MAAM,IAAI,MAAM,uBAAuB;CACrD;CAEA,IAAI;CACJ,MAAM,aAAa,YAAiC;EAClD,MAAM,KAAK;EACX,YAAY,IAAI,WAAW,KAAK,QAAQ,KAAK,cAAc,GAAG,EAAE,QAAQ,CAAC;EACzE,OAAO;CACT;CACA,MAAM,iBAAyB,KAAK,MAAM,QAAQ,IAAI,CAAC;CAEvD,MAAM,WAAW,QAAQ;CACzB,IAAI;CACJ,IAAI;CACJ,MAAM,eAAe,YAAmC;EACtD,MAAM,KAAK;EACX,IAAI,aAAa,KAAA,GAAW,MAAM,IAAI,MAAM,WAAW;EACvD,YAAY,IAAI,YAAY,KAAK,QAAQ,KAAK,gBAAgB,CAAC;EAC/D,WAAW,mBAAmB,SAAS;GAAE;GAAU,OAAO;EAAQ,CAAC;EACnE,OAAO;CACT;CACA,MAAM,SACJ,aAAa,KAAA,IACT,KAAA,IACA;EACE;EACA,MAAM,OAAO,GAAG,iBAAiB,MAAM,aAAa,EAAA,CAAG,KAAK,GAAG,WAAW;EAC1E,YAAY,OAAO,GAAG,MAAM,iBACzB,MAAM,aAAa,EAAA,CAAG,WAAW,GAAG,MAAM,WAAW;EACxD,OAAO,OAAO,kBAAkB,MAAM,aAAa,EAAA,CAAG,MAAM,YAAY;CAC1E;CAEN,IAAI;CAsBJ,OAAO;EACL;EACA,MAAM,MAAM,KAAK,eAAe,CAAC,GAAG;GAClC,MAAM,KAAK;GACX,UAAU,KAAA;GACV,MAAM,SAAS,aAAa,UAAU;GACtC,OAAO,UAAU,SAAS,KAAK;IAC7B,QAAQ,aAAa,UAAU;IAC/B,UAAU,WAAW,SAAS,SAAS,WAAW,MAAM;IACxD,YAAY,CAAC,GAAI,QAAQ,cAAc,CAAC,GAAI,GAAI,aAAa,cAAc,CAAC,CAAE;IAC9E,QAAQ,aAAa;IACrB,OAAO,aAAa;IACpB,QAAQ,aAAa;IACrB,UAAU,aAAa;IACvB,sBAAsB,aAAa;GACrC,CAAC;EACH;EACA,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG;GAC9B,MAAM,KAAK;GACX,MAAM,EAAE,MAAM,GAAG,SAAS;GAC1B,IAAI,SAAS,UAAU,QAAQ,MAAM,aAAa,EAAA,CAAG,KAAK,GAAG,IAAI;GACjE,OAAO,cAAc,SAAS,GAAG;IAAE,GAAG;IAAM,GAAI,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,KAAK;GAAG,CAAC;EACvF;EACA;EACA,MAAM,MAAM,eAAe,CAAC,GAAG;GAC7B,QAAQ,MAAM,aAAa,EAAA,CAAG,MAAM,YAAY;EAClD;EACA,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,GAAG;GAC1C,MAAM,KAAK;GACX,MAAM,EAAE,MAAM,GAAG,SAAS;GAC1B,IAAI,SAAS,UAAU,QAAQ,MAAM,aAAa,EAAA,CAAG,WAAW,GAAG,MAAM,IAAI;GAC7E,OAAO,QAAQ,MAAM,WAAW,GAAG,MAAM,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,KAAK,CAAC;EAC7E;EACA,MAAM,KAAK,IAAI,cAAc,CAAC,GAAG;GAC/B,MAAM,KAAK;GACX,OAAO,aAAa,SAAS,IAAI,WAAW;EAC9C;EACA,MAAM,OAAO,IAAI,OAAO,gBAAgB,CAAC,GAAG;GAC1C,MAAM,KAAK;GACX,OAAO,YAAY,SAAS,IAAI,OAAO,aAAa;EACtD;EAEA,MAAM,QAAQ,IAAI,QAAQ,iBAAiB,CAAC,GAAG;GAC7C,MAAM,KAAK;GACX,OAAO,gBAAgB,SAAS,IAAI,QAAQ,cAAc;EAC5D;EAEA,MAAM,SAAS,IAAI,iBAAiB,CAAC,GAAG;GACtC,MAAM,KAAK;GAEX,MAAM,OAAO,QAAQ,MAAM,IAAI,EAAE,CAAC,EAAE,aAAa;GACjD,OAAO,QAAQ,MACZ,SAAS,IAAI,cAAc,CAAC,CAC5B,KAAK,YACJ,QAAQ,QAAQ,aAAa,OAAO;IAAE,GAAG;IAAS,OAAO;GAAG,IAAI,OAClE;EACJ;EACA,MAAM,SAAS,MAAM,kBAAkB,CAAC,GAAG;GACzC,MAAM,KAAK;GACX,UAAU,KAAA;GACV,OAAO,aAAa,SAAS,MAAM;IACjC,GAAG;IACH,YAAY,CAAC,GAAI,QAAQ,cAAc,CAAC,GAAI,GAAI,gBAAgB,cAAc,CAAC,CAAE;GACnF,CAAC;EACH;EACA,MAAM,OAAO,eAAe,CAAC,GAAG;GAC9B,OAAO,UAAU,SAAS,OAAO;IAC/B,GAAG;IACH,YAAY,CAAC,GAAI,QAAQ,cAAc,CAAC,GAAI,GAAI,aAAa,cAAc,CAAC,CAAE;IAC9E,OAAO,YAAY;KACjB,MAAM,KAAK;KACX,UAAU,KAAA;IACZ;GACF,CAAC;EACH;EACA,MAAM,UAAU;GACd,MAAM,KAAK;GAEX,MAAM,OAAO;GACb,UAAU,KAAA;GACV,OAAO,QAAQ,QAAQ,OAAO;EAChC;EACA,MAAM,KAAK,IAAI,cAAc,CAAC,GAAG;GAC/B,MAAM,KAAK;GACX,OAAO,aAAa,SAAS,IAAI,WAAW;EAC9C;EACA,OAAA;GA1GA,MAAM,IAAI,KAAK,eAAe,CAAC,GAAG;IAChC,QAAQ,MAAM,WAAW,EAAA,CAAG,IAAI,KAAK,cAAc,SAAS,CAAC;GAC/D;GACA,MAAM,OAAO,KAAK;IAChB,QAAQ,MAAM,WAAW,EAAA,CAAG,OAAO,GAAG;GACxC;GACA,MAAM,OAAO;IACX,QAAQ,MAAM,WAAW,EAAA,CAAG,KAAK,SAAS,CAAC;GAC7C;GACA,IAAI,aAAa,CAAC,GAAG;IACnB,OAAO,SAAS,SAAS,YAAY;KACnC,GAAG;KACH,YAAY,QAAQ;KACpB,aAAa;MACX,UAAU,KAAA;MACV,OAAO,QAAQ,QAAQ;KACzB;IACF,CAAC;GACH;EAwFI;EACJ,QAAQ;GACN,SAAS;GAET,aAAa,YAAY;IACvB,MAAM;IACN,MAAM,QAAQ,MAAM;IACpB,SAAS,MAAM;IACf,SAAS,MAAM;IACf,MAAM,UAAU,QAAQ;GAC1B,EAAA,CAAG;GACH,OAAO;EACT;CACF;AACF"}