polite-crawler 0.0.0-stage → 0.1.0

package/build/html-oNpGAHwi.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-oNpGAHwi.mjs","names":[],"sources":["../src/urls.ts","../src/text.ts","../src/html.ts"],"sourcesContent":["/**\n * This module gives the canonical form of a URL. The document id of a page and the cache key of the\n * crawler use it. Thus two spellings of one page are one document and one cached answer.\n */\n\nconst TRACKING =\n  /^(utm_[a-z_]+|fbclid|gclid|gclsrc|dclid|gbraid|wbraid|msclkid|yclid|ttclid|twclid|li_fat_id|srsltid|mc_[a-z]+|_ga|_gl|_hsenc|_hsmi|__hstc|__hssc|__hsfp|mkt_tok|ref_src|igshid)$/i;\n\n/** This function decodes the escapes of unreserved characters (`%7E` is `~`), and writes the other escapes in capitals. */\nconst normalizeEscapes = (value: string): string =>\n  value.replace(/%[0-9a-f]{2}/gi, (escape) => {\n    const char = String.fromCharCode(Number.parseInt(escape.slice(1), 16));\n    return /^[A-Za-z0-9\\-._~]$/.test(char) ? char : escape.toUpperCase();\n  });\n\n/**\n * This function gives one URL for each page: a lowercase host, and no fragment, tracking\n * parameters, empty query or default port. It decodes the escapes of unreserved characters.\n */\nexport const canonicalUrl = (value: string): string => {\n  const url = new URL(value);\n  url.hash = \"\";\n  url.hostname = url.hostname.toLowerCase();\n  if (\n    (url.protocol === \"https:\" && url.port === \"443\") ||\n    (url.protocol === \"http:\" && url.port === \"80\")\n  ) {\n    url.port = \"\";\n  }\n  // a copy of the keys, because the loop deletes from the live list\n  for (const key of Array.from(url.searchParams.keys())) {\n    if (TRACKING.test(key)) url.searchParams.delete(key);\n  }\n  if (url.pathname.includes(\"%\")) url.pathname = normalizeEscapes(url.pathname);\n  // \"/p?\" is \"/p\"\n  url.search = url.search.includes(\"%\") ? normalizeEscapes(url.search) : url.search;\n  return url.href;\n};\n\n/**\n * This function gives the cache key of a URL: its canonical form. A text that is not a URL stays as\n * it is. The crawler and `reindex` use the same key, so a rebuilt index answers the same requests.\n */\nexport const cacheKeyOf = (value: string): string => {\n  try {\n    return canonicalUrl(value);\n  } catch {\n    return value;\n  }\n};\n","/**\n * The body decoder changes the bytes of a response into text. It finds the encoding in this order,\n * as the WHATWG encoding rules and the browsers do:\n *\n * 1. A byte order mark.\n * 2. The `charset` of the `Content-Type` header.\n * 3. A `<meta charset>` or `http-equiv` declaration in the first 2,048 bytes of an HTML page, or the\n *    encoding of an XML declaration.\n * 4. UTF-8 when the bytes are UTF-8, and windows-1252 when they are not.\n *\n * It uses only `TextDecoder`, so it operates in a browser too. A byte that the encoding does not\n * know becomes U+FFFD. The decoder does not throw.\n */\n\nconst PRESCAN_BYTES = 2048;\n\n/** This function gives the WHATWG name of an encoding label, or undefined when the runtime does not know the label. */\nconst encodingOf = (label: string): string | undefined => {\n  try {\n    return new TextDecoder(label.trim().toLowerCase()).encoding;\n  } catch {\n    return undefined;\n  }\n};\n\nconst bomOf = (bytes: Uint8Array): string | undefined => {\n  if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) return \"utf-8\";\n  if (bytes[0] === 0xfe && bytes[1] === 0xff) return \"utf-16be\";\n  if (bytes[0] === 0xff && bytes[1] === 0xfe) return \"utf-16le\";\n  return undefined;\n};\n\n/** This function gives the `charset` parameter of a media type, or undefined. */\nconst charsetOf = (contentType: string | null): string | undefined => {\n  if (contentType === null) return undefined;\n  const match = /;\\s*charset\\s*=\\s*(?:\"([^\"]*)\"|([^\\s;\"]+))/i.exec(contentType.slice(0, 512));\n  return match?.[1] ?? match?.[2] ?? undefined;\n};\n\n/** This function reads the encoding that the start of an HTML or XML document declares. */\nconst declaredCharset = (bytes: Uint8Array): string | undefined => {\n  let head = \"\";\n  const end = Math.min(bytes.length, PRESCAN_BYTES);\n  for (let index = 0; index < end; index += 1) head += String.fromCharCode(bytes[index]!);\n  const xml = /^\\s*<\\?xml[^>]*?\\bencoding\\s*=\\s*[\"']([^\"']+)[\"']/i.exec(head);\n  if (xml !== null) return xml[1];\n  // a declaration inside a comment declares nothing\n  const markup = head.replace(/<!--[\\s\\S]*?-->/g, \"\");\n  // `<meta charset=x>`, and `<meta http-equiv=\"Content-Type\" content=\"text/html; charset=x\">`\n  const meta = /<meta\\b[^>]*?\\bcharset\\s*=\\s*[\"']?\\s*([A-Za-z0-9_:.+-]+)/i.exec(markup);\n  return meta?.[1];\n};\n\n/**\n * This function counts the UTF-8 sequences of more than one byte, and the bytes that no UTF-8\n * sequence explains. An incomplete sequence at the end is no error: a body cut at its size limit\n * ends so.\n */\nconst utf8Evidence = (\n  bytes: Uint8Array,\n): { readonly multibyte: number; readonly invalid: number } => {\n  let multibyte = 0;\n  let invalid = 0;\n  let index = 0;\n  while (index < bytes.length) {\n    const lead = bytes[index]!;\n    if (lead < 0x80) {\n      index += 1;\n      continue;\n    }\n    const need =\n      lead >= 0xc2 && lead <= 0xdf\n        ? 1\n        : lead >= 0xe0 && lead <= 0xef\n          ? 2\n          : lead >= 0xf0 && lead <= 0xf4\n            ? 3\n            : 0;\n    if (need === 0) {\n      invalid += 1;\n      index += 1;\n      continue;\n    }\n    // the second byte has a narrower range after E0, ED, F0 and F4\n    const low = lead === 0xe0 ? 0xa0 : lead === 0xf0 ? 0x90 : 0x80;\n    const high = lead === 0xed ? 0x9f : lead === 0xf4 ? 0x8f : 0xbf;\n    let ok = true;\n    let at = 1;\n    while (at <= need && index + at < bytes.length) {\n      const next = bytes[index + at]!;\n      if (at === 1 ? next < low || next > high : next < 0x80 || next > 0xbf) {\n        ok = false;\n        break;\n      }\n      at += 1;\n    }\n    // the body ends inside the sequence\n    if (ok && at <= need) break;\n    if (ok) {\n      multibyte += 1;\n      index += need + 1;\n    } else {\n      invalid += 1;\n      index += 1;\n    }\n  }\n  return { multibyte, invalid };\n};\n\n/**\n * This function gives the WHATWG name of the encoding of a body, for example `utf-8` or\n * `windows-1252`. The function `decodeBody` uses it.\n */\nexport const bodyEncoding = (bytes: Uint8Array, contentType: string | null): string => {\n  const bom = bomOf(bytes);\n  if (bom !== undefined) return bom;\n  const header = charsetOf(contentType);\n  const fromHeader = header === undefined ? undefined : encodingOf(header);\n  if (fromHeader !== undefined) return fromHeader;\n  if (contentType === null || /html|xml/i.test(contentType)) {\n    const label = declaredCharset(bytes);\n    let declared = label === undefined ? undefined : encodingOf(label);\n    // the HTML standard reads a UTF-16 declaration in ASCII-compatible bytes as UTF-8\n    if (declared?.startsWith(\"utf-16\") === true) declared = \"utf-8\";\n    if (declared === \"x-user-defined\") declared = \"windows-1252\";\n    if (declared === \"utf-8\") return declared;\n    if (declared !== undefined) {\n      // A browser serialises the page it rendered as UTF-8, and the old declaration stays. UTF-8\n      // with no error is almost never a legacy encoding by chance.\n      const evidence = utf8Evidence(bytes);\n      return evidence.invalid === 0 && evidence.multibyte > 0 ? \"utf-8\" : declared;\n    }\n  }\n  if (/json/i.test(contentType ?? \"\")) return \"utf-8\";\n  const evidence = utf8Evidence(bytes);\n  return evidence.invalid > 0 && evidence.invalid >= evidence.multibyte ? \"windows-1252\" : \"utf-8\";\n};\n\n/**\n * These are the characters of the bytes 0x80 to 0x9F in windows-1252, from the WHATWG index.\n * Node.js 22 decodes windows-1252 as ISO-8859-1, which gives a control character for each of\n * these bytes. A later runtime gives the correct characters, and then the map changes nothing.\n */\nconst WINDOWS_1252_HIGH = [\n  0x20ac, 0x81, 0x201a, 0x192, 0x201e, 0x2026, 0x2020, 0x2021, 0x2c6, 0x2030, 0x160, 0x2039, 0x152,\n  0x8d, 0x17d, 0x8f, 0x90, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x2dc, 0x2122,\n  0x161, 0x203a, 0x153, 0x9d, 0x17e, 0x178,\n];\nconst C1_CONTROLS = new RegExp(`[${String.fromCharCode(0x80)}-${String.fromCharCode(0x9f)}]`, \"g\");\nconst repairWindows1252 = (text: string): string =>\n  text.replace(C1_CONTROLS, (char) =>\n    String.fromCharCode(WINDOWS_1252_HIGH[char.charCodeAt(0) - 0x80]!),\n  );\n\n/**\n * This function decodes the body of a response. It reads the encoding from the byte order mark,\n * the `Content-Type` header and the document, with `bodyEncoding`. It does not throw.\n */\nexport const decodeBody = (bytes: Uint8Array, contentType: string | null): string => {\n  const encoding = bodyEncoding(bytes, contentType);\n  try {\n    const text = new TextDecoder(encoding, { fatal: false }).decode(bytes);\n    return encoding === \"windows-1252\" ? repairWindows1252(text) : text;\n  } catch {\n    // a runtime with a small ICU knows only a few encodings\n    return new TextDecoder(\"utf-8\", { fatal: false }).decode(bytes);\n  }\n};\n","import { Readability } from \"@mozilla/readability\";\nimport { parseHTML } from \"linkedom\";\nimport { canonicalUrl } from \"./urls.ts\";\n\nexport { bodyEncoding, decodeBody } from \"./text.ts\";\n\n/**\n * The page reader reads a web page with no browser first. It gives:\n *\n * 1. The JSON-LD, the structured data that 41% of pages have. For a venue, it gives the name, the\n *    address, the phone number and sometimes the hours.\n * 2. The visible text of the page.\n * 3. A decision: does the page need JavaScript to show its content?\n *\n * On a short page, the text is all of the visible text. Readability is for articles, and it\n * removes footers, sidebars and tables, where a venue keeps its hours. Thus on a long page, the\n * main text of Readability comes first. A blank line follows it, and then each other line of the\n * page.\n *\n * The cost of the reader grows in proportion to the size of the page. It walks the document with\n * no recursion, so deep markup cannot overflow the stack. It does not use Readability on a page\n * that is too large or too deep for it.\n */\n\nexport interface PageLink {\n  readonly href: string;\n  readonly text: string;\n}\n\nexport interface PageExtract {\n  readonly title: string;\n  readonly description: string | null;\n  readonly text: string;\n  /**\n   * When the reader cut `text` to its maximum, this text is the whole visible text in the order of\n   * the page. A quote past the cut is still in it. Otherwise this value is undefined.\n   */\n  readonly fullText?: string | undefined;\n  /** These are the JSON-LD objects of the page, with each `@graph` made flat. */\n  readonly jsonld: readonly Record<string, unknown>[];\n  readonly links: readonly PageLink[];\n  readonly lang: string | null;\n  readonly visibleChars: number;\n  /** These are the names of the application states that the HTML holds (Next.js, Nuxt, Apollo). */\n  readonly embedded: readonly string[];\n  /** This value is true when the page shows almost nothing without its scripts. */\n  readonly needsRender: boolean;\n  readonly renderReason: string | null;\n  /** This URL is the `<link rel=\"canonical\">` of the page when it is on the same site, or null. */\n  readonly canonical: string | null;\n  /** This URL is the target of a `<meta http-equiv=\"refresh\">` that sends the reader on in 5 seconds or less. */\n  readonly redirect: string | null;\n  /**\n   * This value names the bot check or the firewall that sent its page in place of the content, for\n   * example \"Cloudflare\". A crawler must not open a browser to get past it.\n   */\n  readonly blocked: string | null;\n  /** These notes tell what the reader left out, for example a JSON-LD block that it cannot read. */\n  readonly notes: readonly string[];\n}\n\nconst SKIP = new Set([\n  \"script\",\n  \"style\",\n  \"noscript\",\n  // a template is inert; a declarative shadow root is read (see `isShadowRoot`)\n  \"template\",\n  \"svg\",\n  \"math\",\n  // never shown, and a page without a <head> leaves it among the content\n  \"title\",\n  \"meta\",\n  \"link\",\n  \"base\",\n  \"iframe\",\n  \"frame\",\n  \"object\",\n  \"embed\",\n  \"canvas\",\n  \"audio\",\n  \"video\",\n  \"datalist\",\n  \"param\",\n  \"source\",\n  \"track\",\n]);\n/** These elements show as blocks: each starts a line. The list follows the default style sheet of HTML. */\nconst BLOCK = new Set([\n  \"address\",\n  \"article\",\n  \"aside\",\n  \"blockquote\",\n  \"body\",\n  \"br\",\n  \"caption\",\n  \"center\",\n  \"dd\",\n  \"details\",\n  \"dialog\",\n  \"dir\",\n  \"div\",\n  \"dl\",\n  \"dt\",\n  \"fieldset\",\n  \"figcaption\",\n  \"figure\",\n  \"footer\",\n  \"form\",\n  \"frameset\",\n  \"h1\",\n  \"h2\",\n  \"h3\",\n  \"h4\",\n  \"h5\",\n  \"h6\",\n  // linkedom leaves the content of a page that never closes its <head> inside it\n  \"head\",\n  \"header\",\n  \"hgroup\",\n  \"hr\",\n  \"html\",\n  \"legend\",\n  \"li\",\n  \"listing\",\n  \"main\",\n  \"marquee\",\n  \"menu\",\n  \"nav\",\n  \"noframes\",\n  \"ol\",\n  // a dropdown's choices, one to a line\n  \"optgroup\",\n  \"option\",\n  \"p\",\n  \"plaintext\",\n  \"pre\",\n  \"search\",\n  \"section\",\n  \"summary\",\n  \"table\",\n  \"tbody\",\n  \"tfoot\",\n  \"thead\",\n  \"tr\",\n  \"ul\",\n  \"xmp\",\n]);\n/**\n * These elements run inline with the text around them. Any other element, a custom element too,\n * ends a word, so its text stays apart from the next word.\n */\nconst INLINE = new Set([\n  \"a\",\n  \"abbr\",\n  \"acronym\",\n  \"b\",\n  \"bdi\",\n  \"bdo\",\n  \"big\",\n  \"blink\",\n  \"cite\",\n  \"code\",\n  \"data\",\n  \"del\",\n  \"dfn\",\n  \"em\",\n  \"font\",\n  \"i\",\n  \"img\",\n  \"ins\",\n  \"kbd\",\n  \"label\",\n  \"mark\",\n  \"nobr\",\n  \"picture\",\n  \"q\",\n  \"rb\",\n  \"rp\",\n  \"rt\",\n  \"rtc\",\n  \"ruby\",\n  \"s\",\n  \"samp\",\n  \"slot\",\n  \"small\",\n  \"span\",\n  \"strike\",\n  \"strong\",\n  \"sub\",\n  \"sup\",\n  \"time\",\n  \"tt\",\n  \"u\",\n  \"var\",\n  \"wbr\",\n]);\n/** A table row becomes one line (\"Sunday 10 am–5 pm\"), with a space between its cells. */\nconst CELL = new Set([\"td\", \"th\"]);\nconst WORD_END = new Set([\"a\", \"button\"]);\n\n/** A page shorter than this keeps all its text. On a longer page, the text of Readability comes first. */\nconst SHORT_PAGE_CHARS = 6000;\nconst MAX_TEXT_CHARS = 50_000;\n/** On a long page, this part of the text is kept for the lines after the article: a footer keeps its hours. */\nconst REST_RESERVE_CHARS = 12_000;\nconst MAX_LINKS = 300;\n/** Readability takes seconds on a page with more elements or links than this, and more time again on deep markup. */\nconst READABILITY_MAX_ELEMENTS = 10_000;\nconst READABILITY_MAX_LINKS = 1500;\nconst READABILITY_MAX_DEPTH = 150;\n/**\n * The parser of linkedom takes time in proportion to the square of the depth of the markup: 200,000\n * levels take 30 seconds. The reader reads the tags below this depth as text.\n */\nconst MAX_DEPTH = 1000;\n\n/** This is the small part of the DOM that this file uses. Node has no DOM types, and linkedom supplies the DOM. */\ninterface DomNode {\n  readonly nodeType: number;\n  readonly nodeName: string;\n  readonly textContent: string | null;\n  readonly childNodes: ArrayLike<DomNode>;\n  readonly firstChild?: DomNode | null | undefined;\n  readonly nextSibling?: DomNode | null | undefined;\n  readonly parentNode?: DomNode | null | undefined;\n  /** This method takes the node out of its parent. */\n  readonly remove?: (() => void) | undefined;\n}\n\ninterface DomElement extends DomNode {\n  getAttribute(name: string): string | null;\n  querySelector(selector: string): DomElement | null;\n  /** This is the content of a `<template>`. */\n  readonly content?: DomNode | null | undefined;\n}\n\n/**\n * This type has no `body` or `head`, on purpose. When an element comes before `<head>`, linkedom\n * makes a new, empty pair. Then `document.body` reads nothing, but a browser shows the full page.\n */\ninterface DomDocument extends DomNode {\n  querySelector(selector: string): DomElement | null;\n  querySelectorAll(selector: string): ArrayLike<DomElement> & Iterable<DomElement>;\n  readonly documentElement: DomElement | null;\n}\n\nconst parse = (html: string): DomDocument =>\n  (parseHTML(html) as unknown as { document: DomDocument }).document;\n\n/** These elements have no end tag. */\nconst VOID = new Set([\n  \"area\",\n  \"base\",\n  \"basefont\",\n  \"bgsound\",\n  \"br\",\n  \"col\",\n  \"embed\",\n  \"frame\",\n  \"hr\",\n  \"img\",\n  \"input\",\n  \"keygen\",\n  \"link\",\n  \"meta\",\n  \"param\",\n  \"source\",\n  \"track\",\n  \"wbr\",\n]);\n/** The content of these elements is text up to their end tag, not markup. */\nconst RAW_TEXT = new Set([\n  \"iframe\",\n  \"noembed\",\n  \"noframes\",\n  \"noscript\",\n  \"plaintext\",\n  \"script\",\n  \"style\",\n  \"textarea\",\n  \"title\",\n  \"xmp\",\n]);\n/** A new element of one of these kinds closes the open one (\"<li>one<li>two\"). */\nconst IMPLIED_END = new Set([\n  \"dd\",\n  \"dt\",\n  \"li\",\n  \"optgroup\",\n  \"option\",\n  \"p\",\n  \"rb\",\n  \"rp\",\n  \"rt\",\n  \"rtc\",\n  \"tbody\",\n  \"td\",\n  \"tfoot\",\n  \"th\",\n  \"thead\",\n  \"tr\",\n]);\n\nconst isLetter = (code: number): boolean =>\n  (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a);\nconst isNameChar = (code: number): boolean =>\n  isLetter(code) ||\n  (code >= 0x30 && code <= 0x39) ||\n  code === 0x2d ||\n  code === 0x3a ||\n  code === 0x5f ||\n  code === 0x2e;\n\n/** This function gives the index after the `>` of a tag. A `>` inside a quoted attribute value does not end it. */\nconst endOfTag = (html: string, from: number): number => {\n  let quote = 0;\n  let afterEquals = false;\n  for (let at = from; at < html.length; at += 1) {\n    const code = html.charCodeAt(at);\n    if (quote !== 0) {\n      if (code === quote) quote = 0;\n    } else if ((code === 0x22 || code === 0x27) && afterEquals) {\n      quote = code;\n    } else if (code === 0x3e) {\n      return at + 1;\n    }\n    if (code === 0x3d) afterEquals = true;\n    else if (code !== 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d) afterEquals = false;\n  }\n  return html.length;\n};\n\n/**\n * This function reads the tags below `MAX_DEPTH` levels as text: a block tag becomes a line break\n * and another tag becomes nothing. It counts the open elements roughly, in one pass. When it\n * removes nothing, it gives the same string.\n */\nconst flattenDeepMarkup = (html: string): { readonly html: string; readonly dropped: number } => {\n  const stack: string[] = [];\n  const open = new Map<string, number>();\n  const extra = new Map<string, number>();\n  const out: string[] = [];\n  let copied = 0;\n  let dropped = 0;\n  const popTo = (name: string): void => {\n    if ((open.get(name) ?? 0) === 0) return;\n    for (;;) {\n      const top = stack.pop()!;\n      open.set(top, (open.get(top) ?? 1) - 1);\n      if (top === name) return;\n    }\n  };\n  const replace = (start: number, end: number, name: string): void => {\n    out.push(html.slice(copied, start), BLOCK.has(name) ? \"\\n\" : INLINE.has(name) ? \"\" : \" \");\n    copied = end;\n  };\n  let index = 0;\n  while (index < html.length) {\n    const lt = html.indexOf(\"<\", index);\n    if (lt === -1) break;\n    if (html.startsWith(\"<!--\", lt)) {\n      const end = html.indexOf(\"-->\", lt + 4);\n      index = end === -1 ? html.length : end + 3;\n      continue;\n    }\n    const next = html.charCodeAt(lt + 1);\n    if (next === 0x21 || next === 0x3f) {\n      const end = html.indexOf(\">\", lt);\n      index = end === -1 ? html.length : end + 1;\n      continue;\n    }\n    const closing = next === 0x2f;\n    const nameStart = lt + (closing ? 2 : 1);\n    if (!isLetter(html.charCodeAt(nameStart))) {\n      index = lt + 1;\n      continue;\n    }\n    let nameEnd = nameStart + 1;\n    while (nameEnd < html.length && isNameChar(html.charCodeAt(nameEnd))) nameEnd += 1;\n    const name = html.slice(nameStart, nameEnd).toLowerCase();\n    const tagEnd = endOfTag(html, nameEnd);\n    index = tagEnd;\n    if (closing) {\n      // the deepest elements close first, and those are the ones that were removed\n      const removed = extra.get(name) ?? 0;\n      if (removed > 0) {\n        extra.set(name, removed - 1);\n        replace(lt, tagEnd, name);\n      } else {\n        popTo(name);\n      }\n      continue;\n    }\n    if (RAW_TEXT.has(name)) {\n      const end = new RegExp(`</${name}[\\\\s/>]`, \"gi\");\n      end.lastIndex = tagEnd;\n      index = end.exec(html)?.index ?? html.length;\n      continue;\n    }\n    if (VOID.has(name) || html.charCodeAt(tagEnd - 2) === 0x2f) continue;\n    if (IMPLIED_END.has(name)) popTo(name);\n    if (stack.length >= MAX_DEPTH) {\n      extra.set(name, (extra.get(name) ?? 0) + 1);\n      dropped += 1;\n      replace(lt, tagEnd, name);\n      continue;\n    }\n    stack.push(name);\n    open.set(name, (open.get(name) ?? 0) + 1);\n  }\n  if (dropped === 0) return { html, dropped };\n  out.push(html.slice(copied));\n  return { html: out.join(\"\"), dropped };\n};\n\nconst attribute = (node: DomNode, name: string): string | null =>\n  typeof (node as Partial<DomElement>).getAttribute === \"function\"\n    ? (node as DomElement).getAttribute(name)\n    : null;\n\n/** A `<template shadowrootmode>` holds a declarative shadow root: the browser shows its content. */\nconst isShadowRoot = (node: DomNode): boolean =>\n  attribute(node, \"shadowrootmode\") !== null || attribute(node, \"shadowroot\") !== null;\n\n/** The reader reads only these properties of an inline style. */\nconst STYLE_PROPERTIES = new Set([\"display\", \"font-size\"]);\n\n/**\n * This function reads `display` and `font-size` from the `style` attribute of an element. The last\n * declaration wins, and an `!important` declaration wins over a declaration that is not important.\n * It reads no style sheet. Its time grows in proportion to the length of the attribute.\n */\nconst inlineStyle = (node: DomNode): ReadonlyMap<string, string> => {\n  const found = new Map<string, string>();\n  const style = attribute(node, \"style\");\n  if (style === null || style.length === 0) return found;\n  const important = new Set<string>();\n  for (const declaration of style.split(\";\")) {\n    const colon = declaration.indexOf(\":\");\n    if (colon === -1) continue;\n    const property = declaration.slice(0, colon).trim().toLowerCase();\n    if (!STYLE_PROPERTIES.has(property)) continue;\n    let value = declaration\n      .slice(colon + 1)\n      .trim()\n      .toLowerCase();\n    const bang = value.lastIndexOf(\"!\");\n    const strong = bang !== -1 && value.slice(bang + 1).trim() === \"important\";\n    if (strong) value = value.slice(0, bang).trim();\n    if (important.has(property) && !strong) continue;\n    if (strong) important.add(property);\n    found.set(property, value);\n  }\n  return found;\n};\n\n/**\n * This function tells if a reader of the page does not get an element and its content. A browser\n * does not show an element with `display: none` or with the attribute `hidden` (unless its inline\n * style sets a different display). A screen reader skips an element with `aria-hidden=\"true\"`, and\n * a model reads a page as a screen reader does.\n */\nconst notRendered = (node: DomNode, style: ReadonlyMap<string, string>): boolean => {\n  const display = style.get(\"display\");\n  if (display === \"none\") return true;\n  if (display === undefined && attribute(node, \"hidden\") !== null) return true;\n  return attribute(node, \"aria-hidden\")?.trim().toLowerCase() === \"true\";\n};\n\n/** A font size of zero: `0`, `0px`, `0em` or `0%`. */\nconst ZERO_SIZE = /^\\+?(?:0+(?:\\.0*)?|\\.0+)(?:[a-z]{1,4}|%)?$/;\n/** These sizes are a part of the size of the parent, so a part of zero stays zero. */\nconst RELATIVE_SIZE =\n  /^(?:\\+?[\\d.]+(?:em|ex|ch|cap|ic|lh|%)|larger|smaller|inherit|unset|revert|revert-layer)$/;\n\n/** This function tells if the text of an element has a zero size. A child that sets a size of its own shows its text again. */\nconst zeroSize = (fontSize: string | undefined, parentZero: boolean): boolean => {\n  if (fontSize === undefined) return parentZero;\n  if (ZERO_SIZE.test(fontSize)) return true;\n  return parentZero && RELATIVE_SIZE.test(fontSize);\n};\n\ninterface Walked {\n  readonly text: string;\n  readonly elements: number;\n  readonly links: number;\n  readonly depth: number;\n  /** These nodes are not text for a reader: the elements that `notRendered` refuses, and the text of a zero size. */\n  readonly unseen: readonly DomNode[];\n}\n\ninterface Frame {\n  readonly name: string;\n  /** The children of an element are read with `nextSibling`, which costs nothing in linkedom. */\n  next: DomNode | null | undefined;\n  /** linkedom gives a document type no `nextSibling`, so the few children of a document are read as a list. */\n  readonly list: ArrayLike<DomNode> | undefined;\n  index: number;\n  /** This value is true when the text of the element has a zero size. */\n  readonly zero: boolean;\n}\n\n/**\n * This function walks a tree with a stack of its own. Thus the depth of the markup cannot overflow\n * the stack. It reads only the text that a reader gets (refer to `notRendered` and `zeroSize`), and it\n * gives the nodes that it did not read.\n */\nconst walkText = (root: DomNode): Walked => {\n  const parts: string[] = [];\n  const unseen: DomNode[] = [];\n  // a link or a button ends a word even where the markup runs it into the\n  // next (\"Offers from AmexGlobal Dining Access\"), though not before a stop\n  let seam = false;\n  let elements = 0;\n  let links = 0;\n  let depth = 0;\n  const stack: Frame[] = [];\n  const enter = (node: DomNode): void => {\n    if (node.nodeType === 3) {\n      if (stack[stack.length - 1]?.zero === true) {\n        unseen.push(node);\n        return;\n      }\n      const text = node.textContent ?? \"\";\n      if (seam && /^[^\\s.,;:!?)\\]–—-]/.test(text)) parts.push(\" \");\n      if (text.length > 0) seam = false;\n      parts.push(text);\n      return;\n    }\n    if (node.nodeType !== 1 && node.nodeType !== 9 && node.nodeType !== 11) return;\n    const name = node.nodeName.toLowerCase();\n    let container = node;\n    if (name === \"template\") {\n      if (!isShadowRoot(node)) return;\n      container = (node as DomElement).content ?? node;\n    } else if (SKIP.has(name)) {\n      return;\n    }\n    let zero = stack[stack.length - 1]?.zero ?? false;\n    if (node.nodeType === 1) {\n      const style = inlineStyle(node);\n      if (notRendered(node, style)) {\n        unseen.push(node);\n        return;\n      }\n      zero = zeroSize(style.get(\"font-size\"), zero);\n      elements += 1;\n      if (name === \"a\") links += 1;\n      if (BLOCK.has(name)) parts.push(\"\\n\");\n      else if (!INLINE.has(name) && !CELL.has(name)) seam = true;\n    }\n    stack.push(\n      container.nodeType === 1\n        ? { name, next: container.firstChild, list: undefined, index: 0, zero }\n        : { name, next: undefined, list: container.childNodes, index: 0, zero },\n    );\n    if (stack.length > depth) depth = stack.length;\n  };\n  const leave = (name: string): void => {\n    const block = BLOCK.has(name);\n    if (block) parts.push(\"\\n\");\n    else if (CELL.has(name)) parts.push(\" \");\n    if (block) seam = false;\n    else if (WORD_END.has(name) || !INLINE.has(name)) seam = true;\n  };\n  enter(root);\n  while (stack.length > 0) {\n    const frame = stack[stack.length - 1]!;\n    let child: DomNode | null | undefined;\n    if (frame.list !== undefined) {\n      child = frame.index < frame.list.length ? frame.list[frame.index] : undefined;\n      frame.index += 1;\n    } else {\n      child = frame.next;\n      if (child !== null && child !== undefined) frame.next = child.nextSibling;\n    }\n    if (child === null || child === undefined) {\n      stack.pop();\n      leave(frame.name);\n    } else {\n      enter(child);\n    }\n  }\n  const text = parts\n    .join(\"\")\n    .split(\"\\n\")\n    .map((line) =>\n      line\n        .replace(/[​-‍⁠­]/g, \"\")\n        .replace(/\\s+/g, \" \")\n        .trim(),\n    )\n    .filter((line) => line.length > 0)\n    .join(\"\\n\");\n  return { text, elements, links, depth, unseen };\n};\n\n/** This function gives the text that a reader sees, with one line for each block. */\nexport const visibleText = (root: DomNode): string => walkText(root).text;\n\n// ── JSON-LD ─────────────────────────────────────────────────────────────────\n\nconst WRAPPERS: ReadonlyArray<readonly [RegExp, RegExp]> = [\n  [/^<!--/, /-->$/],\n  [/^\\/\\/\\s{0,8}<!\\[CDATA\\[/, /\\/\\/\\s{0,8}\\]\\]>$/],\n  [/^\\/\\*\\s{0,8}<!\\[CDATA\\[\\s{0,8}\\*\\//, /\\/\\*\\s{0,8}\\]\\]>\\s{0,8}\\*\\/$/],\n  [/^<!\\[CDATA\\[/, /\\]\\]>$/],\n];\n\nconst unwrap = (source: string): string => {\n  let text = source.replace(/^/, \"\").trim();\n  for (const [open, close] of WRAPPERS) {\n    text = text.replace(open, \"\").trim();\n    text = text.replace(close, \"\").trim();\n  }\n  return text;\n};\n\n/**\n * This function cuts a script into its top-level JSON values. It also repairs two common faults: a\n * comma before a closing bracket, and a raw control character inside a string.\n */\nconst jsonValues = (source: string): string[] => {\n  const values: string[] = [];\n  let out: string[] = [];\n  let depth = 0;\n  let inString = false;\n  let escaped = false;\n  let pendingComma = false;\n  for (let index = 0; index < source.length; index += 1) {\n    const char = source[index]!;\n    const code = source.charCodeAt(index);\n    if (inString) {\n      if (escaped) escaped = false;\n      else if (char === \"\\\\\") escaped = true;\n      else if (char === '\"') inString = false;\n      // a string between two values is skipped with them\n      if (depth === 0) continue;\n      out.push(\n        code >= 0x20\n          ? char\n          : char === \"\\n\"\n            ? \"\\\\n\"\n            : char === \"\\r\"\n              ? \"\\\\r\"\n              : char === \"\\t\"\n                ? \"\\\\t\"\n                : `\\\\u${code.toString(16).padStart(4, \"0\")}`,\n      );\n      continue;\n    }\n    if (depth === 0) {\n      // between values only an object or an array starts a value\n      if (char === \"{\" || char === \"[\") {\n        depth = 1;\n        out = [char];\n        pendingComma = false;\n      } else if (char === '\"') {\n        inString = true;\n      }\n      continue;\n    }\n    if (code === 0x20 || code === 0x09 || code === 0x0a || code === 0x0d) {\n      out.push(char);\n      continue;\n    }\n    if (char === \",\") {\n      if (pendingComma) out.push(\",\");\n      pendingComma = true;\n      continue;\n    }\n    if (pendingComma) {\n      // a comma before a closing bracket is dropped\n      if (char !== \"}\" && char !== \"]\") out.push(\",\");\n      pendingComma = false;\n    }\n    out.push(char);\n    if (char === '\"') inString = true;\n    else if (char === \"{\" || char === \"[\") depth += 1;\n    else if (char === \"}\" || char === \"]\") {\n      depth -= 1;\n      if (depth === 0) {\n        values.push(out.join(\"\"));\n        out = [];\n      }\n    }\n  }\n  // a value that never closes is given whole, and its parse fails\n  if (depth > 0) values.push(out.join(\"\"));\n  return values;\n};\n\nconst parseJsonLd = (\n  sources: readonly string[],\n): { readonly records: Record<string, unknown>[]; readonly unreadable: number } => {\n  const records: Record<string, unknown>[] = [];\n  let unreadable = 0;\n  // an explicit stack: JSON nested deeply in arrays cannot overflow the call stack\n  const push = (first: unknown): void => {\n    const stack: unknown[] = [first];\n    const found: Record<string, unknown>[] = [];\n    while (stack.length > 0) {\n      const value = stack.pop();\n      if (Array.isArray(value)) {\n        for (let index = value.length - 1; index >= 0; index -= 1) stack.push(value[index]);\n        continue;\n      }\n      if (value === null || typeof value !== \"object\") continue;\n      const record = value as Record<string, unknown>;\n      const graph = record[\"@graph\"];\n      if (graph !== undefined && graph !== null && typeof graph === \"object\") {\n        const { \"@graph\": _graph, ...rest } = record;\n        // the record itself first, then its graph, in the order of the page\n        stack.push(graph);\n        if (Object.keys(rest).some((key) => key !== \"@context\")) found.push(rest);\n        continue;\n      }\n      found.push(record);\n    }\n    records.push(...found);\n  };\n  for (const source of sources) {\n    const text = unwrap(source);\n    if (text.length === 0) continue;\n    try {\n      push(JSON.parse(text));\n      continue;\n    } catch {\n      // repaired below\n    }\n    let failed = false;\n    let any = false;\n    for (const value of jsonValues(text)) {\n      try {\n        push(JSON.parse(value));\n        any = true;\n      } catch {\n        failed = true;\n      }\n    }\n    if (failed || !any) unreadable += 1;\n  }\n  return { records, unreadable };\n};\n\n/**\n * This function reads the JSON-LD blocks and makes arrays and `@graph` flat. It repairs a comma\n * before a closing bracket, a raw line break in a string and two values in one block. It ignores\n * a block that it cannot repair.\n */\nexport const readJsonLd = (sources: readonly string[]): Record<string, unknown>[] =>\n  parseJsonLd(sources).records;\n\n/** This function gives the types that a JSON-LD object declares, without the schema.org prefix. */\nexport const typesOf = (record: Record<string, unknown>): string[] => {\n  const raw = record[\"@type\"];\n  const list = Array.isArray(raw) ? raw : raw === undefined ? [] : [raw];\n  return list\n    .filter((value): value is string => typeof value === \"string\")\n    .map((value) => value.replace(/^.*[/#]/, \"\").replace(/^[A-Za-z][\\w.+-]*:/, \"\"));\n};\n\n// ── URLs ────────────────────────────────────────────────────────────────────\n\nexport { canonicalUrl };\n\nconst isWeb = (url: URL): boolean => url.protocol === \"http:\" || url.protocol === \"https:\";\nconst siteOf = (host: string): string => host.toLowerCase().replace(/^www\\./, \"\");\n\n// ── what the page needs ─────────────────────────────────────────────────────\n\n/** The marks of the pages that a bot check or a firewall sends in place of the content. */\nconst CHALLENGES: ReadonlyArray<readonly [string, RegExp]> = [\n  [\"Incapsula\", /_Incapsula_Resource|Incapsula incident ID/i],\n  [\"Cloudflare\", /\\/cdn-cgi\\/challenge-platform\\/|cf-chl-|cf_chl_opt|cf-browser-verification/i],\n  [\"Sucuri\", /sucuri_cloudproxy|Sucuri WebSite Firewall/i],\n  [\"DataDome\", /captcha-delivery\\.com/i],\n  [\"PerimeterX\", /_pxCaptcha|px-captcha|captcha\\.px-cdn\\.net/i],\n  [\"AWS WAF\", /AwsWafIntegration|token\\.awswaf\\.com|\\/challenge\\.js.{0,80}awswaf/i],\n  [\"DDoS-Guard\", /ddos-guard\\.net\\/|__ddg1_/i],\n];\nconst CHALLENGE_TITLE =\n  /^(?:just a moment\\.*|attention required!? \\| cloudflare|access to this page has been denied\\.?|please verify you are a human|are you a robot\\??|ddos-guard|checking your browser.{0,40})$/i;\n\nconst ASKS_FOR_SCRIPT =\n  /(?:enable|turn on|activate|switch on|allow)\\s+(?:your\\s+)?javascript|javascript\\s+(?:is\\s+)?(?:required|disabled|turned off|not enabled|must be enabled|needs to be enabled)|(?:requires?|needs?)\\s+javascript|javascript\\s+aktivieren|activ(?:er|ez)\\s+(?:le\\s+)?javascript|habilit(?:ar|e)\\s+(?:el\\s+)?javascript/i;\n/** The noscript of a comment widget (Disqus) asks for JavaScript for the comments, not for the page. */\nconst COMMENTS = /comments|disqus/i;\n\n/** These elements hold the whole page of an application that its scripts draw. */\nconst APP_ROOTS = \"#root, #app, #__next, #__nuxt, #___gatsby, app-root, my-app, [data-reactroot]\";\n\nconst EMBEDDED: ReadonlyArray<readonly [string, (html: string, doc: DomDocument) => boolean]> = [\n  [\"next\", (_html, doc) => doc.querySelector('script[id=\"__NEXT_DATA__\"]') !== null],\n  [\"next-app\", (html) => html.includes(\"self.__next_f.push\")],\n  [\"nuxt\", (html) => html.includes(\"__NUXT_DATA__\") || html.includes(\"window.__NUXT__\")],\n  [\"apollo\", (html) => html.includes(\"__APOLLO_STATE__\")],\n];\n\n/** This function reads `content=\"0; url=/home\"`. A refresh with no URL reloads the page, and is no redirect. */\nconst refreshTarget = (\n  content: string,\n): { readonly delay: number; readonly url: string } | undefined => {\n  const text = content.slice(0, 2000).trim();\n  const delay = /^\\d+(?:\\.\\d*)?/.exec(text)?.[0] ?? \"\";\n  const url = text\n    .slice(delay.length)\n    .replace(/^[\\s;,]+/, \"\")\n    .replace(/^url\\s*=\\s*/i, \"\")\n    .replace(/^([\"'])(.*)\\1$/s, \"$2\")\n    .trim();\n  return url.length === 0 ? undefined : { delay: Number(delay || 0), url };\n};\n\n// ── composing the text ──────────────────────────────────────────────────────\n\n/** This function cuts text to a length at the end of a line where it can. It does not cut a surrogate pair. */\nconst cutHead = (text: string, length: number): string => {\n  if (text.length <= length) return text;\n  if (length <= 0) return \"\";\n  const line = text.lastIndexOf(\"\\n\", length);\n  let end = line > length - 2000 && line > 0 ? line : length;\n  const code = text.charCodeAt(end - 1);\n  if (code >= 0xd800 && code <= 0xdbff) end -= 1;\n  return text.slice(0, end);\n};\n\n/** This function keeps the end of the text, from the start of a line where it can. */\nconst cutTail = (text: string, length: number): string => {\n  if (text.length <= length) return text;\n  if (length <= 0) return \"\";\n  let start = text.length - length;\n  const line = text.indexOf(\"\\n\", start);\n  if (line !== -1 && line < start + 2000) start = line + 1;\n  const code = text.charCodeAt(start);\n  if (code >= 0xdc00 && code <= 0xdfff) start += 1;\n  return text.slice(start);\n};\n\n/** This function keeps the start and the end of a text that is too long: a footer keeps its hours. */\nconst capText = (text: string): string =>\n  text.length <= MAX_TEXT_CHARS\n    ? text\n    : `${cutHead(text, MAX_TEXT_CHARS - REST_RESERVE_CHARS - 2)}\\n\\n${cutTail(text, REST_RESERVE_CHARS)}`;\n\n/** A line that can label the lines below it: short, with no number, and not a sentence (\"Opening hours\"). */\nconst isHeading = (line: string): boolean =>\n  line.length <= 40 && !/[.!?]$/.test(line) && !/[0-9]/.test(line);\n\n/**\n * This function finds the lines of the article in the lines of the page, one for one and in order.\n * Thus a line that the article and a sidebar share stays in the sidebar (\"10 am – 5 pm\").\n */\nconst articleLines = (all: readonly string[], main: readonly string[]): Uint8Array => {\n  const taken = new Uint8Array(all.length);\n  const positions = new Map<string, number[]>();\n  all.forEach((line, index) => {\n    const list = positions.get(line);\n    if (list === undefined) positions.set(line, [index]);\n    else list.push(index);\n  });\n  const cursors = new Map<string, number>();\n  let at = 0;\n  let matched = 0;\n  for (const line of main) {\n    const list = positions.get(line);\n    if (list === undefined) continue;\n    let cursor = cursors.get(line) ?? 0;\n    while (cursor < list.length && list[cursor]! < at) cursor += 1;\n    cursors.set(line, cursor);\n    const position = list[cursor];\n    if (position === undefined) continue;\n    // a line far ahead is the same line only when the page has it once\n    if (matched > 0 && position - at > 400 && list.length > 1) continue;\n    taken[position] = 1;\n    cursors.set(line, cursor + 1);\n    at = position + 1;\n    matched += 1;\n  }\n  return taken;\n};\n\n/** This function gives the lines of the page that are not in the article. Each run keeps the heading above it. */\nconst restOf = (all: readonly string[], taken: Uint8Array): string[] => {\n  const rest: string[] = [];\n  for (let index = 0; index < all.length; index += 1) {\n    if (taken[index] === 1) continue;\n    const above = all[index - 1];\n    if (index > 0 && taken[index - 1] === 1 && above !== undefined && isHeading(above))\n      rest.push(above);\n    rest.push(all[index]!);\n  }\n  return rest;\n};\n\nexport const extractPage = (html: string, pageUrl: string): PageExtract => {\n  const notes: string[] = [];\n  const flat = flattenDeepMarkup(html);\n  if (flat.dropped > 0) {\n    notes.push(\n      `the markup nests deeper than ${String(MAX_DEPTH)} elements, so the reader read the deeper tags as text`,\n    );\n  }\n  const doc = parse(flat.html);\n  const meta = (selector: string): string | null =>\n    doc.querySelector(selector)?.getAttribute(\"content\")?.trim() || null;\n\n  const scripts = [...doc.querySelectorAll(\"script\")];\n  const parsed = parseJsonLd(\n    scripts\n      .filter((node) =>\n        /^\\s*application\\/ld\\+json\\s*(?:;|$)/i.test(node.getAttribute(\"type\") ?? \"\"),\n      )\n      .map((node) => node.textContent ?? \"\"),\n  );\n  const jsonld = parsed.records;\n  if (parsed.unreadable > 0) {\n    notes.push(\n      `${String(parsed.unreadable)} JSON-LD ${parsed.unreadable === 1 ? \"block\" : \"blocks\"} could not be read`,\n    );\n  }\n\n  const page = (() => {\n    try {\n      return new URL(pageUrl);\n    } catch {\n      return undefined;\n    }\n  })();\n  const baseHref = doc.querySelector(\"base[href]\")?.getAttribute(\"href\");\n  const base = (() => {\n    try {\n      const resolved = new URL(baseHref ?? pageUrl, pageUrl);\n      // a <base href=\"javascript:...\"> would take every relative link with it\n      return isWeb(resolved) ? resolved.href : pageUrl;\n    } catch {\n      return pageUrl;\n    }\n  })();\n  const resolve = (raw: string): URL | undefined => {\n    try {\n      const url = new URL(raw.trim(), base);\n      if (!isWeb(url)) return undefined;\n      url.username = \"\";\n      url.password = \"\";\n      return url;\n    } catch {\n      return undefined;\n    }\n  };\n  const self = page === undefined ? undefined : canonicalUrl(page.href);\n\n  const refresh = [...doc.querySelectorAll(\"meta[http-equiv]\")]\n    .filter((node) => /^\\s*refresh\\s*$/i.test(node.getAttribute(\"http-equiv\") ?? \"\"))\n    .map((node) => refreshTarget(node.getAttribute(\"content\") ?? \"\"))\n    .find((target) => target !== undefined);\n  const refreshUrl = refresh === undefined ? undefined : resolve(refresh.url);\n  const redirect =\n    refresh !== undefined &&\n    refresh.delay <= 5 &&\n    refreshUrl !== undefined &&\n    canonicalUrl(refreshUrl.href) !== self\n      ? canonicalUrl(refreshUrl.href)\n      : null;\n\n  const canonical = (() => {\n    const node = [...doc.querySelectorAll(\"link[rel][href]\")].find((link) =>\n      (link.getAttribute(\"rel\") ?? \"\").toLowerCase().split(/\\s+/).includes(\"canonical\"),\n    );\n    const url = node === undefined ? undefined : resolve(node.getAttribute(\"href\") ?? \"\");\n    // a canonical on another site could take the page's identity: it is not trusted\n    if (url === undefined || page === undefined || siteOf(url.hostname) !== siteOf(page.hostname)) {\n      return null;\n    }\n    return canonicalUrl(url.href);\n  })();\n\n  const clean = (value: string | null | undefined): string =>\n    (value ?? \"\").replace(/\\s+/g, \" \").trim();\n  const titles = [...doc.querySelectorAll(\"title\")];\n  // an <svg> keeps a <title> of its own (\"Icon\"), which names no page\n  const inSvg = (node: DomNode): boolean => {\n    for (let parent = node.parentNode; parent; parent = parent.parentNode) {\n      if (parent.nodeName.toLowerCase() === \"svg\") return true;\n    }\n    return false;\n  };\n  const title =\n    meta('meta[property=\"og:title\"]') ||\n    clean(titles.find((node) => !inSvg(node))?.textContent) ||\n    clean(doc.querySelector(\"h1\")?.textContent) ||\n    pageUrl;\n  const description = meta('meta[name=\"description\"]') ?? meta('meta[property=\"og:description\"]');\n  const lang = doc.documentElement?.getAttribute(\"lang\")?.trim() || null;\n  const noscript = [...doc.querySelectorAll(\"noscript\")]\n    .map((node) => node.textContent ?? \"\")\n    .filter((text) => !COMMENTS.test(text))\n    .join(\" \");\n  const embedded = EMBEDDED.filter(([, found]) => found(html, doc)).map(([name]) => name);\n  const appShell = [...doc.querySelectorAll(APP_ROOTS)].some(\n    (node) => (node.textContent ?? \"\").trim().length === 0,\n  );\n\n  // the whole document: a page that omits <html> is not under\n  // documentElement at all, and document.body can be a decoy (see DomDocument;\n  // a homepage read as 0 characters and went to the browser for nothing)\n  const walked = walkText(doc);\n  // the links and Readability read only what a reader gets: text that the page hides goes now\n  for (const node of walked.unseen) node.remove?.();\n\n  const seen = new Set<string>(self === undefined ? [] : [self]);\n  const links: PageLink[] = [];\n  if (redirect !== null) {\n    seen.add(redirect);\n    links.push({ href: redirect, text: \"\" });\n  }\n  for (const anchor of doc.querySelectorAll(\"a[href]\")) {\n    if (links.length >= MAX_LINKS) break;\n    const url = resolve(anchor.getAttribute(\"href\") ?? \"\");\n    if (url === undefined) continue;\n    const href = canonicalUrl(url.href);\n    const text =\n      (anchor.textContent ?? \"\").replace(/\\s+/g, \" \").trim() ||\n      anchor.getAttribute(\"aria-label\")?.trim() ||\n      anchor.getAttribute(\"title\")?.trim() ||\n      anchor.querySelector(\"img[alt]\")?.getAttribute(\"alt\")?.trim() ||\n      \"\";\n    // the link to the target of a meta refresh gets the text of the first anchor that names it\n    if (href === redirect && links[0]?.href === redirect && links[0].text === \"\") {\n      links[0] = { href, text: text.slice(0, 120) };\n      continue;\n    }\n    // a link to the page itself (\"#menu\") is no link\n    if (seen.has(href)) continue;\n    seen.add(href);\n    links.push({ href, text: text.slice(0, 120) });\n  }\n\n  const all = walked.text;\n  let text = capText(all);\n  let cut = text.length < all.length;\n  if (all.length > SHORT_PAGE_CHARS) {\n    const main = (() => {\n      if (\n        walked.elements > READABILITY_MAX_ELEMENTS ||\n        walked.links > READABILITY_MAX_LINKS ||\n        walked.depth > READABILITY_MAX_DEPTH\n      ) {\n        notes.push(\"the page is too large for Readability, so its text is in page order\");\n        return \"\";\n      }\n      try {\n        // Readability rewrites the document, and the reader has no more use for it. It gives its\n        // article as an element, read here so that blocks stay lines\n        const article = new Readability<DomNode>(\n          doc as unknown as ConstructorParameters<typeof Readability>[0],\n          {\n            charThreshold: 500,\n            disableJSONLD: true,\n            serializer: (node) => node as unknown as DomNode,\n          },\n        ).parse();\n        return article?.content === null || article?.content === undefined\n          ? \"\"\n          : visibleText(article.content);\n      } catch {\n        notes.push(\"Readability could not read the page, so its text is in page order\");\n        return \"\";\n      }\n    })();\n    if (main.length > 500) {\n      // the article first, then every other line the page shows: a venue's\n      // hours sit in a table or a sidebar Readability drops (a museum's visit\n      // page), not only in its footer\n      const lines = all.split(\"\\n\");\n      const rest = restOf(lines, articleLines(lines, main.split(\"\\n\"))).join(\"\\n\");\n      const head = cutHead(main, MAX_TEXT_CHARS - Math.min(rest.length, REST_RESERVE_CHARS) - 2);\n      const tail = cutHead(rest, MAX_TEXT_CHARS - head.length - 2);\n      text = rest.length > 0 ? `${head}\\n\\n${tail}` : head;\n      cut = head.length < main.length || tail.length < rest.length;\n    }\n  }\n  if (cut) notes.push(`the text is cut to ${String(MAX_TEXT_CHARS)} characters`);\n\n  const visibleChars = all.length;\n  const pageTitle = clean(titles[0]?.textContent);\n  const blocked =\n    visibleChars < 2000\n      ? CHALLENGE_TITLE.test(pageTitle)\n        ? (CHALLENGES.find(([, pattern]) => pattern.test(html))?.[0] ?? \"a bot check\")\n        : (CHALLENGES.find(([, pattern]) => pattern.test(html))?.[0] ?? null)\n      : null;\n  const reason =\n    // a browser would only go past a bot check or follow the redirect\n    blocked !== null || redirect !== null\n      ? null\n      : // an app shell shows next to nothing; a small venue page can still say\n        // its hours and address in a hundred characters, and needs no browser\n        visibleChars < 80\n        ? `only ${String(visibleChars)} characters of visible text`\n        : visibleChars < 1200 && ASKS_FOR_SCRIPT.test(noscript)\n          ? \"the page asks for JavaScript\"\n          : visibleChars < 1200 && appShell\n            ? \"an empty application root\"\n            : null;\n\n  return {\n    title: title.replace(/\\s+/g, \" \").slice(0, 300),\n    description,\n    text,\n    ...(cut ? { fullText: all } : {}),\n    jsonld,\n    links,\n    lang,\n    visibleChars,\n    embedded,\n    needsRender: reason !== null,\n    renderReason: reason,\n    canonical,\n    redirect,\n    blocked,\n    notes,\n  };\n};\n"],"mappings":";;;;;;;AAKA,MAAM,WACJ;;AAGF,MAAM,oBAAoB,UACxB,MAAM,QAAQ,mBAAmB,WAAW;CAC1C,MAAM,OAAO,OAAO,aAAa,OAAO,SAAS,OAAO,MAAM,CAAC,GAAG,EAAE,CAAC;CACrE,OAAO,qBAAqB,KAAK,IAAI,IAAI,OAAO,OAAO,YAAY;AACrE,CAAC;;;;;AAMH,MAAa,gBAAgB,UAA0B;CACrD,MAAM,MAAM,IAAI,IAAI,KAAK;CACzB,IAAI,OAAO;CACX,IAAI,WAAW,IAAI,SAAS,YAAY;CACxC,IACG,IAAI,aAAa,YAAY,IAAI,SAAS,SAC1C,IAAI,aAAa,WAAW,IAAI,SAAS,MAE1C,IAAI,OAAO;CAGb,KAAK,MAAM,OAAO,MAAM,KAAK,IAAI,aAAa,KAAK,CAAC,GAClD,IAAI,SAAS,KAAK,GAAG,GAAG,IAAI,aAAa,OAAO,GAAG;CAErD,IAAI,IAAI,SAAS,SAAS,GAAG,GAAG,IAAI,WAAW,iBAAiB,IAAI,QAAQ;CAE5E,IAAI,SAAS,IAAI,OAAO,SAAS,GAAG,IAAI,iBAAiB,IAAI,MAAM,IAAI,IAAI;CAC3E,OAAO,IAAI;AACb;;;;;AAMA,MAAa,cAAc,UAA0B;CACnD,IAAI;EACF,OAAO,aAAa,KAAK;CAC3B,QAAQ;EACN,OAAO;CACT;AACF;;;;;;;;;;;;;;;;ACnCA,MAAM,gBAAgB;;AAGtB,MAAM,cAAc,UAAsC;CACxD,IAAI;EACF,OAAO,IAAI,YAAY,MAAM,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;CACrD,QAAQ;EACN;CACF;AACF;AAEA,MAAM,SAAS,UAA0C;CACvD,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,OAAQ,MAAM,OAAO,KAAM,OAAO;CACxE,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,KAAM,OAAO;CACnD,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,KAAM,OAAO;AAErD;;AAGA,MAAM,aAAa,gBAAmD;CACpE,IAAI,gBAAgB,MAAM,OAAO,KAAA;CACjC,MAAM,QAAQ,8CAA8C,KAAK,YAAY,MAAM,GAAG,GAAG,CAAC;CAC1F,OAAO,QAAQ,MAAM,QAAQ,MAAM,KAAA;AACrC;;AAGA,MAAM,mBAAmB,UAA0C;CACjE,IAAI,OAAO;CACX,MAAM,MAAM,KAAK,IAAI,MAAM,QAAQ,aAAa;CAChD,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,SAAS,GAAG,QAAQ,OAAO,aAAa,MAAM,MAAO;CACtF,MAAM,MAAM,qDAAqD,KAAK,IAAI;CAC1E,IAAI,QAAQ,MAAM,OAAO,IAAI;CAE7B,MAAM,SAAS,KAAK,QAAQ,oBAAoB,EAAE;CAGlD,OADa,4DAA4D,KAAK,MACpE,CAAC,GAAG;AAChB;;;;;;AAOA,MAAM,gBACJ,UAC6D;CAC7D,IAAI,YAAY;CAChB,IAAI,UAAU;CACd,IAAI,QAAQ;CACZ,OAAO,QAAQ,MAAM,QAAQ;EAC3B,MAAM,OAAO,MAAM;EACnB,IAAI,OAAO,KAAM;GACf,SAAS;GACT;EACF;EACA,MAAM,OACJ,QAAQ,OAAQ,QAAQ,MACpB,IACA,QAAQ,OAAQ,QAAQ,MACtB,IACA,QAAQ,OAAQ,QAAQ,MACtB,IACA;EACV,IAAI,SAAS,GAAG;GACd,WAAW;GACX,SAAS;GACT;EACF;EAEA,MAAM,MAAM,SAAS,MAAO,MAAO,SAAS,MAAO,MAAO;EAC1D,MAAM,OAAO,SAAS,MAAO,MAAO,SAAS,MAAO,MAAO;EAC3D,IAAI,KAAK;EACT,IAAI,KAAK;EACT,OAAO,MAAM,QAAQ,QAAQ,KAAK,MAAM,QAAQ;GAC9C,MAAM,OAAO,MAAM,QAAQ;GAC3B,IAAI,OAAO,IAAI,OAAO,OAAO,OAAO,OAAO,OAAO,OAAQ,OAAO,KAAM;IACrE,KAAK;IACL;GACF;GACA,MAAM;EACR;EAEA,IAAI,MAAM,MAAM,MAAM;EACtB,IAAI,IAAI;GACN,aAAa;GACb,SAAS,OAAO;EAClB,OAAO;GACL,WAAW;GACX,SAAS;EACX;CACF;CACA,OAAO;EAAE;EAAW;CAAQ;AAC9B;;;;;AAMA,MAAa,gBAAgB,OAAmB,gBAAuC;CACrF,MAAM,MAAM,MAAM,KAAK;CACvB,IAAI,QAAQ,KAAA,GAAW,OAAO;CAC9B,MAAM,SAAS,UAAU,WAAW;CACpC,MAAM,aAAa,WAAW,KAAA,IAAY,KAAA,IAAY,WAAW,MAAM;CACvE,IAAI,eAAe,KAAA,GAAW,OAAO;CACrC,IAAI,gBAAgB,QAAQ,YAAY,KAAK,WAAW,GAAG;EACzD,MAAM,QAAQ,gBAAgB,KAAK;EACnC,IAAI,WAAW,UAAU,KAAA,IAAY,KAAA,IAAY,WAAW,KAAK;EAEjE,IAAI,UAAU,WAAW,QAAQ,MAAM,MAAM,WAAW;EACxD,IAAI,aAAa,kBAAkB,WAAW;EAC9C,IAAI,aAAa,SAAS,OAAO;EACjC,IAAI,aAAa,KAAA,GAAW;GAG1B,MAAM,WAAW,aAAa,KAAK;GACnC,OAAO,SAAS,YAAY,KAAK,SAAS,YAAY,IAAI,UAAU;EACtE;CACF;CACA,IAAI,QAAQ,KAAK,eAAe,EAAE,GAAG,OAAO;CAC5C,MAAM,WAAW,aAAa,KAAK;CACnC,OAAO,SAAS,UAAU,KAAK,SAAS,WAAW,SAAS,YAAY,iBAAiB;AAC3F;;;;;;AAOA,MAAM,oBAAoB;CACxB;CAAQ;CAAM;CAAQ;CAAO;CAAQ;CAAQ;CAAQ;CAAQ;CAAO;CAAQ;CAAO;CAAQ;CAC3F;CAAM;CAAO;CAAM;CAAM;CAAQ;CAAQ;CAAQ;CAAQ;CAAQ;CAAQ;CAAQ;CAAO;CACxF;CAAO;CAAQ;CAAO;CAAM;CAAO;AACrC;AACA,MAAM,cAAc,IAAI,OAAO,IAAI,OAAO,aAAa,GAAI,EAAE,GAAG,OAAO,aAAa,GAAI,EAAE,IAAI,GAAG;AACjG,MAAM,qBAAqB,SACzB,KAAK,QAAQ,cAAc,SACzB,OAAO,aAAa,kBAAkB,KAAK,WAAW,CAAC,IAAI,IAAM,CACnE;;;;;AAMF,MAAa,cAAc,OAAmB,gBAAuC;CACnF,MAAM,WAAW,aAAa,OAAO,WAAW;CAChD,IAAI;EACF,MAAM,OAAO,IAAI,YAAY,UAAU,EAAE,OAAO,MAAM,CAAC,CAAC,CAAC,OAAO,KAAK;EACrE,OAAO,aAAa,iBAAiB,kBAAkB,IAAI,IAAI;CACjE,QAAQ;EAEN,OAAO,IAAI,YAAY,SAAS,EAAE,OAAO,MAAM,CAAC,CAAC,CAAC,OAAO,KAAK;CAChE;AACF;;;AC1GA,MAAM,uBAAO,IAAI,IAAI;CACnB;CACA;CACA;CAEA;CACA;CACA;CAEA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;AAED,MAAM,wBAAQ,IAAI,IAAI;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;AAKD,MAAM,yBAAS,IAAI,IAAI;CACrB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;AAED,MAAM,uBAAO,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC;AACjC,MAAM,2BAAW,IAAI,IAAI,CAAC,KAAK,QAAQ,CAAC;;AAGxC,MAAM,mBAAmB;AACzB,MAAM,iBAAiB;;AAEvB,MAAM,qBAAqB;AAC3B,MAAM,YAAY;;AAElB,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAC9B,MAAM,wBAAwB;;;;;AAK9B,MAAM,YAAY;AAgClB,MAAM,SAAS,SACZ,UAAU,IAAI,CAAC,CAA0C;;AAG5D,MAAM,uBAAO,IAAI,IAAI;CACnB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;AAED,MAAM,2BAAW,IAAI,IAAI;CACvB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;AAED,MAAM,8BAAc,IAAI,IAAI;CAC1B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,YAAY,SACf,QAAQ,MAAQ,QAAQ,OAAU,QAAQ,MAAQ,QAAQ;AAC7D,MAAM,cAAc,SAClB,SAAS,IAAI,KACZ,QAAQ,MAAQ,QAAQ,MACzB,SAAS,MACT,SAAS,MACT,SAAS,MACT,SAAS;;AAGX,MAAM,YAAY,MAAc,SAAyB;CACvD,IAAI,QAAQ;CACZ,IAAI,cAAc;CAClB,KAAK,IAAI,KAAK,MAAM,KAAK,KAAK,QAAQ,MAAM,GAAG;EAC7C,MAAM,OAAO,KAAK,WAAW,EAAE;EAC/B,IAAI,UAAU,GACR;OAAA,SAAS,OAAO,QAAQ;EAAA,OACvB,KAAK,SAAS,MAAQ,SAAS,OAAS,aAC7C,QAAQ;OACH,IAAI,SAAS,IAClB,OAAO,KAAK;EAEd,IAAI,SAAS,IAAM,cAAc;OAC5B,IAAI,SAAS,MAAQ,SAAS,KAAQ,SAAS,MAAQ,SAAS,IAAM,cAAc;CAC3F;CACA,OAAO,KAAK;AACd;;;;;;AAOA,MAAM,qBAAqB,SAAsE;CAC/F,MAAM,QAAkB,CAAC;CACzB,MAAM,uBAAO,IAAI,IAAoB;CACrC,MAAM,wBAAQ,IAAI,IAAoB;CACtC,MAAM,MAAgB,CAAC;CACvB,IAAI,SAAS;CACb,IAAI,UAAU;CACd,MAAM,SAAS,SAAuB;EACpC,KAAK,KAAK,IAAI,IAAI,KAAK,OAAO,GAAG;EACjC,SAAS;GACP,MAAM,MAAM,MAAM,IAAI;GACtB,KAAK,IAAI,MAAM,KAAK,IAAI,GAAG,KAAK,KAAK,CAAC;GACtC,IAAI,QAAQ,MAAM;EACpB;CACF;CACA,MAAM,WAAW,OAAe,KAAa,SAAuB;EAClE,IAAI,KAAK,KAAK,MAAM,QAAQ,KAAK,GAAG,MAAM,IAAI,IAAI,IAAI,OAAO,OAAO,IAAI,IAAI,IAAI,KAAK,GAAG;EACxF,SAAS;CACX;CACA,IAAI,QAAQ;CACZ,OAAO,QAAQ,KAAK,QAAQ;EAC1B,MAAM,KAAK,KAAK,QAAQ,KAAK,KAAK;EAClC,IAAI,OAAO,IAAI;EACf,IAAI,KAAK,WAAW,QAAQ,EAAE,GAAG;GAC/B,MAAM,MAAM,KAAK,QAAQ,OAAO,KAAK,CAAC;GACtC,QAAQ,QAAQ,KAAK,KAAK,SAAS,MAAM;GACzC;EACF;EACA,MAAM,OAAO,KAAK,WAAW,KAAK,CAAC;EACnC,IAAI,SAAS,MAAQ,SAAS,IAAM;GAClC,MAAM,MAAM,KAAK,QAAQ,KAAK,EAAE;GAChC,QAAQ,QAAQ,KAAK,KAAK,SAAS,MAAM;GACzC;EACF;EACA,MAAM,UAAU,SAAS;EACzB,MAAM,YAAY,MAAM,UAAU,IAAI;EACtC,IAAI,CAAC,SAAS,KAAK,WAAW,SAAS,CAAC,GAAG;GACzC,QAAQ,KAAK;GACb;EACF;EACA,IAAI,UAAU,YAAY;EAC1B,OAAO,UAAU,KAAK,UAAU,WAAW,KAAK,WAAW,OAAO,CAAC,GAAG,WAAW;EACjF,MAAM,OAAO,KAAK,MAAM,WAAW,OAAO,CAAC,CAAC,YAAY;EACxD,MAAM,SAAS,SAAS,MAAM,OAAO;EACrC,QAAQ;EACR,IAAI,SAAS;GAEX,MAAM,UAAU,MAAM,IAAI,IAAI,KAAK;GACnC,IAAI,UAAU,GAAG;IACf,MAAM,IAAI,MAAM,UAAU,CAAC;IAC3B,QAAQ,IAAI,QAAQ,IAAI;GAC1B,OACE,MAAM,IAAI;GAEZ;EACF;EACA,IAAI,SAAS,IAAI,IAAI,GAAG;GACtB,MAAM,MAAM,IAAI,OAAO,KAAK,KAAK,UAAU,IAAI;GAC/C,IAAI,YAAY;GAChB,QAAQ,IAAI,KAAK,IAAI,CAAC,EAAE,SAAS,KAAK;GACtC;EACF;EACA,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK,WAAW,SAAS,CAAC,MAAM,IAAM;EAC5D,IAAI,YAAY,IAAI,IAAI,GAAG,MAAM,IAAI;EACrC,IAAI,MAAM,UAAU,WAAW;GAC7B,MAAM,IAAI,OAAO,MAAM,IAAI,IAAI,KAAK,KAAK,CAAC;GAC1C,WAAW;GACX,QAAQ,IAAI,QAAQ,IAAI;GACxB;EACF;EACA,MAAM,KAAK,IAAI;EACf,KAAK,IAAI,OAAO,KAAK,IAAI,IAAI,KAAK,KAAK,CAAC;CAC1C;CACA,IAAI,YAAY,GAAG,OAAO;EAAE;EAAM;CAAQ;CAC1C,IAAI,KAAK,KAAK,MAAM,MAAM,CAAC;CAC3B,OAAO;EAAE,MAAM,IAAI,KAAK,EAAE;EAAG;CAAQ;AACvC;AAEA,MAAM,aAAa,MAAe,SAChC,OAAQ,KAA6B,iBAAiB,aACjD,KAAoB,aAAa,IAAI,IACtC;;AAGN,MAAM,gBAAgB,SACpB,UAAU,MAAM,gBAAgB,MAAM,QAAQ,UAAU,MAAM,YAAY,MAAM;;AAGlF,MAAM,mCAAmB,IAAI,IAAI,CAAC,WAAW,WAAW,CAAC;;;;;;AAOzD,MAAM,eAAe,SAA+C;CAClE,MAAM,wBAAQ,IAAI,IAAoB;CACtC,MAAM,QAAQ,UAAU,MAAM,OAAO;CACrC,IAAI,UAAU,QAAQ,MAAM,WAAW,GAAG,OAAO;CACjD,MAAM,4BAAY,IAAI,IAAY;CAClC,KAAK,MAAM,eAAe,MAAM,MAAM,GAAG,GAAG;EAC1C,MAAM,QAAQ,YAAY,QAAQ,GAAG;EACrC,IAAI,UAAU,IAAI;EAClB,MAAM,WAAW,YAAY,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,YAAY;EAChE,IAAI,CAAC,iBAAiB,IAAI,QAAQ,GAAG;EACrC,IAAI,QAAQ,YACT,MAAM,QAAQ,CAAC,CAAC,CAChB,KAAK,CAAC,CACN,YAAY;EACf,MAAM,OAAO,MAAM,YAAY,GAAG;EAClC,MAAM,SAAS,SAAS,MAAM,MAAM,MAAM,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM;EAC/D,IAAI,QAAQ,QAAQ,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK;EAC9C,IAAI,UAAU,IAAI,QAAQ,KAAK,CAAC,QAAQ;EACxC,IAAI,QAAQ,UAAU,IAAI,QAAQ;EAClC,MAAM,IAAI,UAAU,KAAK;CAC3B;CACA,OAAO;AACT;;;;;;;AAQA,MAAM,eAAe,MAAe,UAAgD;CAClF,MAAM,UAAU,MAAM,IAAI,SAAS;CACnC,IAAI,YAAY,QAAQ,OAAO;CAC/B,IAAI,YAAY,KAAA,KAAa,UAAU,MAAM,QAAQ,MAAM,MAAM,OAAO;CACxE,OAAO,UAAU,MAAM,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC,YAAY,MAAM;AAClE;;AAGA,MAAM,YAAY;;AAElB,MAAM,gBACJ;;AAGF,MAAM,YAAY,UAA8B,eAAiC;CAC/E,IAAI,aAAa,KAAA,GAAW,OAAO;CACnC,IAAI,UAAU,KAAK,QAAQ,GAAG,OAAO;CACrC,OAAO,cAAc,cAAc,KAAK,QAAQ;AAClD;;;;;;AA2BA,MAAM,YAAY,SAA0B;CAC1C,MAAM,QAAkB,CAAC;CACzB,MAAM,SAAoB,CAAC;CAG3B,IAAI,OAAO;CACX,IAAI,WAAW;CACf,IAAI,QAAQ;CACZ,IAAI,QAAQ;CACZ,MAAM,QAAiB,CAAC;CACxB,MAAM,SAAS,SAAwB;EACrC,IAAI,KAAK,aAAa,GAAG;GACvB,IAAI,MAAM,MAAM,SAAS,EAAE,EAAE,SAAS,MAAM;IAC1C,OAAO,KAAK,IAAI;IAChB;GACF;GACA,MAAM,OAAO,KAAK,eAAe;GACjC,IAAI,QAAQ,qBAAqB,KAAK,IAAI,GAAG,MAAM,KAAK,GAAG;GAC3D,IAAI,KAAK,SAAS,GAAG,OAAO;GAC5B,MAAM,KAAK,IAAI;GACf;EACF;EACA,IAAI,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,aAAa,IAAI;EACxE,MAAM,OAAO,KAAK,SAAS,YAAY;EACvC,IAAI,YAAY;EAChB,IAAI,SAAS,YAAY;GACvB,IAAI,CAAC,aAAa,IAAI,GAAG;GACzB,YAAa,KAAoB,WAAW;EAC9C,OAAO,IAAI,KAAK,IAAI,IAAI,GACtB;EAEF,IAAI,OAAO,MAAM,MAAM,SAAS,EAAE,EAAE,QAAQ;EAC5C,IAAI,KAAK,aAAa,GAAG;GACvB,MAAM,QAAQ,YAAY,IAAI;GAC9B,IAAI,YAAY,MAAM,KAAK,GAAG;IAC5B,OAAO,KAAK,IAAI;IAChB;GACF;GACA,OAAO,SAAS,MAAM,IAAI,WAAW,GAAG,IAAI;GAC5C,YAAY;GACZ,IAAI,SAAS,KAAK,SAAS;GAC3B,IAAI,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,IAAI;QAC/B,IAAI,CAAC,OAAO,IAAI,IAAI,KAAK,CAAC,KAAK,IAAI,IAAI,GAAG,OAAO;EACxD;EACA,MAAM,KACJ,UAAU,aAAa,IACnB;GAAE;GAAM,MAAM,UAAU;GAAY,MAAM,KAAA;GAAW,OAAO;GAAG;EAAK,IACpE;GAAE;GAAM,MAAM,KAAA;GAAW,MAAM,UAAU;GAAY,OAAO;GAAG;EAAK,CAC1E;EACA,IAAI,MAAM,SAAS,OAAO,QAAQ,MAAM;CAC1C;CACA,MAAM,SAAS,SAAuB;EACpC,MAAM,QAAQ,MAAM,IAAI,IAAI;EAC5B,IAAI,OAAO,MAAM,KAAK,IAAI;OACrB,IAAI,KAAK,IAAI,IAAI,GAAG,MAAM,KAAK,GAAG;EACvC,IAAI,OAAO,OAAO;OACb,IAAI,SAAS,IAAI,IAAI,KAAK,CAAC,OAAO,IAAI,IAAI,GAAG,OAAO;CAC3D;CACA,MAAM,IAAI;CACV,OAAO,MAAM,SAAS,GAAG;EACvB,MAAM,QAAQ,MAAM,MAAM,SAAS;EACnC,IAAI;EACJ,IAAI,MAAM,SAAS,KAAA,GAAW;GAC5B,QAAQ,MAAM,QAAQ,MAAM,KAAK,SAAS,MAAM,KAAK,MAAM,SAAS,KAAA;GACpE,MAAM,SAAS;EACjB,OAAO;GACL,QAAQ,MAAM;GACd,IAAI,UAAU,QAAQ,UAAU,KAAA,GAAW,MAAM,OAAO,MAAM;EAChE;EACA,IAAI,UAAU,QAAQ,UAAU,KAAA,GAAW;GACzC,MAAM,IAAI;GACV,MAAM,MAAM,IAAI;EAClB,OACE,MAAM,KAAK;CAEf;CAYA,OAAO;EAAE,MAXI,MACV,KAAK,EAAE,CAAC,CACR,MAAM,IAAI,CAAC,CACX,KAAK,SACJ,KACG,QAAQ,aAAa,EAAE,CAAC,CACxB,QAAQ,QAAQ,GAAG,CAAC,CACpB,KAAK,CACV,CAAC,CACA,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,IACI;EAAG;EAAU;EAAO;EAAO;CAAO;AAChD;;AAGA,MAAa,eAAe,SAA0B,SAAS,IAAI,CAAC,CAAC;AAIrE,MAAM,WAAqD;CACzD,CAAC,SAAS,MAAM;CAChB,CAAC,2BAA2B,mBAAmB;CAC/C,CAAC,sCAAsC,8BAA8B;CACrE,CAAC,gBAAgB,QAAQ;AAC3B;AAEA,MAAM,UAAU,WAA2B;CACzC,IAAI,OAAO,OAAO,QAAQ,MAAM,EAAE,CAAC,CAAC,KAAK;CACzC,KAAK,MAAM,CAAC,MAAM,UAAU,UAAU;EACpC,OAAO,KAAK,QAAQ,MAAM,EAAE,CAAC,CAAC,KAAK;EACnC,OAAO,KAAK,QAAQ,OAAO,EAAE,CAAC,CAAC,KAAK;CACtC;CACA,OAAO;AACT;;;;;AAMA,MAAM,cAAc,WAA6B;CAC/C,MAAM,SAAmB,CAAC;CAC1B,IAAI,MAAgB,CAAC;CACrB,IAAI,QAAQ;CACZ,IAAI,WAAW;CACf,IAAI,UAAU;CACd,IAAI,eAAe;CACnB,KAAK,IAAI,QAAQ,GAAG,QAAQ,OAAO,QAAQ,SAAS,GAAG;EACrD,MAAM,OAAO,OAAO;EACpB,MAAM,OAAO,OAAO,WAAW,KAAK;EACpC,IAAI,UAAU;GACZ,IAAI,SAAS,UAAU;QAClB,IAAI,SAAS,MAAM,UAAU;QAC7B,IAAI,SAAS,MAAK,WAAW;GAElC,IAAI,UAAU,GAAG;GACjB,IAAI,KACF,QAAQ,KACJ,OACA,SAAS,OACP,QACA,SAAS,OACP,QACA,SAAS,MACP,QACA,MAAM,KAAK,SAAS,EAAE,CAAC,CAAC,SAAS,GAAG,GAAG,GACnD;GACA;EACF;EACA,IAAI,UAAU,GAAG;GAEf,IAAI,SAAS,OAAO,SAAS,KAAK;IAChC,QAAQ;IACR,MAAM,CAAC,IAAI;IACX,eAAe;GACjB,OAAO,IAAI,SAAS,MAClB,WAAW;GAEb;EACF;EACA,IAAI,SAAS,MAAQ,SAAS,KAAQ,SAAS,MAAQ,SAAS,IAAM;GACpE,IAAI,KAAK,IAAI;GACb;EACF;EACA,IAAI,SAAS,KAAK;GAChB,IAAI,cAAc,IAAI,KAAK,GAAG;GAC9B,eAAe;GACf;EACF;EACA,IAAI,cAAc;GAEhB,IAAI,SAAS,OAAO,SAAS,KAAK,IAAI,KAAK,GAAG;GAC9C,eAAe;EACjB;EACA,IAAI,KAAK,IAAI;EACb,IAAI,SAAS,MAAK,WAAW;OACxB,IAAI,SAAS,OAAO,SAAS,KAAK,SAAS;OAC3C,IAAI,SAAS,OAAO,SAAS,KAAK;GACrC,SAAS;GACT,IAAI,UAAU,GAAG;IACf,OAAO,KAAK,IAAI,KAAK,EAAE,CAAC;IACxB,MAAM,CAAC;GACT;EACF;CACF;CAEA,IAAI,QAAQ,GAAG,OAAO,KAAK,IAAI,KAAK,EAAE,CAAC;CACvC,OAAO;AACT;AAEA,MAAM,eACJ,YACiF;CACjF,MAAM,UAAqC,CAAC;CAC5C,IAAI,aAAa;CAEjB,MAAM,QAAQ,UAAyB;EACrC,MAAM,QAAmB,CAAC,KAAK;EAC/B,MAAM,QAAmC,CAAC;EAC1C,OAAO,MAAM,SAAS,GAAG;GACvB,MAAM,QAAQ,MAAM,IAAI;GACxB,IAAI,MAAM,QAAQ,KAAK,GAAG;IACxB,KAAK,IAAI,QAAQ,MAAM,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,KAAK,MAAM,MAAM;IAClF;GACF;GACA,IAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;GACjD,MAAM,SAAS;GACf,MAAM,QAAQ,OAAO;GACrB,IAAI,UAAU,KAAA,KAAa,UAAU,QAAQ,OAAO,UAAU,UAAU;IACtE,MAAM,EAAE,UAAU,QAAQ,GAAG,SAAS;IAEtC,MAAM,KAAK,KAAK;IAChB,IAAI,OAAO,KAAK,IAAI,CAAC,CAAC,MAAM,QAAQ,QAAQ,UAAU,GAAG,MAAM,KAAK,IAAI;IACxE;GACF;GACA,MAAM,KAAK,MAAM;EACnB;EACA,QAAQ,KAAK,GAAG,KAAK;CACvB;CACA,KAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,OAAO,OAAO,MAAM;EAC1B,IAAI,KAAK,WAAW,GAAG;EACvB,IAAI;GACF,KAAK,KAAK,MAAM,IAAI,CAAC;GACrB;EACF,QAAQ,CAER;EACA,IAAI,SAAS;EACb,IAAI,MAAM;EACV,KAAK,MAAM,SAAS,WAAW,IAAI,GACjC,IAAI;GACF,KAAK,KAAK,MAAM,KAAK,CAAC;GACtB,MAAM;EACR,QAAQ;GACN,SAAS;EACX;EAEF,IAAI,UAAU,CAAC,KAAK,cAAc;CACpC;CACA,OAAO;EAAE;EAAS;CAAW;AAC/B;;;;;;AAOA,MAAa,cAAc,YACzB,YAAY,OAAO,CAAC,CAAC;;AAGvB,MAAa,WAAW,WAA8C;CACpE,MAAM,MAAM,OAAO;CAEnB,QADa,MAAM,QAAQ,GAAG,IAAI,MAAM,QAAQ,KAAA,IAAY,CAAC,IAAI,CAAC,GAAG,EAAA,CAElE,QAAQ,UAA2B,OAAO,UAAU,QAAQ,CAAC,CAC7D,KAAK,UAAU,MAAM,QAAQ,WAAW,EAAE,CAAC,CAAC,QAAQ,sBAAsB,EAAE,CAAC;AAClF;AAMA,MAAM,SAAS,QAAsB,IAAI,aAAa,WAAW,IAAI,aAAa;AAClF,MAAM,UAAU,SAAyB,KAAK,YAAY,CAAC,CAAC,QAAQ,UAAU,EAAE;;AAKhF,MAAM,aAAuD;CAC3D,CAAC,aAAa,4CAA4C;CAC1D,CAAC,cAAc,6EAA6E;CAC5F,CAAC,UAAU,4CAA4C;CACvD,CAAC,YAAY,wBAAwB;CACrC,CAAC,cAAc,6CAA6C;CAC5D,CAAC,WAAW,oEAAoE;CAChF,CAAC,cAAc,4BAA4B;AAC7C;AACA,MAAM,kBACJ;AAEF,MAAM,kBACJ;;AAEF,MAAM,WAAW;;AAGjB,MAAM,YAAY;AAElB,MAAM,WAA0F;CAC9F,CAAC,SAAS,OAAO,QAAQ,IAAI,cAAc,8BAA4B,MAAM,IAAI;CACjF,CAAC,aAAa,SAAS,KAAK,SAAS,oBAAoB,CAAC;CAC1D,CAAC,SAAS,SAAS,KAAK,SAAS,eAAe,KAAK,KAAK,SAAS,iBAAiB,CAAC;CACrF,CAAC,WAAW,SAAS,KAAK,SAAS,kBAAkB,CAAC;AACxD;;AAGA,MAAM,iBACJ,YACiE;CACjE,MAAM,OAAO,QAAQ,MAAM,GAAG,GAAI,CAAC,CAAC,KAAK;CACzC,MAAM,QAAQ,iBAAiB,KAAK,IAAI,CAAC,GAAG,MAAM;CAClD,MAAM,MAAM,KACT,MAAM,MAAM,MAAM,CAAC,CACnB,QAAQ,YAAY,EAAE,CAAC,CACvB,QAAQ,gBAAgB,EAAE,CAAC,CAC3B,QAAQ,mBAAmB,IAAI,CAAC,CAChC,KAAK;CACR,OAAO,IAAI,WAAW,IAAI,KAAA,IAAY;EAAE,OAAO,OAAO,SAAS,CAAC;EAAG;CAAI;AACzE;;AAKA,MAAM,WAAW,MAAc,WAA2B;CACxD,IAAI,KAAK,UAAU,QAAQ,OAAO;CAClC,IAAI,UAAU,GAAG,OAAO;CACxB,MAAM,OAAO,KAAK,YAAY,MAAM,MAAM;CAC1C,IAAI,MAAM,OAAO,SAAS,OAAQ,OAAO,IAAI,OAAO;CACpD,MAAM,OAAO,KAAK,WAAW,MAAM,CAAC;CACpC,IAAI,QAAQ,SAAU,QAAQ,OAAQ,OAAO;CAC7C,OAAO,KAAK,MAAM,GAAG,GAAG;AAC1B;;AAGA,MAAM,WAAW,MAAc,WAA2B;CACxD,IAAI,KAAK,UAAU,QAAQ,OAAO;CAClC,IAAI,UAAU,GAAG,OAAO;CACxB,IAAI,QAAQ,KAAK,SAAS;CAC1B,MAAM,OAAO,KAAK,QAAQ,MAAM,KAAK;CACrC,IAAI,SAAS,MAAM,OAAO,QAAQ,KAAM,QAAQ,OAAO;CACvD,MAAM,OAAO,KAAK,WAAW,KAAK;CAClC,IAAI,QAAQ,SAAU,QAAQ,OAAQ,SAAS;CAC/C,OAAO,KAAK,MAAM,KAAK;AACzB;;AAGA,MAAM,WAAW,SACf,KAAK,UAAU,iBACX,OACA,GAAG,QAAQ,MAAM,KAAuC,EAAE,MAAM,QAAQ,MAAM,kBAAkB;;AAGtG,MAAM,aAAa,SACjB,KAAK,UAAU,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;;;;;AAMjE,MAAM,gBAAgB,KAAwB,SAAwC;CACpF,MAAM,QAAQ,IAAI,WAAW,IAAI,MAAM;CACvC,MAAM,4BAAY,IAAI,IAAsB;CAC5C,IAAI,SAAS,MAAM,UAAU;EAC3B,MAAM,OAAO,UAAU,IAAI,IAAI;EAC/B,IAAI,SAAS,KAAA,GAAW,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC;OAC9C,KAAK,KAAK,KAAK;CACtB,CAAC;CACD,MAAM,0BAAU,IAAI,IAAoB;CACxC,IAAI,KAAK;CACT,IAAI,UAAU;CACd,KAAK,MAAM,QAAQ,MAAM;EACvB,MAAM,OAAO,UAAU,IAAI,IAAI;EAC/B,IAAI,SAAS,KAAA,GAAW;EACxB,IAAI,SAAS,QAAQ,IAAI,IAAI,KAAK;EAClC,OAAO,SAAS,KAAK,UAAU,KAAK,UAAW,IAAI,UAAU;EAC7D,QAAQ,IAAI,MAAM,MAAM;EACxB,MAAM,WAAW,KAAK;EACtB,IAAI,aAAa,KAAA,GAAW;EAE5B,IAAI,UAAU,KAAK,WAAW,KAAK,OAAO,KAAK,SAAS,GAAG;EAC3D,MAAM,YAAY;EAClB,QAAQ,IAAI,MAAM,SAAS,CAAC;EAC5B,KAAK,WAAW;EAChB,WAAW;CACb;CACA,OAAO;AACT;;AAGA,MAAM,UAAU,KAAwB,UAAgC;CACtE,MAAM,OAAiB,CAAC;CACxB,KAAK,IAAI,QAAQ,GAAG,QAAQ,IAAI,QAAQ,SAAS,GAAG;EAClD,IAAI,MAAM,WAAW,GAAG;EACxB,MAAM,QAAQ,IAAI,QAAQ;EAC1B,IAAI,QAAQ,KAAK,MAAM,QAAQ,OAAO,KAAK,UAAU,KAAA,KAAa,UAAU,KAAK,GAC/E,KAAK,KAAK,KAAK;EACjB,KAAK,KAAK,IAAI,MAAO;CACvB;CACA,OAAO;AACT;AAEA,MAAa,eAAe,MAAc,YAAiC;CACzE,MAAM,QAAkB,CAAC;CACzB,MAAM,OAAO,kBAAkB,IAAI;CACnC,IAAI,KAAK,UAAU,GACjB,MAAM,KACJ,gCAAgC,OAAO,SAAS,EAAE,sDACpD;CAEF,MAAM,MAAM,MAAM,KAAK,IAAI;CAC3B,MAAM,QAAQ,aACZ,IAAI,cAAc,QAAQ,CAAC,EAAE,aAAa,SAAS,CAAC,EAAE,KAAK,KAAK;CAElE,MAAM,UAAU,CAAC,GAAG,IAAI,iBAAiB,QAAQ,CAAC;CAClD,MAAM,SAAS,YACb,QACG,QAAQ,SACP,uCAAuC,KAAK,KAAK,aAAa,MAAM,KAAK,EAAE,CAC7E,CAAC,CACA,KAAK,SAAS,KAAK,eAAe,EAAE,CACzC;CACA,MAAM,SAAS,OAAO;CACtB,IAAI,OAAO,aAAa,GACtB,MAAM,KACJ,GAAG,OAAO,OAAO,UAAU,EAAE,WAAW,OAAO,eAAe,IAAI,UAAU,SAAS,mBACvF;CAGF,MAAM,cAAc;EAClB,IAAI;GACF,OAAO,IAAI,IAAI,OAAO;EACxB,QAAQ;GACN;EACF;CACF,EAAA,CAAG;CACH,MAAM,WAAW,IAAI,cAAc,YAAY,CAAC,EAAE,aAAa,MAAM;CACrE,MAAM,cAAc;EAClB,IAAI;GACF,MAAM,WAAW,IAAI,IAAI,YAAY,SAAS,OAAO;GAErD,OAAO,MAAM,QAAQ,IAAI,SAAS,OAAO;EAC3C,QAAQ;GACN,OAAO;EACT;CACF,EAAA,CAAG;CACH,MAAM,WAAW,QAAiC;EAChD,IAAI;GACF,MAAM,MAAM,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI;GACpC,IAAI,CAAC,MAAM,GAAG,GAAG,OAAO,KAAA;GACxB,IAAI,WAAW;GACf,IAAI,WAAW;GACf,OAAO;EACT,QAAQ;GACN;EACF;CACF;CACA,MAAM,OAAO,SAAS,KAAA,IAAY,KAAA,IAAY,aAAa,KAAK,IAAI;CAEpE,MAAM,UAAU,CAAC,GAAG,IAAI,iBAAiB,kBAAkB,CAAC,CAAC,CAC1D,QAAQ,SAAS,mBAAmB,KAAK,KAAK,aAAa,YAAY,KAAK,EAAE,CAAC,CAAC,CAChF,KAAK,SAAS,cAAc,KAAK,aAAa,SAAS,KAAK,EAAE,CAAC,CAAC,CAChE,MAAM,WAAW,WAAW,KAAA,CAAS;CACxC,MAAM,aAAa,YAAY,KAAA,IAAY,KAAA,IAAY,QAAQ,QAAQ,GAAG;CAC1E,MAAM,WACJ,YAAY,KAAA,KACZ,QAAQ,SAAS,KACjB,eAAe,KAAA,KACf,aAAa,WAAW,IAAI,MAAM,OAC9B,aAAa,WAAW,IAAI,IAC5B;CAEN,MAAM,mBAAmB;EACvB,MAAM,OAAO,CAAC,GAAG,IAAI,iBAAiB,iBAAiB,CAAC,CAAC,CAAC,MAAM,UAC7D,KAAK,aAAa,KAAK,KAAK,GAAA,CAAI,YAAY,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,SAAS,WAAW,CAClF;EACA,MAAM,MAAM,SAAS,KAAA,IAAY,KAAA,IAAY,QAAQ,KAAK,aAAa,MAAM,KAAK,EAAE;EAEpF,IAAI,QAAQ,KAAA,KAAa,SAAS,KAAA,KAAa,OAAO,IAAI,QAAQ,MAAM,OAAO,KAAK,QAAQ,GAC1F,OAAO;EAET,OAAO,aAAa,IAAI,IAAI;CAC9B,EAAA,CAAG;CAEH,MAAM,SAAS,WACZ,SAAS,GAAA,CAAI,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;CAC1C,MAAM,SAAS,CAAC,GAAG,IAAI,iBAAiB,OAAO,CAAC;CAEhD,MAAM,SAAS,SAA2B;EACxC,KAAK,IAAI,SAAS,KAAK,YAAY,QAAQ,SAAS,OAAO,YACzD,IAAI,OAAO,SAAS,YAAY,MAAM,OAAO,OAAO;EAEtD,OAAO;CACT;CACA,MAAM,QACJ,KAAK,6BAA2B,KAChC,MAAM,OAAO,MAAM,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,WAAW,KACtD,MAAM,IAAI,cAAc,IAAI,CAAC,EAAE,WAAW,KAC1C;CACF,MAAM,cAAc,KAAK,4BAA0B,KAAK,KAAK,mCAAiC;CAC9F,MAAM,OAAO,IAAI,iBAAiB,aAAa,MAAM,CAAC,EAAE,KAAK,KAAK;CAClE,MAAM,WAAW,CAAC,GAAG,IAAI,iBAAiB,UAAU,CAAC,CAAC,CACnD,KAAK,SAAS,KAAK,eAAe,EAAE,CAAC,CACrC,QAAQ,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CACtC,KAAK,GAAG;CACX,MAAM,WAAW,SAAS,QAAQ,GAAG,WAAW,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,IAAI;CACtF,MAAM,WAAW,CAAC,GAAG,IAAI,iBAAiB,SAAS,CAAC,CAAC,CAAC,MACnD,UAAU,KAAK,eAAe,GAAA,CAAI,KAAK,CAAC,CAAC,WAAW,CACvD;CAKA,MAAM,SAAS,SAAS,GAAG;CAE3B,KAAK,MAAM,QAAQ,OAAO,QAAQ,KAAK,SAAS;CAEhD,MAAM,OAAO,IAAI,IAAY,SAAS,KAAA,IAAY,CAAC,IAAI,CAAC,IAAI,CAAC;CAC7D,MAAM,QAAoB,CAAC;CAC3B,IAAI,aAAa,MAAM;EACrB,KAAK,IAAI,QAAQ;EACjB,MAAM,KAAK;GAAE,MAAM;GAAU,MAAM;EAAG,CAAC;CACzC;CACA,KAAK,MAAM,UAAU,IAAI,iBAAiB,SAAS,GAAG;EACpD,IAAI,MAAM,UAAU,WAAW;EAC/B,MAAM,MAAM,QAAQ,OAAO,aAAa,MAAM,KAAK,EAAE;EACrD,IAAI,QAAQ,KAAA,GAAW;EACvB,MAAM,OAAO,aAAa,IAAI,IAAI;EAClC,MAAM,QACH,OAAO,eAAe,GAAA,CAAI,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK,KACrD,OAAO,aAAa,YAAY,CAAC,EAAE,KAAK,KACxC,OAAO,aAAa,OAAO,CAAC,EAAE,KAAK,KACnC,OAAO,cAAc,UAAU,CAAC,EAAE,aAAa,KAAK,CAAC,EAAE,KAAK,KAC5D;EAEF,IAAI,SAAS,YAAY,MAAM,EAAE,EAAE,SAAS,YAAY,MAAM,EAAE,CAAC,SAAS,IAAI;GAC5E,MAAM,KAAK;IAAE;IAAM,MAAM,KAAK,MAAM,GAAG,GAAG;GAAE;GAC5C;EACF;EAEA,IAAI,KAAK,IAAI,IAAI,GAAG;EACpB,KAAK,IAAI,IAAI;EACb,MAAM,KAAK;GAAE;GAAM,MAAM,KAAK,MAAM,GAAG,GAAG;EAAE,CAAC;CAC/C;CAEA,MAAM,MAAM,OAAO;CACnB,IAAI,OAAO,QAAQ,GAAG;CACtB,IAAI,MAAM,KAAK,SAAS,IAAI;CAC5B,IAAI,IAAI,SAAS,kBAAkB;EACjC,MAAM,cAAc;GAClB,IACE,OAAO,WAAW,4BAClB,OAAO,QAAQ,yBACf,OAAO,QAAQ,uBACf;IACA,MAAM,KAAK,qEAAqE;IAChF,OAAO;GACT;GACA,IAAI;IAGF,MAAM,UAAU,IAAI,YAClB,KACA;KACE,eAAe;KACf,eAAe;KACf,aAAa,SAAS;IACxB,CACF,CAAC,CAAC,MAAM;IACR,OAAO,SAAS,YAAY,QAAQ,SAAS,YAAY,KAAA,IACrD,KACA,YAAY,QAAQ,OAAO;GACjC,QAAQ;IACN,MAAM,KAAK,mEAAmE;IAC9E,OAAO;GACT;EACF,EAAA,CAAG;EACH,IAAI,KAAK,SAAS,KAAK;GAIrB,MAAM,QAAQ,IAAI,MAAM,IAAI;GAC5B,MAAM,OAAO,OAAO,OAAO,aAAa,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI;GAC3E,MAAM,OAAO,QAAQ,MAAM,iBAAiB,KAAK,IAAI,KAAK,QAAQ,kBAAkB,IAAI,CAAC;GACzF,MAAM,OAAO,QAAQ,MAAM,iBAAiB,KAAK,SAAS,CAAC;GAC3D,OAAO,KAAK,SAAS,IAAI,GAAG,KAAK,MAAM,SAAS;GAChD,MAAM,KAAK,SAAS,KAAK,UAAU,KAAK,SAAS,KAAK;EACxD;CACF;CACA,IAAI,KAAK,MAAM,KAAK,sBAAsB,OAAO,cAAc,EAAE,YAAY;CAE7E,MAAM,eAAe,IAAI;CACzB,MAAM,YAAY,MAAM,OAAO,EAAE,EAAE,WAAW;CAC9C,MAAM,UACJ,eAAe,MACX,gBAAgB,KAAK,SAAS,IAC3B,WAAW,MAAM,GAAG,aAAa,QAAQ,KAAK,IAAI,CAAC,CAAC,GAAG,MAAM,gBAC7D,WAAW,MAAM,GAAG,aAAa,QAAQ,KAAK,IAAI,CAAC,CAAC,GAAG,MAAM,OAChE;CACN,MAAM,SAEJ,YAAY,QAAQ,aAAa,OAC7B,OAGA,eAAe,KACb,QAAQ,OAAO,YAAY,EAAE,+BAC7B,eAAe,QAAQ,gBAAgB,KAAK,QAAQ,IAClD,iCACA,eAAe,QAAQ,WACrB,8BACA;CAEZ,OAAO;EACL,OAAO,MAAM,QAAQ,QAAQ,GAAG,CAAC,CAAC,MAAM,GAAG,GAAG;EAC9C;EACA;EACA,GAAI,MAAM,EAAE,UAAU,IAAI,IAAI,CAAC;EAC/B;EACA;EACA;EACA;EACA;EACA,aAAa,WAAW;EACxB,cAAc;EACd;EACA;EACA;EACA;CACF;AACF"}

package/build/html.d.mts

@@ -0,0 +1,2 @@
+import { a as typesOf, c as decodeBody, i as readJsonLd, l as canonicalUrl, n as PageLink, o as visibleText, r as extractPage, s as bodyEncoding, t as PageExtract } from "./html-nWYFVsRU.mjs";
+export { PageExtract, PageLink, bodyEncoding, canonicalUrl, decodeBody, extractPage, readJsonLd, typesOf, visibleText };

package/build/html.mjs

@@ -0,0 +1 @@
+export { a as bodyEncoding, c as canonicalUrl, o as decodeBody, t as extractPage, n as readJsonLd, r as typesOf, i as visibleText } from "./html-oNpGAHwi.mjs";

package/build/index.d.mts

@@ -0,0 +1,131 @@
+import { DAYS, Day, Hours, Interval, Week, describeWeek, disagreements, fromCompact, fromSchemaOrg, fromText, toHoursWeek } from "./hours.mjs";
+import { a as typesOf, c as decodeBody, n as PageLink, r as extractPage, s as bodyEncoding, t as PageExtract } from "./html-nWYFVsRU.mjs";
+import { $ as FetchOutcome, A as findDocuments, B as CaptureFilter, C as ReadResult, Ct as RenderOptions, D as catchUpIndex, Dt as playwrightRenderer, E as VerifyResult, Et as Renderer, F as readDocument, G as CAPTURE_SCHEMA, H as CaptureReader, I as recordOverride, J as CaptureWriter, K as CacheEntry, L as reindex, M as isHeldBack, N as pageDocId, O as fetchAndIndex, Ot as BlobStore, P as pageExtractor, Q as DEFAULT_SECRET_PARAMS, R as renderAndIndex, S as ReadOptions, St as RefusedReason, T as VerifyOptions, Tt as Rendered, U as DamagedLine, V as CaptureLog, W as ScanOptions, X as Crawler, Y as CrawlEvent, Z as CrawlerOptions, _ as Extractors, _t as ReadingIndex, a as PoliteCrawler, at as RecordInput, b as PageOptions, bt as NO_PLAYWRIGHT, c as createRuntime, ct as EgressRefused, d as isMailbox, dt as DocIndex, et as FetchRequest, f as openCrawler, ft as DocInput, g as Extractor, gt as Latest, h as CrawlRuntime, ht as Found, i as OpenOptions, it as CaptureRecord, j as heldBackMeta, k as fetchPage, kt as FileBlobStore, l as defaultUserAgent, lt as egressLookup, m as CatchUpResult, mt as ExtractedDoc, n as FetchOptions, nt as RenderRequest, o as RuntimeOptions, ot as makeRecord, p as CapturePayload, pt as DocVersion, q as CacheIndex, r as FindOptions, rt as renderedKey, s as StoreRuntime, st as EgressOptions, t as Duration, tt as Fetched, u as durationMs, ut as Doc, v as HELD_BACK, vt as Variant, w as ReindexResult, wt as RenderRefused, x as PageResult, xt as PlaywrightRenderer, y as Indexed, yt as VersionText, z as verifyQuote } from "./open-CZqwY7GH.mjs";
+import { r as Signals, t as AiInput } from "./signals-BHpy2xb2.mjs";
+import { f as PolicyOptions, i as HostResult, n as DEFAULT_POLICY, o as HostStateStore, p as PolitenessPolicy, r as HostCounts, s as HostStop, t as Clock, u as MemoryHostStateStore, x as realClock } from "./politeness-C5bZ1OKM.mjs";
+import { a as Robots, c as RobotsRule, h as parseRobots, l as RobotsSignal, o as RobotsDecision, s as RobotsGroup, t as ALLOW_ALL, u as decide } from "./robots-R-qaEBOb.mjs";
+//#region src/browser.d.ts
+/**
+ * This function gives a renderer that checks one time, before its first page, that the browser is
+ * on the disk (`browserMissing`). With no browser, each page gets a `RenderRefused` with the reason
+ * `browser` at once, and the renderer opens nothing. `createRuntime` puts its default renderer in
+ * it, so a library with no browser costs no wait and no error capture.
+ */
+export declare const checkedRenderer: (renderer: Renderer, check?: () => Promise<string | undefined>) => Renderer;
+//#endregion
+//#region src/fold.d.ts
+interface Verified {
+  readonly found: boolean;
+  /** These values are the character positions in the text of the document, when the quote is there. */
+  readonly start: number | null;
+  readonly end: number | null;
+  /** This text is the exact quote as stored, with some text before and after it. */
+  readonly excerpt: string | null;
+  /** This value is the number of places where the quote is, with no overlap. */
+  readonly count: number;
+}
+/**
+ * This function finds a quote in a text, in the `quote` mode of `foldText`. It ignores case, runs
+ * of spaces, typographic punctuation, soft hyphens, zero-width characters and the differences of
+ * NFKC, NFC and NFD. The positions are in the original text.
+ */
+export declare const locate: (text: string, quote: string) => Verified;
+//#endregion
+//#region src/hoststate.d.ts
+/** These options change a SQLite host state store. */
+interface SqliteHostStateOptions {
+  /** This value is the time in milliseconds that a step waits for the lock of a different process. The default is 10 seconds. */
+  readonly busyTimeoutMs?: number | undefined;
+  /** During the lease of a different process, a request asks again after this wait at most. The default is 1 second. */
+  readonly pollMs?: number | undefined;
+}
+/**
+ * This store keeps the state of each host in a SQLite file. Two or more processes on one store
+ * share it, so they share the pace and the stops of each host. Thus a host gets one request at a
+ * time from all of them. Each method is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock
+ * of a different process (`busy_timeout`).
+ *
+ * The lease of a request belongs to the store that gave it. A different process that asks during
+ * the lease gets a wait of one second or less, and asks again. A process that stops during a
+ * request holds its lease until the lease ends (5 minutes by default).
+ *
+ * The state is not a projection of the capture log. A deletion of the file loses only the pace,
+ * the counts and the stops, and the crawler starts them again.
+ */
+export declare class SqliteHostStateStore implements HostStateStore {
+  readonly path: string;
+  private readonly db;
+  private readonly statements;
+  /** This id names the leases of this store. */
+  private readonly owner;
+  private reservations;
+  private closed;
+  private readonly pollMs;
+  constructor(path: string, options?: SqliteHostStateOptions);
+  private stmt;
+  /** This method does one step in one transaction. */
+  private step;
+  private row;
+  /** This method removes the hosts that have nothing to remember: no rest, no lease, no counts and no stop. */
+  private sweep;
+  reserve(host: string, now: number, leaseMs: number): number;
+  record(host: string, result: HostResult): HostCounts;
+  release(host: string, nextAt: number): void;
+  defer(host: string, until: number): void;
+  stop(host: string, until: number, reason: string): void;
+  stopped(host: string, now: number): HostStop | undefined;
+  reset(host?: string): void;
+  /** This value is the number of hosts that the store holds. */
+  get size(): number;
+  /** This method closes the file. A second close does nothing. */
+  close(): void;
+}
+//#endregion
+//#region src/version.d.ts
+/** This is the version of the package. A test makes sure that it agrees with `package.json`. */
+export declare const VERSION = "0.1.0";
+/** This is the product token that robots.txt rules match when an application gives no token. */
+export declare const DEFAULT_TOKEN = "PoliteCrawler";
+/** This page tells a site owner what the crawler is and how to control it. */
+export declare const BOT_PAGE = "https://mark1russell7.github.io/crawler/bot/";
+//#endregion
+//#region src/pdf.d.ts
+/**
+ * The crawler reads the text of a PDF. Restaurant menus are often PDFs: 29 of 231 surveyed
+ * restaurant sites link one. A menu that the crawler reads one time stays searchable. A paper in a
+ * PDF reads the same way. A scanned PDF has no text, so the result is empty, not a guess.
+ *
+ * A small file can cost much work: a stream of 150 KB can inflate to 100 MB. Thus the reader:
+ *
+ * - Inflates the compressed streams first, with a limit, and refuses a file that inflates past it.
+ * - Reads one page at a time, and lets other work run between the pages.
+ * - Stops at a page limit, a text limit and a time limit, and says so (`truncated`). The time limit
+ *   stops a page in the middle too, and the text limit cuts the text of a page.
+ */
+interface PdfText {
+  /** This is the title in the file, or null when the file names no real title ("Microsoft Word - menu.docx"). */
+  readonly title: string | null;
+  readonly text: string;
+  readonly pages: number;
+  /** This value is true when a limit stopped the reader before the last page. */
+  readonly truncated: boolean;
+  /** This note tells why the reader stopped or refused the file, or it is null. */
+  readonly note: string | null;
+}
+/** These are the limits of one read. */
+interface PdfOptions {
+  /** The reader reads no more pages than this. The default is 500. */
+  readonly maxPages?: number | undefined;
+  /** The reader stops when the text has this many characters, in the middle of a page too. The default is 200,000. */
+  readonly maxChars?: number | undefined;
+  /** The reader stops after this time in milliseconds, in the middle of a page too. The default is 20,000. */
+  readonly timeBudgetMs?: number | undefined;
+  /** The reader refuses a file whose streams inflate to more bytes than this. The default is 8 MB. */
+  readonly maxInflatedBytes?: number | undefined;
+}
+/** This function finds a PDF from its first bytes or its content type. Some servers put bytes before `%PDF-`. */
+export declare const isPdf: (body: Uint8Array, contentType: string | null) => boolean;
+export declare const readPdf: (bytes: Uint8Array, options?: PdfOptions) => Promise<PdfText>;
+//#endregion
+export { ALLOW_ALL, type AiInput, type BlobStore, CAPTURE_SCHEMA, type CacheEntry, type CacheIndex, type CaptureFilter, CaptureLog, type CapturePayload, type CaptureReader, type CaptureRecord, type CaptureWriter, type CatchUpResult, type Clock, type CrawlEvent, type CrawlRuntime, Crawler, type CrawlerOptions, DAYS, DEFAULT_POLICY, DEFAULT_SECRET_PARAMS, type DamagedLine, type Day, type Doc, DocIndex, type DocInput, type DocVersion, type Duration, type EgressOptions, EgressRefused, type ExtractedDoc, type Extractor, Extractors, type FetchOptions, type FetchOutcome, type FetchRequest, type Fetched, FileBlobStore, type FindOptions, type Found, HELD_BACK, type HostCounts, type HostResult, type HostStateStore, type HostStop, type Hours, type Indexed, type Interval, type Latest, MemoryHostStateStore, NO_PLAYWRIGHT, type OpenOptions, type PageExtract, type PageLink, type PageOptions, type PageResult, type PdfOptions, type PdfText, type PlaywrightRenderer, type PolicyOptions, type PoliteCrawler, type PolitenessPolicy, type ReadOptions, type ReadResult, type ReadingIndex, type RecordInput, type RefusedReason, type ReindexResult, type RenderOptions, RenderRefused, type RenderRequest, type Rendered, type Renderer, type Robots, type RobotsDecision, type RobotsGroup, type RobotsRule, type RobotsSignal, type RuntimeOptions, type ScanOptions, type Signals, type StoreRuntime, type Variant, type Verified, type VerifyOptions, type VerifyResult, type VersionText, type Week, bodyEncoding, catchUpIndex, createRuntime, decide, decodeBody, defaultUserAgent, describeWeek, disagreements, durationMs, egressLookup, extractPage, fetchAndIndex, fetchPage, findDocuments, fromCompact, fromSchemaOrg, fromText, heldBackMeta, isHeldBack, isMailbox, makeRecord, openCrawler, pageDocId, pageExtractor, parseRobots, playwrightRenderer, readDocument, realClock, recordOverride, reindex, renderAndIndex, renderedKey, toHoursWeek, typesOf, verifyQuote };
+//# sourceMappingURL=index.d.mts.map

package/build/index.mjs

@@ -0,0 +1,8 @@
+import { a as RenderRefused, i as NO_PLAYWRIGHT, o as playwrightRenderer, r as checkedRenderer } from "./browser-gJ_h4AhB.mjs";
+import { A as DEFAULT_SECRET_PARAMS, C as readPdf, D as locate, E as DocIndex, F as FileBlobStore, M as makeRecord, N as EgressRefused, O as CAPTURE_SCHEMA, P as egressLookup, S as isPdf, T as SqliteHostStateStore, _ as readDocument, a as openCrawler, b as renderAndIndex, c as HELD_BACK, d as fetchPage, f as findDocuments, g as pageExtractor, h as pageDocId, i as isMailbox, j as renderedKey, k as Crawler, l as catchUpIndex, m as isHeldBack, n as defaultUserAgent, p as heldBackMeta, r as durationMs, s as Extractors, t as createRuntime, u as fetchAndIndex, v as recordOverride, w as CaptureLog, x as verifyQuote, y as reindex } from "./open-CJR4b44v.mjs";
+import { DEFAULT_POLICY, MemoryHostStateStore, realClock } from "./politeness.mjs";
+import { ALLOW_ALL, decide, parseRobots } from "./robots.mjs";
+import { n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-rVx8_y1_.mjs";
+import { a as bodyEncoding, o as decodeBody, r as typesOf, t as extractPage } from "./html-oNpGAHwi.mjs";
+import { DAYS, describeWeek, disagreements, fromCompact, fromSchemaOrg, fromText, toHoursWeek } from "./hours.mjs";
+export { ALLOW_ALL, BOT_PAGE, CAPTURE_SCHEMA, CaptureLog, Crawler, DAYS, DEFAULT_POLICY, DEFAULT_SECRET_PARAMS, DEFAULT_TOKEN, DocIndex, EgressRefused, Extractors, FileBlobStore, HELD_BACK, MemoryHostStateStore, NO_PLAYWRIGHT, RenderRefused, SqliteHostStateStore, VERSION, bodyEncoding, catchUpIndex, checkedRenderer, createRuntime, decide, decodeBody, defaultUserAgent, describeWeek, disagreements, durationMs, egressLookup, extractPage, fetchAndIndex, fetchPage, findDocuments, fromCompact, fromSchemaOrg, fromText, heldBackMeta, isHeldBack, isMailbox, isPdf, locate, makeRecord, openCrawler, pageDocId, pageExtractor, parseRobots, playwrightRenderer, readDocument, readPdf, realClock, recordOverride, reindex, renderAndIndex, renderedKey, toHoursWeek, typesOf, verifyQuote };

package/build/mcp.d.mts

@@ -0,0 +1,40 @@
+import { a as PoliteCrawler, i as OpenOptions } from "./open-CZqwY7GH.mjs";
+import { McpServer, McpServer as McpServer$1 } from "@modelcontextprotocol/sdk/server/mcp.js";
+//#region src/mcp.d.ts
+/** These options change the tools of the MCP server. */
+export interface CrawlServerOptions {
+  /**
+   * These product tokens get their robots.txt rules applied to each fetch, in addition to the token
+   * of the crawler. The default is `["Claude-User"]`, because an assistant asks for these pages.
+   */
+  readonly onBehalfOf?: readonly string[] | undefined;
+  /**
+   * The value true lets `user_provided` release held-back text when the client cannot ask the
+   * user. It is the opt-in of the operator. The default is false. When the client can ask the
+   * user (MCP elicitation), the server always asks first.
+   */
+  readonly allowUserProvided?: boolean | undefined;
+  /** This text tells why the server renders no page. The result of a page that needs JavaScript shows it. */
+  readonly renderOff?: string | undefined;
+}
+/** These options start the MCP server on the standard input and output. */
+export interface StartMcpOptions extends OpenOptions, CrawlServerOptions {
+  /** This function writes the messages of the server. The default writes them to the standard error. */
+  readonly log?: ((line: string) => void) | undefined;
+}
+/**
+ * This function makes an MCP server with four tools over one crawler: `crawl_fetch`,
+ * `crawl_find`, `crawl_read` and `crawl_verify`. The server does not connect to a transport. Each
+ * fetch applies the robots.txt rules for `Claude-User` too, unless `onBehalfOf` gives other tokens.
+ */
+export declare const createMcpServer: (crawler: PoliteCrawler, options?: CrawlServerOptions) => McpServer$1;
+/**
+ * This function starts the MCP server on the standard input and output. The fetches apply the
+ * robots.txt rules for `Claude-User` too, because an assistant asks for these pages. When the
+ * default renderer has no browser, the server turns rendering off at start and writes one note.
+ * The server stops when its standard input closes, or at `SIGINT` or `SIGTERM`.
+ */
+export declare const startMcpServer: (options: StartMcpOptions) => Promise<void>;
+//#endregion
+export type { McpServer };
+//# sourceMappingURL=mcp.d.mts.map