polite-crawler 0.1.4 → 0.2.0

package/build/robots.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"robots.mjs","names":[],"sources":["../src/robots.ts"],"sourcesContent":["/**\n * This module reads robots.txt as RFC 9309 (September 2022) specifies it. It is for a crawler that\n * tells who it is and obeys what a site says. The important rules, each with a test:\n *\n * - A group is one or more `user-agent` lines and the rules after them. A `user-agent` line after\n *   an `allow` or `disallow` line starts a new group, as the parser of Google does. A different\n *   line (`crawl-delay`) between two `user-agent` lines does not split the group. A rule before\n *   the first `user-agent` line is in no group.\n * - The product token matches with no attention to case. The matching groups merge. The group\n *   `*` applies only when no group names the crawler.\n * - The longest matching rule wins, and `allow` wins a tie. A `*` matches any sequence, and a `$`\n *   at the end anchors the end. The matcher uses no regular expression, and its time is linear.\n * - `/robots.txt` is always permitted, and an empty `disallow` matches nothing.\n * - The parser reads at least the first 500 KiB.\n * - A usage line (`content-signal`, `content-usage`) belongs to its group, with an optional path\n *   before its value. A line before the first group applies to each crawler.\n *\n * The function `robotsFromStatus` gives the meaning of the status of the fetch. A 4xx is\n * \"unavailable\", so all paths are permitted. A 5xx or no response is \"unreachable\", so no path is\n * permitted. A 429 is unreachable too, as Google does it. A site that limits the rate of its robots\n * file does not ask for a crawl.\n */\n\nexport const ROBOTS_MAX_BYTES: number = 500 * 1024;\n\nexport interface RobotsRule {\n  readonly allow: boolean;\n  /** This value is the path pattern, normalized for comparison. */\n  readonly pattern: string;\n  /** This value is the number of the line in robots.txt, from 1. A rule that the code makes has none. */\n  readonly line?: number | undefined;\n}\n\n/** A usage line states a preference about the use of the content (`content-signal`, `content-usage`). */\nexport interface RobotsSignal {\n  /** This value is the name of the line, in lowercase. */\n  readonly key: string;\n  /** This text is the value of the line, after the path. */\n  readonly value: string;\n  /** This value is the normalized path pattern of the line. A line with no path applies to each path. */\n  readonly path: string | undefined;\n  /** This value is the number of the line in robots.txt, from 1. */\n  readonly line: number;\n}\n\nexport interface RobotsGroup {\n  /** These values are the user agents of the group, in lowercase. */\n  readonly agents: readonly string[];\n  /** This value is the number of the first `user-agent` line of the group, from 1. */\n  readonly line?: number | undefined;\n  readonly rules: readonly RobotsRule[];\n  /** This value is the nonstandard `crawl-delay` in seconds, when the group sets one. */\n  readonly crawlDelaySeconds: number | undefined;\n  /** These usage lines are in the group. They apply when the group applies. */\n  readonly signals?: readonly RobotsSignal[] | undefined;\n}\n\nexport interface Robots {\n  readonly groups: readonly RobotsGroup[];\n  readonly sitemaps: readonly string[];\n  /** These lines state usage preferences (`content-signal: …`, `content-usage: …`), as written, from each group. */\n  readonly signals: readonly string[];\n  /** These usage lines come before the first group, so they apply to each crawler. */\n  readonly fileSignals?: readonly RobotsSignal[] | undefined;\n  /**\n   * This text tells why the crawler could not read robots.txt: a dead domain, a broken certificate\n   * or a server error. Then no path is permitted, and the note of a skip gives this reason. It does\n   * not name a rule of the site.\n   */\n  readonly unreachable?: string | undefined;\n}\n\nexport const ALLOW_ALL: Robots = { groups: [], sitemaps: [], signals: [] };\n\nexport const DISALLOW_ALL: Robots = {\n  groups: [\n    { agents: [\"*\"], rules: [{ allow: false, pattern: \"/\" }], crawlDelaySeconds: undefined },\n  ],\n  sitemaps: [],\n  signals: [],\n};\n\nconst UNRESERVED = /^[A-Za-z0-9\\-._~]$/;\nconst URL_ENCODED = new Set(\"\\\"'<>\\\\^`{|}\");\n/**\n * The URL parser encodes these ASCII characters in a path or a query: the controls, a space, DEL,\n * and `\"'<>\\^`{|}`. A rule can contain them as written.\n */\nconst encodedAscii = (char: string): boolean => {\n  const code = char.charCodeAt(0);\n  return code <= 0x20 || code === 0x7f || URL_ENCODED.has(char);\n};\nconst HEX = /^[0-9A-Fa-f]$/;\n\nconst escapeOf = (char: string): string => {\n  const code = char.codePointAt(0)!;\n  if (code <= 0x7f) return `%${code.toString(16).toUpperCase().padStart(2, \"0\")}`;\n  try {\n    return encodeURIComponent(char);\n  } catch {\n    // a lone surrogate: the URL parser writes the replacement character\n    return \"%EF%BF%BD\";\n  }\n};\n\n/**\n * This function normalizes a path, or the pattern of a rule, for comparison, as §2.2.2 asks. It\n * encodes the characters outside US-ASCII, and the ASCII characters that the URL parser encodes\n * (a space, `\"`, `<`, `{`). It decodes an escape of an unreserved character, so `/%7Efoo` and\n * `/~foo` are equal. The other escapes stay, with uppercase hexadecimal digits.\n */\nexport const normalizePath = (value: string): string => {\n  let out = \"\";\n  const chars = Array.from(value);\n  for (let index = 0; index < chars.length; index += 1) {\n    const char = chars[index]!;\n    if (char === \"%\" && HEX.test(chars[index + 1] ?? \"\") && HEX.test(chars[index + 2] ?? \"\")) {\n      const hex = `${chars[index + 1]!}${chars[index + 2]!}`;\n      const decoded = String.fromCharCode(Number.parseInt(hex, 16));\n      out += UNRESERVED.test(decoded) ? decoded : `%${hex.toUpperCase()}`;\n      index += 2;\n      continue;\n    }\n    out += char.charCodeAt(0) > 0x7f || encodedAscii(char) ? escapeOf(char) : char;\n  }\n  return out;\n};\n\n/** This function tells if a text is a product token: letters, `_` and `-` only, and one or more (RFC 9309, section 2.2.1). */\nexport const isProductToken = (value: string): boolean => /^[A-Za-z_-]+$/.test(value);\n\n/** This function gives the first product token of a user agent: `Googlebot/2.1` gives `googlebot`. */\nconst tokenOf = (value: string): string =>\n  (/^[A-Za-z_-]+/.exec(value.trim())?.[0] ?? \"\").toLowerCase();\n\n/** These names are parts of the header of a browser, not the name of a crawler. */\nconst BROWSER_PARTS: ReadonlySet<string> = new Set([\n  \"mozilla\",\n  \"applewebkit\",\n  \"khtml\",\n  \"gecko\",\n  \"chrome\",\n  \"chromium\",\n  \"safari\",\n  \"version\",\n  \"mobile\",\n  \"firefox\",\n  \"edg\",\n  \"opr\",\n  \"like\",\n]);\n\n/**\n * This function gives the product token that robots.txt rules match, in lowercase. `MyBot/1.0`\n * gives `mybot`. The full header `Mozilla/5.0 (compatible; MyBot/1.0; +https://…)` also gives\n * `mybot`. The token is the name after `compatible;`, or else the first product that is not a part\n * of a browser.\n */\nexport const productToken = (value: string): string => {\n  const text = value.trim();\n  const first = tokenOf(text);\n  if (first !== \"mozilla\") return first;\n  const compatible = /compatible;\\s*([A-Za-z_-]+)/i.exec(text)?.[1];\n  if (compatible !== undefined) return compatible.toLowerCase();\n  const products = [...text.matchAll(/([A-Za-z][A-Za-z_-]*)\\/[\\w.]+/g)].map((match) =>\n    (match[1] ?? \"\").toLowerCase(),\n  );\n  return products.find((product) => !BROWSER_PARTS.has(product)) ?? first;\n};\n\nconst truncateUtf8 = (text: string, maxBytes: number): string => {\n  const bytes = new TextEncoder().encode(text);\n  if (bytes.length <= maxBytes) return text;\n  // cut at the last line break inside the limit, so no rule is half-read\n  const head = new TextDecoder().decode(bytes.slice(0, maxBytes));\n  const lastBreak = Math.max(head.lastIndexOf(\"\\n\"), head.lastIndexOf(\"\\r\"));\n  return lastBreak === -1 ? \"\" : head.slice(0, lastBreak);\n};\n\n/** This function reads a usage line: an optional path, then the value (`/images/ train-ai=n`). */\nconst signalOf = (key: string, value: string, line: number): RobotsSignal => {\n  const path = /^(\\/\\S*)\\s+(.*)$/.exec(value);\n  return path === null\n    ? { key, value, path: undefined, line }\n    : { key, value: path[2]!.trim(), path: normalizePath(path[1]!), line };\n};\n\nexport const parseRobots = (input: string): Robots => {\n  const groups: Array<{\n    agents: string[];\n    line: number;\n    rules: RobotsRule[];\n    crawlDelaySeconds: number | undefined;\n    signals: RobotsSignal[];\n  }> = [];\n  const sitemaps: string[] = [];\n  const signals: string[] = [];\n  const fileSignals: RobotsSignal[] = [];\n  let current: (typeof groups)[number] | undefined;\n  let collectingAgents = false;\n  for (const [index, raw] of truncateUtf8(input, ROBOTS_MAX_BYTES)\n    .split(/\\r\\n|\\r|\\n/)\n    .entries()) {\n    const line = raw.replace(/#.*$/, \"\").trim();\n    const colon = line.indexOf(\":\");\n    if (colon <= 0) continue;\n    const key = line.slice(0, colon).trim().toLowerCase();\n    const value = line.slice(colon + 1).trim();\n    if (key === \"user-agent\") {\n      if (current === undefined || !collectingAgents) {\n        current = {\n          agents: [],\n          line: index + 1,\n          rules: [],\n          crawlDelaySeconds: undefined,\n          signals: [],\n        };\n        groups.push(current);\n      }\n      current.agents.push(value.toLowerCase());\n      collectingAgents = true;\n      continue;\n    }\n    // records that belong to no group\n    if (key === \"sitemap\") {\n      if (value.length > 0) sitemaps.push(value);\n      continue;\n    }\n    // an RSL `License` line names the license of the content: the crawler records it with the other usage lines\n    if (key === \"content-signal\" || key === \"content-usage\" || key === \"license\") {\n      signals.push(`${key}: ${value}`);\n      (current?.signals ?? fileSignals).push(signalOf(key, value, index + 1));\n      continue;\n    }\n    if (current === undefined) continue;\n    if (key === \"allow\" || key === \"disallow\") {\n      // only a rule ends the list of agents, so a crawl-delay between two agents keeps one group\n      collectingAgents = false;\n      if (value.length > 0)\n        current.rules.push({\n          allow: key === \"allow\",\n          pattern: normalizePath(value),\n          line: index + 1,\n        });\n    } else if (key === \"crawl-delay\") {\n      const seconds = Number(value);\n      if (Number.isFinite(seconds) && seconds >= 0) current.crawlDelaySeconds = seconds;\n    }\n  }\n  return { groups, sitemaps, signals, fileSignals };\n};\n\n/**\n * This function tells if a normalized path matches the pattern of a rule. A `*` matches any\n * sequence, and a `$` at the end anchors the end. The function finds the parts between the stars\n * in sequence, each at its first position. No pattern makes it slow: its time is linear in the\n * length of the path for each part, and it uses no regular expression.\n */\nexport const matchesPattern = (pattern: string, path: string): boolean => {\n  const anchored = pattern.endsWith(\"$\");\n  const parts = (anchored ? pattern.slice(0, -1) : pattern).split(\"*\");\n  const first = parts[0]!;\n  if (!path.startsWith(first)) return false;\n  if (parts.length === 1) return !anchored || path.length === first.length;\n  let at = first.length;\n  const last = parts.length - 1;\n  for (let index = 1; index < last; index += 1) {\n    const part = parts[index]!;\n    // two stars in sequence give an empty part, which matches at once\n    if (part.length === 0) continue;\n    const found = path.indexOf(part, at);\n    if (found === -1) return false;\n    at = found + part.length;\n  }\n  const tail = parts[last]!;\n  if (!anchored) return tail.length === 0 || path.indexOf(tail, at) !== -1;\n  return path.length - tail.length >= at && path.endsWith(tail);\n};\n\ninterface Chosen {\n  readonly rules: readonly RobotsRule[];\n  readonly groups: readonly RobotsGroup[];\n  readonly crawlDelaySeconds: number | undefined;\n  /** This value is true when a group names this token, and `*` did not apply. */\n  readonly explicit: boolean;\n}\n\nconst groupsFor = (robots: Robots, token: string): Chosen => {\n  const wanted = token === \"*\" ? \"*\" : productToken(token);\n  const named = robots.groups.filter((group) =>\n    group.agents.some((agent) => agent !== \"*\" && tokenOf(agent) === wanted),\n  );\n  const chosen =\n    named.length > 0 ? named : robots.groups.filter((group) => group.agents.includes(\"*\"));\n  const delays = chosen.flatMap((group) =>\n    group.crawlDelaySeconds === undefined ? [] : [group.crawlDelaySeconds],\n  );\n  return {\n    rules: chosen.flatMap((group) => group.rules),\n    groups: chosen,\n    crawlDelaySeconds: delays.length > 0 ? Math.max(...delays) : undefined,\n    explicit: named.length > 0,\n  };\n};\n\n/** This function gives the rule that decides for the path: the longest match, and `allow` in a tie. */\nconst decidingRule = (rules: readonly RobotsRule[], path: string): RobotsRule | undefined => {\n  let best: RobotsRule | undefined;\n  for (const rule of rules) {\n    if (!matchesPattern(rule.pattern, path)) continue;\n    if (\n      best === undefined ||\n      rule.pattern.length > best.pattern.length ||\n      (rule.pattern.length === best.pattern.length && rule.allow && !best.allow)\n    ) {\n      best = rule;\n    }\n  }\n  return best;\n};\n\nexport interface RobotsDecision {\n  readonly allowed: boolean;\n  /** This value is the product token whose rules refused the path. */\n  readonly by: string | undefined;\n  readonly crawlDelaySeconds: number | undefined;\n  /** This rule decided. When no rule matches the path, the path is permitted and this value is empty. */\n  readonly rule?: RobotsRule | undefined;\n  /** These groups applied to the decision: the groups that name the token, or the group `*`. */\n  readonly groups?: readonly RobotsGroup[] | undefined;\n}\n\n/**\n * This function tells if these tokens can fetch `pathAndQuery`. Each token becomes its product\n * token (`productToken`), so `MyBot/1.0` matches the group `mybot`. The first token is the token of\n * the crawler, and `*` applies when no group names it. A further token, for example `Claude-User`,\n * applies only when the site names it. A site that writes rules for an agent means them for the\n * fetches of that agent.\n */\nexport const decide = (\n  robots: Robots,\n  tokens: readonly string[],\n  pathAndQuery: string,\n): RobotsDecision => {\n  const path = normalizePath(pathAndQuery.length === 0 ? \"/\" : pathAndQuery);\n  const primary = groupsFor(robots, tokens[0] ?? \"*\");\n  // the crawl delay of a group that names a further agent applies to the fetches of that agent too\n  const delays = [primary.crawlDelaySeconds];\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    if (extra.explicit) delays.push(extra.crawlDelaySeconds);\n  }\n  const known = delays.filter((delay): delay is number => delay !== undefined);\n  const crawlDelaySeconds = known.length > 0 ? Math.max(...known) : undefined;\n  if (path === \"/robots.txt\") {\n    return { allowed: true, by: undefined, crawlDelaySeconds, rule: undefined, groups: [] };\n  }\n  const rule = decidingRule(primary.rules, path);\n  if (rule !== undefined && !rule.allow) {\n    return { allowed: false, by: tokens[0], crawlDelaySeconds, rule, groups: primary.groups };\n  }\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    const refusal = extra.explicit ? decidingRule(extra.rules, path) : undefined;\n    if (refusal !== undefined && !refusal.allow) {\n      return { allowed: false, by: token, crawlDelaySeconds, rule: refusal, groups: extra.groups };\n    }\n  }\n  return { allowed: true, by: undefined, crawlDelaySeconds, rule, groups: primary.groups };\n};\n\n/**\n * This function gives the usage lines that apply to these tokens and this path, as\n * `key: value` texts. The lines come from three places:\n *\n * - the lines before the first group\n * - the groups of the first token, or `*`\n * - the groups that name a further token\n *\n * In one group, the line with the longest matching path wins for each key.\n */\nexport const signalsFor = (\n  robots: Robots,\n  tokens: readonly string[],\n  pathAndQuery: string,\n): string[] => {\n  const path = normalizePath(pathAndQuery.length === 0 ? \"/\" : pathAndQuery);\n  const groups = new Set(groupsFor(robots, tokens[0] ?? \"*\").groups);\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    if (extra.explicit) for (const group of extra.groups) groups.add(group);\n  }\n  const chosen: RobotsSignal[] = [];\n  const pick = (lines: readonly RobotsSignal[]): void => {\n    // for each key: the length of the longest matching path, and the lines with that length\n    const best = new Map<string, { length: number; lines: RobotsSignal[] }>();\n    for (const line of lines) {\n      if (line.path !== undefined && !matchesPattern(line.path, path)) continue;\n      const length = line.path?.length ?? 0;\n      const before = best.get(line.key);\n      if (before === undefined || length > before.length)\n        best.set(line.key, { length, lines: [line] });\n      else if (length === before.length) before.lines.push(line);\n    }\n    for (const { lines: kept } of best.values()) chosen.push(...kept);\n  };\n  pick(robots.fileSignals ?? []);\n  for (const group of groups) pick(group.signals ?? []);\n  return chosen.sort((a, b) => a.line - b.line).map((line) => `${line.key}: ${line.value}`);\n};\n\n/** This function gives the robots of a site whose robots.txt the crawler cannot read. No path is permitted, and the reason stays. */\nexport const unreachableRobots = (reason: string): Robots => ({\n  ...DISALLOW_ALL,\n  unreachable: reason,\n});\n\n/** This text is the reason when a robots.txt that answered 200 is a web page, not a robots.txt file. */\nexport const HTML_ROBOTS =\n  \"robots.txt is a web page, not a robots.txt file (a login or challenge page)\";\n\n/**\n * This function tells if the body of a robots.txt is a web page. A server can send a challenge, a\n * login or an error page with a 200. Such a body starts with `<!doctype`, `<html`, `<head` or\n * `<body`. With the content type `text/html`, a tag of a page in the first 4 KB is enough. A plain\n * robots.txt that a server sends as `text/html` stays a file.\n */\nexport const isHtmlRobots = (contentType: string | null | undefined, text: string): boolean => {\n  const head = (text.charCodeAt(0) === 0xfeff ? text.slice(1) : text).slice(0, 4096);\n  if (/^\\s*<(!doctype|html|head|body)[\\s>]/i.test(head)) return true;\n  return (\n    /html/i.test(contentType ?? \"\") &&\n    /<(html|head|body|script|meta|title|div|form)[\\s>/]/i.test(head)\n  );\n};\n\n/**\n * This function reads the answer of a robots.txt fetch: the status, the content type and the body.\n * A 2xx with a web page in place of the file is unreadable. Thus no path is permitted (fail\n * closed), as for a body that does not decode. The other statuses follow `robotsFromStatus`.\n */\nexport const robotsFromResponse = (\n  status: number,\n  contentType: string | null | undefined,\n  body: Uint8Array | string,\n): Robots => {\n  if (status < 200 || status >= 300) return robotsFromStatus(status, undefined);\n  const text = typeof body === \"string\" ? body : new TextDecoder(\"utf-8\").decode(body);\n  if (isHtmlRobots(contentType, text)) return unreachableRobots(HTML_ROBOTS);\n  return parseRobots(text);\n};\n\n/** This function gives the meaning of the status of a robots.txt fetch (RFC 9309 §2.3.1). */\nexport const robotsFromStatus = (status: number, body: string | undefined): Robots => {\n  if (status >= 200 && status < 300) return parseRobots(body ?? \"\");\n  if (status === 429) return unreachableRobots(\"status 429, rate-limited\");\n  if (status >= 400 && status < 500) return ALLOW_ALL;\n  return unreachableRobots(status === 0 ? \"no response\" : `status ${String(status)}`);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAuBA,MAAa,mBAA2B;AAiDxC,MAAa,YAAoB;CAAE,QAAQ,CAAC;CAAG,UAAU,CAAC;CAAG,SAAS,CAAC;AAAE;AAEzE,MAAa,eAAuB;CAClC,QAAQ,CACN;EAAE,QAAQ,CAAC,GAAG;EAAG,OAAO,CAAC;GAAE,OAAO;GAAO,SAAS;EAAI,CAAC;EAAG,mBAAmB,KAAA;CAAU,CACzF;CACA,UAAU,CAAC;CACX,SAAS,CAAC;AACZ;AAEA,MAAM,aAAa;AACnB,MAAM,8BAAc,IAAI,IAAI,cAAc;;;;;AAK1C,MAAM,gBAAgB,SAA0B;CAC9C,MAAM,OAAO,KAAK,WAAW,CAAC;CAC9B,OAAO,QAAQ,MAAQ,SAAS,OAAQ,YAAY,IAAI,IAAI;AAC9D;AACA,MAAM,MAAM;AAEZ,MAAM,YAAY,SAAyB;CACzC,MAAM,OAAO,KAAK,YAAY,CAAC;CAC/B,IAAI,QAAQ,KAAM,OAAO,IAAI,KAAK,SAAS,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,GAAG,GAAG;CAC5E,IAAI;EACF,OAAO,mBAAmB,IAAI;CAChC,QAAQ;EAEN,OAAO;CACT;AACF;;;;;;;AAQA,MAAa,iBAAiB,UAA0B;CACtD,IAAI,MAAM;CACV,MAAM,QAAQ,MAAM,KAAK,KAAK;CAC9B,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS,GAAG;EACpD,MAAM,OAAO,MAAM;EACnB,IAAI,SAAS,OAAO,IAAI,KAAK,MAAM,QAAQ,MAAM,EAAE,KAAK,IAAI,KAAK,MAAM,QAAQ,MAAM,EAAE,GAAG;GACxF,MAAM,MAAM,GAAG,MAAM,QAAQ,KAAM,MAAM,QAAQ;GACjD,MAAM,UAAU,OAAO,aAAa,OAAO,SAAS,KAAK,EAAE,CAAC;GAC5D,OAAO,WAAW,KAAK,OAAO,IAAI,UAAU,IAAI,IAAI,YAAY;GAChE,SAAS;GACT;EACF;EACA,OAAO,KAAK,WAAW,CAAC,IAAI,OAAQ,aAAa,IAAI,IAAI,SAAS,IAAI,IAAI;CAC5E;CACA,OAAO;AACT;;AAGA,MAAa,kBAAkB,UAA2B,gBAAgB,KAAK,KAAK;;AAGpF,MAAM,WAAW,WACd,eAAe,KAAK,MAAM,KAAK,CAAC,CAAC,GAAG,MAAM,GAAA,CAAI,YAAY;;AAG7D,MAAM,gCAAqC,IAAI,IAAI;CACjD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;;AAQD,MAAa,gBAAgB,UAA0B;CACrD,MAAM,OAAO,MAAM,KAAK;CACxB,MAAM,QAAQ,QAAQ,IAAI;CAC1B,IAAI,UAAU,WAAW,OAAO;CAChC,MAAM,aAAa,+BAA+B,KAAK,IAAI,CAAC,GAAG;CAC/D,IAAI,eAAe,KAAA,GAAW,OAAO,WAAW,YAAY;CAI5D,OAHiB,CAAC,GAAG,KAAK,SAAS,gCAAgC,CAAC,CAAC,CAAC,KAAK,WACxE,MAAM,MAAM,GAAA,CAAI,YAAY,CAEjB,CAAC,CAAC,MAAM,YAAY,CAAC,cAAc,IAAI,OAAO,CAAC,KAAK;AACpE;AAEA,MAAM,gBAAgB,MAAc,aAA6B;CAC/D,MAAM,QAAQ,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI;CAC3C,IAAI,MAAM,UAAU,UAAU,OAAO;CAErC,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,MAAM,MAAM,GAAG,QAAQ,CAAC;CAC9D,MAAM,YAAY,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG,KAAK,YAAY,IAAI,CAAC;CACzE,OAAO,cAAc,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS;AACxD;;AAGA,MAAM,YAAY,KAAa,OAAe,SAA+B;CAC3E,MAAM,OAAO,mBAAmB,KAAK,KAAK;CAC1C,OAAO,SAAS,OACZ;EAAE;EAAK;EAAO,MAAM,KAAA;EAAW;CAAK,IACpC;EAAE;EAAK,OAAO,KAAK,EAAE,CAAE,KAAK;EAAG,MAAM,cAAc,KAAK,EAAG;EAAG;CAAK;AACzE;AAEA,MAAa,eAAe,UAA0B;CACpD,MAAM,SAMD,CAAC;CACN,MAAM,WAAqB,CAAC;CAC5B,MAAM,UAAoB,CAAC;CAC3B,MAAM,cAA8B,CAAC;CACrC,IAAI;CACJ,IAAI,mBAAmB;CACvB,KAAK,MAAM,CAAC,OAAO,QAAQ,aAAa,OAAO,gBAAgB,CAAC,CAC7D,MAAM,YAAY,CAAC,CACnB,QAAQ,GAAG;EACZ,MAAM,OAAO,IAAI,QAAQ,QAAQ,EAAE,CAAC,CAAC,KAAK;EAC1C,MAAM,QAAQ,KAAK,QAAQ,GAAG;EAC9B,IAAI,SAAS,GAAG;EAChB,MAAM,MAAM,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,YAAY;EACpD,MAAM,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC,CAAC,KAAK;EACzC,IAAI,QAAQ,cAAc;GACxB,IAAI,YAAY,KAAA,KAAa,CAAC,kBAAkB;IAC9C,UAAU;KACR,QAAQ,CAAC;KACT,MAAM,QAAQ;KACd,OAAO,CAAC;KACR,mBAAmB,KAAA;KACnB,SAAS,CAAC;IACZ;IACA,OAAO,KAAK,OAAO;GACrB;GACA,QAAQ,OAAO,KAAK,MAAM,YAAY,CAAC;GACvC,mBAAmB;GACnB;EACF;EAEA,IAAI,QAAQ,WAAW;GACrB,IAAI,MAAM,SAAS,GAAG,SAAS,KAAK,KAAK;GACzC;EACF;EAEA,IAAI,QAAQ,oBAAoB,QAAQ,mBAAmB,QAAQ,WAAW;GAC5E,QAAQ,KAAK,GAAG,IAAI,IAAI,OAAO;GAC/B,CAAC,SAAS,WAAW,YAAA,CAAa,KAAK,SAAS,KAAK,OAAO,QAAQ,CAAC,CAAC;GACtE;EACF;EACA,IAAI,YAAY,KAAA,GAAW;EAC3B,IAAI,QAAQ,WAAW,QAAQ,YAAY;GAEzC,mBAAmB;GACnB,IAAI,MAAM,SAAS,GACjB,QAAQ,MAAM,KAAK;IACjB,OAAO,QAAQ;IACf,SAAS,cAAc,KAAK;IAC5B,MAAM,QAAQ;GAChB,CAAC;EACL,OAAO,IAAI,QAAQ,eAAe;GAChC,MAAM,UAAU,OAAO,KAAK;GAC5B,IAAI,OAAO,SAAS,OAAO,KAAK,WAAW,GAAG,QAAQ,oBAAoB;EAC5E;CACF;CACA,OAAO;EAAE;EAAQ;EAAU;EAAS;CAAY;AAClD;;;;;;;AAQA,MAAa,kBAAkB,SAAiB,SAA0B;CACxE,MAAM,WAAW,QAAQ,SAAS,GAAG;CACrC,MAAM,SAAS,WAAW,QAAQ,MAAM,GAAG,EAAE,IAAI,QAAA,CAAS,MAAM,GAAG;CACnE,MAAM,QAAQ,MAAM;CACpB,IAAI,CAAC,KAAK,WAAW,KAAK,GAAG,OAAO;CACpC,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,KAAK,WAAW,MAAM;CAClE,IAAI,KAAK,MAAM;CACf,MAAM,OAAO,MAAM,SAAS;CAC5B,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,SAAS,GAAG;EAC5C,MAAM,OAAO,MAAM;EAEnB,IAAI,KAAK,WAAW,GAAG;EACvB,MAAM,QAAQ,KAAK,QAAQ,MAAM,EAAE;EACnC,IAAI,UAAU,IAAI,OAAO;EACzB,KAAK,QAAQ,KAAK;CACpB;CACA,MAAM,OAAO,MAAM;CACnB,IAAI,CAAC,UAAU,OAAO,KAAK,WAAW,KAAK,KAAK,QAAQ,MAAM,EAAE,MAAM;CACtE,OAAO,KAAK,SAAS,KAAK,UAAU,MAAM,KAAK,SAAS,IAAI;AAC9D;AAUA,MAAM,aAAa,QAAgB,UAA0B;CAC3D,MAAM,SAAS,UAAU,MAAM,MAAM,aAAa,KAAK;CACvD,MAAM,QAAQ,OAAO,OAAO,QAAQ,UAClC,MAAM,OAAO,MAAM,UAAU,UAAU,OAAO,QAAQ,KAAK,MAAM,MAAM,CACzE;CACA,MAAM,SACJ,MAAM,SAAS,IAAI,QAAQ,OAAO,OAAO,QAAQ,UAAU,MAAM,OAAO,SAAS,GAAG,CAAC;CACvF,MAAM,SAAS,OAAO,SAAS,UAC7B,MAAM,sBAAsB,KAAA,IAAY,CAAC,IAAI,CAAC,MAAM,iBAAiB,CACvE;CACA,OAAO;EACL,OAAO,OAAO,SAAS,UAAU,MAAM,KAAK;EAC5C,QAAQ;EACR,mBAAmB,OAAO,SAAS,IAAI,KAAK,IAAI,GAAG,MAAM,IAAI,KAAA;EAC7D,UAAU,MAAM,SAAS;CAC3B;AACF;;AAGA,MAAM,gBAAgB,OAA8B,SAAyC;CAC3F,IAAI;CACJ,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,GAAG;EACzC,IACE,SAAS,KAAA,KACT,KAAK,QAAQ,SAAS,KAAK,QAAQ,UAClC,KAAK,QAAQ,WAAW,KAAK,QAAQ,UAAU,KAAK,SAAS,CAAC,KAAK,OAEpE,OAAO;CAEX;CACA,OAAO;AACT;;;;;;;;AAoBA,MAAa,UACX,QACA,QACA,iBACmB;CACnB,MAAM,OAAO,cAAc,aAAa,WAAW,IAAI,MAAM,YAAY;CACzE,MAAM,UAAU,UAAU,QAAQ,OAAO,MAAM,GAAG;CAElD,MAAM,SAAS,CAAC,QAAQ,iBAAiB;CACzC,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,IAAI,MAAM,UAAU,OAAO,KAAK,MAAM,iBAAiB;CACzD;CACA,MAAM,QAAQ,OAAO,QAAQ,UAA2B,UAAU,KAAA,CAAS;CAC3E,MAAM,oBAAoB,MAAM,SAAS,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,KAAA;CAClE,IAAI,SAAS,eACX,OAAO;EAAE,SAAS;EAAM,IAAI,KAAA;EAAW;EAAmB,MAAM,KAAA;EAAW,QAAQ,CAAC;CAAE;CAExF,MAAM,OAAO,aAAa,QAAQ,OAAO,IAAI;CAC7C,IAAI,SAAS,KAAA,KAAa,CAAC,KAAK,OAC9B,OAAO;EAAE,SAAS;EAAO,IAAI,OAAO;EAAI;EAAmB;EAAM,QAAQ,QAAQ;CAAO;CAE1F,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,MAAM,UAAU,MAAM,WAAW,aAAa,MAAM,OAAO,IAAI,IAAI,KAAA;EACnE,IAAI,YAAY,KAAA,KAAa,CAAC,QAAQ,OACpC,OAAO;GAAE,SAAS;GAAO,IAAI;GAAO;GAAmB,MAAM;GAAS,QAAQ,MAAM;EAAO;CAE/F;CACA,OAAO;EAAE,SAAS;EAAM,IAAI,KAAA;EAAW;EAAmB;EAAM,QAAQ,QAAQ;CAAO;AACzF;;;;;;;;;;;AAYA,MAAa,cACX,QACA,QACA,iBACa;CACb,MAAM,OAAO,cAAc,aAAa,WAAW,IAAI,MAAM,YAAY;CACzE,MAAM,SAAS,IAAI,IAAI,UAAU,QAAQ,OAAO,MAAM,GAAG,CAAC,CAAC,MAAM;CACjE,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,IAAI,MAAM,UAAU,KAAK,MAAM,SAAS,MAAM,QAAQ,OAAO,IAAI,KAAK;CACxE;CACA,MAAM,SAAyB,CAAC;CAChC,MAAM,QAAQ,UAAyC;EAErD,MAAM,uBAAO,IAAI,IAAuD;EACxE,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,KAAK,SAAS,KAAA,KAAa,CAAC,eAAe,KAAK,MAAM,IAAI,GAAG;GACjE,MAAM,SAAS,KAAK,MAAM,UAAU;GACpC,MAAM,SAAS,KAAK,IAAI,KAAK,GAAG;GAChC,IAAI,WAAW,KAAA,KAAa,SAAS,OAAO,QAC1C,KAAK,IAAI,KAAK,KAAK;IAAE;IAAQ,OAAO,CAAC,IAAI;GAAE,CAAC;QACzC,IAAI,WAAW,OAAO,QAAQ,OAAO,MAAM,KAAK,IAAI;EAC3D;EACA,KAAK,MAAM,EAAE,OAAO,UAAU,KAAK,OAAO,GAAG,OAAO,KAAK,GAAG,IAAI;CAClE;CACA,KAAK,OAAO,eAAe,CAAC,CAAC;CAC7B,KAAK,MAAM,SAAS,QAAQ,KAAK,MAAM,WAAW,CAAC,CAAC;CACpD,OAAO,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,SAAS,GAAG,KAAK,IAAI,IAAI,KAAK,OAAO;AAC1F;;AAGA,MAAa,qBAAqB,YAA4B;CAC5D,GAAG;CACH,aAAa;AACf;;AAGA,MAAa,cACX;;;;;;;AAQF,MAAa,gBAAgB,aAAwC,SAA0B;CAC7F,MAAM,QAAQ,KAAK,WAAW,CAAC,MAAM,QAAS,KAAK,MAAM,CAAC,IAAI,KAAA,CAAM,MAAM,GAAG,IAAI;CACjF,IAAI,uCAAuC,KAAK,IAAI,GAAG,OAAO;CAC9D,OACE,QAAQ,KAAK,eAAe,EAAE,KAC9B,sDAAsD,KAAK,IAAI;AAEnE;;;;;;AAOA,MAAa,sBACX,QACA,aACA,SACW;CACX,IAAI,SAAS,OAAO,UAAU,KAAK,OAAO,iBAAiB,QAAQ,KAAA,CAAS;CAC5E,MAAM,OAAO,OAAO,SAAS,WAAW,OAAO,IAAI,YAAY,OAAO,CAAC,CAAC,OAAO,IAAI;CACnF,IAAI,aAAa,aAAa,IAAI,GAAG,OAAO,kBAAkB,WAAW;CACzE,OAAO,YAAY,IAAI;AACzB;;AAGA,MAAa,oBAAoB,QAAgB,SAAqC;CACpF,IAAI,UAAU,OAAO,SAAS,KAAK,OAAO,YAAY,QAAQ,EAAE;CAChE,IAAI,WAAW,KAAK,OAAO,kBAAkB,0BAA0B;CACvE,IAAI,UAAU,OAAO,SAAS,KAAK,OAAO;CAC1C,OAAO,kBAAkB,WAAW,IAAI,gBAAgB,UAAU,OAAO,MAAM,GAAG;AACpF"}
+{"version":3,"file":"robots.mjs","names":[],"sources":["../src/robots.ts"],"sourcesContent":["/**\n * This module reads robots.txt as RFC 9309 (September 2022) specifies it. It is for a crawler that\n * tells who it is and obeys what a site says. The important rules, each with a test:\n *\n * - A group is one or more `user-agent` lines and the rules after them. A `user-agent` line after\n *   an `allow` or `disallow` line starts a new group, as the parser of Google does. A different\n *   line (`crawl-delay`) between two `user-agent` lines does not split the group. A rule before\n *   the first `user-agent` line is in no group.\n * - The product token matches with no attention to case. The matching groups merge. The group\n *   `*` applies only when no group names the crawler.\n * - The longest matching rule wins, and `allow` wins a tie. A `*` matches any sequence, and a `$`\n *   at the end anchors the end. The matcher uses no regular expression, and its time is linear.\n * - `/robots.txt` is always permitted, and an empty `disallow` matches nothing.\n * - The parser reads at least the first 500 KiB.\n * - A usage line (`content-signal`, `content-usage`) belongs to its group, with an optional path\n *   before its value. A line before the first group applies to each crawler.\n *\n * The function `robotsFromStatus` gives the meaning of the status of the fetch. A 4xx is\n * \"unavailable\", so all paths are permitted. A 5xx or no response is \"unreachable\", so no path is\n * permitted. A 429 is unreachable too, as Google does it. A site that limits the rate of its robots\n * file does not ask for a crawl.\n */\n\nexport const ROBOTS_MAX_BYTES: number = 500 * 1024;\n\nexport interface RobotsRule {\n  readonly allow: boolean;\n  /** This value is the path pattern, normalized for comparison. */\n  readonly pattern: string;\n  /** This value is the number of the line in robots.txt, from 1. A rule that the code makes has none. */\n  readonly line?: number | undefined;\n}\n\n/** A usage line states a preference about the use of the content (`content-signal`, `content-usage`). */\nexport interface RobotsSignal {\n  /** This value is the name of the line, in lowercase. */\n  readonly key: string;\n  /** This text is the value of the line, after the path. */\n  readonly value: string;\n  /** This value is the normalized path pattern of the line. A line with no path applies to each path. */\n  readonly path: string | undefined;\n  /** This value is the number of the line in robots.txt, from 1. */\n  readonly line: number;\n}\n\nexport interface RobotsGroup {\n  /** These values are the user agents of the group, in lowercase. */\n  readonly agents: readonly string[];\n  /** This value is the number of the first `user-agent` line of the group, from 1. */\n  readonly line?: number | undefined;\n  readonly rules: readonly RobotsRule[];\n  /** This value is the nonstandard `crawl-delay` in seconds, when the group sets one. */\n  readonly crawlDelaySeconds: number | undefined;\n  /** These usage lines are in the group. They apply when the group applies. */\n  readonly signals?: readonly RobotsSignal[] | undefined;\n}\n\nexport interface Robots {\n  readonly groups: readonly RobotsGroup[];\n  readonly sitemaps: readonly string[];\n  /** These lines state usage preferences (`content-signal: …`, `content-usage: …`), as written, from each group. */\n  readonly signals: readonly string[];\n  /** These usage lines come before the first group, so they apply to each crawler. */\n  readonly fileSignals?: readonly RobotsSignal[] | undefined;\n  /**\n   * This text tells why the crawler could not read robots.txt: a dead domain, a broken certificate\n   * or a server error. Then no path is permitted, and the note of a skip gives this reason. It does\n   * not name a rule of the site.\n   */\n  readonly unreachable?: string | undefined;\n}\n\nexport const ALLOW_ALL: Robots = { groups: [], sitemaps: [], signals: [] };\n\nexport const DISALLOW_ALL: Robots = {\n  groups: [\n    { agents: [\"*\"], rules: [{ allow: false, pattern: \"/\" }], crawlDelaySeconds: undefined },\n  ],\n  sitemaps: [],\n  signals: [],\n};\n\nconst UNRESERVED = /^[A-Za-z0-9\\-._~]$/;\nconst URL_ENCODED = new Set(\"\\\"'<>\\\\^`{|}\");\n/**\n * The URL parser encodes these ASCII characters in a path or a query: the controls, a space, DEL,\n * and `\"'<>\\^`{|}`. A rule can contain them as written.\n */\nconst encodedAscii = (char: string): boolean => {\n  const code = char.charCodeAt(0);\n  return code <= 0x20 || code === 0x7f || URL_ENCODED.has(char);\n};\nconst HEX = /^[0-9A-Fa-f]$/;\n\nconst escapeOf = (char: string): string => {\n  const code = char.codePointAt(0)!;\n  if (code <= 0x7f) return `%${code.toString(16).toUpperCase().padStart(2, \"0\")}`;\n  try {\n    return encodeURIComponent(char);\n  } catch {\n    // a lone surrogate: the URL parser writes the replacement character\n    return \"%EF%BF%BD\";\n  }\n};\n\n/**\n * This function normalizes a path, or the pattern of a rule, for comparison, as §2.2.2 asks. It\n * encodes the characters outside US-ASCII, and the ASCII characters that the URL parser encodes\n * (a space, `\"`, `<`, `{`). It decodes an escape of an unreserved character, so `/%7Efoo` and\n * `/~foo` are equal. The other escapes stay, with uppercase hexadecimal digits.\n */\nexport const normalizePath = (value: string): string => {\n  let out = \"\";\n  const chars = Array.from(value);\n  for (let index = 0; index < chars.length; index += 1) {\n    const char = chars[index]!;\n    if (char === \"%\" && HEX.test(chars[index + 1] ?? \"\") && HEX.test(chars[index + 2] ?? \"\")) {\n      const hex = `${chars[index + 1]!}${chars[index + 2]!}`;\n      const decoded = String.fromCharCode(Number.parseInt(hex, 16));\n      out += UNRESERVED.test(decoded) ? decoded : `%${hex.toUpperCase()}`;\n      index += 2;\n      continue;\n    }\n    out += char.charCodeAt(0) > 0x7f || encodedAscii(char) ? escapeOf(char) : char;\n  }\n  return out;\n};\n\n/** This function tells if a text is a product token: letters, `_` and `-` only, and one or more (RFC 9309, section 2.2.1). */\nexport const isProductToken = (value: string): boolean => /^[A-Za-z_-]+$/.test(value);\n\n/** This function gives the first product token of a user agent: `Googlebot/2.1` gives `googlebot`. */\nconst tokenOf = (value: string): string =>\n  (/^[A-Za-z_-]+/.exec(value.trim())?.[0] ?? \"\").toLowerCase();\n\n/** These names are parts of the header of a browser, not the name of a crawler. */\nconst BROWSER_PARTS: ReadonlySet<string> = new Set([\n  \"mozilla\",\n  \"applewebkit\",\n  \"khtml\",\n  \"gecko\",\n  \"chrome\",\n  \"chromium\",\n  \"safari\",\n  \"version\",\n  \"mobile\",\n  \"firefox\",\n  \"edg\",\n  \"opr\",\n  \"like\",\n]);\n\n/**\n * This function gives the product token that robots.txt rules match, in lowercase. `MyBot/1.0`\n * gives `mybot`. The full header `Mozilla/5.0 (compatible; MyBot/1.0; +https://…)` also gives\n * `mybot`. The token is the name after `compatible;`, or else the first product that is not a part\n * of a browser.\n */\nexport const productToken = (value: string): string => {\n  const text = value.trim();\n  const first = tokenOf(text);\n  if (first !== \"mozilla\") return first;\n  const compatible = /compatible;\\s*([A-Za-z_-]+)/i.exec(text)?.[1];\n  if (compatible !== undefined) return compatible.toLowerCase();\n  // One pass in linear time: each match is a name that starts with a letter, with an optional `/` and\n  // version after it, or a run of other characters. A run of other characters takes no version, so\n  // `_/a/a` gives the product `a`, as the scan before 0.2.0 did.\n  const products = [...text.matchAll(/([A-Za-z][A-Za-z_-]*)(\\/[\\w.]+)?|[^A-Za-z]+/g)].flatMap(\n    (match) => (match[1] !== undefined && match[2] !== undefined ? [match[1].toLowerCase()] : []),\n  );\n  return products.find((product) => !BROWSER_PARTS.has(product)) ?? first;\n};\n\nconst truncateUtf8 = (text: string, maxBytes: number): string => {\n  const bytes = new TextEncoder().encode(text);\n  if (bytes.length <= maxBytes) return text;\n  // cut at the last line break inside the limit, so no rule is half-read\n  const head = new TextDecoder().decode(bytes.slice(0, maxBytes));\n  const lastBreak = Math.max(head.lastIndexOf(\"\\n\"), head.lastIndexOf(\"\\r\"));\n  return lastBreak === -1 ? \"\" : head.slice(0, lastBreak);\n};\n\n/** This function reads a usage line: an optional path, then the value (`/images/ train-ai=n`). */\nconst signalOf = (key: string, value: string, line: number): RobotsSignal => {\n  const path = /^(\\/\\S*)\\s+(.*)$/.exec(value);\n  return path === null\n    ? { key, value, path: undefined, line }\n    : { key, value: path[2]!.trim(), path: normalizePath(path[1]!), line };\n};\n\nexport const parseRobots = (input: string): Robots => {\n  const groups: Array<{\n    agents: string[];\n    line: number;\n    rules: RobotsRule[];\n    crawlDelaySeconds: number | undefined;\n    signals: RobotsSignal[];\n  }> = [];\n  const sitemaps: string[] = [];\n  const signals: string[] = [];\n  const fileSignals: RobotsSignal[] = [];\n  let current: (typeof groups)[number] | undefined;\n  let collectingAgents = false;\n  for (const [index, raw] of truncateUtf8(input, ROBOTS_MAX_BYTES)\n    .split(/\\r\\n|\\r|\\n/)\n    .entries()) {\n    const line = raw.replace(/#.*$/, \"\").trim();\n    const colon = line.indexOf(\":\");\n    if (colon <= 0) continue;\n    const key = line.slice(0, colon).trim().toLowerCase();\n    const value = line.slice(colon + 1).trim();\n    if (key === \"user-agent\") {\n      if (current === undefined || !collectingAgents) {\n        current = {\n          agents: [],\n          line: index + 1,\n          rules: [],\n          crawlDelaySeconds: undefined,\n          signals: [],\n        };\n        groups.push(current);\n      }\n      current.agents.push(value.toLowerCase());\n      collectingAgents = true;\n      continue;\n    }\n    // records that belong to no group\n    if (key === \"sitemap\") {\n      if (value.length > 0) sitemaps.push(value);\n      continue;\n    }\n    // an RSL `License` line names the license of the content: the crawler records it with the other usage lines\n    if (key === \"content-signal\" || key === \"content-usage\" || key === \"license\") {\n      signals.push(`${key}: ${value}`);\n      (current?.signals ?? fileSignals).push(signalOf(key, value, index + 1));\n      continue;\n    }\n    if (current === undefined) continue;\n    if (key === \"allow\" || key === \"disallow\") {\n      // only a rule ends the list of agents, so a crawl-delay between two agents keeps one group\n      collectingAgents = false;\n      if (value.length > 0)\n        current.rules.push({\n          allow: key === \"allow\",\n          pattern: normalizePath(value),\n          line: index + 1,\n        });\n    } else if (key === \"crawl-delay\") {\n      const seconds = Number(value);\n      if (Number.isFinite(seconds) && seconds >= 0) current.crawlDelaySeconds = seconds;\n    }\n  }\n  return { groups, sitemaps, signals, fileSignals };\n};\n\n/**\n * This function tells if a normalized path matches the pattern of a rule. A `*` matches any\n * sequence, and a `$` at the end anchors the end. The function finds the parts between the stars\n * in sequence, each at its first position. No pattern makes it slow: its time is linear in the\n * length of the path for each part, and it uses no regular expression.\n */\nexport const matchesPattern = (pattern: string, path: string): boolean => {\n  const anchored = pattern.endsWith(\"$\");\n  const parts = (anchored ? pattern.slice(0, -1) : pattern).split(\"*\");\n  const first = parts[0]!;\n  if (!path.startsWith(first)) return false;\n  if (parts.length === 1) return !anchored || path.length === first.length;\n  let at = first.length;\n  const last = parts.length - 1;\n  for (let index = 1; index < last; index += 1) {\n    const part = parts[index]!;\n    // two stars in sequence give an empty part, which matches at once\n    if (part.length === 0) continue;\n    const found = path.indexOf(part, at);\n    if (found === -1) return false;\n    at = found + part.length;\n  }\n  const tail = parts[last]!;\n  if (!anchored) return tail.length === 0 || path.indexOf(tail, at) !== -1;\n  return path.length - tail.length >= at && path.endsWith(tail);\n};\n\ninterface Chosen {\n  readonly rules: readonly RobotsRule[];\n  readonly groups: readonly RobotsGroup[];\n  readonly crawlDelaySeconds: number | undefined;\n  /** This value is true when a group names this token, and `*` did not apply. */\n  readonly explicit: boolean;\n}\n\nconst groupsFor = (robots: Robots, token: string): Chosen => {\n  const wanted = token === \"*\" ? \"*\" : productToken(token);\n  const named = robots.groups.filter((group) =>\n    group.agents.some((agent) => agent !== \"*\" && tokenOf(agent) === wanted),\n  );\n  const chosen =\n    named.length > 0 ? named : robots.groups.filter((group) => group.agents.includes(\"*\"));\n  const delays = chosen.flatMap((group) =>\n    group.crawlDelaySeconds === undefined ? [] : [group.crawlDelaySeconds],\n  );\n  return {\n    rules: chosen.flatMap((group) => group.rules),\n    groups: chosen,\n    crawlDelaySeconds: delays.length > 0 ? Math.max(...delays) : undefined,\n    explicit: named.length > 0,\n  };\n};\n\n/** This function gives the rule that decides for the path: the longest match, and `allow` in a tie. */\nconst decidingRule = (rules: readonly RobotsRule[], path: string): RobotsRule | undefined => {\n  let best: RobotsRule | undefined;\n  for (const rule of rules) {\n    if (!matchesPattern(rule.pattern, path)) continue;\n    if (\n      best === undefined ||\n      rule.pattern.length > best.pattern.length ||\n      (rule.pattern.length === best.pattern.length && rule.allow && !best.allow)\n    ) {\n      best = rule;\n    }\n  }\n  return best;\n};\n\nexport interface RobotsDecision {\n  readonly allowed: boolean;\n  /** This value is the product token whose rules refused the path. */\n  readonly by: string | undefined;\n  readonly crawlDelaySeconds: number | undefined;\n  /** This rule decided. When no rule matches the path, the path is permitted and this value is empty. */\n  readonly rule?: RobotsRule | undefined;\n  /** These groups applied to the decision: the groups that name the token, or the group `*`. */\n  readonly groups?: readonly RobotsGroup[] | undefined;\n}\n\n/**\n * This function tells if these tokens can fetch `pathAndQuery`. Each token becomes its product\n * token (`productToken`), so `MyBot/1.0` matches the group `mybot`. The first token is the token of\n * the crawler, and `*` applies when no group names it. A further token, for example `Claude-User`,\n * applies only when the site names it. A site that writes rules for an agent means them for the\n * fetches of that agent.\n */\nexport const decide = (\n  robots: Robots,\n  tokens: readonly string[],\n  pathAndQuery: string,\n): RobotsDecision => {\n  const path = normalizePath(pathAndQuery.length === 0 ? \"/\" : pathAndQuery);\n  const primary = groupsFor(robots, tokens[0] ?? \"*\");\n  // the crawl delay of a group that names a further agent applies to the fetches of that agent too\n  const delays = [primary.crawlDelaySeconds];\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    if (extra.explicit) delays.push(extra.crawlDelaySeconds);\n  }\n  const known = delays.filter((delay): delay is number => delay !== undefined);\n  const crawlDelaySeconds = known.length > 0 ? Math.max(...known) : undefined;\n  if (path === \"/robots.txt\") {\n    return { allowed: true, by: undefined, crawlDelaySeconds, rule: undefined, groups: [] };\n  }\n  const rule = decidingRule(primary.rules, path);\n  if (rule !== undefined && !rule.allow) {\n    return { allowed: false, by: tokens[0], crawlDelaySeconds, rule, groups: primary.groups };\n  }\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    const refusal = extra.explicit ? decidingRule(extra.rules, path) : undefined;\n    if (refusal !== undefined && !refusal.allow) {\n      return { allowed: false, by: token, crawlDelaySeconds, rule: refusal, groups: extra.groups };\n    }\n  }\n  return { allowed: true, by: undefined, crawlDelaySeconds, rule, groups: primary.groups };\n};\n\n/**\n * This function gives the usage lines that apply to these tokens and this path, as\n * `key: value` texts. The lines come from three places:\n *\n * - the lines before the first group\n * - the groups of the first token, or `*`\n * - the groups that name a further token\n *\n * In one group, the line with the longest matching path wins for each key.\n */\nexport const signalsFor = (\n  robots: Robots,\n  tokens: readonly string[],\n  pathAndQuery: string,\n): string[] => {\n  const path = normalizePath(pathAndQuery.length === 0 ? \"/\" : pathAndQuery);\n  const groups = new Set(groupsFor(robots, tokens[0] ?? \"*\").groups);\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    if (extra.explicit) for (const group of extra.groups) groups.add(group);\n  }\n  const chosen: RobotsSignal[] = [];\n  const pick = (lines: readonly RobotsSignal[]): void => {\n    // for each key: the length of the longest matching path, and the lines with that length\n    const best = new Map<string, { length: number; lines: RobotsSignal[] }>();\n    for (const line of lines) {\n      if (line.path !== undefined && !matchesPattern(line.path, path)) continue;\n      const length = line.path?.length ?? 0;\n      const before = best.get(line.key);\n      if (before === undefined || length > before.length)\n        best.set(line.key, { length, lines: [line] });\n      else if (length === before.length) before.lines.push(line);\n    }\n    for (const { lines: kept } of best.values()) chosen.push(...kept);\n  };\n  pick(robots.fileSignals ?? []);\n  for (const group of groups) pick(group.signals ?? []);\n  return chosen.sort((a, b) => a.line - b.line).map((line) => `${line.key}: ${line.value}`);\n};\n\n/** This function gives the robots of a site whose robots.txt the crawler cannot read. No path is permitted, and the reason stays. */\nexport const unreachableRobots = (reason: string): Robots => ({\n  ...DISALLOW_ALL,\n  unreachable: reason,\n});\n\n/** This text is the reason when a robots.txt that answered 200 is a web page, not a robots.txt file. */\nexport const HTML_ROBOTS =\n  \"robots.txt is a web page, not a robots.txt file (a login or challenge page)\";\n\n/**\n * This function tells if the body of a robots.txt is a web page. A server can send a challenge, a\n * login or an error page with a 200. Such a body starts with `<!doctype`, `<html`, `<head` or\n * `<body`. With the content type `text/html`, a tag of a page in the first 4 KB is enough. A plain\n * robots.txt that a server sends as `text/html` stays a file.\n */\nexport const isHtmlRobots = (contentType: string | null | undefined, text: string): boolean => {\n  const head = (text.charCodeAt(0) === 0xfeff ? text.slice(1) : text).slice(0, 4096);\n  if (/^\\s*<(!doctype|html|head|body)[\\s>]/i.test(head)) return true;\n  return (\n    /html/i.test(contentType ?? \"\") &&\n    /<(html|head|body|script|meta|title|div|form)[\\s>/]/i.test(head)\n  );\n};\n\n/**\n * This function reads the answer of a robots.txt fetch: the status, the content type and the body.\n * A 2xx with a web page in place of the file is unreadable. Thus no path is permitted (fail\n * closed), as for a body that does not decode. The other statuses follow `robotsFromStatus`.\n */\nexport const robotsFromResponse = (\n  status: number,\n  contentType: string | null | undefined,\n  body: Uint8Array | string,\n): Robots => {\n  if (status < 200 || status >= 300) return robotsFromStatus(status, undefined);\n  const text = typeof body === \"string\" ? body : new TextDecoder(\"utf-8\").decode(body);\n  if (isHtmlRobots(contentType, text)) return unreachableRobots(HTML_ROBOTS);\n  return parseRobots(text);\n};\n\n/** This function gives the meaning of the status of a robots.txt fetch (RFC 9309 §2.3.1). */\nexport const robotsFromStatus = (status: number, body: string | undefined): Robots => {\n  if (status >= 200 && status < 300) return parseRobots(body ?? \"\");\n  if (status === 429) return unreachableRobots(\"status 429, rate-limited\");\n  if (status >= 400 && status < 500) return ALLOW_ALL;\n  return unreachableRobots(status === 0 ? \"no response\" : `status ${String(status)}`);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAuBA,MAAa,mBAA2B;AAiDxC,MAAa,YAAoB;CAAE,QAAQ,CAAC;CAAG,UAAU,CAAC;CAAG,SAAS,CAAC;AAAE;AAEzE,MAAa,eAAuB;CAClC,QAAQ,CACN;EAAE,QAAQ,CAAC,GAAG;EAAG,OAAO,CAAC;GAAE,OAAO;GAAO,SAAS;EAAI,CAAC;EAAG,mBAAmB,KAAA;CAAU,CACzF;CACA,UAAU,CAAC;CACX,SAAS,CAAC;AACZ;AAEA,MAAM,aAAa;AACnB,MAAM,8BAAc,IAAI,IAAI,cAAc;;;;;AAK1C,MAAM,gBAAgB,SAA0B;CAC9C,MAAM,OAAO,KAAK,WAAW,CAAC;CAC9B,OAAO,QAAQ,MAAQ,SAAS,OAAQ,YAAY,IAAI,IAAI;AAC9D;AACA,MAAM,MAAM;AAEZ,MAAM,YAAY,SAAyB;CACzC,MAAM,OAAO,KAAK,YAAY,CAAC;CAC/B,IAAI,QAAQ,KAAM,OAAO,IAAI,KAAK,SAAS,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,GAAG,GAAG;CAC5E,IAAI;EACF,OAAO,mBAAmB,IAAI;CAChC,QAAQ;EAEN,OAAO;CACT;AACF;;;;;;;AAQA,MAAa,iBAAiB,UAA0B;CACtD,IAAI,MAAM;CACV,MAAM,QAAQ,MAAM,KAAK,KAAK;CAC9B,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS,GAAG;EACpD,MAAM,OAAO,MAAM;EACnB,IAAI,SAAS,OAAO,IAAI,KAAK,MAAM,QAAQ,MAAM,EAAE,KAAK,IAAI,KAAK,MAAM,QAAQ,MAAM,EAAE,GAAG;GACxF,MAAM,MAAM,GAAG,MAAM,QAAQ,KAAM,MAAM,QAAQ;GACjD,MAAM,UAAU,OAAO,aAAa,OAAO,SAAS,KAAK,EAAE,CAAC;GAC5D,OAAO,WAAW,KAAK,OAAO,IAAI,UAAU,IAAI,IAAI,YAAY;GAChE,SAAS;GACT;EACF;EACA,OAAO,KAAK,WAAW,CAAC,IAAI,OAAQ,aAAa,IAAI,IAAI,SAAS,IAAI,IAAI;CAC5E;CACA,OAAO;AACT;;AAGA,MAAa,kBAAkB,UAA2B,gBAAgB,KAAK,KAAK;;AAGpF,MAAM,WAAW,WACd,eAAe,KAAK,MAAM,KAAK,CAAC,CAAC,GAAG,MAAM,GAAA,CAAI,YAAY;;AAG7D,MAAM,gCAAqC,IAAI,IAAI;CACjD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;;AAQD,MAAa,gBAAgB,UAA0B;CACrD,MAAM,OAAO,MAAM,KAAK;CACxB,MAAM,QAAQ,QAAQ,IAAI;CAC1B,IAAI,UAAU,WAAW,OAAO;CAChC,MAAM,aAAa,+BAA+B,KAAK,IAAI,CAAC,GAAG;CAC/D,IAAI,eAAe,KAAA,GAAW,OAAO,WAAW,YAAY;CAO5D,OAHiB,CAAC,GAAG,KAAK,SAAS,8CAA8C,CAAC,CAAC,CAAC,SACjF,UAAW,MAAM,OAAO,KAAA,KAAa,MAAM,OAAO,KAAA,IAAY,CAAC,MAAM,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAE/E,CAAC,CAAC,MAAM,YAAY,CAAC,cAAc,IAAI,OAAO,CAAC,KAAK;AACpE;AAEA,MAAM,gBAAgB,MAAc,aAA6B;CAC/D,MAAM,QAAQ,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI;CAC3C,IAAI,MAAM,UAAU,UAAU,OAAO;CAErC,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,MAAM,MAAM,GAAG,QAAQ,CAAC;CAC9D,MAAM,YAAY,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG,KAAK,YAAY,IAAI,CAAC;CACzE,OAAO,cAAc,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS;AACxD;;AAGA,MAAM,YAAY,KAAa,OAAe,SAA+B;CAC3E,MAAM,OAAO,mBAAmB,KAAK,KAAK;CAC1C,OAAO,SAAS,OACZ;EAAE;EAAK;EAAO,MAAM,KAAA;EAAW;CAAK,IACpC;EAAE;EAAK,OAAO,KAAK,EAAE,CAAE,KAAK;EAAG,MAAM,cAAc,KAAK,EAAG;EAAG;CAAK;AACzE;AAEA,MAAa,eAAe,UAA0B;CACpD,MAAM,SAMD,CAAC;CACN,MAAM,WAAqB,CAAC;CAC5B,MAAM,UAAoB,CAAC;CAC3B,MAAM,cAA8B,CAAC;CACrC,IAAI;CACJ,IAAI,mBAAmB;CACvB,KAAK,MAAM,CAAC,OAAO,QAAQ,aAAa,OAAO,gBAAgB,CAAC,CAC7D,MAAM,YAAY,CAAC,CACnB,QAAQ,GAAG;EACZ,MAAM,OAAO,IAAI,QAAQ,QAAQ,EAAE,CAAC,CAAC,KAAK;EAC1C,MAAM,QAAQ,KAAK,QAAQ,GAAG;EAC9B,IAAI,SAAS,GAAG;EAChB,MAAM,MAAM,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,YAAY;EACpD,MAAM,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC,CAAC,KAAK;EACzC,IAAI,QAAQ,cAAc;GACxB,IAAI,YAAY,KAAA,KAAa,CAAC,kBAAkB;IAC9C,UAAU;KACR,QAAQ,CAAC;KACT,MAAM,QAAQ;KACd,OAAO,CAAC;KACR,mBAAmB,KAAA;KACnB,SAAS,CAAC;IACZ;IACA,OAAO,KAAK,OAAO;GACrB;GACA,QAAQ,OAAO,KAAK,MAAM,YAAY,CAAC;GACvC,mBAAmB;GACnB;EACF;EAEA,IAAI,QAAQ,WAAW;GACrB,IAAI,MAAM,SAAS,GAAG,SAAS,KAAK,KAAK;GACzC;EACF;EAEA,IAAI,QAAQ,oBAAoB,QAAQ,mBAAmB,QAAQ,WAAW;GAC5E,QAAQ,KAAK,GAAG,IAAI,IAAI,OAAO;GAC/B,CAAC,SAAS,WAAW,YAAA,CAAa,KAAK,SAAS,KAAK,OAAO,QAAQ,CAAC,CAAC;GACtE;EACF;EACA,IAAI,YAAY,KAAA,GAAW;EAC3B,IAAI,QAAQ,WAAW,QAAQ,YAAY;GAEzC,mBAAmB;GACnB,IAAI,MAAM,SAAS,GACjB,QAAQ,MAAM,KAAK;IACjB,OAAO,QAAQ;IACf,SAAS,cAAc,KAAK;IAC5B,MAAM,QAAQ;GAChB,CAAC;EACL,OAAO,IAAI,QAAQ,eAAe;GAChC,MAAM,UAAU,OAAO,KAAK;GAC5B,IAAI,OAAO,SAAS,OAAO,KAAK,WAAW,GAAG,QAAQ,oBAAoB;EAC5E;CACF;CACA,OAAO;EAAE;EAAQ;EAAU;EAAS;CAAY;AAClD;;;;;;;AAQA,MAAa,kBAAkB,SAAiB,SAA0B;CACxE,MAAM,WAAW,QAAQ,SAAS,GAAG;CACrC,MAAM,SAAS,WAAW,QAAQ,MAAM,GAAG,EAAE,IAAI,QAAA,CAAS,MAAM,GAAG;CACnE,MAAM,QAAQ,MAAM;CACpB,IAAI,CAAC,KAAK,WAAW,KAAK,GAAG,OAAO;CACpC,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,KAAK,WAAW,MAAM;CAClE,IAAI,KAAK,MAAM;CACf,MAAM,OAAO,MAAM,SAAS;CAC5B,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,SAAS,GAAG;EAC5C,MAAM,OAAO,MAAM;EAEnB,IAAI,KAAK,WAAW,GAAG;EACvB,MAAM,QAAQ,KAAK,QAAQ,MAAM,EAAE;EACnC,IAAI,UAAU,IAAI,OAAO;EACzB,KAAK,QAAQ,KAAK;CACpB;CACA,MAAM,OAAO,MAAM;CACnB,IAAI,CAAC,UAAU,OAAO,KAAK,WAAW,KAAK,KAAK,QAAQ,MAAM,EAAE,MAAM;CACtE,OAAO,KAAK,SAAS,KAAK,UAAU,MAAM,KAAK,SAAS,IAAI;AAC9D;AAUA,MAAM,aAAa,QAAgB,UAA0B;CAC3D,MAAM,SAAS,UAAU,MAAM,MAAM,aAAa,KAAK;CACvD,MAAM,QAAQ,OAAO,OAAO,QAAQ,UAClC,MAAM,OAAO,MAAM,UAAU,UAAU,OAAO,QAAQ,KAAK,MAAM,MAAM,CACzE;CACA,MAAM,SACJ,MAAM,SAAS,IAAI,QAAQ,OAAO,OAAO,QAAQ,UAAU,MAAM,OAAO,SAAS,GAAG,CAAC;CACvF,MAAM,SAAS,OAAO,SAAS,UAC7B,MAAM,sBAAsB,KAAA,IAAY,CAAC,IAAI,CAAC,MAAM,iBAAiB,CACvE;CACA,OAAO;EACL,OAAO,OAAO,SAAS,UAAU,MAAM,KAAK;EAC5C,QAAQ;EACR,mBAAmB,OAAO,SAAS,IAAI,KAAK,IAAI,GAAG,MAAM,IAAI,KAAA;EAC7D,UAAU,MAAM,SAAS;CAC3B;AACF;;AAGA,MAAM,gBAAgB,OAA8B,SAAyC;CAC3F,IAAI;CACJ,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,GAAG;EACzC,IACE,SAAS,KAAA,KACT,KAAK,QAAQ,SAAS,KAAK,QAAQ,UAClC,KAAK,QAAQ,WAAW,KAAK,QAAQ,UAAU,KAAK,SAAS,CAAC,KAAK,OAEpE,OAAO;CAEX;CACA,OAAO;AACT;;;;;;;;AAoBA,MAAa,UACX,QACA,QACA,iBACmB;CACnB,MAAM,OAAO,cAAc,aAAa,WAAW,IAAI,MAAM,YAAY;CACzE,MAAM,UAAU,UAAU,QAAQ,OAAO,MAAM,GAAG;CAElD,MAAM,SAAS,CAAC,QAAQ,iBAAiB;CACzC,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,IAAI,MAAM,UAAU,OAAO,KAAK,MAAM,iBAAiB;CACzD;CACA,MAAM,QAAQ,OAAO,QAAQ,UAA2B,UAAU,KAAA,CAAS;CAC3E,MAAM,oBAAoB,MAAM,SAAS,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,KAAA;CAClE,IAAI,SAAS,eACX,OAAO;EAAE,SAAS;EAAM,IAAI,KAAA;EAAW;EAAmB,MAAM,KAAA;EAAW,QAAQ,CAAC;CAAE;CAExF,MAAM,OAAO,aAAa,QAAQ,OAAO,IAAI;CAC7C,IAAI,SAAS,KAAA,KAAa,CAAC,KAAK,OAC9B,OAAO;EAAE,SAAS;EAAO,IAAI,OAAO;EAAI;EAAmB;EAAM,QAAQ,QAAQ;CAAO;CAE1F,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,MAAM,UAAU,MAAM,WAAW,aAAa,MAAM,OAAO,IAAI,IAAI,KAAA;EACnE,IAAI,YAAY,KAAA,KAAa,CAAC,QAAQ,OACpC,OAAO;GAAE,SAAS;GAAO,IAAI;GAAO;GAAmB,MAAM;GAAS,QAAQ,MAAM;EAAO;CAE/F;CACA,OAAO;EAAE,SAAS;EAAM,IAAI,KAAA;EAAW;EAAmB;EAAM,QAAQ,QAAQ;CAAO;AACzF;;;;;;;;;;;AAYA,MAAa,cACX,QACA,QACA,iBACa;CACb,MAAM,OAAO,cAAc,aAAa,WAAW,IAAI,MAAM,YAAY;CACzE,MAAM,SAAS,IAAI,IAAI,UAAU,QAAQ,OAAO,MAAM,GAAG,CAAC,CAAC,MAAM;CACjE,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,IAAI,MAAM,UAAU,KAAK,MAAM,SAAS,MAAM,QAAQ,OAAO,IAAI,KAAK;CACxE;CACA,MAAM,SAAyB,CAAC;CAChC,MAAM,QAAQ,UAAyC;EAErD,MAAM,uBAAO,IAAI,IAAuD;EACxE,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,KAAK,SAAS,KAAA,KAAa,CAAC,eAAe,KAAK,MAAM,IAAI,GAAG;GACjE,MAAM,SAAS,KAAK,MAAM,UAAU;GACpC,MAAM,SAAS,KAAK,IAAI,KAAK,GAAG;GAChC,IAAI,WAAW,KAAA,KAAa,SAAS,OAAO,QAC1C,KAAK,IAAI,KAAK,KAAK;IAAE;IAAQ,OAAO,CAAC,IAAI;GAAE,CAAC;QACzC,IAAI,WAAW,OAAO,QAAQ,OAAO,MAAM,KAAK,IAAI;EAC3D;EACA,KAAK,MAAM,EAAE,OAAO,UAAU,KAAK,OAAO,GAAG,OAAO,KAAK,GAAG,IAAI;CAClE;CACA,KAAK,OAAO,eAAe,CAAC,CAAC;CAC7B,KAAK,MAAM,SAAS,QAAQ,KAAK,MAAM,WAAW,CAAC,CAAC;CACpD,OAAO,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,SAAS,GAAG,KAAK,IAAI,IAAI,KAAK,OAAO;AAC1F;;AAGA,MAAa,qBAAqB,YAA4B;CAC5D,GAAG;CACH,aAAa;AACf;;AAGA,MAAa,cACX;;;;;;;AAQF,MAAa,gBAAgB,aAAwC,SAA0B;CAC7F,MAAM,QAAQ,KAAK,WAAW,CAAC,MAAM,QAAS,KAAK,MAAM,CAAC,IAAI,KAAA,CAAM,MAAM,GAAG,IAAI;CACjF,IAAI,uCAAuC,KAAK,IAAI,GAAG,OAAO;CAC9D,OACE,QAAQ,KAAK,eAAe,EAAE,KAC9B,sDAAsD,KAAK,IAAI;AAEnE;;;;;;AAOA,MAAa,sBACX,QACA,aACA,SACW;CACX,IAAI,SAAS,OAAO,UAAU,KAAK,OAAO,iBAAiB,QAAQ,KAAA,CAAS;CAC5E,MAAM,OAAO,OAAO,SAAS,WAAW,OAAO,IAAI,YAAY,OAAO,CAAC,CAAC,OAAO,IAAI;CACnF,IAAI,aAAa,aAAa,IAAI,GAAG,OAAO,kBAAkB,WAAW;CACzE,OAAO,YAAY,IAAI;AACzB;;AAGA,MAAa,oBAAoB,QAAgB,SAAqC;CACpF,IAAI,UAAU,OAAO,SAAS,KAAK,OAAO,YAAY,QAAQ,EAAE;CAChE,IAAI,WAAW,KAAK,OAAO,kBAAkB,0BAA0B;CACvE,IAAI,UAAU,OAAO,SAAS,KAAK,OAAO;CAC1C,OAAO,kBAAkB,WAAW,IAAI,gBAAgB,UAAU,OAAO,MAAM,GAAG;AACpF"}

package/build/{rolldown-runtime-D7D4PA-g.mjs → rolldown-runtime-8H4AJuhK.mjs}

@@ -1,3 +1,4 @@
+import "node:module";
 //#region \0rolldown/runtime.js
 var __defProp = Object.defineProperty;
 var __exportAll = (all, no_symbols) => {

package/build/{runtime-D_r2SYge.mjs → runtime-C_2BQibH.mjs}

@@ -1,11 +1,11 @@
-import { n as RenderRefused } from "./render-C1CUraWF.mjs";
-import { n as DEFAULT_TOKEN } from "./version-BPem-XZd.mjs";
+import { n as RenderRefused } from "./render-CKE2LILH.mjs";
+import { n as DEFAULT_TOKEN } from "./version-Djva7JkN.mjs";
 import { classify, isClientError, parseHttpDate } from "./classify.mjs";
 import { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, checkPolicy, equalJitterMs, policyFor, realClock, restAfter, validatorsToSend } from "./politeness.mjs";
 import { decide, isProductToken, robotsFromResponse, robotsFromStatus, signalsFor, unreachableRobots } from "./robots.mjs";
 import { NO_SIGNALS, metaSignals, readSignals } from "./signals.mjs";
 import { n as canonicalUrl, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
-import { a as signerOf } from "./webbotauth-CjJdTAfE.mjs";
+import { a as signerOf } from "./webbotauth-Dq_dA_u3.mjs";
 import { o as decodeBody, r as typesOf, t as extractPage } from "./html-DkQUAE_d.mjs";
 import { createHash } from "node:crypto";
 import { Transform } from "node:stream";
@@ -15,7 +15,11 @@ import { lookup } from "node:dns";
 import { isIP } from "node:net";
 import { getDocumentProxy, getMeta } from "unpdf";
 //#region src/egress.ts
-/** The connection hook gives this error when the policy refuses each address of a host. */
+/**
+* The connection hook gives this error when the policy refuses each address of a host.
+*
+* @beta
+*/
 var EgressRefused = class extends Error {
 	address;
 	kind;
@@ -335,6 +339,8 @@ const screen = (policy, host, addresses) => {
 * This function gives a `lookup` hook for the connection options of an undici `Agent`. The hook
 * resolves the name, removes each address that the policy refuses, and gives the rest to the
 * socket. When no address stays, the connection fails with `EgressRefused`.
+*
+* @beta
 */
 const egressLookup = (policy, resolve = resolveAll) => (host, options, callback) => {
 	resolve(host, {
@@ -407,6 +413,8 @@ const sha256 = (text) => createHash("sha256").update(text, "utf8").digest("hex")
 * This function makes a capture record. The hash covers the source, the kind, the external id
 * and the payload. The id is the source and the first 12 digits of the hash. The times, `scope`
 * and `labels` are not part of the hash.
+*
+* @beta
 */
 const makeRecord = (input) => {
 	const content_hash = sha256(stableStringify({
@@ -430,20 +438,25 @@ const makeRecord = (input) => {
 };
 //#endregion
 //#region src/crawler.ts
-/** The note of a request starts with this text when its host was not free before its deadline, so the request did not start. */
+/**
+* The note of a request starts with this text when its host was not free before its deadline, so the request did not start.
+*
+* @beta
+*/
 const DEADLINE_NOTE = "deadline reached";
 /**
-* This function gives the key of the cache entry of a rendered page in the form of version 0.1.
-* The index keeps the variant in a column of its own, and it reads this form too.
+* This value is the version of the payload of the records that the crawler writes. A record with no `schema` is version 1.
+*
+* @beta
 */
-const renderedKey = (url) => `${url}#rendered`;
-/** This value is the version of the payload of the records that the crawler writes. A record with no `schema` is version 1. */
 const CAPTURE_SCHEMA = 2;
 /** The content type of a rendered page: the browser gives its text as UTF-8. */
 const RENDERED_TYPE = "text/html; charset=utf-8";
 /**
 * These query parameters hold secrets in many APIs. The crawler writes `REDACTED` in place of
 * their values in each capture, in addition to the names of `secretParams`.
+*
+* @beta
 */
 const DEFAULT_SECRET_PARAMS = [
 	"access_token",
@@ -606,7 +619,7 @@ var Slots = class {
 	}
 };
 /** This function tells if an error of a renderer says that it has no browser: no `playwright-core`, or no browser on the disk. */
-const missingBrowser = (message) => message.startsWith("rendering needs the package playwright-core and a browser: npm install playwright-core, then npx playwright-core install chromium-headless-shell. For the MCP server with npx, start it with npx -y -p polite-crawler -p playwright-core polite-crawler mcp") || /Executable doesn't exist|Cannot find (?:module|package) 'playwright-core'/.test(message);
+const missingBrowser = (message) => message.startsWith("rendering needs the package playwright-core and a browser: npm install playwright-core, then npx playwright-core install chromium-headless-shell. For the MCP server with npx, start it with npx -y -p polite-crawler-mcp -p playwright-core polite-crawler-mcp") || /Executable doesn't exist|Cannot find (?:module|package) 'playwright-core'/.test(message);
 /** This text tells why the crawler did not fetch a page: a rule of the site, or no robots.txt to read. */
 const robotsRefusal = (robots, by) => robots.unreachable === void 0 ? `robots.txt disallows it for ${by}` : `robots.txt could not be read (${robots.unreachable}), so nothing on the site is fetched`;
 const messageOf$1 = (error) => error instanceof Error ? error.message.split(/\r?\n/)[0] ?? "" : String(error);
@@ -912,6 +925,7 @@ const raceAbort = (work, signal) => {
 		});
 	});
 };
+/** @beta */
 var Crawler = class {
 	options;
 	clock;
@@ -2600,6 +2614,8 @@ const atBoundary = (text, position, forward) => {
 * This function finds a quote in a text, in the `quote` mode of `foldText`. It ignores case, runs
 * of spaces, typographic punctuation, soft hyphens, zero-width characters and the differences of
 * NFKC, NFC and NFD. The positions are in the original text.
+*
+* @beta
 */
 const locate = (text, quote) => {
 	const needle = fold(quote, "quote");
@@ -2749,7 +2765,11 @@ const TIME_BUDGET_MS = 2e4;
 * approximately one second.
 */
 const MAX_INFLATED_BYTES$1 = 8388608;
-/** This function finds a PDF from its first bytes or its content type. Some servers put bytes before `%PDF-`. */
+/**
+* This function finds a PDF from its first bytes or its content type. Some servers put bytes before `%PDF-`.
+*
+* @beta
+*/
 const isPdf = (body, contentType) => {
 	if (/\bpdf\b/i.test(contentType ?? "")) return true;
 	const end = Math.min(body.length - 4, 1024);
@@ -2834,6 +2854,7 @@ const cutText = (text, length) => {
 	if (end > 0 && end < text.length && code >= 56320 && code <= 57343) end -= 1;
 	return text.slice(0, end).trimEnd();
 };
+/** @beta */
 const readPdf = async (bytes, options = {}) => {
 	const maxPages = options.maxPages ?? MAX_PAGES;
 	const maxChars = options.maxChars ?? MAX_CHARS;
@@ -2906,7 +2927,11 @@ const ARCHIVE_COPY_RECIPE = "archive-copy";
 const ARCHIVE_INDEX_RECIPE = "archive-index";
 /** The kind of the document of an archived copy. */
 const ARCHIVED_KIND = "archived";
-/** This function gives the document id of the archived copy of a page. */
+/**
+* This function gives the document id of the archived copy of a page.
+*
+* @beta
+*/
 const archivedDocId = (url) => `archive:${canonicalUrl(url)}`;
 /** The addresses of the archives. */
 const WAYBACK_AVAILABLE = "https://archive.org/wayback/available";
@@ -3108,6 +3133,8 @@ const asArchived = (docs, archived, aiInput) => docs.filter((doc) => typeof doc.
 *
 * It reads the body with the page reader, and gives a document of the kind `archived` with
 * `archived` in its metadata. The usage signals of the HTTP headers in a WARC record apply too.
+*
+* @beta
 */
 const archiveExtractor = async (body, capture) => {
 	let url;
@@ -3405,7 +3432,11 @@ const archivedPage = async (runtime, url, live, options) => {
 };
 //#endregion
 //#region src/runtime.ts
-/** This function gives the document id of a web page. Each page has one id. */
+/**
+* This function gives the document id of a web page. Each page has one id.
+*
+* This export is internal: the root entry point does not have it since 0.2.0.
+*/
 const pageDocId = (url) => `url:${canonicalUrl(url)}`;
 /** The JSON-LD that a document keeps, in characters of JSON. */
 const MAX_JSONLD_CHARS = 3e4;
@@ -3468,6 +3499,8 @@ const keepJsonLd = (records) => {
 * encoding that the page declares, with `decodeBody`. It does not throw: a page that it cannot read
 * gives a document that names the error. When the reader cut the text, the document keeps the whole
 * text in `full_text`, so `verify` finds a quote past the cut.
+*
+* This export is internal: the root entry point does not have it since 0.2.0.
 */
 const pageExtractor = (body, capture) => {
 	if (isPdf(body, capture.contentType)) return pdfDocs(body, capture.url);
@@ -3564,6 +3597,7 @@ const pdfDocs = async (body, url) => {
 		}
 	}];
 };
+/** @beta */
 var Extractors = class {
 	byRecipe = /* @__PURE__ */ new Map([["page", pageExtractor], ["archive-copy", archiveExtractor]]);
 	replaceable = /* @__PURE__ */ new Set(["archive-copy"]);
@@ -3598,14 +3632,24 @@ const STRUCTURAL_META = [
 	"archived",
 	"archived_kind"
 ];
-/** The note of a document that is held back. */
+/**
+* The note of a document that is held back.
+*
+* @beta
+*/
 const HELD_BACK = "This page asked not to be used as AI input, so its text is held back. Open the URL to read it, or read again with userProvided when a person gave this URL directly (the crawler logs that).";
 /**
 * This function tells if a document is held back: the page asked not to be AI input, and no person
 * gave its URL directly. The only way to read a held-back document is `userProvided`.
+*
+* @beta
 */
 const isHeldBack = (doc, userProvided = false) => doc.ai_input === "no" && !userProvided;
-/** This function gives the metadata that a held-back document shows: only the structural fields, which hold no text of the page. */
+/**
+* This function gives the metadata that a held-back document shows: only the structural fields, which hold no text of the page.
+*
+* @beta
+*/
 const heldBackMeta = (meta) => Object.fromEntries(Object.entries(meta).filter(([key]) => STRUCTURAL_META.includes(key)));
 const messageOf = (error) => error instanceof Error ? error.message.split(/\r?\n/)[0] ?? "" : String(error);
 /** This function tells which bot check or firewall sent its page in place of the content, or gives undefined. */
@@ -3663,6 +3707,8 @@ const indexCapture = async (runtime, fetched, recipe, rendered) => {
 * from the cache gives the documents that the index has for that capture. Thus a repeated
 * question sends no request and reads nothing again. An extractor that throws gives no document and an
 * `error`, and the fetch still succeeds.
+*
+* @beta
 */
 const fetchAndIndex = async (runtime, request) => {
 	const fetched = await runtime.crawler.fetch(request);
@@ -3675,7 +3721,11 @@ const fetchAndIndex = async (runtime, request) => {
 		...await indexCapture(runtime, fetched, request.recipe, false)
 	};
 };
-/** This function renders a page in the headless browser and adds what the page shows to the index. */
+/**
+* This function renders a page in the headless browser and adds what the page shows to the index.
+*
+* This export is internal: the root entry point does not have it since 0.2.0.
+*/
 const renderAndIndex = async (runtime, request) => {
 	const fetched = await runtime.crawler.render(request);
 	if (fetched.outcome !== "ok") return {
@@ -3693,6 +3743,8 @@ const HTML_TYPE = /html|xml/i;
 * the page shows almost nothing without JavaScript, or when the caller asks for it. It renders
 * only an HTML page: not a PDF, a different file or a bot check. A page that sends the reader on
 * with a `<meta http-equiv="refresh">` gets one more plain fetch, of its target.
+*
+* @beta
 */
 const fetchPage = async (runtime, url, options) => {
 	const plainRequest = (target) => ({
@@ -3797,6 +3849,8 @@ const isoTime = (name, value) => {
 * This function gives the version of a document that was current at a time (ISO 8601). It uses
 * `ReadingIndex.versionAt` when the index has it. Else it takes the newest version at or before
 * the time.
+*
+* @beta
 */
 const versionAtTime = (runtime, id, at) => {
 	const index = runtime.index;
@@ -3830,6 +3884,8 @@ const onBoundary = (text, position) => {
 * This function writes an `override` record to the capture log. A person gave the URL of a
 * held-back document directly, and the hold stops for one action (`read` or `verify`). The record
 * takes the scope and the labels of `marks`, or else those of the document, as its captures do.
+*
+* @beta
 */
 const recordOverride = async (runtime, doc, action, marks = {}) => {
 	const at = runtime.now();
@@ -3855,6 +3911,8 @@ const recordOverride = async (runtime, doc, action, marks = {}) => {
 * This function reads a range of characters of one stored document. It applies the rule for a
 * held-back document: no title, no text, and only the structural metadata, unless `userProvided`.
 * The positions do not cut a character in two.
+*
+* @beta
 */
 const readDocument = async (runtime, id, options = {}) => {
 	const from = wholeNumber("from", options.from, 0);
@@ -4066,6 +4124,8 @@ const examineQuote = (runtime, id, quote, options) => {
 * unless `userProvided`. The same rule applies to each version. A version that asked not to be AI
 * input stays held back after a later capture drops the signal. The crawler logs an override when
 * `userProvided` opens a held-back version.
+*
+* @beta
 */
 const verifyQuote = async (runtime, id, quote, options = {}) => {
 	const { result, override } = examineQuote(runtime, id, quote, options);
@@ -4076,6 +4136,8 @@ const verifyQuote = async (runtime, id, quote, options = {}) => {
 * This function searches the stored documents. The syntax of the query is the syntax of
 * `parseQuery`. A held-back document is not in the results. The option `rank` chooses the
 * ranking (`RankMode`), and the default is `exact`.
+*
+* @beta
 */
 const findDocuments = async (runtime, q, options = {}) => {
 	await Promise.resolve();
@@ -4234,6 +4296,8 @@ const replayRecord = async (runtime, target, envelope, tally) => {
 * - It counts and skips a damaged line, a missing blob and an extractor that throws.
 * - The documents of a recipe that has no extractor in this runtime stay, from the old index.
 * - A capture that a different writer adds during the rebuild goes into the new index.
+*
+* @beta
 */
 const reindex = async (runtime, options = {}) => {
 	const since = runtime.now();
@@ -4288,6 +4352,8 @@ const missed = (runtime, envelope) => {
 *   function reads the log from a short time before the newest cache entry.
 *
 * It gives the counts, or undefined when the index missed nothing.
+*
+* @beta
 */
 const catchUpIndex = async (runtime) => {
 	if (runtime.index.stale) return {
@@ -4319,6 +4385,6 @@ const catchUpIndex = async (runtime) => {
 	};
 };
 //#endregion
-export { snippetOf as A, egressLookup as B, fallbackReason as C, foldText as D, fold as E, renderedKey as F, makeRecord as I, sha256 as L, Crawler as M, DEADLINE_NOTE as N, locate as O, DEFAULT_SECRET_PARAMS as P, stableStringify as R, archivedOf as S, readPdf as T, versionAtTime as _, fetchPage as a, archivedDocId as b, isHeldBack as c, pageExtractor as d, readDocument as f, verifyQuote as g, renderAndIndex as h, fetchAndIndex as i, CAPTURE_SCHEMA as j, parseQuery as k, isoTime as l, reindex as m, HELD_BACK as n, findDocuments as o, recordOverride as p, catchUpIndex as r, heldBackMeta as s, Extractors as t, pageDocId as u, archiveExtractor as v, isPdf as w, archivedNote as x, archivedCopy as y, EgressRefused as z };
+export { DEADLINE_NOTE as A, fold as C, snippetOf as D, parseQuery as E, EgressRefused as F, egressLookup as I, makeRecord as M, sha256 as N, CAPTURE_SCHEMA as O, stableStringify as P, readPdf as S, locate as T, archivedDocId as _, fetchPage as a, fallbackReason as b, isHeldBack as c, recordOverride as d, reindex as f, archivedCopy as g, archiveExtractor as h, fetchAndIndex as i, DEFAULT_SECRET_PARAMS as j, Crawler as k, isoTime as l, versionAtTime as m, HELD_BACK as n, findDocuments as o, verifyQuote as p, catchUpIndex as r, heldBackMeta as s, Extractors as t, readDocument as u, archivedNote as v, foldText as w, isPdf as x, archivedOf as y };
 
-//# sourceMappingURL=runtime-D_r2SYge.mjs.map
+//# sourceMappingURL=runtime-C_2BQibH.mjs.map