polite-crawler 0.2.2 → 0.2.4

package/build/robots.mjs

@@ -122,18 +122,22 @@ const truncateUtf8 = (text, maxBytes) => {
 	const lastBreak = Math.max(head.lastIndexOf("\n"), head.lastIndexOf("\r"));
 	return lastBreak === -1 ? "" : head.slice(0, lastBreak);
 };
-/** This function reads a usage line: an optional path, then the value (`/images/ train-ai=n`). */
+/**
+* This function reads a usage line: an optional path, then the value (`/images/ train-ai=n`). The
+* path starts with `/` and stops at the first space. The function uses no backtracking, so its
+* time is linear.
+*/
 const signalOf = (key, value, line) => {
-	const path = /^(\/\S*)\s+(.*)$/.exec(value);
-	return path === null ? {
+	const end = value.startsWith("/") ? value.search(/\s/) : -1;
+	return end === -1 ? {
 		key,
 		value,
 		path: void 0,
 		line
 	} : {
 		key,
-		value: path[2].trim(),
-		path: normalizePath(path[1]),
+		value: value.slice(end).trim(),
+		path: normalizePath(value.slice(0, end)),
 		line
 	};
 };
@@ -145,7 +149,8 @@ const parseRobots = (input) => {
 	let current;
 	let collectingAgents = false;
 	for (const [index, raw] of truncateUtf8(input, ROBOTS_MAX_BYTES).split(/\r\n|\r|\n/).entries()) {
-		const line = raw.replace(/#.*$/, "").trim();
+		const hash = raw.indexOf("#");
+		const line = (hash === -1 ? raw : raw.slice(0, hash)).trim();
 		const colon = line.indexOf(":");
 		if (colon <= 0) continue;
 		const key = line.slice(0, colon).trim().toLowerCase();

package/build/robots.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"robots.mjs","names":[],"sources":["../src/robots.ts"],"sourcesContent":["/**\n * This module reads robots.txt as RFC 9309 (September 2022) specifies it. It is for a crawler that\n * tells who it is and obeys what a site says. The important rules, each with a test:\n *\n * - A group is one or more `user-agent` lines and the rules after them. A `user-agent` line after\n *   an `allow` or `disallow` line starts a new group, as the parser of Google does. A different\n *   line (`crawl-delay`) between two `user-agent` lines does not split the group. A rule before\n *   the first `user-agent` line is in no group.\n * - The product token matches with no attention to case. The matching groups merge. The group\n *   `*` applies only when no group names the crawler.\n * - The longest matching rule wins, and `allow` wins a tie. A `*` matches any sequence, and a `$`\n *   at the end anchors the end. The matcher uses no regular expression, and its time is linear.\n * - `/robots.txt` is always permitted, and an empty `disallow` matches nothing.\n * - The parser reads at least the first 500 KiB.\n * - A usage line (`content-signal`, `content-usage`) belongs to its group, with an optional path\n *   before its value. A line before the first group applies to each crawler.\n *\n * The function `robotsFromStatus` gives the meaning of the status of the fetch. A 4xx is\n * \"unavailable\", so all paths are permitted. A 5xx or no response is \"unreachable\", so no path is\n * permitted. A 429 is unreachable too, as Google does it. A site that limits the rate of its robots\n * file does not ask for a crawl.\n */\n\nexport const ROBOTS_MAX_BYTES: number = 500 * 1024;\n\nexport interface RobotsRule {\n  readonly allow: boolean;\n  /** This value is the path pattern, normalized for comparison. */\n  readonly pattern: string;\n  /** This value is the number of the line in robots.txt, from 1. A rule that the code makes has none. */\n  readonly line?: number | undefined;\n}\n\n/** A usage line states a preference about the use of the content (`content-signal`, `content-usage`). */\nexport interface RobotsSignal {\n  /** This value is the name of the line, in lowercase. */\n  readonly key: string;\n  /** This text is the value of the line, after the path. */\n  readonly value: string;\n  /** This value is the normalized path pattern of the line. A line with no path applies to each path. */\n  readonly path: string | undefined;\n  /** This value is the number of the line in robots.txt, from 1. */\n  readonly line: number;\n}\n\nexport interface RobotsGroup {\n  /** These values are the user agents of the group, in lowercase. */\n  readonly agents: readonly string[];\n  /** This value is the number of the first `user-agent` line of the group, from 1. */\n  readonly line?: number | undefined;\n  readonly rules: readonly RobotsRule[];\n  /** This value is the nonstandard `crawl-delay` in seconds, when the group sets one. */\n  readonly crawlDelaySeconds: number | undefined;\n  /** These usage lines are in the group. They apply when the group applies. */\n  readonly signals?: readonly RobotsSignal[] | undefined;\n}\n\nexport interface Robots {\n  readonly groups: readonly RobotsGroup[];\n  readonly sitemaps: readonly string[];\n  /** These lines state usage preferences (`content-signal: …`, `content-usage: …`), as written, from each group. */\n  readonly signals: readonly string[];\n  /** These usage lines come before the first group, so they apply to each crawler. */\n  readonly fileSignals?: readonly RobotsSignal[] | undefined;\n  /**\n   * This text tells why the crawler could not read robots.txt: a dead domain, a broken certificate\n   * or a server error. Then no path is permitted, and the note of a skip gives this reason. It does\n   * not name a rule of the site.\n   */\n  readonly unreachable?: string | undefined;\n}\n\nexport const ALLOW_ALL: Robots = { groups: [], sitemaps: [], signals: [] };\n\nexport const DISALLOW_ALL: Robots = {\n  groups: [\n    { agents: [\"*\"], rules: [{ allow: false, pattern: \"/\" }], crawlDelaySeconds: undefined },\n  ],\n  sitemaps: [],\n  signals: [],\n};\n\nconst UNRESERVED = /^[A-Za-z0-9\\-._~]$/;\nconst URL_ENCODED = new Set(\"\\\"'<>\\\\^`{|}\");\n/**\n * The URL parser encodes these ASCII characters in a path or a query: the controls, a space, DEL,\n * and `\"'<>\\^`{|}`. A rule can contain them as written.\n */\nconst encodedAscii = (char: string): boolean => {\n  const code = char.charCodeAt(0);\n  return code <= 0x20 || code === 0x7f || URL_ENCODED.has(char);\n};\nconst HEX = /^[0-9A-Fa-f]$/;\n\nconst escapeOf = (char: string): string => {\n  const code = char.codePointAt(0)!;\n  if (code <= 0x7f) return `%${code.toString(16).toUpperCase().padStart(2, \"0\")}`;\n  try {\n    return encodeURIComponent(char);\n  } catch {\n    // a lone surrogate: the URL parser writes the replacement character\n    return \"%EF%BF%BD\";\n  }\n};\n\n/**\n * This function normalizes a path, or the pattern of a rule, for comparison, as §2.2.2 asks. It\n * encodes the characters outside US-ASCII, and the ASCII characters that the URL parser encodes\n * (a space, `\"`, `<`, `{`). It decodes an escape of an unreserved character, so `/%7Efoo` and\n * `/~foo` are equal. The other escapes stay, with uppercase hexadecimal digits.\n */\nexport const normalizePath = (value: string): string => {\n  let out = \"\";\n  const chars = Array.from(value);\n  for (let index = 0; index < chars.length; index += 1) {\n    const char = chars[index]!;\n    if (char === \"%\" && HEX.test(chars[index + 1] ?? \"\") && HEX.test(chars[index + 2] ?? \"\")) {\n      const hex = `${chars[index + 1]!}${chars[index + 2]!}`;\n      const decoded = String.fromCharCode(Number.parseInt(hex, 16));\n      out += UNRESERVED.test(decoded) ? decoded : `%${hex.toUpperCase()}`;\n      index += 2;\n      continue;\n    }\n    out += char.charCodeAt(0) > 0x7f || encodedAscii(char) ? escapeOf(char) : char;\n  }\n  return out;\n};\n\n/** This function tells if a text is a product token: letters, `_` and `-` only, and one or more (RFC 9309, section 2.2.1). */\nexport const isProductToken = (value: string): boolean => /^[A-Za-z_-]+$/.test(value);\n\n/** This function gives the first product token of a user agent: `Googlebot/2.1` gives `googlebot`. */\nconst tokenOf = (value: string): string =>\n  (/^[A-Za-z_-]+/.exec(value.trim())?.[0] ?? \"\").toLowerCase();\n\n/** These names are parts of the header of a browser, not the name of a crawler. */\nconst BROWSER_PARTS: ReadonlySet<string> = new Set([\n  \"mozilla\",\n  \"applewebkit\",\n  \"khtml\",\n  \"gecko\",\n  \"chrome\",\n  \"chromium\",\n  \"safari\",\n  \"version\",\n  \"mobile\",\n  \"firefox\",\n  \"edg\",\n  \"opr\",\n  \"like\",\n]);\n\n/**\n * This function gives the product token that robots.txt rules match, in lowercase. `MyBot/1.0`\n * gives `mybot`. The full header `Mozilla/5.0 (compatible; MyBot/1.0; +https://…)` also gives\n * `mybot`. The token is the name after `compatible;`, or else the first product that is not a part\n * of a browser.\n */\nexport const productToken = (value: string): string => {\n  const text = value.trim();\n  const first = tokenOf(text);\n  if (first !== \"mozilla\") return first;\n  const compatible = /compatible;\\s*([A-Za-z_-]+)/i.exec(text)?.[1];\n  if (compatible !== undefined) return compatible.toLowerCase();\n  // One pass in linear time: each match is a name that starts with a letter, with an optional `/` and\n  // version after it, or a run of other characters. A run of other characters takes no version, so\n  // `_/a/a` gives the product `a`, as the scan before 0.2.0 did.\n  const products = [...text.matchAll(/([A-Za-z][A-Za-z_-]*)(\\/[\\w.]+)?|[^A-Za-z]+/g)].flatMap(\n    (match) => (match[1] !== undefined && match[2] !== undefined ? [match[1].toLowerCase()] : []),\n  );\n  return products.find((product) => !BROWSER_PARTS.has(product)) ?? first;\n};\n\nconst truncateUtf8 = (text: string, maxBytes: number): string => {\n  const bytes = new TextEncoder().encode(text);\n  if (bytes.length <= maxBytes) return text;\n  // cut at the last line break inside the limit, so no rule is half-read\n  const head = new TextDecoder().decode(bytes.slice(0, maxBytes));\n  const lastBreak = Math.max(head.lastIndexOf(\"\\n\"), head.lastIndexOf(\"\\r\"));\n  return lastBreak === -1 ? \"\" : head.slice(0, lastBreak);\n};\n\n/** This function reads a usage line: an optional path, then the value (`/images/ train-ai=n`). */\nconst signalOf = (key: string, value: string, line: number): RobotsSignal => {\n  const path = /^(\\/\\S*)\\s+(.*)$/.exec(value);\n  return path === null\n    ? { key, value, path: undefined, line }\n    : { key, value: path[2]!.trim(), path: normalizePath(path[1]!), line };\n};\n\nexport const parseRobots = (input: string): Robots => {\n  const groups: Array<{\n    agents: string[];\n    line: number;\n    rules: RobotsRule[];\n    crawlDelaySeconds: number | undefined;\n    signals: RobotsSignal[];\n  }> = [];\n  const sitemaps: string[] = [];\n  const signals: string[] = [];\n  const fileSignals: RobotsSignal[] = [];\n  let current: (typeof groups)[number] | undefined;\n  let collectingAgents = false;\n  for (const [index, raw] of truncateUtf8(input, ROBOTS_MAX_BYTES)\n    .split(/\\r\\n|\\r|\\n/)\n    .entries()) {\n    const line = raw.replace(/#.*$/, \"\").trim();\n    const colon = line.indexOf(\":\");\n    if (colon <= 0) continue;\n    const key = line.slice(0, colon).trim().toLowerCase();\n    const value = line.slice(colon + 1).trim();\n    if (key === \"user-agent\") {\n      if (current === undefined || !collectingAgents) {\n        current = {\n          agents: [],\n          line: index + 1,\n          rules: [],\n          crawlDelaySeconds: undefined,\n          signals: [],\n        };\n        groups.push(current);\n      }\n      current.agents.push(value.toLowerCase());\n      collectingAgents = true;\n      continue;\n    }\n    // records that belong to no group\n    if (key === \"sitemap\") {\n      if (value.length > 0) sitemaps.push(value);\n      continue;\n    }\n    // an RSL `License` line names the license of the content: the crawler records it with the other usage lines\n    if (key === \"content-signal\" || key === \"content-usage\" || key === \"license\") {\n      signals.push(`${key}: ${value}`);\n      (current?.signals ?? fileSignals).push(signalOf(key, value, index + 1));\n      continue;\n    }\n    if (current === undefined) continue;\n    if (key === \"allow\" || key === \"disallow\") {\n      // only a rule ends the list of agents, so a crawl-delay between two agents keeps one group\n      collectingAgents = false;\n      if (value.length > 0)\n        current.rules.push({\n          allow: key === \"allow\",\n          pattern: normalizePath(value),\n          line: index + 1,\n        });\n    } else if (key === \"crawl-delay\") {\n      const seconds = Number(value);\n      if (Number.isFinite(seconds) && seconds >= 0) current.crawlDelaySeconds = seconds;\n    }\n  }\n  return { groups, sitemaps, signals, fileSignals };\n};\n\n/**\n * This function tells if a normalized path matches the pattern of a rule. A `*` matches any\n * sequence, and a `$` at the end anchors the end. The function finds the parts between the stars\n * in sequence, each at its first position. No pattern makes it slow: its time is linear in the\n * length of the path for each part, and it uses no regular expression.\n */\nexport const matchesPattern = (pattern: string, path: string): boolean => {\n  const anchored = pattern.endsWith(\"$\");\n  const parts = (anchored ? pattern.slice(0, -1) : pattern).split(\"*\");\n  const first = parts[0]!;\n  if (!path.startsWith(first)) return false;\n  if (parts.length === 1) return !anchored || path.length === first.length;\n  let at = first.length;\n  const last = parts.length - 1;\n  for (let index = 1; index < last; index += 1) {\n    const part = parts[index]!;\n    // two stars in sequence give an empty part, which matches at once\n    if (part.length === 0) continue;\n    const found = path.indexOf(part, at);\n    if (found === -1) return false;\n    at = found + part.length;\n  }\n  const tail = parts[last]!;\n  if (!anchored) return tail.length === 0 || path.indexOf(tail, at) !== -1;\n  return path.length - tail.length >= at && path.endsWith(tail);\n};\n\ninterface Chosen {\n  readonly rules: readonly RobotsRule[];\n  readonly groups: readonly RobotsGroup[];\n  readonly crawlDelaySeconds: number | undefined;\n  /** This value is true when a group names this token, and `*` did not apply. */\n  readonly explicit: boolean;\n}\n\nconst groupsFor = (robots: Robots, token: string): Chosen => {\n  const wanted = token === \"*\" ? \"*\" : productToken(token);\n  const named = robots.groups.filter((group) =>\n    group.agents.some((agent) => agent !== \"*\" && tokenOf(agent) === wanted),\n  );\n  const chosen =\n    named.length > 0 ? named : robots.groups.filter((group) => group.agents.includes(\"*\"));\n  const delays = chosen.flatMap((group) =>\n    group.crawlDelaySeconds === undefined ? [] : [group.crawlDelaySeconds],\n  );\n  return {\n    rules: chosen.flatMap((group) => group.rules),\n    groups: chosen,\n    crawlDelaySeconds: delays.length > 0 ? Math.max(...delays) : undefined,\n    explicit: named.length > 0,\n  };\n};\n\n/** This function gives the rule that decides for the path: the longest match, and `allow` in a tie. */\nconst decidingRule = (rules: readonly RobotsRule[], path: string): RobotsRule | undefined => {\n  let best: RobotsRule | undefined;\n  for (const rule of rules) {\n    if (!matchesPattern(rule.pattern, path)) continue;\n    if (\n      best === undefined ||\n      rule.pattern.length > best.pattern.length ||\n      (rule.pattern.length === best.pattern.length && rule.allow && !best.allow)\n    ) {\n      best = rule;\n    }\n  }\n  return best;\n};\n\nexport interface RobotsDecision {\n  readonly allowed: boolean;\n  /** This value is the product token whose rules refused the path. */\n  readonly by: string | undefined;\n  readonly crawlDelaySeconds: number | undefined;\n  /** This rule decided. When no rule matches the path, the path is permitted and this value is empty. */\n  readonly rule?: RobotsRule | undefined;\n  /** These groups applied to the decision: the groups that name the token, or the group `*`. */\n  readonly groups?: readonly RobotsGroup[] | undefined;\n}\n\n/**\n * This function tells if these tokens can fetch `pathAndQuery`. Each token becomes its product\n * token (`productToken`), so `MyBot/1.0` matches the group `mybot`. The first token is the token of\n * the crawler, and `*` applies when no group names it. A further token, for example `Claude-User`,\n * applies only when the site names it. A site that writes rules for an agent means them for the\n * fetches of that agent.\n */\nexport const decide = (\n  robots: Robots,\n  tokens: readonly string[],\n  pathAndQuery: string,\n): RobotsDecision => {\n  const path = normalizePath(pathAndQuery.length === 0 ? \"/\" : pathAndQuery);\n  const primary = groupsFor(robots, tokens[0] ?? \"*\");\n  // the crawl delay of a group that names a further agent applies to the fetches of that agent too\n  const delays = [primary.crawlDelaySeconds];\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    if (extra.explicit) delays.push(extra.crawlDelaySeconds);\n  }\n  const known = delays.filter((delay): delay is number => delay !== undefined);\n  const crawlDelaySeconds = known.length > 0 ? Math.max(...known) : undefined;\n  if (path === \"/robots.txt\") {\n    return { allowed: true, by: undefined, crawlDelaySeconds, rule: undefined, groups: [] };\n  }\n  const rule = decidingRule(primary.rules, path);\n  if (rule !== undefined && !rule.allow) {\n    return { allowed: false, by: tokens[0], crawlDelaySeconds, rule, groups: primary.groups };\n  }\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    const refusal = extra.explicit ? decidingRule(extra.rules, path) : undefined;\n    if (refusal !== undefined && !refusal.allow) {\n      return { allowed: false, by: token, crawlDelaySeconds, rule: refusal, groups: extra.groups };\n    }\n  }\n  return { allowed: true, by: undefined, crawlDelaySeconds, rule, groups: primary.groups };\n};\n\n/**\n * This function gives the usage lines that apply to these tokens and this path, as\n * `key: value` texts. The lines come from three places:\n *\n * - the lines before the first group\n * - the groups of the first token, or `*`\n * - the groups that name a further token\n *\n * In one group, the line with the longest matching path wins for each key.\n */\nexport const signalsFor = (\n  robots: Robots,\n  tokens: readonly string[],\n  pathAndQuery: string,\n): string[] => {\n  const path = normalizePath(pathAndQuery.length === 0 ? \"/\" : pathAndQuery);\n  const groups = new Set(groupsFor(robots, tokens[0] ?? \"*\").groups);\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    if (extra.explicit) for (const group of extra.groups) groups.add(group);\n  }\n  const chosen: RobotsSignal[] = [];\n  const pick = (lines: readonly RobotsSignal[]): void => {\n    // for each key: the length of the longest matching path, and the lines with that length\n    const best = new Map<string, { length: number; lines: RobotsSignal[] }>();\n    for (const line of lines) {\n      if (line.path !== undefined && !matchesPattern(line.path, path)) continue;\n      const length = line.path?.length ?? 0;\n      const before = best.get(line.key);\n      if (before === undefined || length > before.length)\n        best.set(line.key, { length, lines: [line] });\n      else if (length === before.length) before.lines.push(line);\n    }\n    for (const { lines: kept } of best.values()) chosen.push(...kept);\n  };\n  pick(robots.fileSignals ?? []);\n  for (const group of groups) pick(group.signals ?? []);\n  return chosen.sort((a, b) => a.line - b.line).map((line) => `${line.key}: ${line.value}`);\n};\n\n/** This function gives the robots of a site whose robots.txt the crawler cannot read. No path is permitted, and the reason stays. */\nexport const unreachableRobots = (reason: string): Robots => ({\n  ...DISALLOW_ALL,\n  unreachable: reason,\n});\n\n/** This text is the reason when a robots.txt that answered 200 is a web page, not a robots.txt file. */\nexport const HTML_ROBOTS =\n  \"robots.txt is a web page, not a robots.txt file (a login or challenge page)\";\n\n/**\n * This function tells if the body of a robots.txt is a web page. A server can send a challenge, a\n * login or an error page with a 200. Such a body starts with `<!doctype`, `<html`, `<head` or\n * `<body`. With the content type `text/html`, a tag of a page in the first 4 KB is enough. A plain\n * robots.txt that a server sends as `text/html` stays a file.\n */\nexport const isHtmlRobots = (contentType: string | null | undefined, text: string): boolean => {\n  const head = (text.charCodeAt(0) === 0xfeff ? text.slice(1) : text).slice(0, 4096);\n  if (/^\\s*<(!doctype|html|head|body)[\\s>]/i.test(head)) return true;\n  return (\n    /html/i.test(contentType ?? \"\") &&\n    /<(html|head|body|script|meta|title|div|form)[\\s>/]/i.test(head)\n  );\n};\n\n/**\n * This function reads the answer of a robots.txt fetch: the status, the content type and the body.\n * A 2xx with a web page in place of the file is unreadable. Thus no path is permitted (fail\n * closed), as for a body that does not decode. The other statuses follow `robotsFromStatus`.\n */\nexport const robotsFromResponse = (\n  status: number,\n  contentType: string | null | undefined,\n  body: Uint8Array | string,\n): Robots => {\n  if (status < 200 || status >= 300) return robotsFromStatus(status, undefined);\n  const text = typeof body === \"string\" ? body : new TextDecoder(\"utf-8\").decode(body);\n  if (isHtmlRobots(contentType, text)) return unreachableRobots(HTML_ROBOTS);\n  return parseRobots(text);\n};\n\n/** This function gives the meaning of the status of a robots.txt fetch (RFC 9309 §2.3.1). */\nexport const robotsFromStatus = (status: number, body: string | undefined): Robots => {\n  if (status >= 200 && status < 300) return parseRobots(body ?? \"\");\n  if (status === 429) return unreachableRobots(\"status 429, rate-limited\");\n  if (status >= 400 && status < 500) return ALLOW_ALL;\n  return unreachableRobots(status === 0 ? \"no response\" : `status ${String(status)}`);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAuBA,MAAa,mBAA2B;AAiDxC,MAAa,YAAoB;CAAE,QAAQ,CAAC;CAAG,UAAU,CAAC;CAAG,SAAS,CAAC;AAAE;AAEzE,MAAa,eAAuB;CAClC,QAAQ,CACN;EAAE,QAAQ,CAAC,GAAG;EAAG,OAAO,CAAC;GAAE,OAAO;GAAO,SAAS;EAAI,CAAC;EAAG,mBAAmB,KAAA;CAAU,CACzF;CACA,UAAU,CAAC;CACX,SAAS,CAAC;AACZ;AAEA,MAAM,aAAa;AACnB,MAAM,8BAAc,IAAI,IAAI,cAAc;;;;;AAK1C,MAAM,gBAAgB,SAA0B;CAC9C,MAAM,OAAO,KAAK,WAAW,CAAC;CAC9B,OAAO,QAAQ,MAAQ,SAAS,OAAQ,YAAY,IAAI,IAAI;AAC9D;AACA,MAAM,MAAM;AAEZ,MAAM,YAAY,SAAyB;CACzC,MAAM,OAAO,KAAK,YAAY,CAAC;CAC/B,IAAI,QAAQ,KAAM,OAAO,IAAI,KAAK,SAAS,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,GAAG,GAAG;CAC5E,IAAI;EACF,OAAO,mBAAmB,IAAI;CAChC,QAAQ;EAEN,OAAO;CACT;AACF;;;;;;;AAQA,MAAa,iBAAiB,UAA0B;CACtD,IAAI,MAAM;CACV,MAAM,QAAQ,MAAM,KAAK,KAAK;CAC9B,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS,GAAG;EACpD,MAAM,OAAO,MAAM;EACnB,IAAI,SAAS,OAAO,IAAI,KAAK,MAAM,QAAQ,MAAM,EAAE,KAAK,IAAI,KAAK,MAAM,QAAQ,MAAM,EAAE,GAAG;GACxF,MAAM,MAAM,GAAG,MAAM,QAAQ,KAAM,MAAM,QAAQ;GACjD,MAAM,UAAU,OAAO,aAAa,OAAO,SAAS,KAAK,EAAE,CAAC;GAC5D,OAAO,WAAW,KAAK,OAAO,IAAI,UAAU,IAAI,IAAI,YAAY;GAChE,SAAS;GACT;EACF;EACA,OAAO,KAAK,WAAW,CAAC,IAAI,OAAQ,aAAa,IAAI,IAAI,SAAS,IAAI,IAAI;CAC5E;CACA,OAAO;AACT;;AAGA,MAAa,kBAAkB,UAA2B,gBAAgB,KAAK,KAAK;;AAGpF,MAAM,WAAW,WACd,eAAe,KAAK,MAAM,KAAK,CAAC,CAAC,GAAG,MAAM,GAAA,CAAI,YAAY;;AAG7D,MAAM,gCAAqC,IAAI,IAAI;CACjD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;;AAQD,MAAa,gBAAgB,UAA0B;CACrD,MAAM,OAAO,MAAM,KAAK;CACxB,MAAM,QAAQ,QAAQ,IAAI;CAC1B,IAAI,UAAU,WAAW,OAAO;CAChC,MAAM,aAAa,+BAA+B,KAAK,IAAI,CAAC,GAAG;CAC/D,IAAI,eAAe,KAAA,GAAW,OAAO,WAAW,YAAY;CAO5D,OAHiB,CAAC,GAAG,KAAK,SAAS,8CAA8C,CAAC,CAAC,CAAC,SACjF,UAAW,MAAM,OAAO,KAAA,KAAa,MAAM,OAAO,KAAA,IAAY,CAAC,MAAM,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAE/E,CAAC,CAAC,MAAM,YAAY,CAAC,cAAc,IAAI,OAAO,CAAC,KAAK;AACpE;AAEA,MAAM,gBAAgB,MAAc,aAA6B;CAC/D,MAAM,QAAQ,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI;CAC3C,IAAI,MAAM,UAAU,UAAU,OAAO;CAErC,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,MAAM,MAAM,GAAG,QAAQ,CAAC;CAC9D,MAAM,YAAY,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG,KAAK,YAAY,IAAI,CAAC;CACzE,OAAO,cAAc,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS;AACxD;;AAGA,MAAM,YAAY,KAAa,OAAe,SAA+B;CAC3E,MAAM,OAAO,mBAAmB,KAAK,KAAK;CAC1C,OAAO,SAAS,OACZ;EAAE;EAAK;EAAO,MAAM,KAAA;EAAW;CAAK,IACpC;EAAE;EAAK,OAAO,KAAK,EAAE,CAAE,KAAK;EAAG,MAAM,cAAc,KAAK,EAAG;EAAG;CAAK;AACzE;AAEA,MAAa,eAAe,UAA0B;CACpD,MAAM,SAMD,CAAC;CACN,MAAM,WAAqB,CAAC;CAC5B,MAAM,UAAoB,CAAC;CAC3B,MAAM,cAA8B,CAAC;CACrC,IAAI;CACJ,IAAI,mBAAmB;CACvB,KAAK,MAAM,CAAC,OAAO,QAAQ,aAAa,OAAO,gBAAgB,CAAC,CAC7D,MAAM,YAAY,CAAC,CACnB,QAAQ,GAAG;EACZ,MAAM,OAAO,IAAI,QAAQ,QAAQ,EAAE,CAAC,CAAC,KAAK;EAC1C,MAAM,QAAQ,KAAK,QAAQ,GAAG;EAC9B,IAAI,SAAS,GAAG;EAChB,MAAM,MAAM,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,YAAY;EACpD,MAAM,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC,CAAC,KAAK;EACzC,IAAI,QAAQ,cAAc;GACxB,IAAI,YAAY,KAAA,KAAa,CAAC,kBAAkB;IAC9C,UAAU;KACR,QAAQ,CAAC;KACT,MAAM,QAAQ;KACd,OAAO,CAAC;KACR,mBAAmB,KAAA;KACnB,SAAS,CAAC;IACZ;IACA,OAAO,KAAK,OAAO;GACrB;GACA,QAAQ,OAAO,KAAK,MAAM,YAAY,CAAC;GACvC,mBAAmB;GACnB;EACF;EAEA,IAAI,QAAQ,WAAW;GACrB,IAAI,MAAM,SAAS,GAAG,SAAS,KAAK,KAAK;GACzC;EACF;EAEA,IAAI,QAAQ,oBAAoB,QAAQ,mBAAmB,QAAQ,WAAW;GAC5E,QAAQ,KAAK,GAAG,IAAI,IAAI,OAAO;GAC/B,CAAC,SAAS,WAAW,YAAA,CAAa,KAAK,SAAS,KAAK,OAAO,QAAQ,CAAC,CAAC;GACtE;EACF;EACA,IAAI,YAAY,KAAA,GAAW;EAC3B,IAAI,QAAQ,WAAW,QAAQ,YAAY;GAEzC,mBAAmB;GACnB,IAAI,MAAM,SAAS,GACjB,QAAQ,MAAM,KAAK;IACjB,OAAO,QAAQ;IACf,SAAS,cAAc,KAAK;IAC5B,MAAM,QAAQ;GAChB,CAAC;EACL,OAAO,IAAI,QAAQ,eAAe;GAChC,MAAM,UAAU,OAAO,KAAK;GAC5B,IAAI,OAAO,SAAS,OAAO,KAAK,WAAW,GAAG,QAAQ,oBAAoB;EAC5E;CACF;CACA,OAAO;EAAE;EAAQ;EAAU;EAAS;CAAY;AAClD;;;;;;;AAQA,MAAa,kBAAkB,SAAiB,SAA0B;CACxE,MAAM,WAAW,QAAQ,SAAS,GAAG;CACrC,MAAM,SAAS,WAAW,QAAQ,MAAM,GAAG,EAAE,IAAI,QAAA,CAAS,MAAM,GAAG;CACnE,MAAM,QAAQ,MAAM;CACpB,IAAI,CAAC,KAAK,WAAW,KAAK,GAAG,OAAO;CACpC,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,KAAK,WAAW,MAAM;CAClE,IAAI,KAAK,MAAM;CACf,MAAM,OAAO,MAAM,SAAS;CAC5B,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,SAAS,GAAG;EAC5C,MAAM,OAAO,MAAM;EAEnB,IAAI,KAAK,WAAW,GAAG;EACvB,MAAM,QAAQ,KAAK,QAAQ,MAAM,EAAE;EACnC,IAAI,UAAU,IAAI,OAAO;EACzB,KAAK,QAAQ,KAAK;CACpB;CACA,MAAM,OAAO,MAAM;CACnB,IAAI,CAAC,UAAU,OAAO,KAAK,WAAW,KAAK,KAAK,QAAQ,MAAM,EAAE,MAAM;CACtE,OAAO,KAAK,SAAS,KAAK,UAAU,MAAM,KAAK,SAAS,IAAI;AAC9D;AAUA,MAAM,aAAa,QAAgB,UAA0B;CAC3D,MAAM,SAAS,UAAU,MAAM,MAAM,aAAa,KAAK;CACvD,MAAM,QAAQ,OAAO,OAAO,QAAQ,UAClC,MAAM,OAAO,MAAM,UAAU,UAAU,OAAO,QAAQ,KAAK,MAAM,MAAM,CACzE;CACA,MAAM,SACJ,MAAM,SAAS,IAAI,QAAQ,OAAO,OAAO,QAAQ,UAAU,MAAM,OAAO,SAAS,GAAG,CAAC;CACvF,MAAM,SAAS,OAAO,SAAS,UAC7B,MAAM,sBAAsB,KAAA,IAAY,CAAC,IAAI,CAAC,MAAM,iBAAiB,CACvE;CACA,OAAO;EACL,OAAO,OAAO,SAAS,UAAU,MAAM,KAAK;EAC5C,QAAQ;EACR,mBAAmB,OAAO,SAAS,IAAI,KAAK,IAAI,GAAG,MAAM,IAAI,KAAA;EAC7D,UAAU,MAAM,SAAS;CAC3B;AACF;;AAGA,MAAM,gBAAgB,OAA8B,SAAyC;CAC3F,IAAI;CACJ,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,GAAG;EACzC,IACE,SAAS,KAAA,KACT,KAAK,QAAQ,SAAS,KAAK,QAAQ,UAClC,KAAK,QAAQ,WAAW,KAAK,QAAQ,UAAU,KAAK,SAAS,CAAC,KAAK,OAEpE,OAAO;CAEX;CACA,OAAO;AACT;;;;;;;;AAoBA,MAAa,UACX,QACA,QACA,iBACmB;CACnB,MAAM,OAAO,cAAc,aAAa,WAAW,IAAI,MAAM,YAAY;CACzE,MAAM,UAAU,UAAU,QAAQ,OAAO,MAAM,GAAG;CAElD,MAAM,SAAS,CAAC,QAAQ,iBAAiB;CACzC,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,IAAI,MAAM,UAAU,OAAO,KAAK,MAAM,iBAAiB;CACzD;CACA,MAAM,QAAQ,OAAO,QAAQ,UAA2B,UAAU,KAAA,CAAS;CAC3E,MAAM,oBAAoB,MAAM,SAAS,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,KAAA;CAClE,IAAI,SAAS,eACX,OAAO;EAAE,SAAS;EAAM,IAAI,KAAA;EAAW;EAAmB,MAAM,KAAA;EAAW,QAAQ,CAAC;CAAE;CAExF,MAAM,OAAO,aAAa,QAAQ,OAAO,IAAI;CAC7C,IAAI,SAAS,KAAA,KAAa,CAAC,KAAK,OAC9B,OAAO;EAAE,SAAS;EAAO,IAAI,OAAO;EAAI;EAAmB;EAAM,QAAQ,QAAQ;CAAO;CAE1F,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,MAAM,UAAU,MAAM,WAAW,aAAa,MAAM,OAAO,IAAI,IAAI,KAAA;EACnE,IAAI,YAAY,KAAA,KAAa,CAAC,QAAQ,OACpC,OAAO;GAAE,SAAS;GAAO,IAAI;GAAO;GAAmB,MAAM;GAAS,QAAQ,MAAM;EAAO;CAE/F;CACA,OAAO;EAAE,SAAS;EAAM,IAAI,KAAA;EAAW;EAAmB;EAAM,QAAQ,QAAQ;CAAO;AACzF;;;;;;;;;;;AAYA,MAAa,cACX,QACA,QACA,iBACa;CACb,MAAM,OAAO,cAAc,aAAa,WAAW,IAAI,MAAM,YAAY;CACzE,MAAM,SAAS,IAAI,IAAI,UAAU,QAAQ,OAAO,MAAM,GAAG,CAAC,CAAC,MAAM;CACjE,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,IAAI,MAAM,UAAU,KAAK,MAAM,SAAS,MAAM,QAAQ,OAAO,IAAI,KAAK;CACxE;CACA,MAAM,SAAyB,CAAC;CAChC,MAAM,QAAQ,UAAyC;EAErD,MAAM,uBAAO,IAAI,IAAuD;EACxE,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,KAAK,SAAS,KAAA,KAAa,CAAC,eAAe,KAAK,MAAM,IAAI,GAAG;GACjE,MAAM,SAAS,KAAK,MAAM,UAAU;GACpC,MAAM,SAAS,KAAK,IAAI,KAAK,GAAG;GAChC,IAAI,WAAW,KAAA,KAAa,SAAS,OAAO,QAC1C,KAAK,IAAI,KAAK,KAAK;IAAE;IAAQ,OAAO,CAAC,IAAI;GAAE,CAAC;QACzC,IAAI,WAAW,OAAO,QAAQ,OAAO,MAAM,KAAK,IAAI;EAC3D;EACA,KAAK,MAAM,EAAE,OAAO,UAAU,KAAK,OAAO,GAAG,OAAO,KAAK,GAAG,IAAI;CAClE;CACA,KAAK,OAAO,eAAe,CAAC,CAAC;CAC7B,KAAK,MAAM,SAAS,QAAQ,KAAK,MAAM,WAAW,CAAC,CAAC;CACpD,OAAO,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,SAAS,GAAG,KAAK,IAAI,IAAI,KAAK,OAAO;AAC1F;;AAGA,MAAa,qBAAqB,YAA4B;CAC5D,GAAG;CACH,aAAa;AACf;;AAGA,MAAa,cACX;;;;;;;AAQF,MAAa,gBAAgB,aAAwC,SAA0B;CAC7F,MAAM,QAAQ,KAAK,WAAW,CAAC,MAAM,QAAS,KAAK,MAAM,CAAC,IAAI,KAAA,CAAM,MAAM,GAAG,IAAI;CACjF,IAAI,uCAAuC,KAAK,IAAI,GAAG,OAAO;CAC9D,OACE,QAAQ,KAAK,eAAe,EAAE,KAC9B,sDAAsD,KAAK,IAAI;AAEnE;;;;;;AAOA,MAAa,sBACX,QACA,aACA,SACW;CACX,IAAI,SAAS,OAAO,UAAU,KAAK,OAAO,iBAAiB,QAAQ,KAAA,CAAS;CAC5E,MAAM,OAAO,OAAO,SAAS,WAAW,OAAO,IAAI,YAAY,OAAO,CAAC,CAAC,OAAO,IAAI;CACnF,IAAI,aAAa,aAAa,IAAI,GAAG,OAAO,kBAAkB,WAAW;CACzE,OAAO,YAAY,IAAI;AACzB;;AAGA,MAAa,oBAAoB,QAAgB,SAAqC;CACpF,IAAI,UAAU,OAAO,SAAS,KAAK,OAAO,YAAY,QAAQ,EAAE;CAChE,IAAI,WAAW,KAAK,OAAO,kBAAkB,0BAA0B;CACvE,IAAI,UAAU,OAAO,SAAS,KAAK,OAAO;CAC1C,OAAO,kBAAkB,WAAW,IAAI,gBAAgB,UAAU,OAAO,MAAM,GAAG;AACpF"}
+{"version":3,"file":"robots.mjs","names":[],"sources":["../src/robots.ts"],"sourcesContent":["/**\n * This module reads robots.txt as RFC 9309 (September 2022) specifies it. It is for a crawler that\n * tells who it is and obeys what a site says. The important rules, each with a test:\n *\n * - A group is one or more `user-agent` lines and the rules after them. A `user-agent` line after\n *   an `allow` or `disallow` line starts a new group, as the parser of Google does. A different\n *   line (`crawl-delay`) between two `user-agent` lines does not split the group. A rule before\n *   the first `user-agent` line is in no group.\n * - The product token matches with no attention to case. The matching groups merge. The group\n *   `*` applies only when no group names the crawler.\n * - The longest matching rule wins, and `allow` wins a tie. A `*` matches any sequence, and a `$`\n *   at the end anchors the end. The matcher uses no regular expression, and its time is linear.\n * - `/robots.txt` is always permitted, and an empty `disallow` matches nothing.\n * - The parser reads at least the first 500 KiB.\n * - A usage line (`content-signal`, `content-usage`) belongs to its group, with an optional path\n *   before its value. A line before the first group applies to each crawler.\n *\n * The function `robotsFromStatus` gives the meaning of the status of the fetch. A 4xx is\n * \"unavailable\", so all paths are permitted. A 5xx or no response is \"unreachable\", so no path is\n * permitted. A 429 is unreachable too, as Google does it. A site that limits the rate of its robots\n * file does not ask for a crawl.\n */\n\nexport const ROBOTS_MAX_BYTES: number = 500 * 1024;\n\nexport interface RobotsRule {\n  readonly allow: boolean;\n  /** This value is the path pattern, normalized for comparison. */\n  readonly pattern: string;\n  /** This value is the number of the line in robots.txt, from 1. A rule that the code makes has none. */\n  readonly line?: number | undefined;\n}\n\n/** A usage line states a preference about the use of the content (`content-signal`, `content-usage`). */\nexport interface RobotsSignal {\n  /** This value is the name of the line, in lowercase. */\n  readonly key: string;\n  /** This text is the value of the line, after the path. */\n  readonly value: string;\n  /** This value is the normalized path pattern of the line. A line with no path applies to each path. */\n  readonly path: string | undefined;\n  /** This value is the number of the line in robots.txt, from 1. */\n  readonly line: number;\n}\n\nexport interface RobotsGroup {\n  /** These values are the user agents of the group, in lowercase. */\n  readonly agents: readonly string[];\n  /** This value is the number of the first `user-agent` line of the group, from 1. */\n  readonly line?: number | undefined;\n  readonly rules: readonly RobotsRule[];\n  /** This value is the nonstandard `crawl-delay` in seconds, when the group sets one. */\n  readonly crawlDelaySeconds: number | undefined;\n  /** These usage lines are in the group. They apply when the group applies. */\n  readonly signals?: readonly RobotsSignal[] | undefined;\n}\n\nexport interface Robots {\n  readonly groups: readonly RobotsGroup[];\n  readonly sitemaps: readonly string[];\n  /** These lines state usage preferences (`content-signal: …`, `content-usage: …`), as written, from each group. */\n  readonly signals: readonly string[];\n  /** These usage lines come before the first group, so they apply to each crawler. */\n  readonly fileSignals?: readonly RobotsSignal[] | undefined;\n  /**\n   * This text tells why the crawler could not read robots.txt: a dead domain, a broken certificate\n   * or a server error. Then no path is permitted, and the note of a skip gives this reason. It does\n   * not name a rule of the site.\n   */\n  readonly unreachable?: string | undefined;\n}\n\nexport const ALLOW_ALL: Robots = { groups: [], sitemaps: [], signals: [] };\n\nexport const DISALLOW_ALL: Robots = {\n  groups: [\n    { agents: [\"*\"], rules: [{ allow: false, pattern: \"/\" }], crawlDelaySeconds: undefined },\n  ],\n  sitemaps: [],\n  signals: [],\n};\n\nconst UNRESERVED = /^[A-Za-z0-9\\-._~]$/;\nconst URL_ENCODED = new Set(\"\\\"'<>\\\\^`{|}\");\n/**\n * The URL parser encodes these ASCII characters in a path or a query: the controls, a space, DEL,\n * and `\"'<>\\^`{|}`. A rule can contain them as written.\n */\nconst encodedAscii = (char: string): boolean => {\n  const code = char.charCodeAt(0);\n  return code <= 0x20 || code === 0x7f || URL_ENCODED.has(char);\n};\nconst HEX = /^[0-9A-Fa-f]$/;\n\nconst escapeOf = (char: string): string => {\n  const code = char.codePointAt(0)!;\n  if (code <= 0x7f) return `%${code.toString(16).toUpperCase().padStart(2, \"0\")}`;\n  try {\n    return encodeURIComponent(char);\n  } catch {\n    // a lone surrogate: the URL parser writes the replacement character\n    return \"%EF%BF%BD\";\n  }\n};\n\n/**\n * This function normalizes a path, or the pattern of a rule, for comparison, as §2.2.2 asks. It\n * encodes the characters outside US-ASCII, and the ASCII characters that the URL parser encodes\n * (a space, `\"`, `<`, `{`). It decodes an escape of an unreserved character, so `/%7Efoo` and\n * `/~foo` are equal. The other escapes stay, with uppercase hexadecimal digits.\n */\nexport const normalizePath = (value: string): string => {\n  let out = \"\";\n  const chars = Array.from(value);\n  for (let index = 0; index < chars.length; index += 1) {\n    const char = chars[index]!;\n    if (char === \"%\" && HEX.test(chars[index + 1] ?? \"\") && HEX.test(chars[index + 2] ?? \"\")) {\n      const hex = `${chars[index + 1]!}${chars[index + 2]!}`;\n      const decoded = String.fromCharCode(Number.parseInt(hex, 16));\n      out += UNRESERVED.test(decoded) ? decoded : `%${hex.toUpperCase()}`;\n      index += 2;\n      continue;\n    }\n    out += char.charCodeAt(0) > 0x7f || encodedAscii(char) ? escapeOf(char) : char;\n  }\n  return out;\n};\n\n/** This function tells if a text is a product token: letters, `_` and `-` only, and one or more (RFC 9309, section 2.2.1). */\nexport const isProductToken = (value: string): boolean => /^[A-Za-z_-]+$/.test(value);\n\n/** This function gives the first product token of a user agent: `Googlebot/2.1` gives `googlebot`. */\nconst tokenOf = (value: string): string =>\n  (/^[A-Za-z_-]+/.exec(value.trim())?.[0] ?? \"\").toLowerCase();\n\n/** These names are parts of the header of a browser, not the name of a crawler. */\nconst BROWSER_PARTS: ReadonlySet<string> = new Set([\n  \"mozilla\",\n  \"applewebkit\",\n  \"khtml\",\n  \"gecko\",\n  \"chrome\",\n  \"chromium\",\n  \"safari\",\n  \"version\",\n  \"mobile\",\n  \"firefox\",\n  \"edg\",\n  \"opr\",\n  \"like\",\n]);\n\n/**\n * This function gives the product token that robots.txt rules match, in lowercase. `MyBot/1.0`\n * gives `mybot`. The full header `Mozilla/5.0 (compatible; MyBot/1.0; +https://…)` also gives\n * `mybot`. The token is the name after `compatible;`, or else the first product that is not a part\n * of a browser.\n */\nexport const productToken = (value: string): string => {\n  const text = value.trim();\n  const first = tokenOf(text);\n  if (first !== \"mozilla\") return first;\n  const compatible = /compatible;\\s*([A-Za-z_-]+)/i.exec(text)?.[1];\n  if (compatible !== undefined) return compatible.toLowerCase();\n  // One pass in linear time: each match is a name that starts with a letter, with an optional `/` and\n  // version after it, or a run of other characters. A run of other characters takes no version, so\n  // `_/a/a` gives the product `a`, as the scan before 0.2.0 did.\n  const products = [...text.matchAll(/([A-Za-z][A-Za-z_-]*)(\\/[\\w.]+)?|[^A-Za-z]+/g)].flatMap(\n    (match) => (match[1] !== undefined && match[2] !== undefined ? [match[1].toLowerCase()] : []),\n  );\n  return products.find((product) => !BROWSER_PARTS.has(product)) ?? first;\n};\n\nconst truncateUtf8 = (text: string, maxBytes: number): string => {\n  const bytes = new TextEncoder().encode(text);\n  if (bytes.length <= maxBytes) return text;\n  // cut at the last line break inside the limit, so no rule is half-read\n  const head = new TextDecoder().decode(bytes.slice(0, maxBytes));\n  const lastBreak = Math.max(head.lastIndexOf(\"\\n\"), head.lastIndexOf(\"\\r\"));\n  return lastBreak === -1 ? \"\" : head.slice(0, lastBreak);\n};\n\n/**\n * This function reads a usage line: an optional path, then the value (`/images/ train-ai=n`). The\n * path starts with `/` and stops at the first space. The function uses no backtracking, so its\n * time is linear.\n */\nconst signalOf = (key: string, value: string, line: number): RobotsSignal => {\n  const end = value.startsWith(\"/\") ? value.search(/\\s/) : -1;\n  return end === -1\n    ? { key, value, path: undefined, line }\n    : { key, value: value.slice(end).trim(), path: normalizePath(value.slice(0, end)), line };\n};\n\nexport const parseRobots = (input: string): Robots => {\n  const groups: Array<{\n    agents: string[];\n    line: number;\n    rules: RobotsRule[];\n    crawlDelaySeconds: number | undefined;\n    signals: RobotsSignal[];\n  }> = [];\n  const sitemaps: string[] = [];\n  const signals: string[] = [];\n  const fileSignals: RobotsSignal[] = [];\n  let current: (typeof groups)[number] | undefined;\n  let collectingAgents = false;\n  for (const [index, raw] of truncateUtf8(input, ROBOTS_MAX_BYTES)\n    .split(/\\r\\n|\\r|\\n/)\n    .entries()) {\n    // a comment starts at the first `#` and continues to the end of the line (RFC 9309, section 2.2.3)\n    const hash = raw.indexOf(\"#\");\n    const line = (hash === -1 ? raw : raw.slice(0, hash)).trim();\n    const colon = line.indexOf(\":\");\n    if (colon <= 0) continue;\n    const key = line.slice(0, colon).trim().toLowerCase();\n    const value = line.slice(colon + 1).trim();\n    if (key === \"user-agent\") {\n      if (current === undefined || !collectingAgents) {\n        current = {\n          agents: [],\n          line: index + 1,\n          rules: [],\n          crawlDelaySeconds: undefined,\n          signals: [],\n        };\n        groups.push(current);\n      }\n      current.agents.push(value.toLowerCase());\n      collectingAgents = true;\n      continue;\n    }\n    // records that belong to no group\n    if (key === \"sitemap\") {\n      if (value.length > 0) sitemaps.push(value);\n      continue;\n    }\n    // an RSL `License` line names the license of the content: the crawler records it with the other usage lines\n    if (key === \"content-signal\" || key === \"content-usage\" || key === \"license\") {\n      signals.push(`${key}: ${value}`);\n      (current?.signals ?? fileSignals).push(signalOf(key, value, index + 1));\n      continue;\n    }\n    if (current === undefined) continue;\n    if (key === \"allow\" || key === \"disallow\") {\n      // only a rule ends the list of agents, so a crawl-delay between two agents keeps one group\n      collectingAgents = false;\n      if (value.length > 0)\n        current.rules.push({\n          allow: key === \"allow\",\n          pattern: normalizePath(value),\n          line: index + 1,\n        });\n    } else if (key === \"crawl-delay\") {\n      const seconds = Number(value);\n      if (Number.isFinite(seconds) && seconds >= 0) current.crawlDelaySeconds = seconds;\n    }\n  }\n  return { groups, sitemaps, signals, fileSignals };\n};\n\n/**\n * This function tells if a normalized path matches the pattern of a rule. A `*` matches any\n * sequence, and a `$` at the end anchors the end. The function finds the parts between the stars\n * in sequence, each at its first position. No pattern makes it slow: its time is linear in the\n * length of the path for each part, and it uses no regular expression.\n */\nexport const matchesPattern = (pattern: string, path: string): boolean => {\n  const anchored = pattern.endsWith(\"$\");\n  const parts = (anchored ? pattern.slice(0, -1) : pattern).split(\"*\");\n  const first = parts[0]!;\n  if (!path.startsWith(first)) return false;\n  if (parts.length === 1) return !anchored || path.length === first.length;\n  let at = first.length;\n  const last = parts.length - 1;\n  for (let index = 1; index < last; index += 1) {\n    const part = parts[index]!;\n    // two stars in sequence give an empty part, which matches at once\n    if (part.length === 0) continue;\n    const found = path.indexOf(part, at);\n    if (found === -1) return false;\n    at = found + part.length;\n  }\n  const tail = parts[last]!;\n  if (!anchored) return tail.length === 0 || path.indexOf(tail, at) !== -1;\n  return path.length - tail.length >= at && path.endsWith(tail);\n};\n\ninterface Chosen {\n  readonly rules: readonly RobotsRule[];\n  readonly groups: readonly RobotsGroup[];\n  readonly crawlDelaySeconds: number | undefined;\n  /** This value is true when a group names this token, and `*` did not apply. */\n  readonly explicit: boolean;\n}\n\nconst groupsFor = (robots: Robots, token: string): Chosen => {\n  const wanted = token === \"*\" ? \"*\" : productToken(token);\n  const named = robots.groups.filter((group) =>\n    group.agents.some((agent) => agent !== \"*\" && tokenOf(agent) === wanted),\n  );\n  const chosen =\n    named.length > 0 ? named : robots.groups.filter((group) => group.agents.includes(\"*\"));\n  const delays = chosen.flatMap((group) =>\n    group.crawlDelaySeconds === undefined ? [] : [group.crawlDelaySeconds],\n  );\n  return {\n    rules: chosen.flatMap((group) => group.rules),\n    groups: chosen,\n    crawlDelaySeconds: delays.length > 0 ? Math.max(...delays) : undefined,\n    explicit: named.length > 0,\n  };\n};\n\n/** This function gives the rule that decides for the path: the longest match, and `allow` in a tie. */\nconst decidingRule = (rules: readonly RobotsRule[], path: string): RobotsRule | undefined => {\n  let best: RobotsRule | undefined;\n  for (const rule of rules) {\n    if (!matchesPattern(rule.pattern, path)) continue;\n    if (\n      best === undefined ||\n      rule.pattern.length > best.pattern.length ||\n      (rule.pattern.length === best.pattern.length && rule.allow && !best.allow)\n    ) {\n      best = rule;\n    }\n  }\n  return best;\n};\n\nexport interface RobotsDecision {\n  readonly allowed: boolean;\n  /** This value is the product token whose rules refused the path. */\n  readonly by: string | undefined;\n  readonly crawlDelaySeconds: number | undefined;\n  /** This rule decided. When no rule matches the path, the path is permitted and this value is empty. */\n  readonly rule?: RobotsRule | undefined;\n  /** These groups applied to the decision: the groups that name the token, or the group `*`. */\n  readonly groups?: readonly RobotsGroup[] | undefined;\n}\n\n/**\n * This function tells if these tokens can fetch `pathAndQuery`. Each token becomes its product\n * token (`productToken`), so `MyBot/1.0` matches the group `mybot`. The first token is the token of\n * the crawler, and `*` applies when no group names it. A further token, for example `Claude-User`,\n * applies only when the site names it. A site that writes rules for an agent means them for the\n * fetches of that agent.\n */\nexport const decide = (\n  robots: Robots,\n  tokens: readonly string[],\n  pathAndQuery: string,\n): RobotsDecision => {\n  const path = normalizePath(pathAndQuery.length === 0 ? \"/\" : pathAndQuery);\n  const primary = groupsFor(robots, tokens[0] ?? \"*\");\n  // the crawl delay of a group that names a further agent applies to the fetches of that agent too\n  const delays = [primary.crawlDelaySeconds];\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    if (extra.explicit) delays.push(extra.crawlDelaySeconds);\n  }\n  const known = delays.filter((delay): delay is number => delay !== undefined);\n  const crawlDelaySeconds = known.length > 0 ? Math.max(...known) : undefined;\n  if (path === \"/robots.txt\") {\n    return { allowed: true, by: undefined, crawlDelaySeconds, rule: undefined, groups: [] };\n  }\n  const rule = decidingRule(primary.rules, path);\n  if (rule !== undefined && !rule.allow) {\n    return { allowed: false, by: tokens[0], crawlDelaySeconds, rule, groups: primary.groups };\n  }\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    const refusal = extra.explicit ? decidingRule(extra.rules, path) : undefined;\n    if (refusal !== undefined && !refusal.allow) {\n      return { allowed: false, by: token, crawlDelaySeconds, rule: refusal, groups: extra.groups };\n    }\n  }\n  return { allowed: true, by: undefined, crawlDelaySeconds, rule, groups: primary.groups };\n};\n\n/**\n * This function gives the usage lines that apply to these tokens and this path, as\n * `key: value` texts. The lines come from three places:\n *\n * - the lines before the first group\n * - the groups of the first token, or `*`\n * - the groups that name a further token\n *\n * In one group, the line with the longest matching path wins for each key.\n */\nexport const signalsFor = (\n  robots: Robots,\n  tokens: readonly string[],\n  pathAndQuery: string,\n): string[] => {\n  const path = normalizePath(pathAndQuery.length === 0 ? \"/\" : pathAndQuery);\n  const groups = new Set(groupsFor(robots, tokens[0] ?? \"*\").groups);\n  for (const token of tokens.slice(1)) {\n    const extra = groupsFor(robots, token);\n    if (extra.explicit) for (const group of extra.groups) groups.add(group);\n  }\n  const chosen: RobotsSignal[] = [];\n  const pick = (lines: readonly RobotsSignal[]): void => {\n    // for each key: the length of the longest matching path, and the lines with that length\n    const best = new Map<string, { length: number; lines: RobotsSignal[] }>();\n    for (const line of lines) {\n      if (line.path !== undefined && !matchesPattern(line.path, path)) continue;\n      const length = line.path?.length ?? 0;\n      const before = best.get(line.key);\n      if (before === undefined || length > before.length)\n        best.set(line.key, { length, lines: [line] });\n      else if (length === before.length) before.lines.push(line);\n    }\n    for (const { lines: kept } of best.values()) chosen.push(...kept);\n  };\n  pick(robots.fileSignals ?? []);\n  for (const group of groups) pick(group.signals ?? []);\n  return chosen.sort((a, b) => a.line - b.line).map((line) => `${line.key}: ${line.value}`);\n};\n\n/** This function gives the robots of a site whose robots.txt the crawler cannot read. No path is permitted, and the reason stays. */\nexport const unreachableRobots = (reason: string): Robots => ({\n  ...DISALLOW_ALL,\n  unreachable: reason,\n});\n\n/** This text is the reason when a robots.txt that answered 200 is a web page, not a robots.txt file. */\nexport const HTML_ROBOTS =\n  \"robots.txt is a web page, not a robots.txt file (a login or challenge page)\";\n\n/**\n * This function tells if the body of a robots.txt is a web page. A server can send a challenge, a\n * login or an error page with a 200. Such a body starts with `<!doctype`, `<html`, `<head` or\n * `<body`. With the content type `text/html`, a tag of a page in the first 4 KB is enough. A plain\n * robots.txt that a server sends as `text/html` stays a file.\n */\nexport const isHtmlRobots = (contentType: string | null | undefined, text: string): boolean => {\n  const head = (text.charCodeAt(0) === 0xfeff ? text.slice(1) : text).slice(0, 4096);\n  if (/^\\s*<(!doctype|html|head|body)[\\s>]/i.test(head)) return true;\n  return (\n    /html/i.test(contentType ?? \"\") &&\n    /<(html|head|body|script|meta|title|div|form)[\\s>/]/i.test(head)\n  );\n};\n\n/**\n * This function reads the answer of a robots.txt fetch: the status, the content type and the body.\n * A 2xx with a web page in place of the file is unreadable. Thus no path is permitted (fail\n * closed), as for a body that does not decode. The other statuses follow `robotsFromStatus`.\n */\nexport const robotsFromResponse = (\n  status: number,\n  contentType: string | null | undefined,\n  body: Uint8Array | string,\n): Robots => {\n  if (status < 200 || status >= 300) return robotsFromStatus(status, undefined);\n  const text = typeof body === \"string\" ? body : new TextDecoder(\"utf-8\").decode(body);\n  if (isHtmlRobots(contentType, text)) return unreachableRobots(HTML_ROBOTS);\n  return parseRobots(text);\n};\n\n/** This function gives the meaning of the status of a robots.txt fetch (RFC 9309 §2.3.1). */\nexport const robotsFromStatus = (status: number, body: string | undefined): Robots => {\n  if (status >= 200 && status < 300) return parseRobots(body ?? \"\");\n  if (status === 429) return unreachableRobots(\"status 429, rate-limited\");\n  if (status >= 400 && status < 500) return ALLOW_ALL;\n  return unreachableRobots(status === 0 ? \"no response\" : `status ${String(status)}`);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAuBA,MAAa,mBAA2B;AAiDxC,MAAa,YAAoB;CAAE,QAAQ,CAAC;CAAG,UAAU,CAAC;CAAG,SAAS,CAAC;AAAE;AAEzE,MAAa,eAAuB;CAClC,QAAQ,CACN;EAAE,QAAQ,CAAC,GAAG;EAAG,OAAO,CAAC;GAAE,OAAO;GAAO,SAAS;EAAI,CAAC;EAAG,mBAAmB,KAAA;CAAU,CACzF;CACA,UAAU,CAAC;CACX,SAAS,CAAC;AACZ;AAEA,MAAM,aAAa;AACnB,MAAM,8BAAc,IAAI,IAAI,cAAc;;;;;AAK1C,MAAM,gBAAgB,SAA0B;CAC9C,MAAM,OAAO,KAAK,WAAW,CAAC;CAC9B,OAAO,QAAQ,MAAQ,SAAS,OAAQ,YAAY,IAAI,IAAI;AAC9D;AACA,MAAM,MAAM;AAEZ,MAAM,YAAY,SAAyB;CACzC,MAAM,OAAO,KAAK,YAAY,CAAC;CAC/B,IAAI,QAAQ,KAAM,OAAO,IAAI,KAAK,SAAS,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,GAAG,GAAG;CAC5E,IAAI;EACF,OAAO,mBAAmB,IAAI;CAChC,QAAQ;EAEN,OAAO;CACT;AACF;;;;;;;AAQA,MAAa,iBAAiB,UAA0B;CACtD,IAAI,MAAM;CACV,MAAM,QAAQ,MAAM,KAAK,KAAK;CAC9B,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS,GAAG;EACpD,MAAM,OAAO,MAAM;EACnB,IAAI,SAAS,OAAO,IAAI,KAAK,MAAM,QAAQ,MAAM,EAAE,KAAK,IAAI,KAAK,MAAM,QAAQ,MAAM,EAAE,GAAG;GACxF,MAAM,MAAM,GAAG,MAAM,QAAQ,KAAM,MAAM,QAAQ;GACjD,MAAM,UAAU,OAAO,aAAa,OAAO,SAAS,KAAK,EAAE,CAAC;GAC5D,OAAO,WAAW,KAAK,OAAO,IAAI,UAAU,IAAI,IAAI,YAAY;GAChE,SAAS;GACT;EACF;EACA,OAAO,KAAK,WAAW,CAAC,IAAI,OAAQ,aAAa,IAAI,IAAI,SAAS,IAAI,IAAI;CAC5E;CACA,OAAO;AACT;;AAGA,MAAa,kBAAkB,UAA2B,gBAAgB,KAAK,KAAK;;AAGpF,MAAM,WAAW,WACd,eAAe,KAAK,MAAM,KAAK,CAAC,CAAC,GAAG,MAAM,GAAA,CAAI,YAAY;;AAG7D,MAAM,gCAAqC,IAAI,IAAI;CACjD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;;AAQD,MAAa,gBAAgB,UAA0B;CACrD,MAAM,OAAO,MAAM,KAAK;CACxB,MAAM,QAAQ,QAAQ,IAAI;CAC1B,IAAI,UAAU,WAAW,OAAO;CAChC,MAAM,aAAa,+BAA+B,KAAK,IAAI,CAAC,GAAG;CAC/D,IAAI,eAAe,KAAA,GAAW,OAAO,WAAW,YAAY;CAO5D,OAHiB,CAAC,GAAG,KAAK,SAAS,8CAA8C,CAAC,CAAC,CAAC,SACjF,UAAW,MAAM,OAAO,KAAA,KAAa,MAAM,OAAO,KAAA,IAAY,CAAC,MAAM,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAE/E,CAAC,CAAC,MAAM,YAAY,CAAC,cAAc,IAAI,OAAO,CAAC,KAAK;AACpE;AAEA,MAAM,gBAAgB,MAAc,aAA6B;CAC/D,MAAM,QAAQ,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI;CAC3C,IAAI,MAAM,UAAU,UAAU,OAAO;CAErC,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,MAAM,MAAM,GAAG,QAAQ,CAAC;CAC9D,MAAM,YAAY,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG,KAAK,YAAY,IAAI,CAAC;CACzE,OAAO,cAAc,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS;AACxD;;;;;;AAOA,MAAM,YAAY,KAAa,OAAe,SAA+B;CAC3E,MAAM,MAAM,MAAM,WAAW,GAAG,IAAI,MAAM,OAAO,IAAI,IAAI;CACzD,OAAO,QAAQ,KACX;EAAE;EAAK;EAAO,MAAM,KAAA;EAAW;CAAK,IACpC;EAAE;EAAK,OAAO,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK;EAAG,MAAM,cAAc,MAAM,MAAM,GAAG,GAAG,CAAC;EAAG;CAAK;AAC5F;AAEA,MAAa,eAAe,UAA0B;CACpD,MAAM,SAMD,CAAC;CACN,MAAM,WAAqB,CAAC;CAC5B,MAAM,UAAoB,CAAC;CAC3B,MAAM,cAA8B,CAAC;CACrC,IAAI;CACJ,IAAI,mBAAmB;CACvB,KAAK,MAAM,CAAC,OAAO,QAAQ,aAAa,OAAO,gBAAgB,CAAC,CAC7D,MAAM,YAAY,CAAC,CACnB,QAAQ,GAAG;EAEZ,MAAM,OAAO,IAAI,QAAQ,GAAG;EAC5B,MAAM,QAAQ,SAAS,KAAK,MAAM,IAAI,MAAM,GAAG,IAAI,EAAA,CAAG,KAAK;EAC3D,MAAM,QAAQ,KAAK,QAAQ,GAAG;EAC9B,IAAI,SAAS,GAAG;EAChB,MAAM,MAAM,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,YAAY;EACpD,MAAM,QAAQ,KAAK,MAAM,QAAQ,CAAC,CAAC,CAAC,KAAK;EACzC,IAAI,QAAQ,cAAc;GACxB,IAAI,YAAY,KAAA,KAAa,CAAC,kBAAkB;IAC9C,UAAU;KACR,QAAQ,CAAC;KACT,MAAM,QAAQ;KACd,OAAO,CAAC;KACR,mBAAmB,KAAA;KACnB,SAAS,CAAC;IACZ;IACA,OAAO,KAAK,OAAO;GACrB;GACA,QAAQ,OAAO,KAAK,MAAM,YAAY,CAAC;GACvC,mBAAmB;GACnB;EACF;EAEA,IAAI,QAAQ,WAAW;GACrB,IAAI,MAAM,SAAS,GAAG,SAAS,KAAK,KAAK;GACzC;EACF;EAEA,IAAI,QAAQ,oBAAoB,QAAQ,mBAAmB,QAAQ,WAAW;GAC5E,QAAQ,KAAK,GAAG,IAAI,IAAI,OAAO;GAC/B,CAAC,SAAS,WAAW,YAAA,CAAa,KAAK,SAAS,KAAK,OAAO,QAAQ,CAAC,CAAC;GACtE;EACF;EACA,IAAI,YAAY,KAAA,GAAW;EAC3B,IAAI,QAAQ,WAAW,QAAQ,YAAY;GAEzC,mBAAmB;GACnB,IAAI,MAAM,SAAS,GACjB,QAAQ,MAAM,KAAK;IACjB,OAAO,QAAQ;IACf,SAAS,cAAc,KAAK;IAC5B,MAAM,QAAQ;GAChB,CAAC;EACL,OAAO,IAAI,QAAQ,eAAe;GAChC,MAAM,UAAU,OAAO,KAAK;GAC5B,IAAI,OAAO,SAAS,OAAO,KAAK,WAAW,GAAG,QAAQ,oBAAoB;EAC5E;CACF;CACA,OAAO;EAAE;EAAQ;EAAU;EAAS;CAAY;AAClD;;;;;;;AAQA,MAAa,kBAAkB,SAAiB,SAA0B;CACxE,MAAM,WAAW,QAAQ,SAAS,GAAG;CACrC,MAAM,SAAS,WAAW,QAAQ,MAAM,GAAG,EAAE,IAAI,QAAA,CAAS,MAAM,GAAG;CACnE,MAAM,QAAQ,MAAM;CACpB,IAAI,CAAC,KAAK,WAAW,KAAK,GAAG,OAAO;CACpC,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,KAAK,WAAW,MAAM;CAClE,IAAI,KAAK,MAAM;CACf,MAAM,OAAO,MAAM,SAAS;CAC5B,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,SAAS,GAAG;EAC5C,MAAM,OAAO,MAAM;EAEnB,IAAI,KAAK,WAAW,GAAG;EACvB,MAAM,QAAQ,KAAK,QAAQ,MAAM,EAAE;EACnC,IAAI,UAAU,IAAI,OAAO;EACzB,KAAK,QAAQ,KAAK;CACpB;CACA,MAAM,OAAO,MAAM;CACnB,IAAI,CAAC,UAAU,OAAO,KAAK,WAAW,KAAK,KAAK,QAAQ,MAAM,EAAE,MAAM;CACtE,OAAO,KAAK,SAAS,KAAK,UAAU,MAAM,KAAK,SAAS,IAAI;AAC9D;AAUA,MAAM,aAAa,QAAgB,UAA0B;CAC3D,MAAM,SAAS,UAAU,MAAM,MAAM,aAAa,KAAK;CACvD,MAAM,QAAQ,OAAO,OAAO,QAAQ,UAClC,MAAM,OAAO,MAAM,UAAU,UAAU,OAAO,QAAQ,KAAK,MAAM,MAAM,CACzE;CACA,MAAM,SACJ,MAAM,SAAS,IAAI,QAAQ,OAAO,OAAO,QAAQ,UAAU,MAAM,OAAO,SAAS,GAAG,CAAC;CACvF,MAAM,SAAS,OAAO,SAAS,UAC7B,MAAM,sBAAsB,KAAA,IAAY,CAAC,IAAI,CAAC,MAAM,iBAAiB,CACvE;CACA,OAAO;EACL,OAAO,OAAO,SAAS,UAAU,MAAM,KAAK;EAC5C,QAAQ;EACR,mBAAmB,OAAO,SAAS,IAAI,KAAK,IAAI,GAAG,MAAM,IAAI,KAAA;EAC7D,UAAU,MAAM,SAAS;CAC3B;AACF;;AAGA,MAAM,gBAAgB,OAA8B,SAAyC;CAC3F,IAAI;CACJ,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,GAAG;EACzC,IACE,SAAS,KAAA,KACT,KAAK,QAAQ,SAAS,KAAK,QAAQ,UAClC,KAAK,QAAQ,WAAW,KAAK,QAAQ,UAAU,KAAK,SAAS,CAAC,KAAK,OAEpE,OAAO;CAEX;CACA,OAAO;AACT;;;;;;;;AAoBA,MAAa,UACX,QACA,QACA,iBACmB;CACnB,MAAM,OAAO,cAAc,aAAa,WAAW,IAAI,MAAM,YAAY;CACzE,MAAM,UAAU,UAAU,QAAQ,OAAO,MAAM,GAAG;CAElD,MAAM,SAAS,CAAC,QAAQ,iBAAiB;CACzC,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,IAAI,MAAM,UAAU,OAAO,KAAK,MAAM,iBAAiB;CACzD;CACA,MAAM,QAAQ,OAAO,QAAQ,UAA2B,UAAU,KAAA,CAAS;CAC3E,MAAM,oBAAoB,MAAM,SAAS,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,KAAA;CAClE,IAAI,SAAS,eACX,OAAO;EAAE,SAAS;EAAM,IAAI,KAAA;EAAW;EAAmB,MAAM,KAAA;EAAW,QAAQ,CAAC;CAAE;CAExF,MAAM,OAAO,aAAa,QAAQ,OAAO,IAAI;CAC7C,IAAI,SAAS,KAAA,KAAa,CAAC,KAAK,OAC9B,OAAO;EAAE,SAAS;EAAO,IAAI,OAAO;EAAI;EAAmB;EAAM,QAAQ,QAAQ;CAAO;CAE1F,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,MAAM,UAAU,MAAM,WAAW,aAAa,MAAM,OAAO,IAAI,IAAI,KAAA;EACnE,IAAI,YAAY,KAAA,KAAa,CAAC,QAAQ,OACpC,OAAO;GAAE,SAAS;GAAO,IAAI;GAAO;GAAmB,MAAM;GAAS,QAAQ,MAAM;EAAO;CAE/F;CACA,OAAO;EAAE,SAAS;EAAM,IAAI,KAAA;EAAW;EAAmB;EAAM,QAAQ,QAAQ;CAAO;AACzF;;;;;;;;;;;AAYA,MAAa,cACX,QACA,QACA,iBACa;CACb,MAAM,OAAO,cAAc,aAAa,WAAW,IAAI,MAAM,YAAY;CACzE,MAAM,SAAS,IAAI,IAAI,UAAU,QAAQ,OAAO,MAAM,GAAG,CAAC,CAAC,MAAM;CACjE,KAAK,MAAM,SAAS,OAAO,MAAM,CAAC,GAAG;EACnC,MAAM,QAAQ,UAAU,QAAQ,KAAK;EACrC,IAAI,MAAM,UAAU,KAAK,MAAM,SAAS,MAAM,QAAQ,OAAO,IAAI,KAAK;CACxE;CACA,MAAM,SAAyB,CAAC;CAChC,MAAM,QAAQ,UAAyC;EAErD,MAAM,uBAAO,IAAI,IAAuD;EACxE,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,KAAK,SAAS,KAAA,KAAa,CAAC,eAAe,KAAK,MAAM,IAAI,GAAG;GACjE,MAAM,SAAS,KAAK,MAAM,UAAU;GACpC,MAAM,SAAS,KAAK,IAAI,KAAK,GAAG;GAChC,IAAI,WAAW,KAAA,KAAa,SAAS,OAAO,QAC1C,KAAK,IAAI,KAAK,KAAK;IAAE;IAAQ,OAAO,CAAC,IAAI;GAAE,CAAC;QACzC,IAAI,WAAW,OAAO,QAAQ,OAAO,MAAM,KAAK,IAAI;EAC3D;EACA,KAAK,MAAM,EAAE,OAAO,UAAU,KAAK,OAAO,GAAG,OAAO,KAAK,GAAG,IAAI;CAClE;CACA,KAAK,OAAO,eAAe,CAAC,CAAC;CAC7B,KAAK,MAAM,SAAS,QAAQ,KAAK,MAAM,WAAW,CAAC,CAAC;CACpD,OAAO,OAAO,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,SAAS,GAAG,KAAK,IAAI,IAAI,KAAK,OAAO;AAC1F;;AAGA,MAAa,qBAAqB,YAA4B;CAC5D,GAAG;CACH,aAAa;AACf;;AAGA,MAAa,cACX;;;;;;;AAQF,MAAa,gBAAgB,aAAwC,SAA0B;CAC7F,MAAM,QAAQ,KAAK,WAAW,CAAC,MAAM,QAAS,KAAK,MAAM,CAAC,IAAI,KAAA,CAAM,MAAM,GAAG,IAAI;CACjF,IAAI,uCAAuC,KAAK,IAAI,GAAG,OAAO;CAC9D,OACE,QAAQ,KAAK,eAAe,EAAE,KAC9B,sDAAsD,KAAK,IAAI;AAEnE;;;;;;AAOA,MAAa,sBACX,QACA,aACA,SACW;CACX,IAAI,SAAS,OAAO,UAAU,KAAK,OAAO,iBAAiB,QAAQ,KAAA,CAAS;CAC5E,MAAM,OAAO,OAAO,SAAS,WAAW,OAAO,IAAI,YAAY,OAAO,CAAC,CAAC,OAAO,IAAI;CACnF,IAAI,aAAa,aAAa,IAAI,GAAG,OAAO,kBAAkB,WAAW;CACzE,OAAO,YAAY,IAAI;AACzB;;AAGA,MAAa,oBAAoB,QAAgB,SAAqC;CACpF,IAAI,UAAU,OAAO,SAAS,KAAK,OAAO,YAAY,QAAQ,EAAE;CAChE,IAAI,WAAW,KAAK,OAAO,kBAAkB,0BAA0B;CACvE,IAAI,UAAU,OAAO,SAAS,KAAK,OAAO;CAC1C,OAAO,kBAAkB,WAAW,IAAI,gBAAgB,UAAU,OAAO,MAAM,GAAG;AACpF"}

package/build/{runtime-BwOZJ71n.mjs → runtime-C0gNyPHi.mjs}

@@ -1,12 +1,12 @@
 import { n as RenderRefused } from "./render-CKE2LILH.mjs";
-import { n as DEFAULT_TOKEN } from "./version-CHqkjAh1.mjs";
+import { n as DEFAULT_TOKEN } from "./version-B_Mpbx8R.mjs";
 import { classify, isClientError, parseHttpDate } from "./classify.mjs";
 import { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, checkPolicy, equalJitterMs, policyFor, realClock, restAfter, validatorsToSend } from "./politeness.mjs";
 import { decide, isProductToken, robotsFromResponse, robotsFromStatus, signalsFor, unreachableRobots } from "./robots.mjs";
 import { NO_SIGNALS, metaSignals, readSignals } from "./signals.mjs";
 import { n as canonicalUrl, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
 import { a as signerOf } from "./webbotauth-Dq_dA_u3.mjs";
-import { o as decodeBody, r as typesOf, t as extractPage } from "./html-DkQUAE_d.mjs";
+import { o as decodeBody, r as typesOf, t as extractPage } from "./html-DzobO0fy.mjs";
 import { createHash } from "node:crypto";
 import { Transform } from "node:stream";
 import { brotliDecompressSync, constants, createBrotliDecompress, createGunzip, createInflate, createInflateRaw, gunzipSync, inflateRawSync, inflateSync } from "node:zlib";
@@ -2642,6 +2642,12 @@ const TOKEN_SPLIT = /[^\p{L}\p{N}\p{Co}\p{Mc}]+/u;
 /** This function gives the tokens of a text, as the search index reads them. */
 const tokensOf = (text) => fold(text, "search").split(TOKEN_SPLIT).filter((token) => token.length > 0);
 const MAX_PARTS = 16;
+/** This function removes the stars at the end of a word. It uses no regular expression, so its time is linear. */
+const withoutStars = (word) => {
+	let end = word.length;
+	while (end > 0 && word.charCodeAt(end - 1) === 42) end -= 1;
+	return word.slice(0, end);
+};
 /**
 * These English words are in almost every document. In the form that needs one part, they match
 * each document and cost the most time. Thus that form leaves them out when other words remain.
@@ -2697,7 +2703,7 @@ const parseQuery = (q) => {
 		const phrase = match[1] !== void 0;
 		const raw = phrase ? match[1] : match[3];
 		const prefix = phrase ? match[2] === "*" : raw.endsWith("*");
-		const tokens = tokensOf(phrase ? raw : raw.replace(/\*+$/, ""));
+		const tokens = tokensOf(phrase ? raw : withoutStars(raw));
 		if (tokens.length === 0) continue;
 		parts.push({
 			text: `"${tokens.join(" ")}"${prefix ? " *" : ""}`,
@@ -4387,4 +4393,4 @@ const catchUpIndex = async (runtime) => {
 //#endregion
 export { DEADLINE_NOTE as A, fold as C, snippetOf as D, parseQuery as E, EgressRefused as F, egressLookup as I, makeRecord as M, sha256 as N, CAPTURE_SCHEMA as O, stableStringify as P, readPdf as S, locate as T, archivedDocId as _, fetchPage as a, fallbackReason as b, isHeldBack as c, recordOverride as d, reindex as f, archivedCopy as g, archiveExtractor as h, fetchAndIndex as i, DEFAULT_SECRET_PARAMS as j, Crawler as k, isoTime as l, versionAtTime as m, HELD_BACK as n, findDocuments as o, verifyQuote as p, catchUpIndex as r, heldBackMeta as s, Extractors as t, readDocument as u, archivedNote as v, foldText as w, isPdf as x, archivedOf as y };
 
-//# sourceMappingURL=runtime-BwOZJ71n.mjs.map
+//# sourceMappingURL=runtime-C0gNyPHi.mjs.map