polite-crawler 0.1.3 → 0.2.0

package/build/index.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.mjs","names":["answered"],"sources":["../src/recipe-venue.ts","../src/recipe-product.ts"],"sourcesContent":["/*\n * The venue recipe reads the opening hours of a venue (a restaurant, a shop, a museum) from its own\n * site. It reads the cheap sources first:\n *\n * 1. The home page: its JSON-LD (`openingHoursSpecification` and `openingHours`) and its text.\n * 2. The few pages of the site that most likely hold hours or a menu, by the words of their links.\n * 3. An OpenStreetMap `opening_hours` value, when the caller gives one. The recipe sends no\n *    request to OpenStreetMap.\n *\n * Each reading names the page, the document and the capture that it came from. The recipe gives\n * readings only. To compare them with the hours that an application keeps is the work of the\n * application, and `compareHours` is the pure helper for it.\n */\nimport type { Doc } from \"./docs.ts\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport type { Archived } from \"./fallback.ts\";\nimport { archivedOf } from \"./fallback.ts\";\nimport type { Day, Hours, Week } from \"./hours.ts\";\nimport {\n  DAYS,\n  describeWeek,\n  disagreements,\n  fromCompact,\n  fromSchemaOrg,\n  fromText,\n} from \"./hours.ts\";\nimport type { PageLink } from \"./html.ts\";\nimport { typesOf } from \"./html.ts\";\nimport type { RecipeTarget } from \"./recipe.ts\";\nimport { deadlineOf, runtimeOf } from \"./recipe.ts\";\nimport type { PageOptions, PageResult } from \"./runtime.ts\";\nimport { HELD_BACK, fetchPage, isHeldBack, recordOverride } from \"./runtime.ts\";\n\n/** These are the limits of the venue recipe. */\nexport const VENUE_LIMITS: {\n  /** The pages of the site that a read fetches after the home page, by default. */\n  readonly pages: number;\n  /** The most pages after the home page. */\n  readonly maxPages: number;\n  /** The most menu links of a result. */\n  readonly menus: number;\n  /** The most readings of one page. A page with more is a list of something else. */\n  readonly readingsPerPage: number;\n  /** The most links of one page that the recipe ranks. */\n  readonly links: number;\n  /** The distance in metres within which a JSON-LD location is the venue. */\n  readonly nearMetres: number;\n} = { pages: 3, maxPages: 8, menus: 8, readingsPerPage: 20, links: 200, nearMetres: 150 };\n\n// ── words ───────────────────────────────────────────────────────────────────\n\nconst STOP_WORDS: ReadonlySet<string> = new Set([\"the\", \"a\", \"an\", \"and\", \"of\", \"at\"]);\n\n/** This function gives the words of a text in one form: lower case, no accents, and `&` as `and`. */\nconst wordsOf = (text: string): string[] =>\n  text\n    .toLowerCase()\n    .normalize(\"NFKD\")\n    .replace(/\\p{M}/gu, \"\")\n    .replace(/&/g, \" and \")\n    .replace(/[^\\p{L}\\p{N} ]/gu, \" \")\n    .split(/\\s+/)\n    .filter((word) => word.length > 0 && !STOP_WORDS.has(word));\n\n/**\n * This function tells if a page names a venue: the page has the first word of the name and 60\n * percent of its words. A domain that a venue let go can show a different business. It gives\n * undefined for a name with no word.\n */\nexport const namesVenue = (\n  name: string,\n  page: { readonly title: string; readonly text: string },\n): boolean | undefined => {\n  const words = wordsOf(name);\n  if (words.length === 0) return undefined;\n  const onPage = new Set(wordsOf(`${page.title} ${page.text}`));\n  const named = new Set(words);\n  const found = [...named].filter((word) => onPage.has(word)).length;\n  return onPage.has(words[0]!) && found / named.size >= 0.6;\n};\n\n// ── the links of the site ───────────────────────────────────────────────────\n\n/** The words of a link that tell what its page holds, with their weights. */\nconst LINK_WORDS: ReadonlyArray<readonly [RegExp, number]> = [\n  [/hours|horario|horaires|opening/, 6],\n  [/menu|food|drink|wine|cocktail/, 4],\n  [/contact|location|visit|find.?us|directions/, 3],\n  [/about|info/, 2],\n  [/lunch|dinner|brunch|breakfast/, 2],\n];\n\n/** The hosts of review and social sites: a menu there is a different page about the venue. */\nconst SOCIAL =\n  /(?:^|\\.)(?:yelp|tripadvisor|facebook|instagram|twitter|x|tiktok|google|apple|spotify)\\.[a-z.]+$/i;\n\n/** The files that hold no hours as text. */\nconst FILE = /\\.(?:pdf|jpe?g|png|gif|webp|svg|mp4)(?:[?#]|$)/i;\n\nconst hostOf = (url: string): string | undefined => {\n  try {\n    return new URL(url).hostname.toLowerCase().replace(/^www\\./, \"\");\n  } catch {\n    return undefined;\n  }\n};\n\nconst withoutHash = (url: string): string => url.replace(/#.*$/s, \"\").replace(/\\/$/, \"\");\n\n/**\n * This function ranks the links of a page by the words of their text and path. The links to\n * the hours or a menu come first. It keeps only the links on the site of `home`,\n * and it leaves out the files (a PDF, an image) and the home page itself.\n */\nexport const rankHoursLinks = (links: readonly PageLink[], home: string): PageLink[] => {\n  const site = hostOf(home);\n  const self = withoutHash(home);\n  const seen = new Set<string>();\n  const scored: Array<{ readonly link: PageLink; readonly score: number }> = [];\n  for (const link of links.slice(0, VENUE_LIMITS.links)) {\n    if (typeof link?.href !== \"string\" || hostOf(link.href) !== site || FILE.test(link.href))\n      continue;\n    const key = withoutHash(link.href);\n    if (key === self || seen.has(key)) continue;\n    let path = \"\";\n    try {\n      path = new URL(link.href).pathname;\n    } catch {\n      continue;\n    }\n    const words = `${typeof link.text === \"string\" ? link.text : \"\"} ${path}`.toLowerCase();\n    const score = LINK_WORDS.reduce(\n      (sum, [pattern, weight]) => sum + (pattern.test(words) ? weight : 0),\n      0,\n    );\n    if (score <= 0) continue;\n    seen.add(key);\n    scored.push({ link, score });\n  }\n  // a stable sort: links of equal score keep the order of the page\n  return scored.sort((a, b) => b.score - a.score).map((entry) => entry.link);\n};\n\n/** A link to a menu. */\nexport interface VenueLink {\n  readonly url: string;\n  readonly text: string;\n}\n\n// ── the location of a venue ─────────────────────────────────────────────────\n\n/** The place of a venue, to choose its hours on a page that lists more than one location. */\nexport interface VenueLocation {\n  /** This text is the street address, for example `12 Via Esempio`. The number and the street count. */\n  readonly address?: string | undefined;\n  readonly lat?: number | undefined;\n  readonly lon?: number | undefined;\n}\n\ninterface Point {\n  readonly lat: number;\n  readonly lon: number;\n}\n\nconst EARTH_METRES = 6_371_000;\n\n/** This function gives the distance in metres between two points on the earth (the haversine formula). */\nconst metresBetween = (a: Point, b: Point): number => {\n  const rad = Math.PI / 180;\n  const dLat = (b.lat - a.lat) * rad;\n  const dLon = (b.lon - a.lon) * rad;\n  const h =\n    Math.sin(dLat / 2) ** 2 +\n    Math.cos(a.lat * rad) * Math.cos(b.lat * rad) * Math.sin(dLon / 2) ** 2;\n  return 2 * EARTH_METRES * Math.asin(Math.min(1, Math.sqrt(h)));\n};\n\nconst pointOf = (lat: unknown, lon: unknown): Point | undefined => {\n  const la = typeof lat === \"string\" ? Number.parseFloat(lat) : lat;\n  const lo = typeof lon === \"string\" ? Number.parseFloat(lon) : lon;\n  return typeof la === \"number\" &&\n    typeof lo === \"number\" &&\n    Number.isFinite(la) &&\n    Number.isFinite(lo) &&\n    Math.abs(la) <= 90 &&\n    Math.abs(lo) <= 180\n    ? { lat: la, lon: lo }\n    : undefined;\n};\n\n/** The words of a street that tell nothing about which street: directions and kinds of road. */\nconst STREET_NOISE: ReadonlySet<string> = new Set([\n  \"n\",\n  \"s\",\n  \"e\",\n  \"w\",\n  \"north\",\n  \"south\",\n  \"east\",\n  \"west\",\n  \"st\",\n  \"street\",\n  \"ave\",\n  \"avenue\",\n  \"rd\",\n  \"road\",\n  \"blvd\",\n  \"boulevard\",\n  \"dr\",\n  \"drive\",\n  \"ln\",\n  \"lane\",\n  \"ct\",\n  \"court\",\n  \"pl\",\n  \"place\",\n  \"sq\",\n  \"square\",\n  \"hwy\",\n  \"highway\",\n  \"pkwy\",\n  \"parkway\",\n  \"way\",\n  \"ter\",\n  \"terrace\",\n  \"suite\",\n  \"ste\",\n  \"unit\",\n]);\n\n/** This function gives the number of a street address and the words that name its street. */\nconst streetOf = (\n  address: string,\n): { readonly number: string; readonly words: ReadonlySet<string> } | undefined => {\n  const words = wordsOf((address.split(\",\")[0] ?? address).slice(0, 200));\n  const number = words.find((word) => /^\\d+[a-z]?$/.test(word));\n  if (number === undefined) return undefined;\n  return {\n    number,\n    words: new Set(words.filter((word) => !/^\\d/.test(word) && !STREET_NOISE.has(word))),\n  };\n};\n\nconst addressText = (value: unknown): string | undefined => {\n  if (typeof value === \"string\") return value;\n  if (Array.isArray(value)) return addressText(value[0]);\n  if (typeof value === \"object\" && value !== null) {\n    const street = (value as Record<string, unknown>)[\"streetAddress\"];\n    return typeof street === \"string\" ? street : undefined;\n  }\n  return undefined;\n};\n\n/**\n * This function tells if a JSON-LD location record is the venue. The record must have the same\n * number on a street that shares a word, or a position near the venue (`VENUE_LIMITS.nearMetres`).\n */\nexport const locationMatches = (\n  record: Readonly<Record<string, unknown>>,\n  location: VenueLocation,\n): boolean => {\n  const geo = record[\"geo\"];\n  const at =\n    typeof geo === \"object\" && geo !== null\n      ? pointOf(\n          (geo as Record<string, unknown>)[\"latitude\"],\n          (geo as Record<string, unknown>)[\"longitude\"],\n        )\n      : undefined;\n  const ours = pointOf(location.lat, location.lon);\n  if (at !== undefined && ours !== undefined && metresBetween(at, ours) <= VENUE_LIMITS.nearMetres)\n    return true;\n  const theirs = addressText(record[\"address\"]);\n  if (theirs === undefined || location.address === undefined) return false;\n  const a = streetOf(theirs);\n  const b = streetOf(location.address);\n  return (\n    a !== undefined &&\n    b !== undefined &&\n    a.number === b.number &&\n    [...a.words].some((word) => b.words.has(word))\n  );\n};\n\nconst located = (record: Readonly<Record<string, unknown>>): boolean =>\n  record[\"address\"] !== undefined || record[\"geo\"] !== undefined;\n\n// ── the readings ────────────────────────────────────────────────────────────\n\n/** One reading of the hours, with the source that it came from. */\nexport interface VenueReading {\n  /** The value `jsonld` is the structured data of a page, `text` the text of a page, and `osm` the value of the caller. */\n  readonly from: \"jsonld\" | \"osm\" | \"text\";\n  /**\n   * This value orders the readings, 0 first. JSON-LD is 0, a text under an hours heading is 1 and\n   * OpenStreetMap is 2. A text with no label of a different schedule is 3. A different schedule is\n   * 4, for example a brunch, a kitchen, a happy hour or an office.\n   */\n  readonly rank: number;\n  /** The value `opening` is the opening hours of the venue (a rank of 3 or less). Else it is `other`. */\n  readonly schedule: \"opening\" | \"other\";\n  /** This text is the line above a reading of the text, for example `Hours` or `Happy hour`. */\n  readonly label: string | null;\n  /** This text is the week as words, from `describeWeek`. */\n  readonly hours: string;\n  readonly week: Week;\n  /** This value is true when the reader guessed an am or a pm. */\n  readonly inferred: boolean;\n  /** This value is true when the reader left out a rule or a line (a holiday, a date). */\n  readonly partial: boolean;\n  /** This text names the season or the dates of the hours. */\n  readonly season: string | null;\n  /** This text is the words that the hours came from. A reading of the text can be a quote for `verify`. */\n  readonly raw: string;\n  /** This path names the JSON-LD record of the reading in the metadata of the document, for example `jsonld[2]`. */\n  readonly path: string | null;\n  /** This URL is the page of the reading. */\n  readonly url: string | null;\n  readonly doc: string | null;\n  /** This id is the capture of the text that the reading came from. A citation names it. */\n  readonly capture: string | null;\n  readonly fetched_at: string | null;\n  /** This value tells that the page is an archived copy, with the archive and the date of the copy. */\n  readonly archived?: Archived | undefined;\n}\n\n/** A stored document, or a version of one: what `venueReadingsOf` reads. */\nexport interface VenueSource {\n  readonly doc?: string | null | undefined;\n  readonly url: string | null;\n  readonly text: string;\n  readonly meta: Readonly<Record<string, unknown>>;\n  readonly capture?: string | null | undefined;\n  readonly fetched_at?: string | null | undefined;\n}\n\n/** The result of `venueReadingsOf`: what one document says. */\nexport interface VenueRead {\n  readonly readings: readonly VenueReading[];\n  /** This value is the first `telephone` of the JSON-LD records of the venue. */\n  readonly phone: string | null;\n  readonly menus: readonly VenueLink[];\n  readonly notes: readonly string[];\n  /** This value counts the work of the reader. The tests use it, because it does not change with the computer. */\n  readonly steps: number;\n}\n\n/** The types of JSON-LD records that have the hours of a venue. */\nconst HOURS_TYPES =\n  /Restaurant|LocalBusiness|FoodEstablishment|BarOrPub|CafeOrCoffeeShop|Bakery|Store|Place|Winery|Brewery|NightClub|Museum|TouristAttraction|Library|EntertainmentBusiness/;\n\n/**\n * A label that names a schedule that is not the opening hours. The line above a schedule is often a\n * different thing, for example a phone number, and such a line tells nothing.\n */\nconst OTHER_SCHEDULE =\n  /brunch|lunch|dinner|breakfast|happy|kitchen|\bbar\b|deliver|take.?out|pick.?up|patio|holiday|season|summer|winter|special|late|menu/i;\n\n/**\n * A label of a service beside the opening hours, or of an office. Its hours are a different\n * schedule, also when the label says \"hours\" (\"Happy hour\", \"Kitchen hours\", \"Office hours\").\n */\nconst SERVICE =\n  /happy|kitchen|deliver|take.?out|pick.?up|brunch|holiday|special|patio|\bbar\b|late.?night|office/i;\n\nconst rankOf = (from: VenueReading[\"from\"], label: string | null): number => {\n  if (from === \"jsonld\") return 0;\n  if (from === \"osm\") return 2;\n  const text = label ?? \"\";\n  if (SERVICE.test(text)) return 4;\n  if (/hour|open/i.test(text)) return 1;\n  return OTHER_SCHEDULE.test(text) ? 4 : 3;\n};\n\nconst reading = (\n  from: VenueReading[\"from\"],\n  hours: Hours,\n  label: string | null,\n  source: {\n    readonly url: string | null;\n    readonly doc: string | null;\n    readonly capture: string | null;\n    readonly fetched_at: string | null;\n    readonly path: string | null;\n    readonly archived?: Archived | undefined;\n  },\n): VenueReading => {\n  const rank = rankOf(from, label);\n  return {\n    from,\n    rank,\n    schedule: rank <= 3 ? \"opening\" : \"other\",\n    label,\n    hours: describeWeek(hours.week),\n    week: hours.week,\n    inferred: hours.inferred,\n    partial: hours.partial === true,\n    season: hours.season ?? null,\n    raw: hours.raw,\n    path: source.path,\n    url: source.url,\n    doc: source.doc,\n    capture: source.capture,\n    fetched_at: source.fetched_at,\n    ...(source.archived === undefined ? {} : { archived: source.archived }),\n  };\n};\n\nconst recordsOf = (meta: Readonly<Record<string, unknown>>): Record<string, unknown>[] => {\n  const jsonld = meta[\"jsonld\"];\n  return Array.isArray(jsonld)\n    ? jsonld.filter(\n        (record): record is Record<string, unknown> =>\n          record !== null && typeof record === \"object\" && !Array.isArray(record),\n      )\n    : [];\n};\n\nconst linksOf = (meta: Readonly<Record<string, unknown>>): PageLink[] => {\n  const links = meta[\"links\"];\n  return Array.isArray(links)\n    ? links.filter(\n        (link): link is PageLink =>\n          link !== null && typeof link === \"object\" && typeof (link as PageLink).href === \"string\",\n      )\n    : [];\n};\n\n/**\n * This function reads the hours of one stored document, or of one version of it. It is pure, and\n * it sends no request. Thus it also reads a version at a time (`versionAtTime`) or an archived copy.\n *\n * - It reads each JSON-LD record of a venue type with `fromSchemaOrg`, and the text with `fromText`.\n * - On a page that lists more than one location, it reads only the location that matches\n *   `location`. With no `location`, it reads none of them, and a note says so.\n * - It gives the first `telephone` and the menu links that are not on a review or social site.\n */\nexport const venueReadingsOf = (\n  source: VenueSource,\n  options: { readonly location?: VenueLocation | undefined } = {},\n): VenueRead => {\n  let steps = 0;\n  const notes: string[] = [];\n  const archived = archivedOf(source.meta);\n  const where = {\n    url: archived?.original ?? source.url,\n    doc: source.doc ?? null,\n    capture: source.capture ?? null,\n    fetched_at: source.fetched_at ?? null,\n    ...(archived === undefined ? {} : { archived }),\n  };\n  const all = recordsOf(source.meta);\n  steps += all.length;\n  const venues = all\n    .map((record, index) => ({ record, index }))\n    .filter(({ record }) => typesOf(record).some((type) => HOURS_TYPES.test(type)));\n  // a page of a group lists each of its locations: only the hours of this venue are its hours\n  let usable = venues;\n  const placed = venues.filter(({ record }) => located(record));\n  if (placed.length > 1) {\n    const location = options.location;\n    const ours =\n      location === undefined\n        ? []\n        : placed.filter(({ record }) => locationMatches(record, location));\n    if (ours.length === 0) {\n      notes.push(\n        location === undefined\n          ? `${where.url ?? \"the page\"} lists ${String(placed.length)} locations: give location to read the hours of one`\n          : `${where.url ?? \"the page\"} lists ${String(placed.length)} locations, and none matched the location: their hours were not read`,\n      );\n    }\n    usable = [...ours, ...venues.filter(({ record }) => !located(record))];\n  }\n  const readings: VenueReading[] = [];\n  let phone: string | null = null;\n  for (const { record, index } of usable) {\n    steps += 1;\n    const hours = fromSchemaOrg(record);\n    if (hours !== undefined && readings.length < VENUE_LIMITS.readingsPerPage) {\n      readings.push(reading(\"jsonld\", hours, null, { ...where, path: `jsonld[${String(index)}]` }));\n    }\n    const telephone = record[\"telephone\"];\n    if (phone === null && typeof telephone === \"string\" && telephone.trim().length > 0)\n      phone = telephone.trim().slice(0, 100);\n  }\n  steps += source.text.length;\n  for (const hours of fromText(source.text)) {\n    if (readings.length >= VENUE_LIMITS.readingsPerPage) {\n      notes.push(\n        `${where.url ?? \"the page\"} gives more than ${String(VENUE_LIMITS.readingsPerPage)} readings: the rest were not read`,\n      );\n      break;\n    }\n    readings.push(reading(\"text\", hours, hours.label, { ...where, path: null }));\n  }\n  const menus: VenueLink[] = [];\n  for (const link of linksOf(source.meta).slice(0, VENUE_LIMITS.links)) {\n    steps += 1;\n    if (menus.length >= VENUE_LIMITS.menus) break;\n    const text = typeof link.text === \"string\" ? link.text.slice(0, 200) : \"\";\n    const host = hostOf(link.href);\n    if (host === undefined || SOCIAL.test(host) || !/menu/i.test(`${text} ${link.href}`)) continue;\n    if (menus.some((menu) => menu.url === link.href)) continue;\n    menus.push({ url: link.href, text });\n  }\n  return { readings, phone, menus, notes, steps };\n};\n\n// ── the recipe ──────────────────────────────────────────────────────────────\n\n/** An OpenStreetMap `opening_hours` value, with where the caller got it. */\nexport interface VenueOsm {\n  /** This text is the value of the tag, for example `Mo-Fr 08:00-18:00; Sa 09:00-13:00`. */\n  readonly value: string;\n  readonly url?: string | null | undefined;\n  readonly doc?: string | null | undefined;\n  readonly capture?: string | null | undefined;\n  readonly fetched_at?: string | null | undefined;\n}\n\n/** These options control `readVenue`. */\nexport interface VenueOptions {\n  /** This value is the number of pages after the home page, from 0 to `VENUE_LIMITS.maxPages`. The default is 3. */\n  readonly pages?: number | undefined;\n  /** This text is the name of the venue. The result tells if the home page names it (`names_venue`). */\n  readonly name?: string | undefined;\n  /** With true, a home page that does not name the venue gives no reading, and the recipe fetches no more pages. */\n  readonly requireName?: boolean | undefined;\n  /** This value is the place of the venue, for a page that lists more than one location. */\n  readonly location?: VenueLocation | undefined;\n  /** This value is an OpenStreetMap `opening_hours` value. The recipe reads it, and sends no request for it. */\n  readonly osm?: string | VenueOsm | undefined;\n  /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */\n  readonly render?: \"auto\" | \"never\" | \"always\" | undefined;\n  /** A stored copy younger than this age answers with no request. The default is `24h`. */\n  readonly maxAge?: Duration | undefined;\n  /** These agents get their robots.txt rules applied too, for example `Claude-User`. */\n  readonly onBehalfOf?: readonly string[] | undefined;\n  readonly signal?: AbortSignal | undefined;\n  /** No request starts after this time from the start of the read, for example `\"2m\"`. */\n  readonly budget?: Duration | undefined;\n  readonly scope?: string | undefined;\n  readonly labels?: Readonly<Record<string, string>> | undefined;\n  /** This name connects the captures to the job that asked for them. The default is `venue`. */\n  readonly job?: string | undefined;\n  /** This value is true when a person gave this URL directly. It is the only way to read a held-back page. The crawler logs it. */\n  readonly userProvided?: boolean | undefined;\n  /** The value `archive` reads an archived copy of a page that the site refuses, or when the site is down. */\n  readonly fallback?: \"archive\" | \"none\" | undefined;\n  readonly archiveDespiteRobots?: boolean | undefined;\n}\n\n/** One page that the venue recipe fetched. */\nexport interface VenuePage {\n  readonly url: string;\n  readonly doc: string | null;\n  readonly outcome: string;\n  readonly status: number;\n  readonly rendered: boolean;\n  readonly capture: string | null;\n  readonly fetched_at: string | null;\n  readonly from_cache: boolean;\n  readonly held_back: boolean;\n  readonly archived?: Archived | undefined;\n  readonly note: string | null;\n}\n\n/** The result of `readVenue`. */\nexport interface VenueResult {\n  /** This URL is the site that the caller gave. */\n  readonly url: string;\n  /** This URL is where the home page landed, after the redirects. */\n  readonly home: string | null;\n  /** This URL is the new site, when the home page landed on a different host. */\n  readonly moved: string | null;\n  /** This value tells if the home page names the venue. It is null with no `name`. */\n  readonly names_venue: boolean | null;\n  readonly pages: readonly VenuePage[];\n  /** These are the readings, one for each source, label and week: the best first. */\n  readonly readings: readonly VenueReading[];\n  /** This reading is the best reading of the opening hours, from any source. */\n  readonly best: VenueReading | null;\n  /** This reading is the best reading of the site itself (not OpenStreetMap) with no guess. A check for a change compares it. */\n  readonly from_site: VenueReading | null;\n  /** These days are the days where two readings of the whole schedule (a rank of 2 or less) disagree. */\n  readonly disagree: readonly Day[];\n  readonly menus: readonly VenueLink[];\n  readonly phone: string | null;\n  /** These ids are the captures of the pages that the recipe read. */\n  readonly captures: readonly string[];\n  /** This value is true when a page asked not to be AI input, so the recipe read nothing from it. */\n  readonly held_back: boolean;\n  readonly note: string | null;\n}\n\nconst answered = (page: PageResult): boolean =>\n  page.outcome === \"ok\" || page.outcome === \"not-modified\" || page.archived !== undefined;\n\n/**\n * This function reads the hours of a venue from its site. It fetches the home page and the pages\n * that most likely hold hours or a menu. Each one is a fetch of the crawler. Thus robots.txt, the\n * pace, the stops, the capture log and the rule for a held-back page apply. It renders a page only\n * when the page needs JavaScript, as `fetch` does.\n *\n * It gives each reading with its source, the best reading, and the days where the readings\n * disagree. It writes nothing but the capture log. To choose the true hours is the decision of\n * the application.\n */\nexport const readVenue = async (\n  target: RecipeTarget,\n  url: string,\n  options: VenueOptions = {},\n): Promise<VenueResult> => {\n  const runtime = runtimeOf(target);\n  const extra = options.pages ?? VENUE_LIMITS.pages;\n  if (!Number.isInteger(extra) || extra < 0 || extra > VENUE_LIMITS.maxPages) {\n    throw new RangeError(\n      `pages must be a whole number from 0 to ${String(VENUE_LIMITS.maxPages)}, not ${String(extra)}`,\n    );\n  }\n  let site: URL;\n  try {\n    site = new URL(url);\n  } catch {\n    throw new TypeError(`readVenue needs the URL of a site, not ${JSON.stringify(url)}`);\n  }\n  if (site.protocol !== \"http:\" && site.protocol !== \"https:\")\n    throw new TypeError(`readVenue needs an http or https URL, not ${url}`);\n  const maxAge = options.maxAge ?? \"24h\";\n  const job = options.job ?? \"venue\";\n  const userProvided = options.userProvided ?? false;\n  const pageOptions: PageOptions = {\n    render: options.render ?? \"auto\",\n    maxAgeMs: maxAge === \"auto\" ? \"auto\" : durationMs(maxAge),\n    onBehalfOf: options.onBehalfOf,\n    job,\n    signal: options.signal,\n    scope: options.scope,\n    labels: options.labels,\n    deadline: deadlineOf(runtime, options.budget),\n    fallback: options.fallback,\n    archiveDespiteRobots: options.archiveDespiteRobots,\n  };\n  const notes: string[] = [];\n  const readings: VenueReading[] = [];\n  const pages: VenuePage[] = [];\n  const menus: VenueLink[] = [];\n  const captures = new Set<string>();\n  let phone: string | null = null;\n  let heldBack = false;\n\n  if (options.osm !== undefined) {\n    const osm: VenueOsm = typeof options.osm === \"string\" ? { value: options.osm } : options.osm;\n    const hours = typeof osm.value === \"string\" ? fromCompact(osm.value, \"osm\") : undefined;\n    if (hours === undefined) {\n      notes.push(\n        `the OpenStreetMap hours cannot be read: ${JSON.stringify(String(osm.value).slice(0, 200))}`,\n      );\n    } else {\n      readings.push(\n        reading(\"osm\", hours, null, {\n          url: osm.url ?? null,\n          doc: osm.doc ?? null,\n          capture: osm.capture ?? null,\n          fetched_at: osm.fetched_at ?? null,\n          path: null,\n        }),\n      );\n      if (typeof osm.capture === \"string\") captures.add(osm.capture);\n    }\n  }\n\n  /** This function fetches one page, and gives its document when the recipe can read it. */\n  const readPage = async (pageUrl: string): Promise<Doc | undefined> => {\n    const page = await fetchPage(runtime, pageUrl, pageOptions);\n    const doc = page.doc === null ? undefined : runtime.index.get(page.doc);\n    const held = doc !== undefined && isHeldBack(doc, userProvided);\n    pages.push({\n      url: pageUrl,\n      doc: page.doc,\n      outcome: page.outcome,\n      status: page.status,\n      rendered: page.rendered,\n      capture: page.capture,\n      fetched_at: page.fetched_at,\n      from_cache: page.from_cache,\n      held_back: held,\n      ...(page.archived === undefined ? {} : { archived: page.archived }),\n      note: page.note,\n    });\n    if (!answered(page)) {\n      notes.push(`${pageUrl}: ${page.outcome}${page.note === null ? \"\" : ` (${page.note})`}`);\n    }\n    if (doc === undefined) return undefined;\n    if (held) {\n      heldBack = true;\n      return undefined;\n    }\n    if (doc.ai_input === \"no\") await recordOverride(runtime, doc, job);\n    captures.add(doc.capture);\n    return doc;\n  };\n\n  const readDoc = (doc: Doc): void => {\n    const read = venueReadingsOf(doc, { location: options.location });\n    readings.push(...read.readings);\n    notes.push(...read.notes);\n    phone ??= read.phone;\n    for (const menu of read.menus) {\n      if (menus.length < VENUE_LIMITS.menus && !menus.some((known) => known.url === menu.url))\n        menus.push(menu);\n    }\n  };\n\n  let homeUrl: string | null = null;\n  let namesVenueResult: boolean | null = null;\n  const home = await readPage(site.href);\n  if (home !== undefined) {\n    homeUrl = archivedOf(home.meta)?.original ?? home.url ?? site.href;\n    if (options.name !== undefined) {\n      namesVenueResult = namesVenue(options.name, home) ?? null;\n      if (namesVenueResult === false) {\n        notes.push(\n          `${homeUrl} does not name ${options.name}: the domain can belong to a different business now`,\n        );\n      }\n    }\n    if (namesVenueResult === false && options.requireName === true) {\n      notes.push(\"requireName: the recipe read no hours from a site that does not name the venue\");\n    } else {\n      readDoc(home);\n      for (const link of rankHoursLinks(linksOf(home.meta), homeUrl).slice(0, extra)) {\n        if (options.signal?.aborted === true) break;\n        const doc = await readPage(link.href);\n        if (doc !== undefined) readDoc(doc);\n      }\n    }\n  }\n  const moved = homeUrl !== null && hostOf(homeUrl) !== hostOf(site.href) ? homeUrl : null;\n  if (moved !== null) notes.push(`${site.href} now goes to ${moved}`);\n  if (heldBack) notes.push(HELD_BACK);\n\n  // one reading for each source, label and week: a site that repeats its JSON-LD on each page said it once\n  const firstOf = new Map<string, VenueReading>();\n  for (const one of readings) {\n    const key = `${one.from}|${one.label ?? \"\"}|${one.hours}|${one.season ?? \"\"}`;\n    if (!firstOf.has(key)) firstOf.set(key, one);\n  }\n  // the best first; of two equal readings, the reading as written before a guess\n  const unique = [...firstOf.values()].sort(\n    (a, b) => a.rank - b.rank || Number(a.inferred) - Number(b.inferred),\n  );\n  const disagree = new Set<Day>();\n  const whole = unique.filter((one) => one.rank <= 2);\n  for (let i = 0; i < whole.length; i += 1) {\n    for (let j = i + 1; j < whole.length; j += 1) {\n      for (const day of disagreements(whole[i]!.week, whole[j]!.week)) disagree.add(day);\n    }\n  }\n  const best = unique.find((one) => one.schedule === \"opening\") ?? null;\n  const fromSite =\n    unique.find((one) => one.from !== \"osm\" && one.schedule === \"opening\" && !one.inferred) ?? null;\n  return {\n    url: site.href,\n    home: homeUrl,\n    moved,\n    names_venue: namesVenueResult,\n    pages,\n    readings: unique,\n    best,\n    from_site: fromSite,\n    disagree: DAYS.filter((day) => disagree.has(day)),\n    menus,\n    phone,\n    captures: [...captures],\n    held_back: heldBack,\n    note: notes.length > 0 ? notes.join(\"; \") : null,\n  };\n};\n","/*\n * The product recipe reads a product page: its schema.org Product data. Most shop systems write\n * this data on each product page. The recipe gives the name, the brand, the codes (SKU and GTIN),\n * the price as an amount and a currency, a range of prices across the variants, the availability,\n * the seller, the image, and the variant that the URL names.\n *\n * A shop that refuses the crawler is a clean stop: the result says `wall`, with a note, and the\n * crawler does not try a different identity.\n */\nimport type { Doc } from \"./docs.ts\";\nimport type { Duration } from \"./duration.ts\";\nimport { durationMs } from \"./duration.ts\";\nimport type { Archived } from \"./fallback.ts\";\nimport { archivedOf } from \"./fallback.ts\";\nimport { extractPage, typesOf } from \"./html.ts\";\nimport type { RecipeTarget } from \"./recipe.ts\";\nimport { deadlineOf, runtimeOf } from \"./recipe.ts\";\nimport type { CrawlRuntime, PageOptions, PageResult } from \"./runtime.ts\";\nimport { HELD_BACK, fetchPage, isHeldBack, recordOverride } from \"./runtime.ts\";\nimport { decodeBody } from \"./text.ts\";\n\ntype Json = Readonly<Record<string, unknown>>;\n\n/** These are the limits of the product reader. A page with more is hostile or broken, and the reader stops there. */\nexport const PRODUCT_LIMITS: {\n  /** The most JSON-LD records that the reader examines. */\n  readonly records: number;\n  /** The most offers of one product or variant. */\n  readonly offers: number;\n  /** The most variants of one product. */\n  readonly variants: number;\n  /** The levels of `hasVariant` and of nested offers that the reader follows. */\n  readonly depth: number;\n  /** The longest text of a price. */\n  readonly priceChars: number;\n} = { records: 500, offers: 200, variants: 200, depth: 2, priceChars: 64 };\n\n/** An amount of money, as schema.org Product data gives it. */\nexport interface Money {\n  /** This value is the amount in the unit of the currency, for example 12.5. */\n  readonly amount: number;\n  /** This value is the ISO 4217 code of the currency, for example `EUR`. */\n  readonly currency: string;\n  /** This text is the price as the page wrote it, for a check with `extract`. */\n  readonly raw: string;\n}\n\n/** The variant of a product that the URL names. */\nexport interface ProductVariant {\n  readonly name: string | null;\n  readonly sku: string | null;\n  readonly price: Money | null;\n  readonly availability: string | null;\n}\n\n/** The result of `productOf`: what the Product data of a page says. */\nexport interface ProductRead {\n  /** This value is true when the page has a Product record. */\n  readonly found: boolean;\n  readonly name: string | null;\n  readonly brand: string | null;\n  readonly sku: string | null;\n  readonly gtin: string | null;\n  /** This value is the lowest price of the offers, in the first currency of the page. */\n  readonly price: Money | null;\n  /** This value is the highest price, when the offers give a range (sizes, colors, variants). */\n  readonly price_to: Money | null;\n  /**\n   * This value is the schema.org name of the availability, for example `InStock`. When the offers\n   * do not agree, it is `InStock` if one offer is in stock, and `availabilities` gives each value.\n   */\n  readonly availability: string | null;\n  readonly availabilities: readonly string[];\n  /** This value is the variant that the URL names (`?variant=`, `sku` or `gtin`), read alone. */\n  readonly variant: ProductVariant | null;\n  readonly seller: string | null;\n  readonly image: string | null;\n  readonly notes: readonly string[];\n  /** This value counts the work of the reader. The tests use it, because it does not change with the computer. */\n  readonly steps: number;\n}\n\nconst PRODUCT_TYPES = /^(?:Product|ProductGroup|IndividualProduct|ProductModel|SomeProducts)$/;\n\n/** A list price or a struck-through price is the old price, not the price. */\nconst WAS_PRICE = /ListPrice|StrikethroughPrice|MSRP|SRP/i;\n\nconst GTIN_KEYS = [\"gtin13\", \"gtin12\", \"gtin14\", \"gtin8\", \"gtin\"] as const;\n\n/** Two symbols that name one currency only. The other symbols (`$`, `¥`) name more than one. */\nconst SYMBOLS: Readonly<Record<string, string>> = { \"€\": \"EUR\", \"£\": \"GBP\" };\n\nlet knownCurrencies: ReadonlySet<string> | undefined;\n/** This function tells if a code is an ISO 4217 currency that the runtime knows. */\nconst isCurrency = (code: string): boolean => {\n  if (!/^[A-Z]{3}$/.test(code)) return false;\n  if (knownCurrencies === undefined) {\n    try {\n      knownCurrencies = new Set(Intl.supportedValuesOf(\"currency\"));\n    } catch {\n      // a runtime with no list: each code of three capital letters counts\n      knownCurrencies = new Set();\n    }\n  }\n  return knownCurrencies.size === 0 || knownCurrencies.has(code);\n};\n\n/** This function gives the number of digits after the decimal point of a currency: 2 for EUR, 0 for JPY. */\nconst fractionDigits = (currency: string | undefined): number | undefined => {\n  if (currency === undefined) return undefined;\n  try {\n    return new Intl.NumberFormat(\"en\", { style: \"currency\", currency }).resolvedOptions()\n      .maximumFractionDigits;\n  } catch {\n    return undefined;\n  }\n};\n\n/**\n * This function reads an amount as a shop writes it: `19.99`, `1,299.00`, `1.299,00`, `$19.99` or\n * the number 19.99. The last separator is the decimal point, with one exception. One kind of\n * separator, each followed by three digits, separates thousands: `1.299` is 1299. For a\n * currency with three decimal digits, one point with three digits is the decimal point. A text\n * with two numbers (a range), a negative amount or zero gives undefined.\n */\nexport const readAmount = (value: unknown, currency?: string): number | undefined => {\n  if (typeof value === \"number\") return Number.isFinite(value) && value > 0 ? value : undefined;\n  if (typeof value !== \"string\" || value.length > PRODUCT_LIMITS.priceChars) return undefined;\n  // one number, with the currency and the spaces around it\n  const match = /^[^\\d-]*?(\\d(?:[\\d.,'   ]*\\d)?)[^\\d]*$/.exec(value.trim());\n  if (match === null) return undefined;\n  let digits = match[1]!.replace(/['   ]/g, \"\");\n  const last = Math.max(digits.lastIndexOf(\".\"), digits.lastIndexOf(\",\"));\n  if (last !== -1) {\n    const kinds = new Set(digits.match(/[.,]/g));\n    const groups = kinds.size === 1 && /^[1-9]\\d{0,2}(?:[.,]\\d{3})+$/.test(digits);\n    const decimalThree = groups && fractionDigits(currency) === 3 && /^\\d+\\.\\d{3}$/.test(digits);\n    digits =\n      groups && !decimalThree\n        ? digits.replace(/[.,]/g, \"\")\n        : `${digits.slice(0, last).replace(/[.,]/g, \"\")}.${digits.slice(last + 1)}`;\n  }\n  const amount = Number(digits);\n  return Number.isFinite(amount) && amount > 0 ? amount : undefined;\n};\n\nconst list = (value: unknown, limit: number): unknown[] =>\n  (Array.isArray(value) ? value : value === undefined || value === null ? [] : [value]).slice(\n    0,\n    limit,\n  );\n\nconst objects = (value: unknown, limit: number): Json[] =>\n  list(value, limit).filter(\n    (item): item is Json => typeof item === \"object\" && item !== null && !Array.isArray(item),\n  );\n\n/** This function gives a name, written as a text or as an object that has one. */\nconst nameOf = (value: unknown): string | null => {\n  for (const item of list(value, 20)) {\n    if (typeof item === \"string\" && item.trim().length > 0) return item.trim().slice(0, 500);\n    if (typeof item === \"object\" && item !== null) {\n      const named = (item as Json)[\"name\"];\n      if (typeof named === \"string\" && named.trim().length > 0) return named.trim().slice(0, 500);\n    }\n  }\n  return null;\n};\n\nconst textOf = (value: unknown): string | null =>\n  typeof value === \"string\" && value.trim().length > 0\n    ? value.trim().slice(0, 200)\n    : typeof value === \"number\" && Number.isFinite(value)\n      ? String(value)\n      : null;\n\nconst imageOf = (value: unknown): string | null => {\n  for (const item of list(value, 20)) {\n    if (typeof item === \"string\" && /^https?:/i.test(item)) return item;\n    const url = typeof item === \"object\" && item !== null ? (item as Json)[\"url\"] : undefined;\n    if (typeof url === \"string\" && /^https?:/i.test(url)) return url;\n  }\n  return null;\n};\n\ninterface Offers {\n  readonly prices: Money[];\n  readonly unknownCurrencies: Set<string>;\n  readonly availability: Set<string>;\n  readonly sellers: Set<string>;\n  /** An amount with no currency, which is not a price. */\n  unstated: boolean;\n  steps: number;\n}\n\nconst emptyOffers = (): Offers => ({\n  prices: [],\n  unknownCurrencies: new Set(),\n  availability: new Set(),\n  sellers: new Set(),\n  unstated: false,\n  steps: 0,\n});\n\n/** This function reads the offers of a product into `read`, and the offers of its variants to `depth` levels. */\nconst readOffers = (product: Json, read: Offers, depth: number): void => {\n  const money = (value: unknown, currency: unknown): void => {\n    const written = typeof currency === \"string\" ? currency.trim() : \"\";\n    const code = SYMBOLS[written] ?? written.toUpperCase();\n    const amount = readAmount(value, code.length > 0 ? code : undefined);\n    if (amount === undefined) return;\n    if (code.length === 0) {\n      read.unstated = true;\n      return;\n    }\n    if (!isCurrency(code)) {\n      read.unknownCurrencies.add(written.slice(0, 20));\n      return;\n    }\n    read.prices.push({\n      amount,\n      currency: code,\n      raw: String(value).slice(0, PRODUCT_LIMITS.priceChars),\n    });\n  };\n  const offer = (o: Json, level: number): void => {\n    read.steps += 1;\n    const spec = objects(o[\"priceSpecification\"], PRODUCT_LIMITS.offers).find((entry) => {\n      const kind = entry[\"priceType\"];\n      return !(typeof kind === \"string\" && WAS_PRICE.test(kind));\n    });\n    const currency = o[\"priceCurrency\"] ?? spec?.[\"priceCurrency\"];\n    money(o[\"price\"] ?? spec?.[\"price\"], currency);\n    money(o[\"lowPrice\"], currency);\n    money(o[\"highPrice\"], currency);\n    const availability =\n      typeof o[\"availability\"] === \"string\"\n        ? o[\"availability\"]\n            .trim()\n            .replace(/^.*[/#:]/, \"\")\n            .slice(0, 60)\n        : undefined;\n    if (availability !== undefined && availability.length > 0) read.availability.add(availability);\n    const seller = nameOf(o[\"seller\"]);\n    if (seller !== null) read.sellers.add(seller);\n    // an AggregateOffer can list its offers\n    if (level < PRODUCT_LIMITS.depth) {\n      for (const inner of objects(o[\"offers\"], PRODUCT_LIMITS.offers)) offer(inner, level + 1);\n    }\n  };\n  for (const o of objects(product[\"offers\"], PRODUCT_LIMITS.offers)) offer(o, 0);\n  if (depth <= 0) return;\n  for (const variant of objects(product[\"hasVariant\"], PRODUCT_LIMITS.variants)) {\n    read.steps += 1;\n    readOffers(variant, read, depth - 1);\n  }\n};\n\nconst availabilityOf = (read: Offers): string | null =>\n  read.availability.size === 0\n    ? null\n    : read.availability.size === 1 || !read.availability.has(\"InStock\")\n      ? [...read.availability][0]!\n      : \"InStock\";\n\nconst pathKey = (value: string, page: string): string | undefined => {\n  // \"#product\" names a part of this page: a related product can have such an id too\n  if (value.startsWith(\"#\")) return undefined;\n  try {\n    const url = new URL(value, page);\n    return `${url.hostname.toLowerCase().replace(/^www\\./, \"\")}${url.pathname.replace(/\\/$/, \"\")}`;\n  } catch {\n    return undefined;\n  }\n};\n\n/** This function gives the size of a record as JSON, or 0 for a record that JSON cannot write (a nesting too deep). */\nconst sizeOf = (record: Json): number => {\n  try {\n    return JSON.stringify(record).length;\n  } catch {\n    return 0;\n  }\n};\n\n/**\n * This function picks the product of a page from its JSON-LD records. A page often has more\n * records of products: the related products, the products that others bought. It picks:\n *\n * 1. The product whose `url` or `@id` is the page (the path, with no query).\n * 2. Else the largest product with offers or variants.\n * 3. Else the largest product.\n *\n * It also reads the `mainEntity` of a record (a `WebPage` or an `ItemPage` that names its product).\n */\nexport const pickProduct = (\n  records: readonly Json[],\n  page: string,\n): { readonly product: Json | undefined; readonly steps: number } => {\n  let steps = 0;\n  const products: Json[] = [];\n  for (const record of records.slice(0, PRODUCT_LIMITS.records)) {\n    steps += 1;\n    if (typesOf(record as Record<string, unknown>).some((type) => PRODUCT_TYPES.test(type)))\n      products.push(record);\n    for (const inner of objects(record[\"mainEntity\"], 20)) {\n      steps += 1;\n      if (typesOf(inner as Record<string, unknown>).some((type) => PRODUCT_TYPES.test(type)))\n        products.push(inner);\n    }\n  }\n  const here = pathKey(page, page);\n  const own = products.find((product) =>\n    [product[\"url\"], product[\"@id\"]].some(\n      (value) => typeof value === \"string\" && here !== undefined && pathKey(value, page) === here,\n    ),\n  );\n  if (own !== undefined) return { product: own, steps };\n  const selling = products.filter(\n    (product) => list(product[\"offers\"], 1).length > 0 || list(product[\"hasVariant\"], 1).length > 0,\n  );\n  const pool = selling.length > 0 ? selling : products;\n  let best: Json | undefined;\n  let size = -1;\n  for (const product of pool) {\n    const length = sizeOf(product);\n    steps += length;\n    if (length > size) {\n      best = product;\n      size = length;\n    }\n  }\n  return { product: best, steps };\n};\n\n/** This function gives the variant that a URL names: the `variant` parameter of Shopify, or a `sku` or `gtin` parameter. */\nexport const pickVariant = (product: Json, page: string): Json | undefined => {\n  let params: URLSearchParams;\n  try {\n    params = new URL(page).searchParams;\n  } catch {\n    return undefined;\n  }\n  const variantId = params.get(\"variant\");\n  const code = params.get(\"sku\") ?? params.get(\"gtin\");\n  if (variantId === null && code === null) return undefined;\n  const names = (value: unknown): boolean => {\n    if (typeof value !== \"string\" || variantId === null) return false;\n    try {\n      return (\n        new URL(value, page).searchParams.get(\"variant\") === variantId ||\n        value.endsWith(`/${variantId}`)\n      );\n    } catch {\n      return false;\n    }\n  };\n  return objects(product[\"hasVariant\"], PRODUCT_LIMITS.variants).find((variant) => {\n    const offers = objects(variant[\"offers\"], PRODUCT_LIMITS.offers);\n    const codes = [\n      variant[\"sku\"],\n      variant[\"gtin\"],\n      variant[\"gtin13\"],\n      variant[\"gtin12\"],\n      variant[\"productID\"],\n    ]\n      .map(textOf)\n      .filter((one): one is string => one !== null);\n    return (\n      (code !== null && codes.includes(code)) ||\n      [variant[\"@id\"], variant[\"url\"], ...offers.map((offer) => offer[\"url\"])].some(names) ||\n      (variantId !== null && codes.includes(variantId))\n    );\n  });\n};\n\nconst byAmount = (a: Money, b: Money): number => a.amount - b.amount;\n\n/**\n * This function reads the Product data of a page from its JSON-LD records. It is pure, and it sends\n * no request. Thus it also reads a version at a time, or the stored JSON-LD of a document.\n *\n * - The prices are in the first currency of the offers. A price with no currency is not a price.\n * - A list price or a struck-through price is not the price.\n * - A currency must be an ISO 4217 code (`€` and `£` count too). The notes name each other one.\n */\nexport const productOf = (records: readonly Json[], page: string): ProductRead => {\n  const picked = pickProduct(records, page);\n  const product = picked.product;\n  if (product === undefined) {\n    return {\n      found: false,\n      name: null,\n      brand: null,\n      sku: null,\n      gtin: null,\n      price: null,\n      price_to: null,\n      availability: null,\n      availabilities: [],\n      variant: null,\n      seller: null,\n      image: null,\n      notes: [\n        \"no schema.org Product data on the page: a list of products, or a shop that does not publish it\",\n      ],\n      steps: picked.steps,\n    };\n  }\n  const read = emptyOffers();\n  readOffers(product, read, PRODUCT_LIMITS.depth);\n  const currency = read.prices[0]?.currency;\n  const sorted = read.prices.filter((price) => price.currency === currency).sort(byAmount);\n  const low = sorted[0] ?? null;\n  const high = sorted.at(-1) ?? null;\n  const gtin = GTIN_KEYS.map((key) => textOf(product[key])).find((value) => value !== null);\n  const chosen = pickVariant(product, page);\n  let variant: ProductVariant | null = null;\n  let steps = picked.steps + read.steps;\n  if (chosen !== undefined) {\n    const own = emptyOffers();\n    readOffers(chosen, own, 0);\n    steps += own.steps;\n    const prices = own.prices.filter((price) => price.currency === currency).sort(byAmount);\n    variant = {\n      name: nameOf(chosen[\"name\"]),\n      sku: textOf(chosen[\"sku\"]),\n      price: prices[0] ?? null,\n      availability: availabilityOf(own),\n    };\n  }\n  const notes = [\n    read.unknownCurrencies.size > 0\n      ? `a currency that is not an ISO 4217 code: ${[...read.unknownCurrencies].join(\", \")}`\n      : null,\n    read.unstated && sorted.length === 0\n      ? \"the page gives a price with no currency, so the price is not read as money\"\n      : sorted.length === 0\n        ? \"the page names the product and gives no price\"\n        : null,\n    new Set(read.prices.map((price) => price.currency)).size > 1\n      ? `the offers have more than one currency: the prices are in ${currency ?? \"\"}`\n      : null,\n    /[?&](?:variant|sku|gtin)=/.test(page) && chosen === undefined\n      ? \"the URL names a variant that the Product data does not list: the price is the whole range\"\n      : null,\n  ].filter((note): note is string => note !== null);\n  return {\n    found: true,\n    name: nameOf(product[\"name\"]),\n    brand: nameOf(product[\"brand\"]),\n    sku: textOf(product[\"sku\"]),\n    gtin: gtin ?? null,\n    price: low,\n    price_to: high !== null && low !== null && high.amount > low.amount ? high : null,\n    availability: availabilityOf(read),\n    availabilities: [...read.availability],\n    variant,\n    seller: read.sellers.size === 0 ? null : [...read.sellers].join(\", \"),\n    image: imageOf(product[\"image\"]),\n    notes,\n    steps,\n  };\n};\n\n// ── the recipe ──────────────────────────────────────────────────────────────\n\n/** These options control `readProduct`. */\nexport interface ProductOptions {\n  /** The value `auto` (the default) renders once more when the plain page has no Product data. `never` does not render. */\n  readonly render?: \"auto\" | \"never\" | \"always\" | undefined;\n  /** A stored copy younger than this age answers with no request. The default is `24h`. */\n  readonly maxAge?: Duration | undefined;\n  /** These agents get their robots.txt rules applied too, for example `Claude-User`. */\n  readonly onBehalfOf?: readonly string[] | undefined;\n  readonly signal?: AbortSignal | undefined;\n  /** No request starts after this time from the start of the read, for example `\"1m\"`. */\n  readonly budget?: Duration | undefined;\n  readonly scope?: string | undefined;\n  readonly labels?: Readonly<Record<string, string>> | undefined;\n  /** This name connects the captures to the job that asked for them. The default is `product`. */\n  readonly job?: string | undefined;\n  /** This value is true when a person gave this URL directly. It is the only way to read a held-back page. The crawler logs it. */\n  readonly userProvided?: boolean | undefined;\n  /** The value `archive` reads an archived copy of a page that the shop refuses. Its prices are of the date of the copy. */\n  readonly fallback?: \"archive\" | \"none\" | undefined;\n  readonly archiveDespiteRobots?: boolean | undefined;\n}\n\n/** The result of `readProduct`. */\nexport interface ProductResult extends Omit<ProductRead, \"steps\" | \"notes\"> {\n  readonly url: string;\n  /** This value is true when the page says what it sells, in structured data: the Product has a name. */\n  readonly verified: boolean;\n  /**\n   * This value is true when the site refused the crawler. The causes are a 401, a 402 or a 403, a\n   * challenge, a bot check, a stop of the host or robots.txt. The recipe stops there, and the note\n   * says why. With `fallback: \"archive\"`, it stays true when an archived copy answers.\n   */\n  readonly wall: boolean;\n  readonly outcome: string;\n  readonly status: number;\n  readonly rendered: boolean;\n  readonly doc: string | null;\n  /** This id is the capture of the text that the recipe read. A citation names it. */\n  readonly capture: string | null;\n  readonly fetched_at: string | null;\n  readonly captures: readonly string[];\n  readonly from_cache: boolean;\n  /** This value is true when the page asked not to be AI input, so the recipe read nothing from it. */\n  readonly held_back: boolean;\n  /** This value tells that the page is an archived copy, with the archive and the date of the copy. */\n  readonly archived?: Archived | undefined;\n  readonly note: string | null;\n}\n\nconst WALLS: ReadonlySet<string> = new Set([\"blocked\", \"challenge\", \"payment-required\"]);\n\n/** This function tells if the site refused the crawler. */\nconst wallOf = (page: PageResult): boolean =>\n  WALLS.has(page.outcome) ||\n  (page.outcome === \"skipped\" && /^(?:robots\\.txt disallows|host stopped)/.test(page.note ?? \"\")) ||\n  (page.doc === null && /\\bbot check\\b/.test(page.note ?? \"\"));\n\nconst answered = (page: PageResult): boolean =>\n  page.doc !== null &&\n  (page.outcome === \"ok\" || page.outcome === \"not-modified\" || page.archived !== undefined);\n\n/**\n * This function gives the JSON-LD records of a document from its stored body. The index keeps a\n * part of the JSON-LD of a large page only, and a product with many variants can be larger. An\n * archived copy gives the JSON-LD that the index keeps.\n */\nconst recordsOfDoc = async (runtime: CrawlRuntime, doc: Doc): Promise<readonly Json[]> => {\n  const kept = Array.isArray(doc.meta[\"jsonld\"]) ? (doc.meta[\"jsonld\"] as Json[]) : [];\n  if (doc.blob === null || archivedOf(doc.meta) !== undefined) return kept;\n  const body = await runtime.blobs.get(doc.blob);\n  if (body === undefined) return kept;\n  const at = doc.url ?? \"\";\n  const type = doc.rendered\n    ? \"text/html; charset=utf-8\"\n    : (runtime.index.latest(at, \"plain\")?.content_type ?? null);\n  try {\n    return extractPage(decodeBody(body, type), at).jsonld;\n  } catch {\n    return kept;\n  }\n};\n\n/**\n * This function reads a product page. It fetches the page through the crawler: robots.txt, the\n * pace, the stops, the capture log and the rule for a held-back page apply. When the plain page has\n * no Product data, it renders the page once, because some shops write that data with JavaScript.\n *\n * A shop that refuses the crawler gives `wall: true` and a note. The recipe writes nothing but the\n * capture log. `extract` can check each value against the stored page.\n */\nexport const readProduct = async (\n  target: RecipeTarget,\n  url: string,\n  options: ProductOptions = {},\n): Promise<ProductResult> => {\n  const runtime = runtimeOf(target);\n  try {\n    const parsed = new URL(url);\n    if (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\") throw new TypeError(url);\n  } catch {\n    throw new TypeError(`readProduct needs an http or https URL, not ${JSON.stringify(url)}`);\n  }\n  const maxAge = options.maxAge ?? \"24h\";\n  const job = options.job ?? \"product\";\n  const userProvided = options.userProvided ?? false;\n  const render = options.render ?? \"auto\";\n  const deadline = deadlineOf(runtime, options.budget);\n  const pageOptions = (mode: \"auto\" | \"never\" | \"always\"): PageOptions => ({\n    render: mode,\n    maxAgeMs: maxAge === \"auto\" ? \"auto\" : durationMs(maxAge),\n    onBehalfOf: options.onBehalfOf,\n    job,\n    signal: options.signal,\n    scope: options.scope,\n    labels: options.labels,\n    deadline,\n    fallback: options.fallback,\n    archiveDespiteRobots: options.archiveDespiteRobots,\n  });\n  let page = await fetchPage(runtime, url, pageOptions(render));\n  const notes: string[] = [];\n  const result = (\n    from: PageResult,\n    read: Omit<ProductRead, \"steps\"> | undefined,\n    more: { readonly held_back?: boolean } = {},\n  ): ProductResult => {\n    const all = [from.note, ...notes, ...(read?.notes ?? [])].filter(\n      (part): part is string => part !== null && part !== undefined && part.length > 0,\n    );\n    return {\n      url,\n      found: read?.found ?? false,\n      verified: read?.found === true && read.name !== null,\n      name: read?.name ?? null,\n      brand: read?.brand ?? null,\n      sku: read?.sku ?? null,\n      gtin: read?.gtin ?? null,\n      price: read?.price ?? null,\n      price_to: read?.price_to ?? null,\n      availability: read?.availability ?? null,\n      availabilities: read?.availabilities ?? [],\n      variant: read?.variant ?? null,\n      seller: read?.seller ?? null,\n      image: read?.image ?? null,\n      wall: wallOf(from),\n      outcome: from.outcome,\n      status: from.status,\n      rendered: from.rendered,\n      doc: from.doc,\n      capture: from.capture,\n      fetched_at: from.fetched_at,\n      captures: [...from.captures],\n      from_cache: from.from_cache,\n      held_back: more.held_back ?? false,\n      ...(from.archived === undefined ? {} : { archived: from.archived }),\n      note: all.length > 0 ? [...new Set(all)].join(\"; \") : null,\n    };\n  };\n  if (!answered(page)) {\n    const wall = wallOf(page);\n    notes.push(\n      wall\n        ? `the shop refused the crawler (${page.outcome}): the recipe read nothing, and it does not try a different identity`\n        : `the page gave no document (${page.outcome})`,\n    );\n    return result(page, undefined);\n  }\n  const held = (from: PageResult): Doc | undefined => {\n    const doc = from.doc === null ? undefined : runtime.index.get(from.doc);\n    return doc !== undefined && isHeldBack(doc, userProvided) ? doc : undefined;\n  };\n  if (held(page) !== undefined) {\n    notes.push(HELD_BACK);\n    return result(page, undefined, { held_back: true });\n  }\n  const docOf = (from: PageResult): Doc | undefined =>\n    from.doc === null ? undefined : runtime.index.get(from.doc);\n  let doc = docOf(page);\n  let read = doc === undefined ? undefined : productOf(await recordsOfDoc(runtime, doc), url);\n  // some shops write their Product data with JavaScript: one render before the recipe says that the page has none\n  if (read?.found !== true && !page.rendered && render === \"auto\" && page.archived === undefined) {\n    const rendered = await fetchPage(runtime, url, pageOptions(\"always\"));\n    const shown = docOf(rendered);\n    if (\n      rendered.rendered &&\n      answered(rendered) &&\n      shown !== undefined &&\n      held(rendered) === undefined\n    ) {\n      page = rendered;\n      doc = shown;\n      read = productOf(await recordsOfDoc(runtime, shown), url);\n    } else if (rendered.note !== null && !rendered.rendered) {\n      notes.push(`the render gave nothing more: ${rendered.note}`);\n    }\n  }\n  if (userProvided && doc?.ai_input === \"no\") await recordOverride(runtime, doc, job);\n  return result(page, read);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAmCA,MAAa,eAaT;CAAE,OAAO;CAAG,UAAU;CAAG,OAAO;CAAG,iBAAiB;CAAI,OAAO;CAAK,YAAY;AAAI;AAIxF,MAAM,6BAAkC,IAAI,IAAI;CAAC;CAAO;CAAK;CAAM;CAAO;CAAM;AAAI,CAAC;;AAGrF,MAAM,WAAW,SACf,KACG,YAAY,CAAC,CACb,UAAU,MAAM,CAAC,CACjB,QAAQ,WAAW,EAAE,CAAC,CACtB,QAAQ,MAAM,OAAO,CAAC,CACtB,QAAQ,oBAAoB,GAAG,CAAC,CAChC,MAAM,KAAK,CAAC,CACZ,QAAQ,SAAS,KAAK,SAAS,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC;;;;;;AAO9D,MAAa,cACX,MACA,SACwB;CACxB,MAAM,QAAQ,QAAQ,IAAI;CAC1B,IAAI,MAAM,WAAW,GAAG,OAAO,KAAA;CAC/B,MAAM,SAAS,IAAI,IAAI,QAAQ,GAAG,KAAK,MAAM,GAAG,KAAK,MAAM,CAAC;CAC5D,MAAM,QAAQ,IAAI,IAAI,KAAK;CAC3B,MAAM,QAAQ,CAAC,GAAG,KAAK,CAAC,CAAC,QAAQ,SAAS,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC;CAC5D,OAAO,OAAO,IAAI,MAAM,EAAG,KAAK,QAAQ,MAAM,QAAQ;AACxD;;AAKA,MAAM,aAAuD;CAC3D,CAAC,kCAAkC,CAAC;CACpC,CAAC,iCAAiC,CAAC;CACnC,CAAC,8CAA8C,CAAC;CAChD,CAAC,cAAc,CAAC;CAChB,CAAC,iCAAiC,CAAC;AACrC;;AAGA,MAAM,SACJ;;AAGF,MAAM,OAAO;AAEb,MAAM,UAAU,QAAoC;CAClD,IAAI;EACF,OAAO,IAAI,IAAI,GAAG,CAAC,CAAC,SAAS,YAAY,CAAC,CAAC,QAAQ,UAAU,EAAE;CACjE,QAAQ;EACN;CACF;AACF;AAEA,MAAM,eAAe,QAAwB,IAAI,QAAQ,SAAS,EAAE,CAAC,CAAC,QAAQ,OAAO,EAAE;;;;;;AAOvF,MAAa,kBAAkB,OAA4B,SAA6B;CACtF,MAAM,OAAO,OAAO,IAAI;CACxB,MAAM,OAAO,YAAY,IAAI;CAC7B,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,SAAqE,CAAC;CAC5E,KAAK,MAAM,QAAQ,MAAM,MAAM,GAAG,aAAa,KAAK,GAAG;EACrD,IAAI,OAAO,MAAM,SAAS,YAAY,OAAO,KAAK,IAAI,MAAM,QAAQ,KAAK,KAAK,KAAK,IAAI,GACrF;EACF,MAAM,MAAM,YAAY,KAAK,IAAI;EACjC,IAAI,QAAQ,QAAQ,KAAK,IAAI,GAAG,GAAG;EACnC,IAAI,OAAO;EACX,IAAI;GACF,OAAO,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC;EAC5B,QAAQ;GACN;EACF;EACA,MAAM,QAAQ,GAAG,OAAO,KAAK,SAAS,WAAW,KAAK,OAAO,GAAG,GAAG,OAAO,YAAY;EACtF,MAAM,QAAQ,WAAW,QACtB,KAAK,CAAC,SAAS,YAAY,OAAO,QAAQ,KAAK,KAAK,IAAI,SAAS,IAClE,CACF;EACA,IAAI,SAAS,GAAG;EAChB,KAAK,IAAI,GAAG;EACZ,OAAO,KAAK;GAAE;GAAM;EAAM,CAAC;CAC7B;CAEA,OAAO,OAAO,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,KAAK,UAAU,MAAM,IAAI;AAC3E;AAuBA,MAAM,eAAe;;AAGrB,MAAM,iBAAiB,GAAU,MAAqB;CACpD,MAAM,MAAM,KAAK,KAAK;CACtB,MAAM,QAAQ,EAAE,MAAM,EAAE,OAAO;CAC/B,MAAM,QAAQ,EAAE,MAAM,EAAE,OAAO;CAC/B,MAAM,IACJ,KAAK,IAAI,OAAO,CAAC,KAAK,IACtB,KAAK,IAAI,EAAE,MAAM,GAAG,IAAI,KAAK,IAAI,EAAE,MAAM,GAAG,IAAI,KAAK,IAAI,OAAO,CAAC,KAAK;CACxE,OAAO,IAAI,eAAe,KAAK,KAAK,KAAK,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC;AAC/D;AAEA,MAAM,WAAW,KAAc,QAAoC;CACjE,MAAM,KAAK,OAAO,QAAQ,WAAW,OAAO,WAAW,GAAG,IAAI;CAC9D,MAAM,KAAK,OAAO,QAAQ,WAAW,OAAO,WAAW,GAAG,IAAI;CAC9D,OAAO,OAAO,OAAO,YACnB,OAAO,OAAO,YACd,OAAO,SAAS,EAAE,KAClB,OAAO,SAAS,EAAE,KAClB,KAAK,IAAI,EAAE,KAAK,MAChB,KAAK,IAAI,EAAE,KAAK,MACd;EAAE,KAAK;EAAI,KAAK;CAAG,IACnB,KAAA;AACN;;AAGA,MAAM,+BAAoC,IAAI,IAAI;CAChD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;;AAGD,MAAM,YACJ,YACiF;CACjF,MAAM,QAAQ,SAAS,QAAQ,MAAM,GAAG,CAAC,CAAC,MAAM,QAAA,CAAS,MAAM,GAAG,GAAG,CAAC;CACtE,MAAM,SAAS,MAAM,MAAM,SAAS,cAAc,KAAK,IAAI,CAAC;CAC5D,IAAI,WAAW,KAAA,GAAW,OAAO,KAAA;CACjC,OAAO;EACL;EACA,OAAO,IAAI,IAAI,MAAM,QAAQ,SAAS,CAAC,MAAM,KAAK,IAAI,KAAK,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC;CACrF;AACF;AAEA,MAAM,eAAe,UAAuC;CAC1D,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO,YAAY,MAAM,EAAE;CACrD,IAAI,OAAO,UAAU,YAAY,UAAU,MAAM;EAC/C,MAAM,SAAU,MAAkC;EAClD,OAAO,OAAO,WAAW,WAAW,SAAS,KAAA;CAC/C;AAEF;;;;;AAMA,MAAa,mBACX,QACA,aACY;CACZ,MAAM,MAAM,OAAO;CACnB,MAAM,KACJ,OAAO,QAAQ,YAAY,QAAQ,OAC/B,QACG,IAAgC,aAChC,IAAgC,YACnC,IACA,KAAA;CACN,MAAM,OAAO,QAAQ,SAAS,KAAK,SAAS,GAAG;CAC/C,IAAI,OAAO,KAAA,KAAa,SAAS,KAAA,KAAa,cAAc,IAAI,IAAI,KAAK,aAAa,YACpF,OAAO;CACT,MAAM,SAAS,YAAY,OAAO,UAAU;CAC5C,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,KAAA,GAAW,OAAO;CACnE,MAAM,IAAI,SAAS,MAAM;CACzB,MAAM,IAAI,SAAS,SAAS,OAAO;CACnC,OACE,MAAM,KAAA,KACN,MAAM,KAAA,KACN,EAAE,WAAW,EAAE,UACf,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,MAAM,SAAS,EAAE,MAAM,IAAI,IAAI,CAAC;AAEjD;AAEA,MAAM,WAAW,WACf,OAAO,eAAe,KAAA,KAAa,OAAO,WAAW,KAAA;;AA+DvD,MAAM,cACJ;;;;;AAMF,MAAM,iBACJ;;;;;AAMF,MAAM,UACJ;AAEF,MAAM,UAAU,MAA4B,UAAiC;CAC3E,IAAI,SAAS,UAAU,OAAO;CAC9B,IAAI,SAAS,OAAO,OAAO;CAC3B,MAAM,OAAO,SAAS;CACtB,IAAI,QAAQ,KAAK,IAAI,GAAG,OAAO;CAC/B,IAAI,aAAa,KAAK,IAAI,GAAG,OAAO;CACpC,OAAO,eAAe,KAAK,IAAI,IAAI,IAAI;AACzC;AAEA,MAAM,WACJ,MACA,OACA,OACA,WAQiB;CACjB,MAAM,OAAO,OAAO,MAAM,KAAK;CAC/B,OAAO;EACL;EACA;EACA,UAAU,QAAQ,IAAI,YAAY;EAClC;EACA,OAAO,aAAa,MAAM,IAAI;EAC9B,MAAM,MAAM;EACZ,UAAU,MAAM;EAChB,SAAS,MAAM,YAAY;EAC3B,QAAQ,MAAM,UAAU;EACxB,KAAK,MAAM;EACX,MAAM,OAAO;EACb,KAAK,OAAO;EACZ,KAAK,OAAO;EACZ,SAAS,OAAO;EAChB,YAAY,OAAO;EACnB,GAAI,OAAO,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,OAAO,SAAS;CACvE;AACF;AAEA,MAAM,aAAa,SAAuE;CACxF,MAAM,SAAS,KAAK;CACpB,OAAO,MAAM,QAAQ,MAAM,IACvB,OAAO,QACJ,WACC,WAAW,QAAQ,OAAO,WAAW,YAAY,CAAC,MAAM,QAAQ,MAAM,CAC1E,IACA,CAAC;AACP;AAEA,MAAM,WAAW,SAAwD;CACvE,MAAM,QAAQ,KAAK;CACnB,OAAO,MAAM,QAAQ,KAAK,IACtB,MAAM,QACH,SACC,SAAS,QAAQ,OAAO,SAAS,YAAY,OAAQ,KAAkB,SAAS,QACpF,IACA,CAAC;AACP;;;;;;;;;;AAWA,MAAa,mBACX,QACA,UAA6D,CAAC,MAChD;CACd,IAAI,QAAQ;CACZ,MAAM,QAAkB,CAAC;CACzB,MAAM,WAAW,WAAW,OAAO,IAAI;CACvC,MAAM,QAAQ;EACZ,KAAK,UAAU,YAAY,OAAO;EAClC,KAAK,OAAO,OAAO;EACnB,SAAS,OAAO,WAAW;EAC3B,YAAY,OAAO,cAAc;EACjC,GAAI,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS;CAC/C;CACA,MAAM,MAAM,UAAU,OAAO,IAAI;CACjC,SAAS,IAAI;CACb,MAAM,SAAS,IACZ,KAAK,QAAQ,WAAW;EAAE;EAAQ;CAAM,EAAE,CAAC,CAC3C,QAAQ,EAAE,aAAa,QAAQ,MAAM,CAAC,CAAC,MAAM,SAAS,YAAY,KAAK,IAAI,CAAC,CAAC;CAEhF,IAAI,SAAS;CACb,MAAM,SAAS,OAAO,QAAQ,EAAE,aAAa,QAAQ,MAAM,CAAC;CAC5D,IAAI,OAAO,SAAS,GAAG;EACrB,MAAM,WAAW,QAAQ;EACzB,MAAM,OACJ,aAAa,KAAA,IACT,CAAC,IACD,OAAO,QAAQ,EAAE,aAAa,gBAAgB,QAAQ,QAAQ,CAAC;EACrE,IAAI,KAAK,WAAW,GAClB,MAAM,KACJ,aAAa,KAAA,IACT,GAAG,MAAM,OAAO,WAAW,SAAS,OAAO,OAAO,MAAM,EAAE,sDAC1D,GAAG,MAAM,OAAO,WAAW,SAAS,OAAO,OAAO,MAAM,EAAE,qEAChE;EAEF,SAAS,CAAC,GAAG,MAAM,GAAG,OAAO,QAAQ,EAAE,aAAa,CAAC,QAAQ,MAAM,CAAC,CAAC;CACvE;CACA,MAAM,WAA2B,CAAC;CAClC,IAAI,QAAuB;CAC3B,KAAK,MAAM,EAAE,QAAQ,WAAW,QAAQ;EACtC,SAAS;EACT,MAAM,QAAQ,cAAc,MAAM;EAClC,IAAI,UAAU,KAAA,KAAa,SAAS,SAAS,aAAa,iBACxD,SAAS,KAAK,QAAQ,UAAU,OAAO,MAAM;GAAE,GAAG;GAAO,MAAM,UAAU,OAAO,KAAK,EAAE;EAAG,CAAC,CAAC;EAE9F,MAAM,YAAY,OAAO;EACzB,IAAI,UAAU,QAAQ,OAAO,cAAc,YAAY,UAAU,KAAK,CAAC,CAAC,SAAS,GAC/E,QAAQ,UAAU,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG;CACzC;CACA,SAAS,OAAO,KAAK;CACrB,KAAK,MAAM,SAAS,SAAS,OAAO,IAAI,GAAG;EACzC,IAAI,SAAS,UAAU,aAAa,iBAAiB;GACnD,MAAM,KACJ,GAAG,MAAM,OAAO,WAAW,mBAAmB,OAAO,aAAa,eAAe,EAAE,kCACrF;GACA;EACF;EACA,SAAS,KAAK,QAAQ,QAAQ,OAAO,MAAM,OAAO;GAAE,GAAG;GAAO,MAAM;EAAK,CAAC,CAAC;CAC7E;CACA,MAAM,QAAqB,CAAC;CAC5B,KAAK,MAAM,QAAQ,QAAQ,OAAO,IAAI,CAAC,CAAC,MAAM,GAAG,aAAa,KAAK,GAAG;EACpE,SAAS;EACT,IAAI,MAAM,UAAU,aAAa,OAAO;EACxC,MAAM,OAAO,OAAO,KAAK,SAAS,WAAW,KAAK,KAAK,MAAM,GAAG,GAAG,IAAI;EACvE,MAAM,OAAO,OAAO,KAAK,IAAI;EAC7B,IAAI,SAAS,KAAA,KAAa,OAAO,KAAK,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,KAAK,GAAG,KAAK,MAAM,GAAG;EACtF,IAAI,MAAM,MAAM,SAAS,KAAK,QAAQ,KAAK,IAAI,GAAG;EAClD,MAAM,KAAK;GAAE,KAAK,KAAK;GAAM;EAAK,CAAC;CACrC;CACA,OAAO;EAAE;EAAU;EAAO;EAAO;EAAO;CAAM;AAChD;AAyFA,MAAMA,cAAY,SAChB,KAAK,YAAY,QAAQ,KAAK,YAAY,kBAAkB,KAAK,aAAa,KAAA;;;;;;;;;;;AAYhF,MAAa,YAAY,OACvB,QACA,KACA,UAAwB,CAAC,MACA;CACzB,MAAM,UAAU,UAAU,MAAM;CAChC,MAAM,QAAQ,QAAQ,SAAS,aAAa;CAC5C,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,KAAK,QAAQ,aAAa,UAChE,MAAM,IAAI,WACR,0CAA0C,OAAO,aAAa,QAAQ,EAAE,QAAQ,OAAO,KAAK,GAC9F;CAEF,IAAI;CACJ,IAAI;EACF,OAAO,IAAI,IAAI,GAAG;CACpB,QAAQ;EACN,MAAM,IAAI,UAAU,0CAA0C,KAAK,UAAU,GAAG,GAAG;CACrF;CACA,IAAI,KAAK,aAAa,WAAW,KAAK,aAAa,UACjD,MAAM,IAAI,UAAU,6CAA6C,KAAK;CACxE,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,MAAM,QAAQ,OAAO;CAC3B,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,cAA2B;EAC/B,QAAQ,QAAQ,UAAU;EAC1B,UAAU,WAAW,SAAS,SAAS,WAAW,MAAM;EACxD,YAAY,QAAQ;EACpB;EACA,QAAQ,QAAQ;EAChB,OAAO,QAAQ;EACf,QAAQ,QAAQ;EAChB,UAAU,WAAW,SAAS,QAAQ,MAAM;EAC5C,UAAU,QAAQ;EAClB,sBAAsB,QAAQ;CAChC;CACA,MAAM,QAAkB,CAAC;CACzB,MAAM,WAA2B,CAAC;CAClC,MAAM,QAAqB,CAAC;CAC5B,MAAM,QAAqB,CAAC;CAC5B,MAAM,2BAAW,IAAI,IAAY;CACjC,IAAI,QAAuB;CAC3B,IAAI,WAAW;CAEf,IAAI,QAAQ,QAAQ,KAAA,GAAW;EAC7B,MAAM,MAAgB,OAAO,QAAQ,QAAQ,WAAW,EAAE,OAAO,QAAQ,IAAI,IAAI,QAAQ;EACzF,MAAM,QAAQ,OAAO,IAAI,UAAU,WAAW,YAAY,IAAI,OAAO,KAAK,IAAI,KAAA;EAC9E,IAAI,UAAU,KAAA,GACZ,MAAM,KACJ,2CAA2C,KAAK,UAAU,OAAO,IAAI,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,GAC3F;OACK;GACL,SAAS,KACP,QAAQ,OAAO,OAAO,MAAM;IAC1B,KAAK,IAAI,OAAO;IAChB,KAAK,IAAI,OAAO;IAChB,SAAS,IAAI,WAAW;IACxB,YAAY,IAAI,cAAc;IAC9B,MAAM;GACR,CAAC,CACH;GACA,IAAI,OAAO,IAAI,YAAY,UAAU,SAAS,IAAI,IAAI,OAAO;EAC/D;CACF;;CAGA,MAAM,WAAW,OAAO,YAA8C;EACpE,MAAM,OAAO,MAAM,UAAU,SAAS,SAAS,WAAW;EAC1D,MAAM,MAAM,KAAK,QAAQ,OAAO,KAAA,IAAY,QAAQ,MAAM,IAAI,KAAK,GAAG;EACtE,MAAM,OAAO,QAAQ,KAAA,KAAa,WAAW,KAAK,YAAY;EAC9D,MAAM,KAAK;GACT,KAAK;GACL,KAAK,KAAK;GACV,SAAS,KAAK;GACd,QAAQ,KAAK;GACb,UAAU,KAAK;GACf,SAAS,KAAK;GACd,YAAY,KAAK;GACjB,YAAY,KAAK;GACjB,WAAW;GACX,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;GACjE,MAAM,KAAK;EACb,CAAC;EACD,IAAI,CAACA,WAAS,IAAI,GAChB,MAAM,KAAK,GAAG,QAAQ,IAAI,KAAK,UAAU,KAAK,SAAS,OAAO,KAAK,KAAK,KAAK,KAAK,IAAI;EAExF,IAAI,QAAQ,KAAA,GAAW,OAAO,KAAA;EAC9B,IAAI,MAAM;GACR,WAAW;GACX;EACF;EACA,IAAI,IAAI,aAAa,MAAM,MAAM,eAAe,SAAS,KAAK,GAAG;EACjE,SAAS,IAAI,IAAI,OAAO;EACxB,OAAO;CACT;CAEA,MAAM,WAAW,QAAmB;EAClC,MAAM,OAAO,gBAAgB,KAAK,EAAE,UAAU,QAAQ,SAAS,CAAC;EAChE,SAAS,KAAK,GAAG,KAAK,QAAQ;EAC9B,MAAM,KAAK,GAAG,KAAK,KAAK;EACxB,UAAU,KAAK;EACf,KAAK,MAAM,QAAQ,KAAK,OACtB,IAAI,MAAM,SAAS,aAAa,SAAS,CAAC,MAAM,MAAM,UAAU,MAAM,QAAQ,KAAK,GAAG,GACpF,MAAM,KAAK,IAAI;CAErB;CAEA,IAAI,UAAyB;CAC7B,IAAI,mBAAmC;CACvC,MAAM,OAAO,MAAM,SAAS,KAAK,IAAI;CACrC,IAAI,SAAS,KAAA,GAAW;EACtB,UAAU,WAAW,KAAK,IAAI,CAAC,EAAE,YAAY,KAAK,OAAO,KAAK;EAC9D,IAAI,QAAQ,SAAS,KAAA,GAAW;GAC9B,mBAAmB,WAAW,QAAQ,MAAM,IAAI,KAAK;GACrD,IAAI,qBAAqB,OACvB,MAAM,KACJ,GAAG,QAAQ,iBAAiB,QAAQ,KAAK,oDAC3C;EAEJ;EACA,IAAI,qBAAqB,SAAS,QAAQ,gBAAgB,MACxD,MAAM,KAAK,gFAAgF;OACtF;GACL,QAAQ,IAAI;GACZ,KAAK,MAAM,QAAQ,eAAe,QAAQ,KAAK,IAAI,GAAG,OAAO,CAAC,CAAC,MAAM,GAAG,KAAK,GAAG;IAC9E,IAAI,QAAQ,QAAQ,YAAY,MAAM;IACtC,MAAM,MAAM,MAAM,SAAS,KAAK,IAAI;IACpC,IAAI,QAAQ,KAAA,GAAW,QAAQ,GAAG;GACpC;EACF;CACF;CACA,MAAM,QAAQ,YAAY,QAAQ,OAAO,OAAO,MAAM,OAAO,KAAK,IAAI,IAAI,UAAU;CACpF,IAAI,UAAU,MAAM,MAAM,KAAK,GAAG,KAAK,KAAK,eAAe,OAAO;CAClE,IAAI,UAAU,MAAM,KAAK,SAAS;CAGlC,MAAM,0BAAU,IAAI,IAA0B;CAC9C,KAAK,MAAM,OAAO,UAAU;EAC1B,MAAM,MAAM,GAAG,IAAI,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,IAAI,MAAM,GAAG,IAAI,UAAU;EACzE,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG,QAAQ,IAAI,KAAK,GAAG;CAC7C;CAEA,MAAM,SAAS,CAAC,GAAG,QAAQ,OAAO,CAAC,CAAC,CAAC,MAClC,GAAG,MAAM,EAAE,OAAO,EAAE,QAAQ,OAAO,EAAE,QAAQ,IAAI,OAAO,EAAE,QAAQ,CACrE;CACA,MAAM,2BAAW,IAAI,IAAS;CAC9B,MAAM,QAAQ,OAAO,QAAQ,QAAQ,IAAI,QAAQ,CAAC;CAClD,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GACrC,KAAK,IAAI,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GACzC,KAAK,MAAM,OAAO,cAAc,MAAM,EAAE,CAAE,MAAM,MAAM,EAAE,CAAE,IAAI,GAAG,SAAS,IAAI,GAAG;CAGrF,MAAM,OAAO,OAAO,MAAM,QAAQ,IAAI,aAAa,SAAS,KAAK;CACjE,MAAM,WACJ,OAAO,MAAM,QAAQ,IAAI,SAAS,SAAS,IAAI,aAAa,aAAa,CAAC,IAAI,QAAQ,KAAK;CAC7F,OAAO;EACL,KAAK,KAAK;EACV,MAAM;EACN;EACA,aAAa;EACb;EACA,UAAU;EACV;EACA,WAAW;EACX,UAAU,KAAK,QAAQ,QAAQ,SAAS,IAAI,GAAG,CAAC;EAChD;EACA;EACA,UAAU,CAAC,GAAG,QAAQ;EACtB,WAAW;EACX,MAAM,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI;CAC9C;AACF;;;;ACpvBA,MAAa,iBAWT;CAAE,SAAS;CAAK,QAAQ;CAAK,UAAU;CAAK,OAAO;CAAG,YAAY;AAAG;AA+CzE,MAAM,gBAAgB;;AAGtB,MAAM,YAAY;AAElB,MAAM,YAAY;CAAC;CAAU;CAAU;CAAU;CAAS;AAAM;;AAGhE,MAAM,UAA4C;CAAE,KAAK;CAAO,KAAK;AAAM;AAE3E,IAAI;;AAEJ,MAAM,cAAc,SAA0B;CAC5C,IAAI,CAAC,aAAa,KAAK,IAAI,GAAG,OAAO;CACrC,IAAI,oBAAoB,KAAA,GACtB,IAAI;EACF,kBAAkB,IAAI,IAAI,KAAK,kBAAkB,UAAU,CAAC;CAC9D,QAAQ;EAEN,kCAAkB,IAAI,IAAI;CAC5B;CAEF,OAAO,gBAAgB,SAAS,KAAK,gBAAgB,IAAI,IAAI;AAC/D;;AAGA,MAAM,kBAAkB,aAAqD;CAC3E,IAAI,aAAa,KAAA,GAAW,OAAO,KAAA;CACnC,IAAI;EACF,OAAO,IAAI,KAAK,aAAa,MAAM;GAAE,OAAO;GAAY;EAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAClF;CACL,QAAQ;EACN;CACF;AACF;;;;;;;;AASA,MAAa,cAAc,OAAgB,aAA0C;CACnF,IAAI,OAAO,UAAU,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK,QAAQ,IAAI,QAAQ,KAAA;CACpF,IAAI,OAAO,UAAU,YAAY,MAAM,SAAS,eAAe,YAAY,OAAO,KAAA;CAElF,MAAM,QAAQ,yCAAyC,KAAK,MAAM,KAAK,CAAC;CACxE,IAAI,UAAU,MAAM,OAAO,KAAA;CAC3B,IAAI,SAAS,MAAM,EAAE,CAAE,QAAQ,WAAW,EAAE;CAC5C,MAAM,OAAO,KAAK,IAAI,OAAO,YAAY,GAAG,GAAG,OAAO,YAAY,GAAG,CAAC;CACtE,IAAI,SAAS,IAAI;EAEf,MAAM,SAAS,IADG,IAAI,OAAO,MAAM,OAAO,CACvB,CAAC,CAAC,SAAS,KAAK,+BAA+B,KAAK,MAAM;EAC7E,MAAM,eAAe,UAAU,eAAe,QAAQ,MAAM,KAAK,eAAe,KAAK,MAAM;EAC3F,SACE,UAAU,CAAC,eACP,OAAO,QAAQ,SAAS,EAAE,IAC1B,GAAG,OAAO,MAAM,GAAG,IAAI,CAAC,CAAC,QAAQ,SAAS,EAAE,EAAE,GAAG,OAAO,MAAM,OAAO,CAAC;CAC9E;CACA,MAAM,SAAS,OAAO,MAAM;CAC5B,OAAO,OAAO,SAAS,MAAM,KAAK,SAAS,IAAI,SAAS,KAAA;AAC1D;AAEA,MAAM,QAAQ,OAAgB,WAC3B,MAAM,QAAQ,KAAK,IAAI,QAAQ,UAAU,KAAA,KAAa,UAAU,OAAO,CAAC,IAAI,CAAC,KAAK,EAAA,CAAG,MACpF,GACA,KACF;AAEF,MAAM,WAAW,OAAgB,UAC/B,KAAK,OAAO,KAAK,CAAC,CAAC,QAChB,SAAuB,OAAO,SAAS,YAAY,SAAS,QAAQ,CAAC,MAAM,QAAQ,IAAI,CAC1F;;AAGF,MAAM,UAAU,UAAkC;CAChD,KAAK,MAAM,QAAQ,KAAK,OAAO,EAAE,GAAG;EAClC,IAAI,OAAO,SAAS,YAAY,KAAK,KAAK,CAAC,CAAC,SAAS,GAAG,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG;EACvF,IAAI,OAAO,SAAS,YAAY,SAAS,MAAM;GAC7C,MAAM,QAAS,KAAc;GAC7B,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GAAG,OAAO,MAAM,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG;EAC5F;CACF;CACA,OAAO;AACT;AAEA,MAAM,UAAU,UACd,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,IAC/C,MAAM,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,IACzB,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAChD,OAAO,KAAK,IACZ;AAER,MAAM,WAAW,UAAkC;CACjD,KAAK,MAAM,QAAQ,KAAK,OAAO,EAAE,GAAG;EAClC,IAAI,OAAO,SAAS,YAAY,YAAY,KAAK,IAAI,GAAG,OAAO;EAC/D,MAAM,MAAM,OAAO,SAAS,YAAY,SAAS,OAAQ,KAAc,SAAS,KAAA;EAChF,IAAI,OAAO,QAAQ,YAAY,YAAY,KAAK,GAAG,GAAG,OAAO;CAC/D;CACA,OAAO;AACT;AAYA,MAAM,qBAA6B;CACjC,QAAQ,CAAC;CACT,mCAAmB,IAAI,IAAI;CAC3B,8BAAc,IAAI,IAAI;CACtB,yBAAS,IAAI,IAAI;CACjB,UAAU;CACV,OAAO;AACT;;AAGA,MAAM,cAAc,SAAe,MAAc,UAAwB;CACvE,MAAM,SAAS,OAAgB,aAA4B;EACzD,MAAM,UAAU,OAAO,aAAa,WAAW,SAAS,KAAK,IAAI;EACjE,MAAM,OAAO,QAAQ,YAAY,QAAQ,YAAY;EACrD,MAAM,SAAS,WAAW,OAAO,KAAK,SAAS,IAAI,OAAO,KAAA,CAAS;EACnE,IAAI,WAAW,KAAA,GAAW;EAC1B,IAAI,KAAK,WAAW,GAAG;GACrB,KAAK,WAAW;GAChB;EACF;EACA,IAAI,CAAC,WAAW,IAAI,GAAG;GACrB,KAAK,kBAAkB,IAAI,QAAQ,MAAM,GAAG,EAAE,CAAC;GAC/C;EACF;EACA,KAAK,OAAO,KAAK;GACf;GACA,UAAU;GACV,KAAK,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,eAAe,UAAU;EACvD,CAAC;CACH;CACA,MAAM,SAAS,GAAS,UAAwB;EAC9C,KAAK,SAAS;EACd,MAAM,OAAO,QAAQ,EAAE,uBAAuB,eAAe,MAAM,CAAC,CAAC,MAAM,UAAU;GACnF,MAAM,OAAO,MAAM;GACnB,OAAO,EAAE,OAAO,SAAS,YAAY,UAAU,KAAK,IAAI;EAC1D,CAAC;EACD,MAAM,WAAW,EAAE,oBAAoB,OAAO;EAC9C,MAAM,EAAE,YAAY,OAAO,UAAU,QAAQ;EAC7C,MAAM,EAAE,aAAa,QAAQ;EAC7B,MAAM,EAAE,cAAc,QAAQ;EAC9B,MAAM,eACJ,OAAO,EAAE,oBAAoB,WACzB,EAAE,eAAe,CACd,KAAK,CAAC,CACN,QAAQ,YAAY,EAAE,CAAC,CACvB,MAAM,GAAG,EAAE,IACd,KAAA;EACN,IAAI,iBAAiB,KAAA,KAAa,aAAa,SAAS,GAAG,KAAK,aAAa,IAAI,YAAY;EAC7F,MAAM,SAAS,OAAO,EAAE,SAAS;EACjC,IAAI,WAAW,MAAM,KAAK,QAAQ,IAAI,MAAM;EAE5C,IAAI,QAAQ,eAAe,OACzB,KAAK,MAAM,SAAS,QAAQ,EAAE,WAAW,eAAe,MAAM,GAAG,MAAM,OAAO,QAAQ,CAAC;CAE3F;CACA,KAAK,MAAM,KAAK,QAAQ,QAAQ,WAAW,eAAe,MAAM,GAAG,MAAM,GAAG,CAAC;CAC7E,IAAI,SAAS,GAAG;CAChB,KAAK,MAAM,WAAW,QAAQ,QAAQ,eAAe,eAAe,QAAQ,GAAG;EAC7E,KAAK,SAAS;EACd,WAAW,SAAS,MAAM,QAAQ,CAAC;CACrC;AACF;AAEA,MAAM,kBAAkB,SACtB,KAAK,aAAa,SAAS,IACvB,OACA,KAAK,aAAa,SAAS,KAAK,CAAC,KAAK,aAAa,IAAI,SAAS,IAC9D,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC,KACvB;AAER,MAAM,WAAW,OAAe,SAAqC;CAEnE,IAAI,MAAM,WAAW,GAAG,GAAG,OAAO,KAAA;CAClC,IAAI;EACF,MAAM,MAAM,IAAI,IAAI,OAAO,IAAI;EAC/B,OAAO,GAAG,IAAI,SAAS,YAAY,CAAC,CAAC,QAAQ,UAAU,EAAE,IAAI,IAAI,SAAS,QAAQ,OAAO,EAAE;CAC7F,QAAQ;EACN;CACF;AACF;;AAGA,MAAM,UAAU,WAAyB;CACvC,IAAI;EACF,OAAO,KAAK,UAAU,MAAM,CAAC,CAAC;CAChC,QAAQ;EACN,OAAO;CACT;AACF;;;;;;;;;;;AAYA,MAAa,eACX,SACA,SACmE;CACnE,IAAI,QAAQ;CACZ,MAAM,WAAmB,CAAC;CAC1B,KAAK,MAAM,UAAU,QAAQ,MAAM,GAAG,eAAe,OAAO,GAAG;EAC7D,SAAS;EACT,IAAI,QAAQ,MAAiC,CAAC,CAAC,MAAM,SAAS,cAAc,KAAK,IAAI,CAAC,GACpF,SAAS,KAAK,MAAM;EACtB,KAAK,MAAM,SAAS,QAAQ,OAAO,eAAe,EAAE,GAAG;GACrD,SAAS;GACT,IAAI,QAAQ,KAAgC,CAAC,CAAC,MAAM,SAAS,cAAc,KAAK,IAAI,CAAC,GACnF,SAAS,KAAK,KAAK;EACvB;CACF;CACA,MAAM,OAAO,QAAQ,MAAM,IAAI;CAC/B,MAAM,MAAM,SAAS,MAAM,YACzB,CAAC,QAAQ,QAAQ,QAAQ,MAAM,CAAC,CAAC,MAC9B,UAAU,OAAO,UAAU,YAAY,SAAS,KAAA,KAAa,QAAQ,OAAO,IAAI,MAAM,IACzF,CACF;CACA,IAAI,QAAQ,KAAA,GAAW,OAAO;EAAE,SAAS;EAAK;CAAM;CACpD,MAAM,UAAU,SAAS,QACtB,YAAY,KAAK,QAAQ,WAAW,CAAC,CAAC,CAAC,SAAS,KAAK,KAAK,QAAQ,eAAe,CAAC,CAAC,CAAC,SAAS,CAChG;CACA,MAAM,OAAO,QAAQ,SAAS,IAAI,UAAU;CAC5C,IAAI;CACJ,IAAI,OAAO;CACX,KAAK,MAAM,WAAW,MAAM;EAC1B,MAAM,SAAS,OAAO,OAAO;EAC7B,SAAS;EACT,IAAI,SAAS,MAAM;GACjB,OAAO;GACP,OAAO;EACT;CACF;CACA,OAAO;EAAE,SAAS;EAAM;CAAM;AAChC;;AAGA,MAAa,eAAe,SAAe,SAAmC;CAC5E,IAAI;CACJ,IAAI;EACF,SAAS,IAAI,IAAI,IAAI,CAAC,CAAC;CACzB,QAAQ;EACN;CACF;CACA,MAAM,YAAY,OAAO,IAAI,SAAS;CACtC,MAAM,OAAO,OAAO,IAAI,KAAK,KAAK,OAAO,IAAI,MAAM;CACnD,IAAI,cAAc,QAAQ,SAAS,MAAM,OAAO,KAAA;CAChD,MAAM,SAAS,UAA4B;EACzC,IAAI,OAAO,UAAU,YAAY,cAAc,MAAM,OAAO;EAC5D,IAAI;GACF,OACE,IAAI,IAAI,OAAO,IAAI,CAAC,CAAC,aAAa,IAAI,SAAS,MAAM,aACrD,MAAM,SAAS,IAAI,WAAW;EAElC,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO,QAAQ,QAAQ,eAAe,eAAe,QAAQ,CAAC,CAAC,MAAM,YAAY;EAC/E,MAAM,SAAS,QAAQ,QAAQ,WAAW,eAAe,MAAM;EAC/D,MAAM,QAAQ;GACZ,QAAQ;GACR,QAAQ;GACR,QAAQ;GACR,QAAQ;GACR,QAAQ;EACV,CAAC,CACE,IAAI,MAAM,CAAC,CACX,QAAQ,QAAuB,QAAQ,IAAI;EAC9C,OACG,SAAS,QAAQ,MAAM,SAAS,IAAI,KACrC;GAAC,QAAQ;GAAQ,QAAQ;GAAQ,GAAG,OAAO,KAAK,UAAU,MAAM,MAAM;EAAC,CAAC,CAAC,KAAK,KAAK,KAClF,cAAc,QAAQ,MAAM,SAAS,SAAS;CAEnD,CAAC;AACH;AAEA,MAAM,YAAY,GAAU,MAAqB,EAAE,SAAS,EAAE;;;;;;;;;AAU9D,MAAa,aAAa,SAA0B,SAA8B;CAChF,MAAM,SAAS,YAAY,SAAS,IAAI;CACxC,MAAM,UAAU,OAAO;CACvB,IAAI,YAAY,KAAA,GACd,OAAO;EACL,OAAO;EACP,MAAM;EACN,OAAO;EACP,KAAK;EACL,MAAM;EACN,OAAO;EACP,UAAU;EACV,cAAc;EACd,gBAAgB,CAAC;EACjB,SAAS;EACT,QAAQ;EACR,OAAO;EACP,OAAO,CACL,gGACF;EACA,OAAO,OAAO;CAChB;CAEF,MAAM,OAAO,YAAY;CACzB,WAAW,SAAS,MAAM,eAAe,KAAK;CAC9C,MAAM,WAAW,KAAK,OAAO,EAAE,EAAE;CACjC,MAAM,SAAS,KAAK,OAAO,QAAQ,UAAU,MAAM,aAAa,QAAQ,CAAC,CAAC,KAAK,QAAQ;CACvF,MAAM,MAAM,OAAO,MAAM;CACzB,MAAM,OAAO,OAAO,GAAG,EAAE,KAAK;CAC9B,MAAM,OAAO,UAAU,KAAK,QAAQ,OAAO,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM,UAAU,UAAU,IAAI;CACxF,MAAM,SAAS,YAAY,SAAS,IAAI;CACxC,IAAI,UAAiC;CACrC,IAAI,QAAQ,OAAO,QAAQ,KAAK;CAChC,IAAI,WAAW,KAAA,GAAW;EACxB,MAAM,MAAM,YAAY;EACxB,WAAW,QAAQ,KAAK,CAAC;EACzB,SAAS,IAAI;EACb,MAAM,SAAS,IAAI,OAAO,QAAQ,UAAU,MAAM,aAAa,QAAQ,CAAC,CAAC,KAAK,QAAQ;EACtF,UAAU;GACR,MAAM,OAAO,OAAO,OAAO;GAC3B,KAAK,OAAO,OAAO,MAAM;GACzB,OAAO,OAAO,MAAM;GACpB,cAAc,eAAe,GAAG;EAClC;CACF;CACA,MAAM,QAAQ;EACZ,KAAK,kBAAkB,OAAO,IAC1B,4CAA4C,CAAC,GAAG,KAAK,iBAAiB,CAAC,CAAC,KAAK,IAAI,MACjF;EACJ,KAAK,YAAY,OAAO,WAAW,IAC/B,+EACA,OAAO,WAAW,IAChB,kDACA;EACN,IAAI,IAAI,KAAK,OAAO,KAAK,UAAU,MAAM,QAAQ,CAAC,CAAC,CAAC,OAAO,IACvD,6DAA6D,YAAY,OACzE;EACJ,4BAA4B,KAAK,IAAI,KAAK,WAAW,KAAA,IACjD,8FACA;CACN,CAAC,CAAC,QAAQ,SAAyB,SAAS,IAAI;CAChD,OAAO;EACL,OAAO;EACP,MAAM,OAAO,QAAQ,OAAO;EAC5B,OAAO,OAAO,QAAQ,QAAQ;EAC9B,KAAK,OAAO,QAAQ,MAAM;EAC1B,MAAM,QAAQ;EACd,OAAO;EACP,UAAU,SAAS,QAAQ,QAAQ,QAAQ,KAAK,SAAS,IAAI,SAAS,OAAO;EAC7E,cAAc,eAAe,IAAI;EACjC,gBAAgB,CAAC,GAAG,KAAK,YAAY;EACrC;EACA,QAAQ,KAAK,QAAQ,SAAS,IAAI,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,CAAC,KAAK,IAAI;EACpE,OAAO,QAAQ,QAAQ,QAAQ;EAC/B;EACA;CACF;AACF;AAqDA,MAAM,wBAA6B,IAAI,IAAI;CAAC;CAAW;CAAa;AAAkB,CAAC;;AAGvF,MAAM,UAAU,SACd,MAAM,IAAI,KAAK,OAAO,KACrB,KAAK,YAAY,aAAa,0CAA0C,KAAK,KAAK,QAAQ,EAAE,KAC5F,KAAK,QAAQ,QAAQ,gBAAgB,KAAK,KAAK,QAAQ,EAAE;AAE5D,MAAM,YAAY,SAChB,KAAK,QAAQ,SACZ,KAAK,YAAY,QAAQ,KAAK,YAAY,kBAAkB,KAAK,aAAa,KAAA;;;;;;AAOjF,MAAM,eAAe,OAAO,SAAuB,QAAuC;CACxF,MAAM,OAAO,MAAM,QAAQ,IAAI,KAAK,SAAS,IAAK,IAAI,KAAK,YAAuB,CAAC;CACnF,IAAI,IAAI,SAAS,QAAQ,WAAW,IAAI,IAAI,MAAM,KAAA,GAAW,OAAO;CACpE,MAAM,OAAO,MAAM,QAAQ,MAAM,IAAI,IAAI,IAAI;CAC7C,IAAI,SAAS,KAAA,GAAW,OAAO;CAC/B,MAAM,KAAK,IAAI,OAAO;CACtB,MAAM,OAAO,IAAI,WACb,6BACC,QAAQ,MAAM,OAAO,IAAI,OAAO,CAAC,EAAE,gBAAgB;CACxD,IAAI;EACF,OAAO,YAAY,WAAW,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;CACjD,QAAQ;EACN,OAAO;CACT;AACF;;;;;;;;;AAUA,MAAa,cAAc,OACzB,QACA,KACA,UAA0B,CAAC,MACA;CAC3B,MAAM,UAAU,UAAU,MAAM;CAChC,IAAI;EACF,MAAM,SAAS,IAAI,IAAI,GAAG;EAC1B,IAAI,OAAO,aAAa,WAAW,OAAO,aAAa,UAAU,MAAM,IAAI,UAAU,GAAG;CAC1F,QAAQ;EACN,MAAM,IAAI,UAAU,+CAA+C,KAAK,UAAU,GAAG,GAAG;CAC1F;CACA,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,MAAM,QAAQ,OAAO;CAC3B,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,WAAW,WAAW,SAAS,QAAQ,MAAM;CACnD,MAAM,eAAe,UAAoD;EACvE,QAAQ;EACR,UAAU,WAAW,SAAS,SAAS,WAAW,MAAM;EACxD,YAAY,QAAQ;EACpB;EACA,QAAQ,QAAQ;EAChB,OAAO,QAAQ;EACf,QAAQ,QAAQ;EAChB;EACA,UAAU,QAAQ;EAClB,sBAAsB,QAAQ;CAChC;CACA,IAAI,OAAO,MAAM,UAAU,SAAS,KAAK,YAAY,MAAM,CAAC;CAC5D,MAAM,QAAkB,CAAC;CACzB,MAAM,UACJ,MACA,MACA,OAAyC,CAAC,MACxB;EAClB,MAAM,MAAM;GAAC,KAAK;GAAM,GAAG;GAAO,GAAI,MAAM,SAAS,CAAC;EAAE,CAAC,CAAC,QACvD,SAAyB,SAAS,QAAQ,SAAS,KAAA,KAAa,KAAK,SAAS,CACjF;EACA,OAAO;GACL;GACA,OAAO,MAAM,SAAS;GACtB,UAAU,MAAM,UAAU,QAAQ,KAAK,SAAS;GAChD,MAAM,MAAM,QAAQ;GACpB,OAAO,MAAM,SAAS;GACtB,KAAK,MAAM,OAAO;GAClB,MAAM,MAAM,QAAQ;GACpB,OAAO,MAAM,SAAS;GACtB,UAAU,MAAM,YAAY;GAC5B,cAAc,MAAM,gBAAgB;GACpC,gBAAgB,MAAM,kBAAkB,CAAC;GACzC,SAAS,MAAM,WAAW;GAC1B,QAAQ,MAAM,UAAU;GACxB,OAAO,MAAM,SAAS;GACtB,MAAM,OAAO,IAAI;GACjB,SAAS,KAAK;GACd,QAAQ,KAAK;GACb,UAAU,KAAK;GACf,KAAK,KAAK;GACV,SAAS,KAAK;GACd,YAAY,KAAK;GACjB,UAAU,CAAC,GAAG,KAAK,QAAQ;GAC3B,YAAY,KAAK;GACjB,WAAW,KAAK,aAAa;GAC7B,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;GACjE,MAAM,IAAI,SAAS,IAAI,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI;EACxD;CACF;CACA,IAAI,CAAC,SAAS,IAAI,GAAG;EACnB,MAAM,OAAO,OAAO,IAAI;EACxB,MAAM,KACJ,OACI,iCAAiC,KAAK,QAAQ,wEAC9C,8BAA8B,KAAK,QAAQ,EACjD;EACA,OAAO,OAAO,MAAM,KAAA,CAAS;CAC/B;CACA,MAAM,QAAQ,SAAsC;EAClD,MAAM,MAAM,KAAK,QAAQ,OAAO,KAAA,IAAY,QAAQ,MAAM,IAAI,KAAK,GAAG;EACtE,OAAO,QAAQ,KAAA,KAAa,WAAW,KAAK,YAAY,IAAI,MAAM,KAAA;CACpE;CACA,IAAI,KAAK,IAAI,MAAM,KAAA,GAAW;EAC5B,MAAM,KAAK,SAAS;EACpB,OAAO,OAAO,MAAM,KAAA,GAAW,EAAE,WAAW,KAAK,CAAC;CACpD;CACA,MAAM,SAAS,SACb,KAAK,QAAQ,OAAO,KAAA,IAAY,QAAQ,MAAM,IAAI,KAAK,GAAG;CAC5D,IAAI,MAAM,MAAM,IAAI;CACpB,IAAI,OAAO,QAAQ,KAAA,IAAY,KAAA,IAAY,UAAU,MAAM,aAAa,SAAS,GAAG,GAAG,GAAG;CAE1F,IAAI,MAAM,UAAU,QAAQ,CAAC,KAAK,YAAY,WAAW,UAAU,KAAK,aAAa,KAAA,GAAW;EAC9F,MAAM,WAAW,MAAM,UAAU,SAAS,KAAK,YAAY,QAAQ,CAAC;EACpE,MAAM,QAAQ,MAAM,QAAQ;EAC5B,IACE,SAAS,YACT,SAAS,QAAQ,KACjB,UAAU,KAAA,KACV,KAAK,QAAQ,MAAM,KAAA,GACnB;GACA,OAAO;GACP,MAAM;GACN,OAAO,UAAU,MAAM,aAAa,SAAS,KAAK,GAAG,GAAG;EAC1D,OAAO,IAAI,SAAS,SAAS,QAAQ,CAAC,SAAS,UAC7C,MAAM,KAAK,iCAAiC,SAAS,MAAM;CAE/D;CACA,IAAI,gBAAgB,KAAK,aAAa,MAAM,MAAM,eAAe,SAAS,KAAK,GAAG;CAClF,OAAO,OAAO,MAAM,IAAI;AAC1B"}

package/build/log-Dx-wXjJl.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"log-Dx-wXjJl.mjs","names":[],"sources":["../src/blobs.ts","../src/log.ts"],"sourcesContent":["import { createHash, randomUUID } from \"node:crypto\";\nimport type { Stats } from \"node:fs\";\nimport { mkdir, open, readFile, rename, rm, stat } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\n\n/**\n * A blob store keeps the bytes of each response, and the hash of the bytes is the key. The crawler\n * needs only these two operations, so an application can give its own store.\n */\nexport interface BlobStore {\n  put(bytes: Uint8Array): Promise<{ readonly hash: string }>;\n  get(hash: string): Promise<Uint8Array | undefined>;\n}\n\n/** These options change a file blob store. */\nexport interface FileBlobStoreOptions {\n  /** When this value is true (the default), a blob is on the disk (`fsync`) before `put` finishes. */\n  readonly sync?: boolean | undefined;\n}\n\nconst HASH = /^[0-9a-f]{64}$/;\n\n/** A rename can fail with these codes when a different writer puts the same blob at the same time. */\nconst RACE_CODES: ReadonlySet<string> = new Set([\n  \"EPERM\",\n  \"EEXIST\",\n  \"EBUSY\",\n  \"EACCES\",\n  \"ENOTEMPTY\",\n]);\n\nconst blobPath = (root: string, hash: string): string =>\n  join(root, hash.slice(0, 2), hash.slice(2, 4), hash);\n\nconst hashOf = (bytes: Uint8Array): string => createHash(\"sha256\").update(bytes).digest(\"hex\");\n\nconst codeOf = (error: unknown): string =>\n  typeof error === \"object\" && error !== null && \"code\" in error ? String(error.code) : \"\";\n\nconst statOf = (path: string): Promise<Stats | undefined> => stat(path).catch(() => undefined);\n\n/** This function writes a folder to the disk, so a rename in it stays after a power cut. Windows cannot open a folder, so it skips this step. */\nconst syncFolder = async (path: string): Promise<void> => {\n  if (process.platform === \"win32\") return;\n  try {\n    const handle = await open(path, \"r\");\n    try {\n      await handle.sync();\n    } finally {\n      await handle.close();\n    }\n  } catch {\n    // a file system that cannot sync a folder keeps the blob in any case\n  }\n};\n\n/**\n * This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash\n * of the bytes. These are its rules:\n *\n * - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is\n *   written again.\n * - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.\n *   A crash thus cannot leave half a blob under its name.\n * - Two writers can put the same bytes at the same time. When the rename of one fails because the\n *   other was first, the put succeeds if the stored file has the same hash.\n * - The method `get` examines the hash of the bytes. A damaged file gets a new name\n *   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.\n */\nexport class FileBlobStore implements BlobStore {\n  private readonly writing = new Map<\n    string,\n    Promise<{ readonly hash: string; readonly existed: boolean }>\n  >();\n  private readonly sync: boolean;\n\n  constructor(\n    readonly dir: string,\n    options: FileBlobStoreOptions = {},\n  ) {\n    this.sync = options.sync ?? true;\n  }\n\n  put(bytes: Uint8Array): Promise<{ readonly hash: string; readonly existed: boolean }> {\n    const hash = hashOf(bytes);\n    const running = this.writing.get(hash);\n    if (running !== undefined) return running;\n    const work = this.write(hash, bytes).finally(() => this.writing.delete(hash));\n    this.writing.set(hash, work);\n    return work;\n  }\n\n  /** This method tells if a file holds exactly the bytes of a hash. */\n  private static async holds(path: string, hash: string): Promise<boolean> {\n    try {\n      return hashOf(await readFile(path)) === hash;\n    } catch {\n      return false;\n    }\n  }\n\n  private async write(\n    hash: string,\n    bytes: Uint8Array,\n  ): Promise<{ readonly hash: string; readonly existed: boolean }> {\n    const target = blobPath(this.dir, hash);\n    const existing = await statOf(target);\n    if (existing?.isFile() === true && existing.size === bytes.length)\n      return { hash, existed: true };\n    const folder = dirname(target);\n    await mkdir(folder, { recursive: true, mode: 0o700 });\n    const temporary = `${target}.${randomUUID()}.tmp`;\n    try {\n      const handle = await open(temporary, \"wx\", 0o600);\n      try {\n        await handle.writeFile(bytes);\n        if (this.sync) await handle.sync();\n      } finally {\n        await handle.close();\n      }\n      try {\n        await rename(temporary, target);\n      } catch (error) {\n        // a different writer put the same bytes first, and Windows does not replace a file in use\n        if (!RACE_CODES.has(codeOf(error)) || !(await FileBlobStore.holds(target, hash)))\n          throw error;\n        return { hash, existed: true };\n      }\n      if (this.sync) await syncFolder(folder);\n      return { hash, existed: false };\n    } finally {\n      await rm(temporary, { force: true }).catch(() => undefined);\n    }\n  }\n\n  async get(hash: string): Promise<Uint8Array | undefined> {\n    if (!HASH.test(hash)) return undefined;\n    const target = blobPath(this.dir, hash);\n    let bytes: Buffer;\n    try {\n      bytes = await readFile(target);\n    } catch {\n      return undefined;\n    }\n    if (hashOf(bytes) !== hash) {\n      // a torn or damaged file is never the body: it moves away, and the next put writes it again\n      await rename(target, `${target}.${randomUUID()}.damaged`).catch(() => undefined);\n      return undefined;\n    }\n    return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n  }\n}\n","import { createReadStream, existsSync } from \"node:fs\";\nimport type { FileHandle } from \"node:fs/promises\";\nimport { mkdir, open, readdir } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\nimport type { CaptureRecord } from \"./record.ts\";\n\n/** A scan filter selects the records of one source, one kind, or the records after one time. */\nexport interface CaptureFilter {\n  readonly source?: string | undefined;\n  readonly kind?: string | undefined;\n  /** This value is an ISO 8601 time. The scan gives the records received at this time or later. */\n  readonly since?: string | undefined;\n}\n\n/** A damaged line is a line of the log that is not a record: a torn append, or bytes that a crash left. */\nexport interface DamagedLine {\n  /** This value is the path of the file. */\n  readonly path: string;\n  /** This value is the number of the line in the file, from 1. */\n  readonly line: number;\n}\n\n/** These options change a scan. */\nexport interface ScanOptions {\n  /** The scan skips a damaged line, and it gives the line to this function. */\n  readonly onDamaged?: ((damaged: DamagedLine) => void) | undefined;\n}\n\n/** The read side of the capture log is what `reindex` replays. */\nexport interface CaptureReader {\n  scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;\n}\n\n/** These options change a capture log. */\nexport interface CaptureLogOptions {\n  /**\n   * When this value is true (the default), the log writes each batch of appends to the disk\n   * (`fdatasync`) before the appends finish. Thus a power cut does not lose a record that the\n   * crawler reported.\n   */\n  readonly sync?: boolean | undefined;\n}\n\ninterface Pending {\n  readonly path: string;\n  readonly line: string;\n  readonly resolve: () => void;\n  readonly reject: (error: unknown) => void;\n}\n\nconst MONTH_FILE = /^(\\d{2})(?:\\.[^.]+)?\\.ndjson$/;\nconst YEAR_DIR = /^\\d{4}$/;\nconst NEWLINE = 0x0a;\n\n/** This function writes the whole buffer. A write to a file can write less than the buffer, and the loop writes the rest. */\nconst writeAll = async (handle: FileHandle, bytes: Buffer): Promise<void> => {\n  let at = 0;\n  while (at < bytes.length) {\n    const { bytesWritten } = await handle.write(bytes, at, bytes.length - at);\n    at += bytesWritten;\n  }\n};\n\n/**\n * This function reads a file line by line, as bytes, so a long file does not become one string. It\n * gives each line and tells if a line break ends it.\n */\nasync function* linesOf(\n  path: string,\n): AsyncGenerator<{ readonly text: string; readonly complete: boolean }> {\n  const decoder = new TextDecoder(\"utf-8\");\n  let rest: Buffer | undefined;\n  for await (const chunk of createReadStream(path) as AsyncIterable<Buffer>) {\n    const data = rest === undefined ? chunk : Buffer.concat([rest, chunk]);\n    rest = undefined;\n    let start = 0;\n    for (;;) {\n      const end = data.indexOf(NEWLINE, start);\n      if (end === -1) break;\n      yield { text: decoder.decode(data.subarray(start, end)), complete: true };\n      start = end + 1;\n    }\n    if (start < data.length) rest = Buffer.from(data.subarray(start));\n  }\n  if (rest !== undefined && rest.length > 0) yield { text: decoder.decode(rest), complete: false };\n}\n\nconst monthKey = (year: string, file: string): string =>\n  `${year}-${MONTH_FILE.exec(file)?.[1] ?? \"00\"}`;\n\n/**\n * The capture log keeps each fetch decision as one line of JSON in\n * `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.\n *\n * These are the rules of the log:\n *\n * - The log appends a batch of records with one write, and it writes the batch to the disk before\n *   the appends finish.\n * - An append always starts on a new line. After a crash leaves a partial line, the next append\n *   starts after a line break. The partial line stays as a damaged line, and the record is good.\n * - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.\n * - The log does not remove or change a line.\n *\n * Two processes can append to one log, because each batch is one write to a file in append mode.\n */\nexport class CaptureLog implements CaptureReader {\n  private readonly handles = new Map<string, FileHandle>();\n  private readonly pending: Pending[] = [];\n  private draining: Promise<void> | undefined;\n  private readonly sync: boolean;\n\n  constructor(\n    readonly dir: string,\n    options: CaptureLogOptions = {},\n  ) {\n    this.sync = options.sync ?? true;\n  }\n\n  private pathOf(record: CaptureRecord): string {\n    return join(\n      this.dir,\n      record.source,\n      record.received_at.slice(0, 4),\n      `${record.received_at.slice(5, 7)}.ndjson`,\n    );\n  }\n\n  private async handleFor(path: string): Promise<FileHandle> {\n    const known = this.handles.get(path);\n    if (known !== undefined) return known;\n    // the folders and the files are for the owner only, where the system has modes\n    await mkdir(dirname(path), { recursive: true, mode: 0o700 });\n    const handle = await open(path, \"a+\", 0o600);\n    this.handles.set(path, handle);\n    return handle;\n  }\n\n  /** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */\n  private static async needsBreak(handle: FileHandle): Promise<boolean> {\n    const { size } = await handle.stat();\n    if (size === 0) return false;\n    const last = Buffer.alloc(1);\n    await handle.read(last, 0, 1, size - 1);\n    return last[0] !== NEWLINE;\n  }\n\n  append(record: CaptureRecord): Promise<void> {\n    let line: string;\n    try {\n      line = `${JSON.stringify(record)}\\n`;\n    } catch (error) {\n      return Promise.reject(error instanceof Error ? error : new Error(String(error)));\n    }\n    return new Promise<void>((resolve, reject) => {\n      this.pending.push({ path: this.pathOf(record), line, resolve, reject });\n      this.kick();\n    });\n  }\n\n  /** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */\n  private kick(): void {\n    if (this.draining !== undefined) return;\n    this.draining = this.drain().finally(() => {\n      this.draining = undefined;\n      if (this.pending.length > 0) this.kick();\n    });\n  }\n\n  /** This method writes the waiting records in batches: one write and one sync for each file. */\n  private async drain(): Promise<void> {\n    while (this.pending.length > 0) {\n      const batch = this.pending.splice(0);\n      const byPath = new Map<string, Pending[]>();\n      for (const item of batch) {\n        const list = byPath.get(item.path);\n        if (list === undefined) byPath.set(item.path, [item]);\n        else list.push(item);\n      }\n      for (const [path, items] of byPath) {\n        try {\n          const handle = await this.handleFor(path);\n          const lead = (await CaptureLog.needsBreak(handle)) ? \"\\n\" : \"\";\n          await writeAll(\n            handle,\n            Buffer.from(`${lead}${items.map((item) => item.line).join(\"\")}`, \"utf8\"),\n          );\n          if (this.sync) await handle.datasync();\n          for (const item of items) item.resolve();\n        } catch (error) {\n          // a handle that failed is opened again for the next append\n          const handle = this.handles.get(path);\n          this.handles.delete(path);\n          await handle?.close().catch(() => undefined);\n          for (const item of items) item.reject(error);\n        }\n      }\n    }\n  }\n\n  /** This method waits until each append in progress is on the disk. */\n  async flush(): Promise<void> {\n    while (this.draining !== undefined) await this.draining;\n  }\n\n  /** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */\n  private async filesOf(source: string, since: string | undefined): Promise<string[]> {\n    const sourceDir = join(this.dir, source);\n    if (!existsSync(sourceDir)) return [];\n    const from = since?.slice(0, 7);\n    const years = (await readdir(sourceDir, { withFileTypes: true }))\n      .filter((entry) => entry.isDirectory() && YEAR_DIR.test(entry.name))\n      .map((entry) => entry.name)\n      .sort();\n    const files: string[] = [];\n    for (const year of years) {\n      if (from !== undefined && year < from.slice(0, 4)) continue;\n      const months = (await readdir(join(sourceDir, year)))\n        .filter((name) => MONTH_FILE.test(name))\n        .sort();\n      for (const month of months) {\n        if (from !== undefined && monthKey(year, month) < from) continue;\n        files.push(join(sourceDir, year, month));\n      }\n    }\n    return files;\n  }\n\n  /** This method reads the records of one source in the sequence of the files. */\n  private async *readSource(\n    source: string,\n    filter: CaptureFilter | undefined,\n    options: ScanOptions | undefined,\n  ): AsyncGenerator<CaptureRecord> {\n    for (const path of await this.filesOf(source, filter?.since)) {\n      let number = 0;\n      for await (const { text, complete } of linesOf(path)) {\n        number += 1;\n        const trimmed = text.trim();\n        if (trimmed.length === 0) continue;\n        let row: CaptureRecord;\n        try {\n          row = JSON.parse(trimmed) as CaptureRecord;\n          if (typeof row !== \"object\" || row === null || typeof row.received_at !== \"string\") {\n            throw new Error(\"not a record\");\n          }\n        } catch {\n          // the append of a different process can be in progress at the end of the file\n          if (complete) options?.onDamaged?.({ path, line: number });\n          continue;\n        }\n        if (filter?.kind !== undefined && row.kind !== filter.kind) continue;\n        if (filter?.since !== undefined && row.received_at < filter.since) continue;\n        yield row;\n      }\n    }\n  }\n\n  /**\n   * This method reads the records that agree with the filter. It first waits for the appends that\n   * are in progress. It reads each file as a stream, so its memory does not grow with the log.\n   *\n   * The records of one source come in the sequence of the files. The records of two or more sources\n   * merge in the sequence of `received_at`.\n   */\n  async *scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord> {\n    await this.flush();\n    const sources =\n      filter?.source !== undefined\n        ? [filter.source]\n        : existsSync(this.dir)\n          ? (await readdir(this.dir, { withFileTypes: true }))\n              .filter((entry) => entry.isDirectory())\n              .map((entry) => entry.name)\n              .sort()\n          : [];\n    if (sources.length === 1) {\n      yield* this.readSource(sources[0]!, filter, options);\n      return;\n    }\n    // a merge of the sources: the next record is the earliest head\n    const streams = sources.map((source) =>\n      this.readSource(source, filter, options)[Symbol.asyncIterator](),\n    );\n    const heads: Array<CaptureRecord | undefined> = [];\n    for (const stream of streams)\n      heads.push((await stream.next()).value as CaptureRecord | undefined);\n    for (;;) {\n      let pick = -1;\n      for (const [index, head] of heads.entries()) {\n        if (head !== undefined && (pick === -1 || head.received_at < heads[pick]!.received_at)) {\n          pick = index;\n        }\n      }\n      if (pick === -1) return;\n      yield heads[pick]!;\n      heads[pick] = (await streams[pick]!.next()).value as CaptureRecord | undefined;\n    }\n  }\n\n  /** This method writes the appends in progress, then closes the files. A later append opens them again. */\n  async close(): Promise<void> {\n    await this.flush();\n    const handles = [...this.handles.values()];\n    this.handles.clear();\n    for (const handle of handles) await handle.close();\n  }\n}\n"],"mappings":";;;;;AAoBA,MAAM,OAAO;;AAGb,MAAM,6BAAkC,IAAI,IAAI;CAC9C;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,YAAY,MAAc,SAC9B,KAAK,MAAM,KAAK,MAAM,GAAG,CAAC,GAAG,KAAK,MAAM,GAAG,CAAC,GAAG,IAAI;AAErD,MAAM,UAAU,UAA8B,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK;AAE7F,MAAM,UAAU,UACd,OAAO,UAAU,YAAY,UAAU,QAAQ,UAAU,QAAQ,OAAO,MAAM,IAAI,IAAI;AAExF,MAAM,UAAU,SAA6C,KAAK,IAAI,CAAC,CAAC,YAAY,KAAA,CAAS;;AAG7F,MAAM,aAAa,OAAO,SAAgC;CACxD,IAAI,QAAQ,aAAa,SAAS;CAClC,IAAI;EACF,MAAM,SAAS,MAAM,KAAK,MAAM,GAAG;EACnC,IAAI;GACF,MAAM,OAAO,KAAK;EACpB,UAAU;GACR,MAAM,OAAO,MAAM;EACrB;CACF,QAAQ,CAER;AACF;;;;;;;;;;;;;;AAeA,IAAa,gBAAb,MAAa,cAAmC;CAQnC;CAPX,0BAA2B,IAAI,IAG7B;CACF;CAEA,YACE,KACA,UAAgC,CAAC,GACjC;EAFS,KAAA,MAAA;EAGT,KAAK,OAAO,QAAQ,QAAQ;CAC9B;CAEA,IAAI,OAAkF;EACpF,MAAM,OAAO,OAAO,KAAK;EACzB,MAAM,UAAU,KAAK,QAAQ,IAAI,IAAI;EACrC,IAAI,YAAY,KAAA,GAAW,OAAO;EAClC,MAAM,OAAO,KAAK,MAAM,MAAM,KAAK,CAAC,CAAC,cAAc,KAAK,QAAQ,OAAO,IAAI,CAAC;EAC5E,KAAK,QAAQ,IAAI,MAAM,IAAI;EAC3B,OAAO;CACT;;CAGA,aAAqB,MAAM,MAAc,MAAgC;EACvE,IAAI;GACF,OAAO,OAAO,MAAM,SAAS,IAAI,CAAC,MAAM;EAC1C,QAAQ;GACN,OAAO;EACT;CACF;CAEA,MAAc,MACZ,MACA,OAC+D;EAC/D,MAAM,SAAS,SAAS,KAAK,KAAK,IAAI;EACtC,MAAM,WAAW,MAAM,OAAO,MAAM;EACpC,IAAI,UAAU,OAAO,MAAM,QAAQ,SAAS,SAAS,MAAM,QACzD,OAAO;GAAE;GAAM,SAAS;EAAK;EAC/B,MAAM,SAAS,QAAQ,MAAM;EAC7B,MAAM,MAAM,QAAQ;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EACpD,MAAM,YAAY,GAAG,OAAO,GAAG,WAAW,EAAE;EAC5C,IAAI;GACF,MAAM,SAAS,MAAM,KAAK,WAAW,MAAM,GAAK;GAChD,IAAI;IACF,MAAM,OAAO,UAAU,KAAK;IAC5B,IAAI,KAAK,MAAM,MAAM,OAAO,KAAK;GACnC,UAAU;IACR,MAAM,OAAO,MAAM;GACrB;GACA,IAAI;IACF,MAAM,OAAO,WAAW,MAAM;GAChC,SAAS,OAAO;IAEd,IAAI,CAAC,WAAW,IAAI,OAAO,KAAK,CAAC,KAAK,CAAE,MAAM,cAAc,MAAM,QAAQ,IAAI,GAC5E,MAAM;IACR,OAAO;KAAE;KAAM,SAAS;IAAK;GAC/B;GACA,IAAI,KAAK,MAAM,MAAM,WAAW,MAAM;GACtC,OAAO;IAAE;IAAM,SAAS;GAAM;EAChC,UAAU;GACR,MAAM,GAAG,WAAW,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;EAC5D;CACF;CAEA,MAAM,IAAI,MAA+C;EACvD,IAAI,CAAC,KAAK,KAAK,IAAI,GAAG,OAAO,KAAA;EAC7B,MAAM,SAAS,SAAS,KAAK,KAAK,IAAI;EACtC,IAAI;EACJ,IAAI;GACF,QAAQ,MAAM,SAAS,MAAM;EAC/B,QAAQ;GACN;EACF;EACA,IAAI,OAAO,KAAK,MAAM,MAAM;GAE1B,MAAM,OAAO,QAAQ,GAAG,OAAO,GAAG,WAAW,EAAE,SAAS,CAAC,CAAC,YAAY,KAAA,CAAS;GAC/E;EACF;EACA,OAAO,IAAI,WAAW,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;CACxE;AACF;;;ACrGA,MAAM,aAAa;AACnB,MAAM,WAAW;AACjB,MAAM,UAAU;;AAGhB,MAAM,WAAW,OAAO,QAAoB,UAAiC;CAC3E,IAAI,KAAK;CACT,OAAO,KAAK,MAAM,QAAQ;EACxB,MAAM,EAAE,iBAAiB,MAAM,OAAO,MAAM,OAAO,IAAI,MAAM,SAAS,EAAE;EACxE,MAAM;CACR;AACF;;;;;AAMA,gBAAgB,QACd,MACuE;CACvE,MAAM,UAAU,IAAI,YAAY,OAAO;CACvC,IAAI;CACJ,WAAW,MAAM,SAAS,iBAAiB,IAAI,GAA4B;EACzE,MAAM,OAAO,SAAS,KAAA,IAAY,QAAQ,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC;EACrE,OAAO,KAAA;EACP,IAAI,QAAQ;EACZ,SAAS;GACP,MAAM,MAAM,KAAK,QAAQ,SAAS,KAAK;GACvC,IAAI,QAAQ,IAAI;GAChB,MAAM;IAAE,MAAM,QAAQ,OAAO,KAAK,SAAS,OAAO,GAAG,CAAC;IAAG,UAAU;GAAK;GACxE,QAAQ,MAAM;EAChB;EACA,IAAI,QAAQ,KAAK,QAAQ,OAAO,OAAO,KAAK,KAAK,SAAS,KAAK,CAAC;CAClE;CACA,IAAI,SAAS,KAAA,KAAa,KAAK,SAAS,GAAG,MAAM;EAAE,MAAM,QAAQ,OAAO,IAAI;EAAG,UAAU;CAAM;AACjG;AAEA,MAAM,YAAY,MAAc,SAC9B,GAAG,KAAK,GAAG,WAAW,KAAK,IAAI,CAAC,GAAG,MAAM;;;;;;;;;;;;;;;;AAiB3C,IAAa,aAAb,MAAa,WAAoC;CAOpC;CANX,0BAA2B,IAAI,IAAwB;CACvD,UAAsC,CAAC;CACvC;CACA;CAEA,YACE,KACA,UAA6B,CAAC,GAC9B;EAFS,KAAA,MAAA;EAGT,KAAK,OAAO,QAAQ,QAAQ;CAC9B;CAEA,OAAe,QAA+B;EAC5C,OAAO,KACL,KAAK,KACL,OAAO,QACP,OAAO,YAAY,MAAM,GAAG,CAAC,GAC7B,GAAG,OAAO,YAAY,MAAM,GAAG,CAAC,EAAE,QACpC;CACF;CAEA,MAAc,UAAU,MAAmC;EACzD,MAAM,QAAQ,KAAK,QAAQ,IAAI,IAAI;EACnC,IAAI,UAAU,KAAA,GAAW,OAAO;EAEhC,MAAM,MAAM,QAAQ,IAAI,GAAG;GAAE,WAAW;GAAM,MAAM;EAAM,CAAC;EAC3D,MAAM,SAAS,MAAM,KAAK,MAAM,MAAM,GAAK;EAC3C,KAAK,QAAQ,IAAI,MAAM,MAAM;EAC7B,OAAO;CACT;;CAGA,aAAqB,WAAW,QAAsC;EACpE,MAAM,EAAE,SAAS,MAAM,OAAO,KAAK;EACnC,IAAI,SAAS,GAAG,OAAO;EACvB,MAAM,OAAO,OAAO,MAAM,CAAC;EAC3B,MAAM,OAAO,KAAK,MAAM,GAAG,GAAG,OAAO,CAAC;EACtC,OAAO,KAAK,OAAO;CACrB;CAEA,OAAO,QAAsC;EAC3C,IAAI;EACJ,IAAI;GACF,OAAO,GAAG,KAAK,UAAU,MAAM,EAAE;EACnC,SAAS,OAAO;GACd,OAAO,QAAQ,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC,CAAC;EACjF;EACA,OAAO,IAAI,SAAe,SAAS,WAAW;GAC5C,KAAK,QAAQ,KAAK;IAAE,MAAM,KAAK,OAAO,MAAM;IAAG;IAAM;IAAS;GAAO,CAAC;GACtE,KAAK,KAAK;EACZ,CAAC;CACH;;CAGA,OAAqB;EACnB,IAAI,KAAK,aAAa,KAAA,GAAW;EACjC,KAAK,WAAW,KAAK,MAAM,CAAC,CAAC,cAAc;GACzC,KAAK,WAAW,KAAA;GAChB,IAAI,KAAK,QAAQ,SAAS,GAAG,KAAK,KAAK;EACzC,CAAC;CACH;;CAGA,MAAc,QAAuB;EACnC,OAAO,KAAK,QAAQ,SAAS,GAAG;GAC9B,MAAM,QAAQ,KAAK,QAAQ,OAAO,CAAC;GACnC,MAAM,yBAAS,IAAI,IAAuB;GAC1C,KAAK,MAAM,QAAQ,OAAO;IACxB,MAAM,OAAO,OAAO,IAAI,KAAK,IAAI;IACjC,IAAI,SAAS,KAAA,GAAW,OAAO,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;SAC/C,KAAK,KAAK,IAAI;GACrB;GACA,KAAK,MAAM,CAAC,MAAM,UAAU,QAC1B,IAAI;IACF,MAAM,SAAS,MAAM,KAAK,UAAU,IAAI;IACxC,MAAM,OAAQ,MAAM,WAAW,WAAW,MAAM,IAAK,OAAO;IAC5D,MAAM,SACJ,QACA,OAAO,KAAK,GAAG,OAAO,MAAM,KAAK,SAAS,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,MAAM,CACzE;IACA,IAAI,KAAK,MAAM,MAAM,OAAO,SAAS;IACrC,KAAK,MAAM,QAAQ,OAAO,KAAK,QAAQ;GACzC,SAAS,OAAO;IAEd,MAAM,SAAS,KAAK,QAAQ,IAAI,IAAI;IACpC,KAAK,QAAQ,OAAO,IAAI;IACxB,MAAM,QAAQ,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;IAC3C,KAAK,MAAM,QAAQ,OAAO,KAAK,OAAO,KAAK;GAC7C;EAEJ;CACF;;CAGA,MAAM,QAAuB;EAC3B,OAAO,KAAK,aAAa,KAAA,GAAW,MAAM,KAAK;CACjD;;CAGA,MAAc,QAAQ,QAAgB,OAA8C;EAClF,MAAM,YAAY,KAAK,KAAK,KAAK,MAAM;EACvC,IAAI,CAAC,WAAW,SAAS,GAAG,OAAO,CAAC;EACpC,MAAM,OAAO,OAAO,MAAM,GAAG,CAAC;EAC9B,MAAM,SAAS,MAAM,QAAQ,WAAW,EAAE,eAAe,KAAK,CAAC,EAAA,CAC5D,QAAQ,UAAU,MAAM,YAAY,KAAK,SAAS,KAAK,MAAM,IAAI,CAAC,CAAC,CACnE,KAAK,UAAU,MAAM,IAAI,CAAC,CAC1B,KAAK;EACR,MAAM,QAAkB,CAAC;EACzB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,SAAS,KAAA,KAAa,OAAO,KAAK,MAAM,GAAG,CAAC,GAAG;GACnD,MAAM,UAAU,MAAM,QAAQ,KAAK,WAAW,IAAI,CAAC,EAAA,CAChD,QAAQ,SAAS,WAAW,KAAK,IAAI,CAAC,CAAC,CACvC,KAAK;GACR,KAAK,MAAM,SAAS,QAAQ;IAC1B,IAAI,SAAS,KAAA,KAAa,SAAS,MAAM,KAAK,IAAI,MAAM;IACxD,MAAM,KAAK,KAAK,WAAW,MAAM,KAAK,CAAC;GACzC;EACF;EACA,OAAO;CACT;;CAGA,OAAe,WACb,QACA,QACA,SAC+B;EAC/B,KAAK,MAAM,QAAQ,MAAM,KAAK,QAAQ,QAAQ,QAAQ,KAAK,GAAG;GAC5D,IAAI,SAAS;GACb,WAAW,MAAM,EAAE,MAAM,cAAc,QAAQ,IAAI,GAAG;IACpD,UAAU;IACV,MAAM,UAAU,KAAK,KAAK;IAC1B,IAAI,QAAQ,WAAW,GAAG;IAC1B,IAAI;IACJ,IAAI;KACF,MAAM,KAAK,MAAM,OAAO;KACxB,IAAI,OAAO,QAAQ,YAAY,QAAQ,QAAQ,OAAO,IAAI,gBAAgB,UACxE,MAAM,IAAI,MAAM,cAAc;IAElC,QAAQ;KAEN,IAAI,UAAU,SAAS,YAAY;MAAE;MAAM,MAAM;KAAO,CAAC;KACzD;IACF;IACA,IAAI,QAAQ,SAAS,KAAA,KAAa,IAAI,SAAS,OAAO,MAAM;IAC5D,IAAI,QAAQ,UAAU,KAAA,KAAa,IAAI,cAAc,OAAO,OAAO;IACnE,MAAM;GACR;EACF;CACF;;;;;;;;CASA,OAAO,KAAK,QAAwB,SAAqD;EACvF,MAAM,KAAK,MAAM;EACjB,MAAM,UACJ,QAAQ,WAAW,KAAA,IACf,CAAC,OAAO,MAAM,IACd,WAAW,KAAK,GAAG,KAChB,MAAM,QAAQ,KAAK,KAAK,EAAE,eAAe,KAAK,CAAC,EAAA,CAC7C,QAAQ,UAAU,MAAM,YAAY,CAAC,CAAC,CACtC,KAAK,UAAU,MAAM,IAAI,CAAC,CAC1B,KAAK,IACR,CAAC;EACT,IAAI,QAAQ,WAAW,GAAG;GACxB,OAAO,KAAK,WAAW,QAAQ,IAAK,QAAQ,OAAO;GACnD;EACF;EAEA,MAAM,UAAU,QAAQ,KAAK,WAC3B,KAAK,WAAW,QAAQ,QAAQ,OAAO,CAAC,CAAC,OAAO,cAAc,CAAC,CACjE;EACA,MAAM,QAA0C,CAAC;EACjD,KAAK,MAAM,UAAU,SACnB,MAAM,MAAM,MAAM,OAAO,KAAK,EAAA,CAAG,KAAkC;EACrE,SAAS;GACP,IAAI,OAAO;GACX,KAAK,MAAM,CAAC,OAAO,SAAS,MAAM,QAAQ,GACxC,IAAI,SAAS,KAAA,MAAc,SAAS,MAAM,KAAK,cAAc,MAAM,KAAK,CAAE,cACxE,OAAO;GAGX,IAAI,SAAS,IAAI;GACjB,MAAM,MAAM;GACZ,MAAM,SAAS,MAAM,QAAQ,KAAK,CAAE,KAAK,EAAA,CAAG;EAC9C;CACF;;CAGA,MAAM,QAAuB;EAC3B,MAAM,KAAK,MAAM;EACjB,MAAM,UAAU,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC;EACzC,KAAK,QAAQ,MAAM;EACnB,KAAK,MAAM,UAAU,SAAS,MAAM,OAAO,MAAM;CACnD;AACF"}

package/build/loopback-C_QYowSf.mjs

@@ -1,22 +0,0 @@
-import { isIP } from "node:net";
-//#region src/loopback.ts
-/**
-* This function tells if a host name or an address is a loopback host: `localhost`, an address in
-* `127.0.0.0/8`, or `::1`. The MCP server over HTTP binds to a loopback host unless the operator
-* permits a different host. The function imports only Node.js, so the command line uses it before
-* the store opens.
-*/
-const isLoopbackHost = (host) => {
-	const name = host.trim().toLowerCase().replace(/^\[(.*)\]$/, "$1");
-	if (name === "localhost" || name.endsWith(".localhost")) return true;
-	const mapped = name.startsWith("::ffff:") ? name.slice(7) : name;
-	if (isIP(mapped) === 4) return mapped.startsWith("127.");
-	if (isIP(name) === 6) return /^(?:0{0,4}:){7}0{0,3}1$|^::1$/.test(name);
-	return false;
-};
-/** This function gives a host for a URL: an IPv6 address goes in brackets. */
-const urlHost = (host) => isIP(host) === 6 ? `[${host}]` : host;
-//#endregion
-export { urlHost as n, isLoopbackHost as t };
-
-//# sourceMappingURL=loopback-C_QYowSf.mjs.map

package/build/loopback-C_QYowSf.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"loopback-C_QYowSf.mjs","names":[],"sources":["../src/loopback.ts"],"sourcesContent":["import { isIP } from \"node:net\";\n\n/**\n * This function tells if a host name or an address is a loopback host: `localhost`, an address in\n * `127.0.0.0/8`, or `::1`. The MCP server over HTTP binds to a loopback host unless the operator\n * permits a different host. The function imports only Node.js, so the command line uses it before\n * the store opens.\n */\nexport const isLoopbackHost = (host: string): boolean => {\n  const name = host\n    .trim()\n    .toLowerCase()\n    .replace(/^\\[(.*)\\]$/, \"$1\");\n  if (name === \"localhost\" || name.endsWith(\".localhost\")) return true;\n  const mapped = name.startsWith(\"::ffff:\") ? name.slice(7) : name;\n  if (isIP(mapped) === 4) return mapped.startsWith(\"127.\");\n  if (isIP(name) === 6) return /^(?:0{0,4}:){7}0{0,3}1$|^::1$/.test(name);\n  return false;\n};\n\n/** This function gives a host for a URL: an IPv6 address goes in brackets. */\nexport const urlHost = (host: string): string => (isIP(host) === 6 ? `[${host}]` : host);\n"],"mappings":";;;;;;;;AAQA,MAAa,kBAAkB,SAA0B;CACvD,MAAM,OAAO,KACV,KAAK,CAAC,CACN,YAAY,CAAC,CACb,QAAQ,cAAc,IAAI;CAC7B,IAAI,SAAS,eAAe,KAAK,SAAS,YAAY,GAAG,OAAO;CAChE,MAAM,SAAS,KAAK,WAAW,SAAS,IAAI,KAAK,MAAM,CAAC,IAAI;CAC5D,IAAI,KAAK,MAAM,MAAM,GAAG,OAAO,OAAO,WAAW,MAAM;CACvD,IAAI,KAAK,IAAI,MAAM,GAAG,OAAO,gCAAgC,KAAK,IAAI;CACtE,OAAO;AACT;;AAGA,MAAa,WAAW,SAA0B,KAAK,IAAI,MAAM,IAAI,IAAI,KAAK,KAAK"}

package/build/mcp.d.mts

@@ -1,199 +0,0 @@
-import { E as OpenOptions, O as PoliteCrawler, s as DaemonMcpHost } from "./daemon-DP4EqjFL.mjs";
-import { McpServer, McpServer as McpServer$1 } from "@modelcontextprotocol/sdk/server/mcp.js";
-import { Request, RequestId, Result, Task } from "@modelcontextprotocol/sdk/types.js";
-import { CreateTaskOptions, TaskStore } from "@modelcontextprotocol/sdk/experimental/tasks/interfaces.js";
-import "@modelcontextprotocol/sdk/shared/protocol.js";
-//#region src/mcp-tasks.d.ts
-/** These are the limits of the tasks of one MCP server. */
-export declare const TASK_LIMITS: {
-  /** A task and its result stay for this time after the task starts or ends: 1 hour by default. */
-  readonly ttlMs: 3_600_000;
-  /** The longest time that a client can ask for: 1 day. */
-  readonly maxTtlMs: 86_400_000;
-  /** The time that the server proposes between two polls of the status: 1 second. */
-  readonly pollMs: 1_000;
-  /** The most tasks that run at one time. A new task above the limit is an error. */
-  readonly maxRunning: 8;
-};
-/** This is a task of the server, as the status of the daemon shows it. */
-interface TaskInfo {
-  /** This id is the task id that the client got. */
-  readonly id: string;
-  /** This value is the name of the tool, for example `crawl_site`. */
-  readonly tool: string;
-  /** This value is `working`, `input_required`, `completed`, `failed` or `cancelled`. */
-  readonly status: string;
-  /** This text tells the progress, for example `3 of 10 done`, or null. */
-  readonly message: string | null;
-  readonly created_at: string;
-  readonly updated_at: string;
-}
-/** These options make a task store. */
-interface CrawlTaskStoreOptions {
-  /** This value is the most tasks that run at one time. The default is 8. */
-  readonly maxRunning?: number | undefined;
-  /** This function operates when a task starts, changes its status or ends. */
-  readonly onChange?: (() => void) | undefined;
-}
-/**
- * This store keeps the tasks of the long tools in memory. It wraps the `InMemoryTaskStore` of the
- * SDK, which binds each task to its session. It also keeps an abort signal for each task: the
- * cancel of a task (`tasks/cancel`) aborts the work of its tool. One store serves each session of
- * one server.
- */
-export declare class CrawlTaskStore implements TaskStore {
-  private readonly options;
-  private readonly store;
-  private readonly jobs;
-  private readonly maxRunning;
-  private closed;
-  constructor(options?: CrawlTaskStoreOptions);
-  private changed;
-  /** This method removes the jobs whose tasks the SDK store removed at the end of their time. */
-  private prune;
-  createTask(params: CreateTaskOptions, requestId: RequestId, request: Request, sessionId?: string): Promise<Task>;
-  getTask(taskId: string, sessionId?: string): Promise<Task | null>;
-  storeTaskResult(taskId: string, status: "completed" | "failed", result: Result, sessionId?: string): Promise<void>;
-  getTaskResult(taskId: string, sessionId?: string): Promise<Result>;
-  updateTaskStatus(taskId: string, status: Task["status"], statusMessage?: string, sessionId?: string): Promise<void>;
-  listTasks(cursor?: string, sessionId?: string): Promise<{
-    tasks: Task[];
-    nextCursor?: string;
-  }>;
-  /** This method gives the abort signal of a task. A cancel of the task aborts it. */
-  signal(taskId: string): AbortSignal;
-  /** This method gives the reason that a task did not start, or undefined for a task that started. */
-  refusal(taskId: string): string | undefined;
-  /** This method tells if the client asked for the task. False is the call of a client with no tasks. */
-  augmented(taskId: string): boolean;
-  /** This method gives the tasks that did not end, the oldest first. */
-  running(): TaskInfo[];
-  /** This method gives the sessions that have a task in progress. */
-  busySessions(): Set<string>;
-  /** This method cancels the tasks of a session that closed. Their work stops. */
-  cancelSession(sessionId: string, message?: string): Promise<void>;
-  /** This method cancels each task that runs, and removes all the tasks. A second close does nothing. */
-  close(): Promise<void>;
-}
-//#endregion
-//#region src/mcp-http.d.ts
-/** These are the defaults of the MCP server over HTTP. */
-export declare const MCP_HTTP_DEFAULTS: {
-  /** The port of the server. */
-  readonly port: 9307;
-  /** The host of the server: the IPv4 loopback address. */
-  readonly host: "127.0.0.1";
-  /** The path of the MCP endpoint. */
-  readonly path: "/mcp";
-  /** The path of the status of the daemon. */
-  readonly statusPath: "/status";
-  /** The most sessions at one time. A new session above the limit closes the session that was idle longest. */
-  readonly maxSessions: 32;
-  /** The largest body of a request that starts a session: 4 MiB. */
-  readonly maxBodyBytes: 4_194_304;
-  /** The shortest token: 16 characters. */
-  readonly minTokenLength: 16;
-};
-/** These options serve MCP over HTTP. */
-interface McpHttpOptions {
-  /** This value is the port. The default is 9307. The value 0 takes a free port. */
-  readonly port?: number | undefined;
-  /** This value is the host to bind. The default is `127.0.0.1`. A host that is not loopback needs `allowRemote`. */
-  readonly host?: string | undefined;
-  /** The value true permits a host that is not loopback. Then other computers can try the token. */
-  readonly allowRemote?: boolean | undefined;
-  /** This value is the bearer token. The default reads `tokenFile`, or makes a random token. */
-  readonly token?: string | undefined;
-  /** This file holds the bearer token, when `token` is not given. */
-  readonly tokenFile?: string | undefined;
-  /** This function gives the JSON of `GET /status`. Without it, the path gives 404. */
-  readonly status?: (() => unknown) | undefined;
-  /** This store keeps the tasks of each session. The default is a new `CrawlTaskStore`. */
-  readonly tasks?: CrawlTaskStore | undefined;
-  /** This function writes the messages of the server. The default writes nothing. */
-  readonly log?: ((line: string) => void) | undefined;
-}
-/** This is a server that serves MCP over HTTP. */
-interface McpHttpServer {
-  /** This URL is the MCP endpoint, for example `http://127.0.0.1:9307/mcp`. */
-  readonly url: string;
-  readonly host: string;
-  readonly port: number;
-  /** This value is the bearer token. Give it to the client, and keep it secret. */
-  readonly token: string;
-  /** This store keeps the tasks of the long tools. */
-  readonly tasks: CrawlTaskStore;
-  /** This method counts the open sessions. */
-  sessions(): number;
-  /** This method closes each session, cancels each task and stops the server. A second close does nothing. */
-  close(): Promise<void>;
-}
-/**
- * This function compares a token with the token of the server in constant time. It compares the
- * SHA-256 digests, so the time does not depend on the length or the text of either token.
- */
-export declare const sameToken: (given: string, token: string) => boolean;
-/** This function makes a random token: 32 bytes in base64url, 43 characters. */
-export declare const makeToken: () => string;
-//#endregion
-//#region src/mcp.d.ts
-/** These options change the tools of the MCP server. */
-export interface CrawlServerOptions {
-  /**
-   * These product tokens get their robots.txt rules applied to each fetch, in addition to the token
-   * of the crawler. The default is `["Claude-User"]`, because an assistant asks for these pages.
-   */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /**
-   * The value true lets `user_provided` release held-back text when the client cannot ask the
-   * user. It is the opt-in of the operator. The default is false. When the client can ask the
-   * user (MCP elicitation), the server always asks first.
-   */
-  readonly allowUserProvided?: boolean | undefined;
-  /** This text tells why the server renders no page. The result of a page that needs JavaScript shows it. */
-  readonly renderOff?: string | undefined;
-  /**
-   * This store makes the long tools (`crawl_site` and `crawl_discover`) MCP tasks. A client that
-   * asks for a task gets a task id at once. A client with no tasks gets the result in the call. The
-   * server over HTTP gives the store. Without it, the long tools are normal tools.
-   */
-  readonly tasks?: CrawlTaskStore | undefined;
-}
-/** These options start the MCP server on the standard input and output. */
-export interface StartMcpOptions extends OpenOptions, CrawlServerOptions {
-  /** This function writes the messages of the server. The default writes them to the standard error. */
-  readonly log?: ((line: string) => void) | undefined;
-}
-/**
- * This function makes an MCP server with nine tools over one crawler: `crawl_fetch`, `crawl_find`,
- * `crawl_read`, `crawl_verify`, `crawl_extract`, `crawl_discover`, `crawl_site`, `crawl_diff` and
- * `crawl_watch_list`.
- * The server does not connect to a transport. Each fetch applies the robots.txt rules for
- * `Claude-User` too, unless `onBehalfOf` gives other tokens. No tool adds a watch or an export,
- * because they are actions of the operator.
- */
-export declare const createMcpServer: (crawler: PoliteCrawler, options?: CrawlServerOptions) => McpServer$1;
-/**
- * This function starts the MCP server on the standard input and output. The fetches apply the
- * robots.txt rules for `Claude-User` too, because an assistant asks for these pages. When the
- * default renderer has no browser, the server turns rendering off at start and writes one note.
- * The server stops when its standard input closes, or at `SIGINT` or `SIGTERM`.
- */
-export declare const startMcpServer: (options: StartMcpOptions) => Promise<void>;
-/** These options serve the tools over HTTP: the options of the tools, and of the HTTP server. */
-export interface StartMcpHttpOptions extends McpHttpOptions, Omit<CrawlServerOptions, "tasks"> {}
-/**
- * This function serves the tools of a crawler over the Streamable HTTP transport of MCP. Each
- * session gets an MCP server with the nine tools of the server on the standard input and output.
- * The envelope, the elicitation and the opt-ins are the same. `crawl_site` and `crawl_discover` are MCP tasks.
- * Each request needs the bearer token. The function does not close the crawler.
- */
-export declare const startMcpHttpServer: (crawler: PoliteCrawler, options?: StartMcpHttpOptions) => Promise<McpHttpServer>;
-/**
- * This host serves the tools of the daemon over HTTP. It is the seam between the daemon and its MCP
- * transport: a different MCP host can replace it with `DaemonOptions.mcpHost`.
- */
-export declare const mcpHttpHost: DaemonMcpHost;
-//#endregion
-export type { CrawlTaskStoreOptions, McpHttpOptions, McpHttpServer, McpServer, TaskInfo };
-//# sourceMappingURL=mcp.d.mts.map