polite-crawler 0.1.3 → 0.2.0

package/build/politeness-yI3xBEx9.d.mts

@@ -1,367 +0,0 @@
-//#region src/politeness.d.ts
-/**
- * The scheduler controls the pace of each host. The speed of a polite crawler is the number of
- * hosts in progress divided by the delay for each host. Thus the scheduler keeps one request in
- * progress for each host, and it waits the delay of the host between two requests. It uses its
- * concurrency across many hosts, as the back queues of the Mercator crawler do.
- *
- * The caller decides the delay after each request, and `run` gets it from the task. Only the caller
- * knows the documented rate of the source, the crawl delay of the site and the duration of the
- * request. A wait holds no global slot, so a host that waits does not block a different host.
- *
- * The state of each host (its next time, its counts and its stop) is in a `HostStateStore`. The
- * default store is in memory. A store that processes share keeps one pace for each host across them.
- */
-interface Clock {
-  readonly now: () => number;
-  /** This function waits `ms` milliseconds. When the signal aborts, the wait ends early. */
-  readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
-}
-declare const realClock: Clock;
-/** The crawler does not ask a host again until the stop ends, after a refusal (a challenge, 403 or 402). */
-declare class HostStopped extends Error {
-  readonly host: string;
-  readonly reason: string;
-  /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */
-  readonly until: number;
-  constructor(host: string, reason: string, until?: number);
-}
-interface Paced<T> {
-  readonly value: T;
-  /** This value is the time that the host rests before its next request. */
-  readonly delayMs: number;
-}
-/** A stop keeps the crawler away from a host until a time. */
-interface HostStop {
-  /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */
-  readonly until: number;
-  readonly reason: string;
-}
-/** These are the counts of a host. Each count gives the results of one kind in sequence. */
-interface HostCounts {
-  /** This value counts errors and requests with no answer. At the limit of the policy, the host stops. */
-  readonly failures: number;
-  /** This value counts the requests to wait (429, 502, 503, 504). It sets the backoff and does not stop the host. */
-  readonly waits: number;
-}
-/**
- * This value tells how one result changes the counts of a host:
- *
- * - `success` sets the two counts to zero.
- * - `failure` adds one to `failures`.
- * - `wait` adds one to `waits`.
- * - `neutral` changes nothing. A client error (a 400) is neutral, because a different request can succeed.
- */
-type HostResult = "success" | "failure" | "wait" | "neutral";
-/**
- * A host state store keeps the state of each host for the scheduler. Its methods are synchronous,
- * and each method is one atomic step on the state of one host. Thus a store can be a table of
- * SQLite that processes share. Times are milliseconds since the epoch.
- *
- * The state of a host has five parts:
- *
- * - the next time: the earliest time of the next request
- * - the lease: the end of the slot of the request in progress, when there is one
- * - the counts (`HostCounts`)
- * - the stop: an end time and a reason, when there is one
- */
-interface HostStateStore {
-  /**
-   * This method asks for the slot of a host at the time `now`. When the host has no rest left
-   * and no lease, the store gives the slot to the caller until `now + leaseMs` and gives 0.
-   * Otherwise it changes nothing and gives the time to wait in milliseconds: the time to the next
-   * time, or to the end of the lease. A store that processes share gives a short wait (one second
-   * or less) during a lease, because the holder usually releases it early. The caller asks again after the wait.
-   */
-  reserve(host: string, now: number, leaseMs: number): number;
-  /** This method changes the counts of a host for one result, and it gives the new counts. */
-  record(host: string, result: HostResult): HostCounts;
-  /**
-   * This method ends the lease of the caller. It sets the next time to `nextAt` when `nextAt` is
-   * later than the stored next time. Only the holder of the lease uses it.
-   */
-  release(host: string, nextAt: number): void;
-  /** This method sets the next time to `until` when `until` is later. It does not touch the lease. */
-  defer(host: string, until: number): void;
-  /** This method stops a host until a time. When the host has a stop that ends later, that stop stays. */
-  stop(host: string, until: number, reason: string): void;
-  /** This method gives the stop of a host at the time `now`. A stop that ended gives no value. */
-  stopped(host: string, now: number): HostStop | undefined;
-  /**
-   * This method removes the stops and the counts of one host, or of each host when there is no
-   * host. It keeps the next times and the leases, so a reset cannot make a host go faster than its pace.
-   */
-  reset(host?: string): void;
-  /**
-   * This method gives the validator memo of a host, when the store has one. A store without this
-   * method and `noteValidators` has no memo. The crawler then sends each validator of a stored copy.
-   */
-  validators?(host: string): ValidatorMemo | undefined;
-  /**
-   * This method adds one conditional request to the memo of a host, with the rules of
-   * `nextValidatorMemo`, in one atomic step. It gives the new memo.
-   */
-  noteValidators?(host: string, report: ValidatorReport, now: number): ValidatorMemo;
-}
-/** This value tells the result that the crawler learned for one validator of a host. */
-type ValidatorVerdict = "unknown" | "works" | "fails";
-/**
- * This value tells what one conditional request got:
- *
- * - `not-modified`: a 304.
- * - `same`: a 200 with the body that the store has. Thus the validator did not match.
- * - `changed`: a 200 with a different body. This answer tells nothing about the validator.
- */
-type ValidatorAnswer = "not-modified" | "same" | "changed";
-/** This report tells the validators that one conditional request sent, and the answer that it got. */
-interface ValidatorReport {
-  /** This value is true when the request sent `If-None-Match`. */
-  readonly etag: boolean;
-  /** This value is true when the request sent `If-Modified-Since`. */
-  readonly lastModified: boolean;
-  readonly answer: ValidatorAnswer;
-}
-/**
- * A validator memo holds what the crawler learned about the validators of one host. A server
- * examines `If-None-Match` first, and it ignores `If-Modified-Since` when the request has the two
- * (RFC 9110, section 13.1.3). Thus a host with unstable ETags does not answer 304 to a request with
- * the two validators at any time. Its stable `Last-Modified` date does not help. `nextValidatorMemo`
- * gives the rules.
- */
-interface ValidatorMemo {
-  /** This value is true when the crawler leaves out `If-None-Match` for the host and sends only `If-Modified-Since`. */
-  readonly skipEtag: boolean;
-  /** This value tells if a request with `If-None-Match` got a 304 (`works`) or the stored body again (`fails`). */
-  readonly etag: ValidatorVerdict;
-  /** This value tells the same for a request with only `If-Modified-Since`. */
-  readonly lastModified: ValidatorVerdict;
-  /** This value counts the stored bodies in sequence that came back to a request with `If-None-Match`. */
-  readonly etagMisses: number;
-  /** This value counts the stored bodies in sequence that came back to a request with only `If-Modified-Since`. */
-  readonly dateMisses: number;
-  /** This value counts the requests with only `If-Modified-Since` since the crawler stopped the ETag, or since the last probe. */
-  readonly sinceProbe: number;
-  /** This time (milliseconds since the epoch) is the last time that the crawler stopped or started the ETag, or probed it. */
-  readonly changedAt: number;
-}
-/** These are the limits of the validator memo. */
-interface ValidatorLimits {
-  /** After this number of stored bodies in sequence, a validator fails. The value is 3. */
-  readonly misses: number;
-  /** After this number of requests with only `If-Modified-Since`, the next request sends the ETag again (a probe). The value is 20. */
-  readonly probeEvery: number;
-  /** After this time with no probe, the next request is a probe. The value is 7 days. */
-  readonly probeAgeMs: number;
-}
-/** These are the limits of the validator memo of the crawler. */
-declare const VALIDATOR_MEMO: ValidatorLimits;
-/**
- * This function tells which validators the next conditional request to a host sends. It sends
- * each validator of the stored copy, except when the memo stopped the ETag and the copy has a
- * `Last-Modified` date. Then it sends only `If-Modified-Since`, and the ETag too for a probe.
- * A probe comes after `probeEvery` requests or after `probeAgeMs`.
- */
-declare const validatorsToSend: (memo: ValidatorMemo | undefined, stored: {
-  readonly etag: boolean;
-  readonly lastModified: boolean;
-}, now: number) => {
-  readonly etag: boolean;
-  readonly lastModified: boolean;
-};
-/**
- * This function adds one conditional request to the memo of a host, and gives the new memo.
- * These are the rules, with the limits of `VALIDATOR_MEMO`:
- *
- * 1. A 304 to a request with `If-None-Match` marks the ETag `works`. After a probe, the crawler sends the ETag again.
- * 2. After `misses` stored bodies in sequence with `If-None-Match`, the ETag fails. When the request also sent
- *    `If-Modified-Since`, the crawler stops the ETag for the host.
- * 3. A probe that gets a 200 keeps the ETag stopped, and the count to the next probe starts again.
- * 4. After `misses` stored bodies in sequence with only `If-Modified-Since`, that validator fails too. Then the
- *    crawler sends the two validators again, because they cost nothing. It tries the date alone again after `probeAgeMs`.
- * 5. A changed body tells nothing about a validator.
- */
-declare const nextValidatorMemo: (memo: ValidatorMemo | undefined, report: ValidatorReport, now: number) => ValidatorMemo;
-/**
- * This store keeps the state of each host in memory, for one process. It removes the state of an
- * idle host (no rest left, no lease, no counts and no stop) when it holds more than `maxHosts` hosts.
- * It keeps the validator memos of the `maxHosts` hosts that it used last.
- */
-declare class MemoryHostStateStore implements HostStateStore {
-  private readonly maxHosts;
-  private readonly states;
-  /** The validator memos, the memo used last at the end. */
-  private readonly memos;
-  private sweepAt;
-  constructor(maxHosts?: number);
-  /** This value is the number of hosts that the store holds. */
-  get size(): number;
-  private state;
-  private sweep;
-  reserve(host: string, now: number, leaseMs: number): number;
-  record(host: string, result: HostResult): HostCounts;
-  release(host: string, nextAt: number): void;
-  defer(host: string, until: number): void;
-  stop(host: string, until: number, reason: string): void;
-  stopped(host: string, now: number): HostStop | undefined;
-  reset(host?: string): void;
-  validators(host: string): ValidatorMemo | undefined;
-  noteValidators(host: string, report: ValidatorReport, now: number): ValidatorMemo;
-}
-/** These options change the scheduler. */
-interface SchedulerOptions {
-  /** This store keeps the state of each host. The default is a `MemoryHostStateStore`. */
-  readonly store?: HostStateStore | undefined;
-  /** This value is the longest slot of one request. A slot that a crashed process holds ends after it. The default is 5 minutes. */
-  readonly leaseMs?: number | undefined;
-  /** A host rests this long after a task that throws an error, or a task that gives no valid delay. The default is 1 second. */
-  readonly fallbackDelayMs?: number | undefined;
-}
-declare class HostScheduler {
-  private readonly clock;
-  private readonly maxActive;
-  /** This store keeps the state of each host. */
-  readonly store: HostStateStore;
-  private readonly chains;
-  private active;
-  private readonly queue;
-  private readonly leaseMs;
-  private readonly fallbackDelayMs;
-  constructor(clock: Clock, maxActive: number, options?: SchedulerOptions);
-  private acquire;
-  private release;
-  private sleep;
-  /**
-   * This method starts one request on a host, in sequence, after the rest of the host. When the
-   * signal aborts during the wait, the promise rejects with the reason of the signal.
-   */
-  run<T>(host: string, task: () => Promise<Paced<T>>, signal?: AbortSignal): Promise<T>;
-  /** The server asked for a pause (Retry-After), or a backoff chose one. */
-  park(host: string, untilMs: number): void;
-  /** This method stops a host until `untilMs`. With no time, the stop has no end. */
-  stop(host: string, reason: string, untilMs?: number): void;
-  stoppedReason(host: string): string | undefined;
-  /** This method gives the stop of a host, with its end, when the host is stopped. */
-  stopOf(host: string): HostStop | undefined;
-  /** This value counts the failures of a host in sequence, for the backoff. A success sets it to zero. */
-  failed(host: string): number;
-  /** This value counts the requests to wait (429, 503) of a host in sequence. They do not stop the host. */
-  waited(host: string): number;
-  succeeded(host: string): void;
-  /** This method removes the stops and the counts of one host, or of each host. The pace stays. */
-  reset(host?: string): void;
-}
-/**
- * This function gives an exponential backoff with full jitter, from the analysis of AWS. The wait
- * is random, up to base × 2^(try − 1), with a maximum. The jitter keeps retries apart.
- */
-declare const backoffMs: (attempt: number, baseMs: number, capMs: number, random: () => number) => number;
-/**
- * This function gives an exponential backoff with equal jitter. The wait is half of
- * base × 2^(try − 1), with a maximum, and a random part up to the other half. Thus each backoff
- * is at least half of its step, and it is not lost under the rest of the host.
- */
-declare const equalJitterMs: (attempt: number, baseMs: number, capMs: number, random: () => number) => number;
-/** These are the facts of one request that set the rest of its host. */
-interface RestInput {
-  /** This value is the kind of the request: a web page, a documented API, or a render in the browser. */
-  readonly kind: "page" | "api" | "render";
-  /** This value is the duration of the request, in milliseconds. */
-  readonly elapsedMs: number;
-  /** This value is the `crawl-delay` of robots.txt, in seconds. */
-  readonly crawlDelaySeconds?: number | undefined;
-  /** This value is the documented pace of the source. It replaces the interval of the policy for the kind. */
-  readonly minIntervalMs?: number | undefined;
-  /** This value is the wait that the server asked for (`Retry-After`). */
-  readonly retryAfterMs?: number | undefined;
-}
-/** The rest of a host after a request, and the reason for its length. */
-interface Rest {
-  readonly ms: number;
-  readonly reason: "pace" | "slow" | "retry-after" | "crawl-delay";
-}
-/**
- * This function gives the rest of a host after one request, as the crawler calculates it. These
- * are the parts:
- *
- * - The interval of the kind (3 seconds for a page), or the documented pace of the source.
- * - The `crawl-delay` of robots.txt, up to `crawlDelayCapMs`, when it is longer.
- * - After a page or a render, `elapsedFactor` times the duration of the request, up to `elapsedCapMs`.
- * - The `Retry-After` of the server, when it is longer.
- *
- * The backoff after an error or a request to wait comes on top, with jitter. The function is
- * pure, so a simulation and a study can use the pace of the crawler.
- */
-declare const restAfter: (policy: PolitenessPolicy, input: RestInput) => Rest;
-/**
- * The politeness policy holds each limit of the crawler. Each value has a default, and a host can
- * have values of its own. The times are in milliseconds.
- */
-interface PolitenessPolicy {
-  /** This value is the shortest rest of a host between two page requests. The default is 3 seconds. */
-  readonly pageIntervalMs: number;
-  /** This value is the shortest rest of a host between two API requests. The default is 1 second. */
-  readonly apiIntervalMs: number;
-  /** This value is the shortest rest of a host after a render. The default is 5 seconds. */
-  readonly renderIntervalMs: number;
-  /** After a page, the host rests this number of times the duration of the request. The default is 5. */
-  readonly elapsedFactor: number;
-  /** This value is the maximum of the rest that comes from the duration. The default is 30 seconds. */
-  readonly elapsedCapMs: number;
-  /** This value is the maximum `crawl-delay` that the crawler obeys. The default is 60 seconds. */
-  readonly crawlDelayCapMs: number;
-  /** This value is the maximum number of redirects for one fetch. The default is 5. */
-  readonly maxRedirects: number;
-  /** This value is the maximum size of a body after decompression. The default is 5 MiB. */
-  readonly maxBytes: number;
-  /** This value is the deadline of one request, the body included. The default is 90 seconds. */
-  readonly requestTimeoutMs: number;
-  /** This value is the deadline of one render. The default is 20 seconds. */
-  readonly renderTimeoutMs: number;
-  /** After this number of failures in sequence, the host stops. The default is 5. */
-  readonly maxFailures: number;
-  /** This value is the stop after too many failures. The default is 1 hour. */
-  readonly failureStopMs: number;
-  /** This value is the stop after a 401 or a 403. The default is 1 hour. */
-  readonly blockedStopMs: number;
-  /** This value is the stop after a bot challenge. The default is 24 hours. */
-  readonly challengeStopMs: number;
-  /** This value is the stop after a 402. The default is 24 hours. */
-  readonly paymentStopMs: number;
-  /** The crawler obeys a `Retry-After` up to this value. A longer wait stops the host for that time. The default is 1 hour. */
-  readonly maxRetryAfterMs: number;
-  /** This value is the first step of the backoff after an error or a request to wait. The default is 5 seconds. */
-  readonly backoffBaseMs: number;
-  /** This value is the maximum step of the backoff. The default is 300 seconds. */
-  readonly backoffCapMs: number;
-  /** The crawler keeps a robots.txt for this time. The default is 24 hours. */
-  readonly robotsTtlMs: number;
-  /** The crawler keeps an unreachable robots.txt verdict for this time first, with a backoff. The default is 5 minutes. */
-  readonly robotsRetryMs: number;
-  /** This value is the maximum time for an unreachable robots.txt verdict. The default is 1 hour. */
-  readonly robotsRetryCapMs: number;
-  /** A stored robots.txt younger than this age replaces a robots.txt that the crawler cannot read. The default is 30 days. */
-  readonly robotsStaleMs: number;
-  /** This value is the maximum freshness that the crawler gives a response with no freshness of its own. The default is 24 hours. */
-  readonly heuristicFreshnessCapMs: number;
-}
-/**
- * This value is the longest stop that a `Retry-After` can give: 30 days. A wait longer than
- * `maxRetryAfterMs` stops the host for the time that the server asks, up to this value.
- */
-declare const MAX_STOP_MS: number;
-declare const DEFAULT_POLICY: PolitenessPolicy;
-/**
- * These options change the politeness policy. The values at the top apply to each host. The
- * `hosts` map gives values for one host (`api.example.org`) or for the hosts under a name
- * (`*.example.org`). The most specific entry wins.
- */
-interface PolicyOptions extends Partial<PolitenessPolicy> {
-  readonly hosts?: Readonly<Record<string, Partial<PolitenessPolicy>>> | undefined;
-}
-/** This function examines the options of a policy. It throws a `RangeError` for a value that is not a valid number. */
-declare const checkPolicy: (options: PolicyOptions | undefined) => void;
-/** This function gives the policy of one host: the defaults, then the general options, then the entries for the host. */
-declare const policyFor: (options: PolicyOptions | undefined, host: string) => PolitenessPolicy;
-//#endregion
-export { validatorsToSend as A, backoffMs as C, policyFor as D, nextValidatorMemo as E, realClock as O, ValidatorVerdict as S, equalJitterMs as T, VALIDATOR_MEMO as _, HostScheduler as a, ValidatorMemo as b, HostStopped as c, Paced as d, PolicyOptions as f, SchedulerOptions as g, RestInput as h, HostResult as i, restAfter as k, MAX_STOP_MS as l, Rest as m, DEFAULT_POLICY as n, HostStateStore as o, PolitenessPolicy as p, HostCounts as r, HostStop as s, Clock as t, MemoryHostStateStore as u, ValidatorAnswer as v, checkPolicy as w, ValidatorReport as x, ValidatorLimits as y };
-//# sourceMappingURL=politeness-yI3xBEx9.d.mts.map

package/build/render-C1CUraWF.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"render-C1CUraWF.mjs","names":[],"sources":["../src/render.ts"],"sourcesContent":["import type { APIResponse, Browser, BrowserContext, Page, Request, Route } from \"playwright-core\";\n\n/**\n * The renderer opens the headless browser only for a page that shows almost nothing without\n * JavaScript. The settings:\n *\n * - The default `chromium-headless-shell` of Playwright, the lighter build.\n * - One browser for many pages, and a new context for each page. Thus no cookie or storage goes\n *   from one site to the next. A context has its own HTTP cache, so routing each request costs\n *   no cache across sites.\n * - Each request of the page goes through a route, the first navigation too. The route refuses\n *   images, fonts and media by their resource type. It asks `allowRequest` about each other\n *   request, and it sends `headers` with it.\n * - Playwright routes only the first request of a redirect chain. Thus with `allowRequest`, the\n *   route sends each request itself, follows each redirect itself and asks about each step. The\n *   browser then gets each body whole, not as a stream.\n * - No download, no popup, no service worker and no dialog.\n * - A wait until the visible text stops growing, with a limit. Playwright discourages\n *   `networkidle`. The text includes open shadow roots and the frames of the same origin.\n * - One deadline for the whole render: the navigation, the wait and the reading of the page.\n * - After a crash or a disconnection, the next page launches a new browser.\n * - The browser closes after two idle minutes. An idle Chromium holds hundreds of megabytes, so\n *   it stays open only while pages need it. The open browser keeps the process alive, so a\n *   script uses `close()` when it is done.\n */\n\nexport interface Rendered {\n  readonly html: string;\n  readonly finalUrl: string;\n  readonly status: number | null;\n  readonly headers: Readonly<Record<string, string>>;\n}\n\n/** These are the options of one render. */\nexport interface RenderOptions {\n  readonly userAgent: string;\n  /** This value is the deadline of the whole render, in milliseconds. */\n  readonly timeoutMs: number;\n  /** These headers go with each request of the page, for example the `From` header. */\n  readonly headers?: Readonly<Record<string, string>> | undefined;\n  /**\n   * This function decides each request that the page makes, the first navigation included. When\n   * it gives false, the browser does not send the request. The crawler uses it to apply the\n   * egress policy and robots.txt to the requests of the page.\n   */\n  readonly allowRequest?: ((url: string) => Promise<boolean> | boolean) | undefined;\n  /** This value is the maximum number of redirects of one request of the page. The default is 5, as for a plain fetch. */\n  readonly maxRedirects?: number | undefined;\n  /** When this signal aborts, the renderer stops the page: it closes the context, and the render rejects. */\n  readonly signal?: AbortSignal | undefined;\n  /**\n   * This function gives the signature headers of one request of the page (Web Bot Auth). The route\n   * adds them to the request and to each step of its redirects. When the function gives undefined,\n   * the browser does not send the request.\n   */\n  readonly sign?: ((url: string) => Readonly<Record<string, string>> | undefined) | undefined;\n}\n\nexport interface Renderer {\n  render(url: string, options: RenderOptions): Promise<Rendered>;\n  close(): Promise<void>;\n}\n\n/** The reason of a `RenderRefused`. */\nexport type RefusedReason = \"refused\" | \"download\" | \"scheme\" | \"browser\";\n\n/**\n * This error tells that the renderer did not render the page, and that the page or the site is\n * not at fault. The reason is one of these:\n *\n * - `refused`: `allowRequest` refused the first navigation, a step of its redirects, or a later\n *   navigation of the page that left it on an error page. `url` names the refused URL.\n * - `download`: the URL gives a file, not a page.\n * - `scheme`: the URL is not an http or https URL.\n * - `browser`: the renderer has no browser (`playwright-core` or its browser is not installed). The\n *   crawler then renders no more pages, and gives each one the note at once.\n *\n * A crawler does not count this error as a failure of the host.\n */\nexport class RenderRefused extends Error {\n  readonly reason: RefusedReason;\n  /** This URL is the request that `allowRequest` or the limit of redirects refused, when the reason is `refused`. */\n  readonly url: string | undefined;\n  constructor(reason: RefusedReason, message: string, url?: string) {\n    super(message);\n    this.name = \"RenderRefused\";\n    this.reason = reason;\n    this.url = url;\n  }\n}\n\n/** The browser does not ask for these. The text of a page needs none of them. */\nconst BLOCKED_TYPES = new Set([\"image\", \"font\", \"media\"]);\n/** These statuses send the request on to the URL in `Location` (the Fetch standard). */\nconst REDIRECTS = new Set([301, 302, 303, 307, 308]);\nconst MAX_REDIRECTS = 5;\n\n/** The text is stable when it does not grow for this time. */\nconst SETTLED_MS = 1000;\nconst POLL_MS = 250;\nconst SETTLE_CAP_MS = 12_000;\n/** A page with no text after this time has nothing more to show. */\nconst EMPTY_SETTLE_MS = 3000;\n/** The part of the deadline that is kept to read the page after the wait. */\nconst READ_RESERVE_MS = 2000;\n/** A context that does not close in this time takes its browser with it. */\nconst CLOSE_MS = 5000;\n/** A browser that does not open a context in this time is replaced. */\nconst OPEN_MS = 5000;\n\n/** The text of the error when the optional package `playwright-core` is not installed. */\nexport const NO_PLAYWRIGHT =\n  \"rendering needs the package playwright-core and a browser: npm install playwright-core, then npx playwright-core install chromium-headless-shell. For the MCP server with npx, start it with npx -y -p polite-crawler -p playwright-core polite-crawler mcp\";\n\n// playwright-core is an optional peer: it loads only when a page first needs a browser\nconst launch = async (): Promise<Browser> => {\n  let chromium: typeof import(\"playwright-core\").chromium;\n  try {\n    ({ chromium } = await import(\"playwright-core\"));\n  } catch (cause) {\n    throw new Error(NO_PLAYWRIGHT, { cause });\n  }\n  return chromium.launch({ headless: true });\n};\n\n/** This script gives the length of the text that the page shows: its body, its open shadow roots and its frames of the same origin. */\nconst MEASURE = `(() => {\n  let total = document.body ? document.body.innerText.length : 0;\n  const visit = (root) => {\n    for (const element of root.querySelectorAll(\"*\")) {\n      if (element.shadowRoot) {\n        total += (element.shadowRoot.textContent || \"\").length;\n        visit(element.shadowRoot);\n      }\n    }\n  };\n  visit(document);\n  for (const frame of document.querySelectorAll(\"iframe, frame\")) {\n    try {\n      const inner = frame.contentDocument;\n      if (inner && inner.body) total += inner.body.innerText.length;\n    } catch (error) {}\n  }\n  return total;\n})()`;\n\n/** This script gives the HTML of the page with its open shadow roots, and the body of each frame of the same origin. */\nconst SERIALIZE = `(() => {\n  const roots = [];\n  const visit = (root) => {\n    for (const element of root.querySelectorAll(\"*\")) {\n      if (element.shadowRoot) {\n        roots.push(element.shadowRoot);\n        visit(element.shadowRoot);\n      }\n    }\n  };\n  visit(document);\n  const root = document.documentElement;\n  if (!root) return { html: \"\", frames: [] };\n  const quote = (value) => value.replace(/&/g, \"&amp;\").replace(/\"/g, \"&quot;\");\n  const open = \"<html\" + Array.from(root.attributes, (item) => \" \" + item.name + '=\"' + quote(item.value) + '\"').join(\"\") + \">\";\n  const inner =\n    roots.length > 0 && typeof root.getHTML === \"function\"\n      ? root.getHTML({ serializableShadowRoots: true, shadowRoots: roots })\n      : root.innerHTML;\n  const frames = [];\n  for (const frame of document.querySelectorAll(\"iframe, frame\")) {\n    try {\n      const doc = frame.contentDocument;\n      if (doc && doc.body) frames.push({ src: frame.src || \"\", html: doc.body.innerHTML });\n    } catch (error) {}\n  }\n  return { html: \"<!DOCTYPE html>\" + open + inner + \"</html>\", frames };\n})()`;\n\ninterface Serialized {\n  readonly html: string;\n  readonly frames: ReadonlyArray<{ readonly src: string; readonly html: string }>;\n}\n\n/** This function puts the body of each frame at the end of the page, in a section that names the frame. */\nconst withFrames = ({ html, frames }: Serialized): string => {\n  if (frames.length === 0) return html;\n  const sections = frames\n    .map(\n      ({ src, html: body }) =>\n        `<section data-polite-frame=\"${src.replace(/&/g, \"&amp;\").replace(/\"/g, \"&quot;\")}\">${body}</section>`,\n    )\n    .join(\"\");\n  const end = html.toLowerCase().lastIndexOf(\"</body>\");\n  return end === -1 ? `${html}${sections}` : `${html.slice(0, end)}${sections}${html.slice(end)}`;\n};\n\nconst messageOf = (error: unknown): string =>\n  error instanceof Error ? error.message : String(error);\n\n/** This function gives the headers with their names in lowercase, as the route sends them. */\nconst lowerKeys = (headers: Readonly<Record<string, string>>): Record<string, string> =>\n  Object.fromEntries(Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value]));\n\nconst sleep = (ms: number): Promise<void> =>\n  new Promise((resolve) => {\n    setTimeout(resolve, Math.max(0, ms)).unref();\n  });\n\n/** This function gives the value of a promise, or undefined when the time ends first. */\nconst within = async <T>(promise: Promise<T>, ms: number): Promise<T | undefined> => {\n  let timer: ReturnType<typeof setTimeout> | undefined;\n  const late = new Promise<undefined>((resolve) => {\n    timer = setTimeout(() => resolve(undefined), Math.max(0, ms));\n  });\n  try {\n    return await Promise.race([promise, late]);\n  } finally {\n    clearTimeout(timer);\n  }\n};\n\nexport interface PlaywrightRenderer extends Renderer {\n  /** This function tells if a browser is open. */\n  readonly open: () => boolean;\n}\n\nexport const playwrightRenderer = (\n  options: {\n    readonly idleMs?: number;\n    /**\n     * This function starts the browser: it gives a `Browser` of `playwright-core`. The default is\n     * the headless shell of Playwright. The type names no type of `playwright-core`, so a program\n     * with no `playwright-core` can check its types.\n     */\n    readonly launch?: (() => Promise<object>) | undefined;\n  } = {},\n): PlaywrightRenderer => {\n  const idleMs = options.idleMs ?? 120_000;\n  const start = (options.launch ?? launch) as () => Promise<Browser>;\n  let browser: Promise<Browser> | undefined;\n  let inFlight = 0;\n  let idle: ReturnType<typeof setTimeout> | undefined;\n  const shutSoon = (): void => {\n    if (idle !== undefined) clearTimeout(idle);\n    idle = setTimeout(() => {\n      idle = undefined;\n      if (inFlight > 0 || browser === undefined) return;\n      const closing = browser;\n      browser = undefined;\n      void closing.then((running) => running.close()).catch(() => undefined);\n    }, idleMs);\n    // the timer alone does not keep the process alive; the open browser does\n    idle.unref();\n  };\n  const acquire = async (): Promise<Browser> => {\n    for (let attempt = 0; ; attempt += 1) {\n      if (browser === undefined) {\n        const launching: Promise<Browser> = start().then((running) => {\n          // a crash, a kill or a closed pipe: the next page launches a new browser\n          running.on(\"disconnected\", () => {\n            if (browser === launching) browser = undefined;\n          });\n          return running;\n        });\n        // a failed launch is not kept, so the next page tries again; a newer\n        // launch that replaced this one is not touched\n        launching.catch(() => {\n          if (browser === launching) browser = undefined;\n        });\n        browser = launching;\n      }\n      const held = browser;\n      const running = await held;\n      if (running.isConnected() || attempt > 0) return running;\n      if (browser === held) browser = undefined;\n    }\n  };\n  /** This function gives up on a browser that does not answer. The next page launches a new one. */\n  const forget = (running: Browser): void => {\n    const held = browser;\n    if (held !== undefined) {\n      void held.then(\n        (current) => {\n          if (current === running && browser === held) browser = undefined;\n        },\n        () => undefined,\n      );\n    }\n    void running.close().catch(() => undefined);\n  };\n\n  const renderPage = async (\n    url: string,\n    {\n      userAgent,\n      timeoutMs,\n      headers,\n      allowRequest,\n      maxRedirects = MAX_REDIRECTS,\n      signal,\n      sign,\n    }: RenderOptions,\n  ): Promise<Rendered> => {\n    if (signal?.aborted === true)\n      throw signal.reason instanceof Error ? signal.reason : new Error(\"the render was cancelled\");\n    const deadline = Date.now() + timeoutMs;\n    const left = (): number => Math.max(1, deadline - Date.now());\n    let running: Browser | undefined;\n    let context: BrowserContext | undefined;\n    let shownPage: Page | undefined;\n    /** The last navigation of the page that the route refused: the reason, and the refused URL. */\n    let refused: { readonly message: string; readonly url: string } | undefined;\n    /** The last navigation of the page that the route answered: the URL that the browser asked for, and the URL of the answer. */\n    let navigated: { readonly asked: string; readonly final: string } | undefined;\n    const extra = Object.fromEntries(\n      Object.entries(headers ?? {}).map(([name, value]) => [name.toLowerCase(), value]),\n    );\n    const contextOptions = {\n      userAgent,\n      serviceWorkers: \"block\" as const,\n      acceptDownloads: false,\n      viewport: { width: 1280, height: 900 },\n    };\n    const ask = async (url: string): Promise<boolean> =>\n      allowRequest === undefined\n        ? true\n        : Promise.resolve()\n            .then(() => allowRequest(url))\n            .catch(() => false);\n    /**\n     * This function sends one request of the page and follows its redirects. Each step is a new\n     * decision of `allowRequest`, and a step past `maxRedirects` is refused. It gives the answer,\n     * or the reason of a refusal.\n     *\n     * These rules are the rules of a browser. Each step gets the cookies of its own URL from the\n     * context. A credential does not go to a different origin. A 303, or a 301 or 302 after a POST,\n     * changes the request to a GET.\n     */\n    const follow = async (\n      route: Route,\n      sent: Record<string, string> | undefined,\n    ): Promise<\n      | { readonly response: APIResponse; readonly url: string }\n      | { readonly refusal: string; readonly url: string }\n    > => {\n      const request = route.request();\n      let url = request.url();\n      let method = request.method();\n      let headers: Record<string, string> = { ...request.headers(), ...extra };\n      let response = await route.fetch({\n        maxRedirects: 0,\n        timeout: left(),\n        ...(sent === undefined ? {} : { headers: sent }),\n      });\n      for (let hop = 0; ; hop += 1) {\n        const location = REDIRECTS.has(response.status())\n          ? response.headers()[\"location\"]\n          : undefined;\n        if (location === undefined) return { response, url };\n        let next: URL;\n        try {\n          next = new URL(location, url);\n        } catch {\n          return { refusal: `${url} redirects to a URL that does not parse`, url };\n        }\n        if (hop >= maxRedirects) {\n          return {\n            refusal: `${request.url()} redirects more than ${String(maxRedirects)} times`,\n            url: next.href,\n          };\n        }\n        if (!(await ask(next.href))) {\n          return {\n            refusal: `the request for ${next.href}, a redirect of ${url}, was refused`,\n            url: next.href,\n          };\n        }\n        const status = response.status();\n        const get = status === 303 || ((status === 301 || status === 302) && method === \"POST\");\n        const crossOrigin = next.origin !== new URL(url).origin;\n        headers = Object.fromEntries(\n          Object.entries(headers).filter(\n            ([name]) =>\n              name !== \"cookie\" &&\n              !(crossOrigin && name === \"authorization\") &&\n              !(get && (name === \"content-type\" || name === \"content-length\")),\n          ),\n        );\n        if (get) method = \"GET\";\n        // ── signed requests: each step of the redirects gets a signature for its own URL ──\n        if (sign !== undefined) {\n          const signed = sign(next.href);\n          if (signed === undefined)\n            return {\n              refusal: `the signer did not sign the request for ${next.href}`,\n              url: next.href,\n            };\n          headers = { ...headers, ...lowerKeys(signed) };\n        }\n        const body = get ? undefined : (request.postDataBuffer() ?? undefined);\n        response = await context!.request.fetch(next.href, {\n          method,\n          headers,\n          maxRedirects: 0,\n          timeout: left(),\n          ...(body === undefined ? {} : { data: body }),\n        });\n        url = next.href;\n      }\n    };\n    /** This function tells if a request is a navigation of the page itself, not of a frame or a popup. */\n    const ofPage = (request: Request): boolean => {\n      if (!request.isNavigationRequest()) return false;\n      try {\n        return request.frame().parentFrame() === null && request.frame().page() === shownPage;\n      } catch {\n        // a popup asks for its first page before it has a frame\n        return false;\n      }\n    };\n    const decide = async (route: Route): Promise<void> => {\n      const request = route.request();\n      const main = ofPage(request);\n      try {\n        if (BLOCKED_TYPES.has(request.resourceType())) {\n          await route.abort(\"blockedbyclient\");\n          return;\n        }\n        if (!(await ask(request.url()))) {\n          if (main)\n            refused = {\n              message: `the request for ${request.url()} was refused`,\n              url: request.url(),\n            };\n          await route.abort(\"blockedbyclient\");\n          return;\n        }\n        // ── signed requests: a request that the signer does not sign does not go out ──\n        const signed = sign === undefined ? {} : sign(request.url());\n        if (signed === undefined) {\n          if (main)\n            refused = {\n              message: `the signer did not sign the request for ${request.url()}`,\n              url: request.url(),\n            };\n          await route.abort(\"blockedbyclient\");\n          return;\n        }\n        const added = { ...extra, ...lowerKeys(signed) };\n        const sent =\n          Object.keys(added).length === 0\n            ? undefined\n            : {\n                ...Object.fromEntries(\n                  Object.entries(await request.allHeaders()).filter(\n                    ([name]) => !name.startsWith(\":\") && name !== \"host\",\n                  ),\n                ),\n                ...added,\n              };\n        if (allowRequest === undefined) {\n          await route.continue(sent === undefined ? undefined : { headers: sent });\n          return;\n        }\n        const answer = await follow(route, sent);\n        if (\"refusal\" in answer) {\n          if (main) refused = { message: answer.refusal, url: answer.url };\n          await route.abort(\"blockedbyclient\");\n          return;\n        }\n        if (main) navigated = { asked: request.url(), final: answer.url };\n        await route.fulfill({ response: answer.response });\n      } catch {\n        // the page closed while the request waited: nothing to answer\n      }\n    };\n    const work = (async (): Promise<Rendered> => {\n      running = await acquire();\n      // a browser that was killed can keep its pipe open and answer nothing:\n      // it gets a few seconds, and then a new browser takes its place\n      const opened = await within(\n        running.newContext(contextOptions).catch((error: unknown) => {\n          if (!/closed|disconnected/i.test(messageOf(error))) throw error;\n          return undefined;\n        }),\n        Math.min(OPEN_MS, left()),\n      );\n      if (opened === undefined) {\n        forget(running);\n        running = await acquire();\n        context = await running.newContext(contextOptions);\n      } else {\n        context = opened;\n      }\n      await context.route(\"**/*\", decide);\n      if (allowRequest !== undefined && typeof context.routeWebSocket === \"function\") {\n        await context.routeWebSocket(/.*/, async (socket) => {\n          const allowed = await Promise.resolve()\n            .then(() => allowRequest(socket.url().replace(/^ws/i, \"http\")))\n            .catch(() => false);\n          if (allowed) socket.connectToServer();\n          else await socket.close({ code: 1008, reason: \"refused\" }).catch(() => undefined);\n        });\n      }\n      const page: Page = await context.newPage();\n      shownPage = page;\n      // a popup is not the page, and a dialog waits for no one\n      context.on(\"page\", (other) => {\n        if (other !== page) void other.close().catch(() => undefined);\n      });\n      page.on(\"dialog\", (dialog) => {\n        void dialog.dismiss().catch(() => undefined);\n      });\n      let response;\n      try {\n        response = await page.goto(url, {\n          waitUntil: \"domcontentloaded\",\n          timeout: Math.max(1, left() - READ_RESERVE_MS),\n        });\n      } catch (error) {\n        if (refused !== undefined) throw new RenderRefused(\"refused\", refused.message, refused.url);\n        if (/Download is starting/i.test(messageOf(error))) {\n          throw new RenderRefused(\"download\", `${url} gives a file, not a page`);\n        }\n        throw error;\n      }\n      const settleEnd = Math.min(deadline - READ_RESERVE_MS, Date.now() + SETTLE_CAP_MS);\n      const settleStart = Date.now();\n      let last = -1;\n      let since = Date.now();\n      while (Date.now() < settleEnd) {\n        const measured = (page.evaluate(MEASURE) as Promise<number>).catch(\n          async (error: unknown) => {\n            // a script of the page can start a navigation while the text is measured: the next poll measures the new document\n            if (!/destroyed|navigat/i.test(messageOf(error))) throw error;\n            await page\n              .waitForLoadState(\"domcontentloaded\", { timeout: left() })\n              .catch(() => undefined);\n            // no length equals NaN, so the wait goes on\n            return Number.NaN;\n          },\n        );\n        const length = await within(measured, settleEnd - Date.now());\n        // a page that keeps its thread busy does not answer: the reading below decides\n        if (length === undefined) break;\n        if (length !== last) {\n          last = length;\n          since = Date.now();\n        } else if (\n          Date.now() - since >= SETTLED_MS &&\n          (length > 0 || Date.now() - settleStart >= EMPTY_SETTLE_MS)\n        ) {\n          break;\n        }\n        await sleep(Math.min(POLL_MS, settleEnd - Date.now()));\n      }\n      // a navigation can replace the document while it is read: the read waits for the new document\n      // and tries again, until the deadline (a page can navigate many times in a row)\n      let serialized: Serialized | undefined;\n      while (serialized === undefined) {\n        try {\n          serialized = (await page.evaluate(SERIALIZE)) as Serialized;\n        } catch (error) {\n          if (!/destroyed|navigat/i.test(messageOf(error)) || Date.now() >= deadline) throw error;\n          await page\n            .waitForLoadState(\"domcontentloaded\", { timeout: left() })\n            .catch(() => undefined);\n        }\n      }\n      // a later navigation that the route refused leaves the browser on its error page, which is not the page\n      if (refused !== undefined && page.url().startsWith(\"chrome-error:\")) {\n        throw new RenderRefused(\"refused\", `the page moved: ${refused.message}`, refused.url);\n      }\n      // the route followed the redirects of the navigation, so the browser shows the first URL\n      const shown = page.url().replace(/#.*$/s, \"\");\n      return {\n        html: withFrames(serialized),\n        finalUrl:\n          navigated !== undefined && shown === navigated.asked ? navigated.final : page.url(),\n        status: response?.status() ?? null,\n        headers: response?.headers() ?? {},\n      };\n    })();\n    // the deadline can win the race; the work then ends when its context closes\n    work.catch(() => undefined);\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    let cancel: (() => void) | undefined;\n    const expired = new Promise<never>((_, reject) => {\n      timer = setTimeout(\n        () => reject(new Error(`render timed out after ${String(timeoutMs)} ms`)),\n        timeoutMs,\n      );\n      // a cancelled render ends now; the context closes below, and that stops the page\n      if (signal !== undefined) {\n        cancel = () =>\n          reject(\n            signal.reason instanceof Error ? signal.reason : new Error(\"the render was cancelled\"),\n          );\n        signal.addEventListener(\"abort\", cancel, { once: true });\n      }\n    });\n    const closeContext = async (): Promise<void> => {\n      const open = context;\n      if (open === undefined) return;\n      const closed = await within(\n        open.close().then(\n          () => true,\n          () => true,\n        ),\n        CLOSE_MS,\n      );\n      if (closed === undefined && running !== undefined) forget(running);\n    };\n    try {\n      return await Promise.race([work, expired]);\n    } finally {\n      clearTimeout(timer);\n      if (cancel !== undefined) signal?.removeEventListener(\"abort\", cancel);\n      if (context !== undefined) await closeContext();\n      // a context made after the deadline is closed when it arrives\n      else void work.then(closeContext, closeContext).catch(() => undefined);\n    }\n  };\n\n  return {\n    open: () => browser !== undefined,\n    async render(url, renderOptions) {\n      let target: URL;\n      try {\n        target = new URL(url);\n      } catch {\n        throw new RenderRefused(\"scheme\", `${url} is not a URL`);\n      }\n      if (target.protocol !== \"http:\" && target.protocol !== \"https:\") {\n        throw new RenderRefused(\n          \"scheme\",\n          `the renderer reads http and https pages, not ${target.protocol}`,\n        );\n      }\n      if (idle !== undefined) {\n        clearTimeout(idle);\n        idle = undefined;\n      }\n      inFlight += 1;\n      try {\n        return await renderPage(target.href, renderOptions);\n      } finally {\n        inFlight -= 1;\n        if (inFlight === 0) shutSoon();\n      }\n    },\n    async close() {\n      if (idle !== undefined) clearTimeout(idle);\n      idle = undefined;\n      const launched = browser;\n      browser = undefined;\n      // a launch that failed has nothing to close, and closing never throws:\n      // the idle timer calls this with no one to catch it\n      if (launched !== undefined)\n        await launched.then(\n          (open) => open.close(),\n          () => undefined,\n        );\n    },\n  };\n};\n"],"mappings":";;;;;;;;;;;;;;AA+EA,IAAa,gBAAb,cAAmC,MAAM;CACvC;;CAEA;CACA,YAAY,QAAuB,SAAiB,KAAc;EAChE,MAAM,OAAO;EACb,KAAK,OAAO;EACZ,KAAK,SAAS;EACd,KAAK,MAAM;CACb;AACF;;AAGA,MAAM,gCAAgB,IAAI,IAAI;CAAC;CAAS;CAAQ;AAAO,CAAC;;AAExD,MAAM,4BAAY,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;CAAK;AAAG,CAAC;AACnD,MAAM,gBAAgB;;AAGtB,MAAM,aAAa;AACnB,MAAM,UAAU;AAChB,MAAM,gBAAgB;;AAEtB,MAAM,kBAAkB;;AAExB,MAAM,kBAAkB;;AAExB,MAAM,WAAW;;AAEjB,MAAM,UAAU;;AAGhB,MAAa,gBACX;AAGF,MAAM,SAAS,YAA8B;CAC3C,IAAI;CACJ,IAAI;EACF,CAAC,CAAE,YAAa,MAAM,OAAO;CAC/B,SAAS,OAAO;EACd,MAAM,IAAI,MAAM,eAAe,EAAE,MAAM,CAAC;CAC1C;CACA,OAAO,SAAS,OAAO,EAAE,UAAU,KAAK,CAAC;AAC3C;;AAGA,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;AAqBhB,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmClB,MAAM,cAAc,EAAE,MAAM,aAAiC;CAC3D,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,WAAW,OACd,KACE,EAAE,KAAK,MAAM,WACZ,+BAA+B,IAAI,QAAQ,MAAM,OAAO,CAAC,CAAC,QAAQ,MAAM,QAAQ,EAAE,IAAI,KAAK,WAC/F,CAAC,CACA,KAAK,EAAE;CACV,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,YAAY,SAAS;CACpD,OAAO,QAAQ,KAAK,GAAG,OAAO,aAAa,GAAG,KAAK,MAAM,GAAG,GAAG,IAAI,WAAW,KAAK,MAAM,GAAG;AAC9F;AAEA,MAAM,aAAa,UACjB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;;AAGvD,MAAM,aAAa,YACjB,OAAO,YAAY,OAAO,QAAQ,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,WAAW,CAAC,KAAK,YAAY,GAAG,KAAK,CAAC,CAAC;AAEhG,MAAM,SAAS,OACb,IAAI,SAAS,YAAY;CACvB,WAAW,SAAS,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM;AAC7C,CAAC;;AAGH,MAAM,SAAS,OAAU,SAAqB,OAAuC;CACnF,IAAI;CACJ,MAAM,OAAO,IAAI,SAAoB,YAAY;EAC/C,QAAQ,iBAAiB,QAAQ,KAAA,CAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAC;CAC9D,CAAC;CACD,IAAI;EACF,OAAO,MAAM,QAAQ,KAAK,CAAC,SAAS,IAAI,CAAC;CAC3C,UAAU;EACR,aAAa,KAAK;CACpB;AACF;AAOA,MAAa,sBACX,UAQI,CAAC,MACkB;CACvB,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,QAAS,QAAQ,UAAU;CACjC,IAAI;CACJ,IAAI,WAAW;CACf,IAAI;CACJ,MAAM,iBAAuB;EAC3B,IAAI,SAAS,KAAA,GAAW,aAAa,IAAI;EACzC,OAAO,iBAAiB;GACtB,OAAO,KAAA;GACP,IAAI,WAAW,KAAK,YAAY,KAAA,GAAW;GAC3C,MAAM,UAAU;GAChB,UAAU,KAAA;GACV,QAAa,MAAM,YAAY,QAAQ,MAAM,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;EACvE,GAAG,MAAM;EAET,KAAK,MAAM;CACb;CACA,MAAM,UAAU,YAA8B;EAC5C,KAAK,IAAI,UAAU,IAAK,WAAW,GAAG;GACpC,IAAI,YAAY,KAAA,GAAW;IACzB,MAAM,YAA8B,MAAM,CAAC,CAAC,MAAM,YAAY;KAE5D,QAAQ,GAAG,sBAAsB;MAC/B,IAAI,YAAY,WAAW,UAAU,KAAA;KACvC,CAAC;KACD,OAAO;IACT,CAAC;IAGD,UAAU,YAAY;KACpB,IAAI,YAAY,WAAW,UAAU,KAAA;IACvC,CAAC;IACD,UAAU;GACZ;GACA,MAAM,OAAO;GACb,MAAM,UAAU,MAAM;GACtB,IAAI,QAAQ,YAAY,KAAK,UAAU,GAAG,OAAO;GACjD,IAAI,YAAY,MAAM,UAAU,KAAA;EAClC;CACF;;CAEA,MAAM,UAAU,YAA2B;EACzC,MAAM,OAAO;EACb,IAAI,SAAS,KAAA,GACX,KAAU,MACP,YAAY;GACX,IAAI,YAAY,WAAW,YAAY,MAAM,UAAU,KAAA;EACzD,SACM,KAAA,CACR;EAEF,QAAa,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;CAC5C;CAEA,MAAM,aAAa,OACjB,KACA,EACE,WACA,WACA,SACA,cACA,eAAe,eACf,QACA,WAEoB;EACtB,IAAI,QAAQ,YAAY,MACtB,MAAM,OAAO,kBAAkB,QAAQ,OAAO,yBAAS,IAAI,MAAM,0BAA0B;EAC7F,MAAM,WAAW,KAAK,IAAI,IAAI;EAC9B,MAAM,aAAqB,KAAK,IAAI,GAAG,WAAW,KAAK,IAAI,CAAC;EAC5D,IAAI;EACJ,IAAI;EACJ,IAAI;;EAEJ,IAAI;;EAEJ,IAAI;EACJ,MAAM,QAAQ,OAAO,YACnB,OAAO,QAAQ,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,WAAW,CAAC,KAAK,YAAY,GAAG,KAAK,CAAC,CAClF;EACA,MAAM,iBAAiB;GACrB;GACA,gBAAgB;GAChB,iBAAiB;GACjB,UAAU;IAAE,OAAO;IAAM,QAAQ;GAAI;EACvC;EACA,MAAM,MAAM,OAAO,QACjB,iBAAiB,KAAA,IACb,OACA,QAAQ,QAAQ,CAAC,CACd,WAAW,aAAa,GAAG,CAAC,CAAC,CAC7B,YAAY,KAAK;;;;;;;;;;EAU1B,MAAM,SAAS,OACb,OACA,SAIG;GACH,MAAM,UAAU,MAAM,QAAQ;GAC9B,IAAI,MAAM,QAAQ,IAAI;GACtB,IAAI,SAAS,QAAQ,OAAO;GAC5B,IAAI,UAAkC;IAAE,GAAG,QAAQ,QAAQ;IAAG,GAAG;GAAM;GACvE,IAAI,WAAW,MAAM,MAAM,MAAM;IAC/B,cAAc;IACd,SAAS,KAAK;IACd,GAAI,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,KAAK;GAChD,CAAC;GACD,KAAK,IAAI,MAAM,IAAK,OAAO,GAAG;IAC5B,MAAM,WAAW,UAAU,IAAI,SAAS,OAAO,CAAC,IAC5C,SAAS,QAAQ,CAAC,CAAC,cACnB,KAAA;IACJ,IAAI,aAAa,KAAA,GAAW,OAAO;KAAE;KAAU;IAAI;IACnD,IAAI;IACJ,IAAI;KACF,OAAO,IAAI,IAAI,UAAU,GAAG;IAC9B,QAAQ;KACN,OAAO;MAAE,SAAS,GAAG,IAAI;MAA0C;KAAI;IACzE;IACA,IAAI,OAAO,cACT,OAAO;KACL,SAAS,GAAG,QAAQ,IAAI,EAAE,uBAAuB,OAAO,YAAY,EAAE;KACtE,KAAK,KAAK;IACZ;IAEF,IAAI,CAAE,MAAM,IAAI,KAAK,IAAI,GACvB,OAAO;KACL,SAAS,mBAAmB,KAAK,KAAK,kBAAkB,IAAI;KAC5D,KAAK,KAAK;IACZ;IAEF,MAAM,SAAS,SAAS,OAAO;IAC/B,MAAM,MAAM,WAAW,QAAS,WAAW,OAAO,WAAW,QAAQ,WAAW;IAChF,MAAM,cAAc,KAAK,WAAW,IAAI,IAAI,GAAG,CAAC,CAAC;IACjD,UAAU,OAAO,YACf,OAAO,QAAQ,OAAO,CAAC,CAAC,QACrB,CAAC,UACA,SAAS,YACT,EAAE,eAAe,SAAS,oBAC1B,EAAE,QAAQ,SAAS,kBAAkB,SAAS,kBAClD,CACF;IACA,IAAI,KAAK,SAAS;IAElB,IAAI,SAAS,KAAA,GAAW;KACtB,MAAM,SAAS,KAAK,KAAK,IAAI;KAC7B,IAAI,WAAW,KAAA,GACb,OAAO;MACL,SAAS,2CAA2C,KAAK;MACzD,KAAK,KAAK;KACZ;KACF,UAAU;MAAE,GAAG;MAAS,GAAG,UAAU,MAAM;KAAE;IAC/C;IACA,MAAM,OAAO,MAAM,KAAA,IAAa,QAAQ,eAAe,KAAK,KAAA;IAC5D,WAAW,MAAM,QAAS,QAAQ,MAAM,KAAK,MAAM;KACjD;KACA;KACA,cAAc;KACd,SAAS,KAAK;KACd,GAAI,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM,KAAK;IAC7C,CAAC;IACD,MAAM,KAAK;GACb;EACF;;EAEA,MAAM,UAAU,YAA8B;GAC5C,IAAI,CAAC,QAAQ,oBAAoB,GAAG,OAAO;GAC3C,IAAI;IACF,OAAO,QAAQ,MAAM,CAAC,CAAC,YAAY,MAAM,QAAQ,QAAQ,MAAM,CAAC,CAAC,KAAK,MAAM;GAC9E,QAAQ;IAEN,OAAO;GACT;EACF;EACA,MAAM,SAAS,OAAO,UAAgC;GACpD,MAAM,UAAU,MAAM,QAAQ;GAC9B,MAAM,OAAO,OAAO,OAAO;GAC3B,IAAI;IACF,IAAI,cAAc,IAAI,QAAQ,aAAa,CAAC,GAAG;KAC7C,MAAM,MAAM,MAAM,iBAAiB;KACnC;IACF;IACA,IAAI,CAAE,MAAM,IAAI,QAAQ,IAAI,CAAC,GAAI;KAC/B,IAAI,MACF,UAAU;MACR,SAAS,mBAAmB,QAAQ,IAAI,EAAE;MAC1C,KAAK,QAAQ,IAAI;KACnB;KACF,MAAM,MAAM,MAAM,iBAAiB;KACnC;IACF;IAEA,MAAM,SAAS,SAAS,KAAA,IAAY,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC;IAC3D,IAAI,WAAW,KAAA,GAAW;KACxB,IAAI,MACF,UAAU;MACR,SAAS,2CAA2C,QAAQ,IAAI;MAChE,KAAK,QAAQ,IAAI;KACnB;KACF,MAAM,MAAM,MAAM,iBAAiB;KACnC;IACF;IACA,MAAM,QAAQ;KAAE,GAAG;KAAO,GAAG,UAAU,MAAM;IAAE;IAC/C,MAAM,OACJ,OAAO,KAAK,KAAK,CAAC,CAAC,WAAW,IAC1B,KAAA,IACA;KACE,GAAG,OAAO,YACR,OAAO,QAAQ,MAAM,QAAQ,WAAW,CAAC,CAAC,CAAC,QACxC,CAAC,UAAU,CAAC,KAAK,WAAW,GAAG,KAAK,SAAS,MAChD,CACF;KACA,GAAG;IACL;IACN,IAAI,iBAAiB,KAAA,GAAW;KAC9B,MAAM,MAAM,SAAS,SAAS,KAAA,IAAY,KAAA,IAAY,EAAE,SAAS,KAAK,CAAC;KACvE;IACF;IACA,MAAM,SAAS,MAAM,OAAO,OAAO,IAAI;IACvC,IAAI,aAAa,QAAQ;KACvB,IAAI,MAAM,UAAU;MAAE,SAAS,OAAO;MAAS,KAAK,OAAO;KAAI;KAC/D,MAAM,MAAM,MAAM,iBAAiB;KACnC;IACF;IACA,IAAI,MAAM,YAAY;KAAE,OAAO,QAAQ,IAAI;KAAG,OAAO,OAAO;IAAI;IAChE,MAAM,MAAM,QAAQ,EAAE,UAAU,OAAO,SAAS,CAAC;GACnD,QAAQ,CAER;EACF;EACA,MAAM,QAAQ,YAA+B;GAC3C,UAAU,MAAM,QAAQ;GAGxB,MAAM,SAAS,MAAM,OACnB,QAAQ,WAAW,cAAc,CAAC,CAAC,OAAO,UAAmB;IAC3D,IAAI,CAAC,uBAAuB,KAAK,UAAU,KAAK,CAAC,GAAG,MAAM;GAE5D,CAAC,GACD,KAAK,IAAI,SAAS,KAAK,CAAC,CAC1B;GACA,IAAI,WAAW,KAAA,GAAW;IACxB,OAAO,OAAO;IACd,UAAU,MAAM,QAAQ;IACxB,UAAU,MAAM,QAAQ,WAAW,cAAc;GACnD,OACE,UAAU;GAEZ,MAAM,QAAQ,MAAM,QAAQ,MAAM;GAClC,IAAI,iBAAiB,KAAA,KAAa,OAAO,QAAQ,mBAAmB,YAClE,MAAM,QAAQ,eAAe,MAAM,OAAO,WAAW;IAInD,IAAI,MAHkB,QAAQ,QAAQ,CAAC,CACpC,WAAW,aAAa,OAAO,IAAI,CAAC,CAAC,QAAQ,QAAQ,MAAM,CAAC,CAAC,CAAC,CAC9D,YAAY,KAAK,GACP,OAAO,gBAAgB;SAC/B,MAAM,OAAO,MAAM;KAAE,MAAM;KAAM,QAAQ;IAAU,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAClF,CAAC;GAEH,MAAM,OAAa,MAAM,QAAQ,QAAQ;GACzC,YAAY;GAEZ,QAAQ,GAAG,SAAS,UAAU;IAC5B,IAAI,UAAU,MAAM,MAAW,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;GAC9D,CAAC;GACD,KAAK,GAAG,WAAW,WAAW;IAC5B,OAAY,QAAQ,CAAC,CAAC,YAAY,KAAA,CAAS;GAC7C,CAAC;GACD,IAAI;GACJ,IAAI;IACF,WAAW,MAAM,KAAK,KAAK,KAAK;KAC9B,WAAW;KACX,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,eAAe;IAC/C,CAAC;GACH,SAAS,OAAO;IACd,IAAI,YAAY,KAAA,GAAW,MAAM,IAAI,cAAc,WAAW,QAAQ,SAAS,QAAQ,GAAG;IAC1F,IAAI,wBAAwB,KAAK,UAAU,KAAK,CAAC,GAC/C,MAAM,IAAI,cAAc,YAAY,GAAG,IAAI,0BAA0B;IAEvE,MAAM;GACR;GACA,MAAM,YAAY,KAAK,IAAI,WAAW,iBAAiB,KAAK,IAAI,IAAI,aAAa;GACjF,MAAM,cAAc,KAAK,IAAI;GAC7B,IAAI,OAAO;GACX,IAAI,QAAQ,KAAK,IAAI;GACrB,OAAO,KAAK,IAAI,IAAI,WAAW;IAC7B,MAAM,WAAY,KAAK,SAAS,OAAO,CAAC,CAAqB,MAC3D,OAAO,UAAmB;KAExB,IAAI,CAAC,qBAAqB,KAAK,UAAU,KAAK,CAAC,GAAG,MAAM;KACxD,MAAM,KACH,iBAAiB,oBAAoB,EAAE,SAAS,KAAK,EAAE,CAAC,CAAC,CACzD,YAAY,KAAA,CAAS;KAExB,OAAO;IACT,CACF;IACA,MAAM,SAAS,MAAM,OAAO,UAAU,YAAY,KAAK,IAAI,CAAC;IAE5D,IAAI,WAAW,KAAA,GAAW;IAC1B,IAAI,WAAW,MAAM;KACnB,OAAO;KACP,QAAQ,KAAK,IAAI;IACnB,OAAO,IACL,KAAK,IAAI,IAAI,SAAS,eACrB,SAAS,KAAK,KAAK,IAAI,IAAI,eAAe,kBAE3C;IAEF,MAAM,MAAM,KAAK,IAAI,SAAS,YAAY,KAAK,IAAI,CAAC,CAAC;GACvD;GAGA,IAAI;GACJ,OAAO,eAAe,KAAA,GACpB,IAAI;IACF,aAAc,MAAM,KAAK,SAAS,SAAS;GAC7C,SAAS,OAAO;IACd,IAAI,CAAC,qBAAqB,KAAK,UAAU,KAAK,CAAC,KAAK,KAAK,IAAI,KAAK,UAAU,MAAM;IAClF,MAAM,KACH,iBAAiB,oBAAoB,EAAE,SAAS,KAAK,EAAE,CAAC,CAAC,CACzD,YAAY,KAAA,CAAS;GAC1B;GAGF,IAAI,YAAY,KAAA,KAAa,KAAK,IAAI,CAAC,CAAC,WAAW,eAAe,GAChE,MAAM,IAAI,cAAc,WAAW,mBAAmB,QAAQ,WAAW,QAAQ,GAAG;GAGtF,MAAM,QAAQ,KAAK,IAAI,CAAC,CAAC,QAAQ,SAAS,EAAE;GAC5C,OAAO;IACL,MAAM,WAAW,UAAU;IAC3B,UACE,cAAc,KAAA,KAAa,UAAU,UAAU,QAAQ,UAAU,QAAQ,KAAK,IAAI;IACpF,QAAQ,UAAU,OAAO,KAAK;IAC9B,SAAS,UAAU,QAAQ,KAAK,CAAC;GACnC;EACF,EAAA,CAAG;EAEH,KAAK,YAAY,KAAA,CAAS;EAC1B,IAAI;EACJ,IAAI;EACJ,MAAM,UAAU,IAAI,SAAgB,GAAG,WAAW;GAChD,QAAQ,iBACA,uBAAO,IAAI,MAAM,0BAA0B,OAAO,SAAS,EAAE,IAAI,CAAC,GACxE,SACF;GAEA,IAAI,WAAW,KAAA,GAAW;IACxB,eACE,OACE,OAAO,kBAAkB,QAAQ,OAAO,yBAAS,IAAI,MAAM,0BAA0B,CACvF;IACF,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;GACzD;EACF,CAAC;EACD,MAAM,eAAe,YAA2B;GAC9C,MAAM,OAAO;GACb,IAAI,SAAS,KAAA,GAAW;GAQxB,IAAI,MAPiB,OACnB,KAAK,MAAM,CAAC,CAAC,WACL,YACA,IACR,GACA,QACF,MACe,KAAA,KAAa,YAAY,KAAA,GAAW,OAAO,OAAO;EACnE;EACA,IAAI;GACF,OAAO,MAAM,QAAQ,KAAK,CAAC,MAAM,OAAO,CAAC;EAC3C,UAAU;GACR,aAAa,KAAK;GAClB,IAAI,WAAW,KAAA,GAAW,QAAQ,oBAAoB,SAAS,MAAM;GACrE,IAAI,YAAY,KAAA,GAAW,MAAM,aAAa;QAEzC,KAAU,KAAK,cAAc,YAAY,CAAC,CAAC,YAAY,KAAA,CAAS;EACvE;CACF;CAEA,OAAO;EACL,YAAY,YAAY,KAAA;EACxB,MAAM,OAAO,KAAK,eAAe;GAC/B,IAAI;GACJ,IAAI;IACF,SAAS,IAAI,IAAI,GAAG;GACtB,QAAQ;IACN,MAAM,IAAI,cAAc,UAAU,GAAG,IAAI,cAAc;GACzD;GACA,IAAI,OAAO,aAAa,WAAW,OAAO,aAAa,UACrD,MAAM,IAAI,cACR,UACA,gDAAgD,OAAO,UACzD;GAEF,IAAI,SAAS,KAAA,GAAW;IACtB,aAAa,IAAI;IACjB,OAAO,KAAA;GACT;GACA,YAAY;GACZ,IAAI;IACF,OAAO,MAAM,WAAW,OAAO,MAAM,aAAa;GACpD,UAAU;IACR,YAAY;IACZ,IAAI,aAAa,GAAG,SAAS;GAC/B;EACF;EACA,MAAM,QAAQ;GACZ,IAAI,SAAS,KAAA,GAAW,aAAa,IAAI;GACzC,OAAO,KAAA;GACP,MAAM,WAAW;GACjB,UAAU,KAAA;GAGV,IAAI,aAAa,KAAA,GACf,MAAM,SAAS,MACZ,SAAS,KAAK,MAAM,SACf,KAAA,CACR;EACJ;CACF;AACF"}

package/build/robots-R-qaEBOb.d.mts

@@ -1,144 +0,0 @@
-//#region src/robots.d.ts
-/**
- * This module reads robots.txt as RFC 9309 (September 2022) specifies it. It is for a crawler that
- * tells who it is and obeys what a site says. The important rules, each with a test:
- *
- * - A group is one or more `user-agent` lines and the rules after them. A `user-agent` line after
- *   an `allow` or `disallow` line starts a new group, as the parser of Google does. A different
- *   line (`crawl-delay`) between two `user-agent` lines does not split the group. A rule before
- *   the first `user-agent` line is in no group.
- * - The product token matches with no attention to case. The matching groups merge. The group
- *   `*` applies only when no group names the crawler.
- * - The longest matching rule wins, and `allow` wins a tie. A `*` matches any sequence, and a `$`
- *   at the end anchors the end. The matcher uses no regular expression, and its time is linear.
- * - `/robots.txt` is always permitted, and an empty `disallow` matches nothing.
- * - The parser reads at least the first 500 KiB.
- * - A usage line (`content-signal`, `content-usage`) belongs to its group, with an optional path
- *   before its value. A line before the first group applies to each crawler.
- *
- * The function `robotsFromStatus` gives the meaning of the status of the fetch. A 4xx is
- * "unavailable", so all paths are permitted. A 5xx or no response is "unreachable", so no path is
- * permitted. A 429 is unreachable too, as Google does it. A site that limits the rate of its robots
- * file does not ask for a crawl.
- */
-declare const ROBOTS_MAX_BYTES: number;
-interface RobotsRule {
-  readonly allow: boolean;
-  /** This value is the path pattern, normalized for comparison. */
-  readonly pattern: string;
-  /** This value is the number of the line in robots.txt, from 1. A rule that the code makes has none. */
-  readonly line?: number | undefined;
-}
-/** A usage line states a preference about the use of the content (`content-signal`, `content-usage`). */
-interface RobotsSignal {
-  /** This value is the name of the line, in lowercase. */
-  readonly key: string;
-  /** This text is the value of the line, after the path. */
-  readonly value: string;
-  /** This value is the normalized path pattern of the line. A line with no path applies to each path. */
-  readonly path: string | undefined;
-  /** This value is the number of the line in robots.txt, from 1. */
-  readonly line: number;
-}
-interface RobotsGroup {
-  /** These values are the user agents of the group, in lowercase. */
-  readonly agents: readonly string[];
-  /** This value is the number of the first `user-agent` line of the group, from 1. */
-  readonly line?: number | undefined;
-  readonly rules: readonly RobotsRule[];
-  /** This value is the nonstandard `crawl-delay` in seconds, when the group sets one. */
-  readonly crawlDelaySeconds: number | undefined;
-  /** These usage lines are in the group. They apply when the group applies. */
-  readonly signals?: readonly RobotsSignal[] | undefined;
-}
-interface Robots {
-  readonly groups: readonly RobotsGroup[];
-  readonly sitemaps: readonly string[];
-  /** These lines state usage preferences (`content-signal: …`, `content-usage: …`), as written, from each group. */
-  readonly signals: readonly string[];
-  /** These usage lines come before the first group, so they apply to each crawler. */
-  readonly fileSignals?: readonly RobotsSignal[] | undefined;
-  /**
-   * This text tells why the crawler could not read robots.txt: a dead domain, a broken certificate
-   * or a server error. Then no path is permitted, and the note of a skip gives this reason. It does
-   * not name a rule of the site.
-   */
-  readonly unreachable?: string | undefined;
-}
-declare const ALLOW_ALL: Robots;
-declare const DISALLOW_ALL: Robots;
-/**
- * This function normalizes a path, or the pattern of a rule, for comparison, as §2.2.2 asks. It
- * encodes the characters outside US-ASCII, and the ASCII characters that the URL parser encodes
- * (a space, `"`, `<`, `{`). It decodes an escape of an unreserved character, so `/%7Efoo` and
- * `/~foo` are equal. The other escapes stay, with uppercase hexadecimal digits.
- */
-declare const normalizePath: (value: string) => string;
-/** This function tells if a text is a product token: letters, `_` and `-` only, and one or more (RFC 9309, section 2.2.1). */
-declare const isProductToken: (value: string) => boolean;
-/**
- * This function gives the product token that robots.txt rules match, in lowercase. `MyBot/1.0`
- * gives `mybot`. The full header `Mozilla/5.0 (compatible; MyBot/1.0; +https://…)` also gives
- * `mybot`. The token is the name after `compatible;`, or else the first product that is not a part
- * of a browser.
- */
-declare const productToken: (value: string) => string;
-declare const parseRobots: (input: string) => Robots;
-/**
- * This function tells if a normalized path matches the pattern of a rule. A `*` matches any
- * sequence, and a `$` at the end anchors the end. The function finds the parts between the stars
- * in sequence, each at its first position. No pattern makes it slow: its time is linear in the
- * length of the path for each part, and it uses no regular expression.
- */
-declare const matchesPattern: (pattern: string, path: string) => boolean;
-interface RobotsDecision {
-  readonly allowed: boolean;
-  /** This value is the product token whose rules refused the path. */
-  readonly by: string | undefined;
-  readonly crawlDelaySeconds: number | undefined;
-  /** This rule decided. When no rule matches the path, the path is permitted and this value is empty. */
-  readonly rule?: RobotsRule | undefined;
-  /** These groups applied to the decision: the groups that name the token, or the group `*`. */
-  readonly groups?: readonly RobotsGroup[] | undefined;
-}
-/**
- * This function tells if these tokens can fetch `pathAndQuery`. Each token becomes its product
- * token (`productToken`), so `MyBot/1.0` matches the group `mybot`. The first token is the token of
- * the crawler, and `*` applies when no group names it. A further token, for example `Claude-User`,
- * applies only when the site names it. A site that writes rules for an agent means them for the
- * fetches of that agent.
- */
-declare const decide: (robots: Robots, tokens: readonly string[], pathAndQuery: string) => RobotsDecision;
-/**
- * This function gives the usage lines that apply to these tokens and this path, as
- * `key: value` texts. The lines come from three places:
- *
- * - the lines before the first group
- * - the groups of the first token, or `*`
- * - the groups that name a further token
- *
- * In one group, the line with the longest matching path wins for each key.
- */
-declare const signalsFor: (robots: Robots, tokens: readonly string[], pathAndQuery: string) => string[];
-/** This function gives the robots of a site whose robots.txt the crawler cannot read. No path is permitted, and the reason stays. */
-declare const unreachableRobots: (reason: string) => Robots;
-/** This text is the reason when a robots.txt that answered 200 is a web page, not a robots.txt file. */
-declare const HTML_ROBOTS = "robots.txt is a web page, not a robots.txt file (a login or challenge page)";
-/**
- * This function tells if the body of a robots.txt is a web page. A server can send a challenge, a
- * login or an error page with a 200. Such a body starts with `<!doctype`, `<html`, `<head` or
- * `<body`. With the content type `text/html`, a tag of a page in the first 4 KB is enough. A plain
- * robots.txt that a server sends as `text/html` stays a file.
- */
-declare const isHtmlRobots: (contentType: string | null | undefined, text: string) => boolean;
-/**
- * This function reads the answer of a robots.txt fetch: the status, the content type and the body.
- * A 2xx with a web page in place of the file is unreadable. Thus no path is permitted (fail
- * closed), as for a body that does not decode. The other statuses follow `robotsFromStatus`.
- */
-declare const robotsFromResponse: (status: number, contentType: string | null | undefined, body: Uint8Array | string) => Robots;
-/** This function gives the meaning of the status of a robots.txt fetch (RFC 9309 §2.3.1). */
-declare const robotsFromStatus: (status: number, body: string | undefined) => Robots;
-//#endregion
-export { robotsFromResponse as _, Robots as a, unreachableRobots as b, RobotsRule as c, isHtmlRobots as d, isProductToken as f, productToken as g, parseRobots as h, ROBOTS_MAX_BYTES as i, RobotsSignal as l, normalizePath as m, DISALLOW_ALL as n, RobotsDecision as o, matchesPattern as p, HTML_ROBOTS as r, RobotsGroup as s, ALLOW_ALL as t, decide as u, robotsFromStatus as v, signalsFor as y };
-//# sourceMappingURL=robots-R-qaEBOb.d.mts.map