polite-crawler 0.1.0 → 0.1.2

package/build/{politeness-C5bZ1OKM.d.mts → politeness-yI3xBEx9.d.mts}

@@ -92,14 +92,105 @@ interface HostStateStore {
    * host. It keeps the next times and the leases, so a reset cannot make a host go faster than its pace.
    */
   reset(host?: string): void;
+  /**
+   * This method gives the validator memo of a host, when the store has one. A store without this
+   * method and `noteValidators` has no memo. The crawler then sends each validator of a stored copy.
+   */
+  validators?(host: string): ValidatorMemo | undefined;
+  /**
+   * This method adds one conditional request to the memo of a host, with the rules of
+   * `nextValidatorMemo`, in one atomic step. It gives the new memo.
+   */
+  noteValidators?(host: string, report: ValidatorReport, now: number): ValidatorMemo;
 }
+/** This value tells the result that the crawler learned for one validator of a host. */
+type ValidatorVerdict = "unknown" | "works" | "fails";
+/**
+ * This value tells what one conditional request got:
+ *
+ * - `not-modified`: a 304.
+ * - `same`: a 200 with the body that the store has. Thus the validator did not match.
+ * - `changed`: a 200 with a different body. This answer tells nothing about the validator.
+ */
+type ValidatorAnswer = "not-modified" | "same" | "changed";
+/** This report tells the validators that one conditional request sent, and the answer that it got. */
+interface ValidatorReport {
+  /** This value is true when the request sent `If-None-Match`. */
+  readonly etag: boolean;
+  /** This value is true when the request sent `If-Modified-Since`. */
+  readonly lastModified: boolean;
+  readonly answer: ValidatorAnswer;
+}
+/**
+ * A validator memo holds what the crawler learned about the validators of one host. A server
+ * examines `If-None-Match` first, and it ignores `If-Modified-Since` when the request has the two
+ * (RFC 9110, section 13.1.3). Thus a host with unstable ETags does not answer 304 to a request with
+ * the two validators at any time. Its stable `Last-Modified` date does not help. `nextValidatorMemo`
+ * gives the rules.
+ */
+interface ValidatorMemo {
+  /** This value is true when the crawler leaves out `If-None-Match` for the host and sends only `If-Modified-Since`. */
+  readonly skipEtag: boolean;
+  /** This value tells if a request with `If-None-Match` got a 304 (`works`) or the stored body again (`fails`). */
+  readonly etag: ValidatorVerdict;
+  /** This value tells the same for a request with only `If-Modified-Since`. */
+  readonly lastModified: ValidatorVerdict;
+  /** This value counts the stored bodies in sequence that came back to a request with `If-None-Match`. */
+  readonly etagMisses: number;
+  /** This value counts the stored bodies in sequence that came back to a request with only `If-Modified-Since`. */
+  readonly dateMisses: number;
+  /** This value counts the requests with only `If-Modified-Since` since the crawler stopped the ETag, or since the last probe. */
+  readonly sinceProbe: number;
+  /** This time (milliseconds since the epoch) is the last time that the crawler stopped or started the ETag, or probed it. */
+  readonly changedAt: number;
+}
+/** These are the limits of the validator memo. */
+interface ValidatorLimits {
+  /** After this number of stored bodies in sequence, a validator fails. The value is 3. */
+  readonly misses: number;
+  /** After this number of requests with only `If-Modified-Since`, the next request sends the ETag again (a probe). The value is 20. */
+  readonly probeEvery: number;
+  /** After this time with no probe, the next request is a probe. The value is 7 days. */
+  readonly probeAgeMs: number;
+}
+/** These are the limits of the validator memo of the crawler. */
+declare const VALIDATOR_MEMO: ValidatorLimits;
+/**
+ * This function tells which validators the next conditional request to a host sends. It sends
+ * each validator of the stored copy, except when the memo stopped the ETag and the copy has a
+ * `Last-Modified` date. Then it sends only `If-Modified-Since`, and the ETag too for a probe.
+ * A probe comes after `probeEvery` requests or after `probeAgeMs`.
+ */
+declare const validatorsToSend: (memo: ValidatorMemo | undefined, stored: {
+  readonly etag: boolean;
+  readonly lastModified: boolean;
+}, now: number) => {
+  readonly etag: boolean;
+  readonly lastModified: boolean;
+};
+/**
+ * This function adds one conditional request to the memo of a host, and gives the new memo.
+ * These are the rules, with the limits of `VALIDATOR_MEMO`:
+ *
+ * 1. A 304 to a request with `If-None-Match` marks the ETag `works`. After a probe, the crawler sends the ETag again.
+ * 2. After `misses` stored bodies in sequence with `If-None-Match`, the ETag fails. When the request also sent
+ *    `If-Modified-Since`, the crawler stops the ETag for the host.
+ * 3. A probe that gets a 200 keeps the ETag stopped, and the count to the next probe starts again.
+ * 4. After `misses` stored bodies in sequence with only `If-Modified-Since`, that validator fails too. Then the
+ *    crawler sends the two validators again, because they cost nothing. It tries the date alone again after `probeAgeMs`.
+ * 5. A changed body tells nothing about a validator.
+ */
+declare const nextValidatorMemo: (memo: ValidatorMemo | undefined, report: ValidatorReport, now: number) => ValidatorMemo;
 /**
  * This store keeps the state of each host in memory, for one process. It removes the state of an
  * idle host (no rest left, no lease, no counts and no stop) when it holds more than `maxHosts` hosts.
+ * It keeps the validator memos of the `maxHosts` hosts that it used last.
  */
 declare class MemoryHostStateStore implements HostStateStore {
   private readonly maxHosts;
   private readonly states;
+  /** The validator memos, the memo used last at the end. */
+  private readonly memos;
   private sweepAt;
   constructor(maxHosts?: number);
   /** This value is the number of hosts that the store holds. */
@@ -113,6 +204,8 @@ declare class MemoryHostStateStore implements HostStateStore {
   stop(host: string, until: number, reason: string): void;
   stopped(host: string, now: number): HostStop | undefined;
   reset(host?: string): void;
+  validators(host: string): ValidatorMemo | undefined;
+  noteValidators(host: string, report: ValidatorReport, now: number): ValidatorMemo;
 }
 /** These options change the scheduler. */
 interface SchedulerOptions {
@@ -270,5 +363,5 @@ declare const checkPolicy: (options: PolicyOptions | undefined) => void;
 /** This function gives the policy of one host: the defaults, then the general options, then the entries for the host. */
 declare const policyFor: (options: PolicyOptions | undefined, host: string) => PolitenessPolicy;
 //#endregion
-export { restAfter as S, backoffMs as _, HostScheduler as a, policyFor as b, HostStopped as c, Paced as d, PolicyOptions as f, SchedulerOptions as g, RestInput as h, HostResult as i, MAX_STOP_MS as l, Rest as m, DEFAULT_POLICY as n, HostStateStore as o, PolitenessPolicy as p, HostCounts as r, HostStop as s, Clock as t, MemoryHostStateStore as u, checkPolicy as v, realClock as x, equalJitterMs as y };
-//# sourceMappingURL=politeness-C5bZ1OKM.d.mts.map
+export { validatorsToSend as A, backoffMs as C, policyFor as D, nextValidatorMemo as E, realClock as O, ValidatorVerdict as S, equalJitterMs as T, VALIDATOR_MEMO as _, HostScheduler as a, ValidatorMemo as b, HostStopped as c, Paced as d, PolicyOptions as f, SchedulerOptions as g, RestInput as h, HostResult as i, restAfter as k, MAX_STOP_MS as l, Rest as m, DEFAULT_POLICY as n, HostStateStore as o, PolitenessPolicy as p, HostCounts as r, HostStop as s, Clock as t, MemoryHostStateStore as u, ValidatorAnswer as v, checkPolicy as w, ValidatorReport as x, ValidatorLimits as y };
+//# sourceMappingURL=politeness-yI3xBEx9.d.mts.map

package/build/politeness.d.mts

@@ -1,2 +1,2 @@
-import { S as restAfter, _ as backoffMs, a as HostScheduler, b as policyFor, c as HostStopped, d as Paced, f as PolicyOptions, g as SchedulerOptions, h as RestInput, i as HostResult, l as MAX_STOP_MS, m as Rest, n as DEFAULT_POLICY, o as HostStateStore, p as PolitenessPolicy, r as HostCounts, s as HostStop, t as Clock, u as MemoryHostStateStore, v as checkPolicy, x as realClock, y as equalJitterMs } from "./politeness-C5bZ1OKM.mjs";
-export { Clock, DEFAULT_POLICY, HostCounts, HostResult, HostScheduler, HostStateStore, HostStop, HostStopped, MAX_STOP_MS, MemoryHostStateStore, Paced, PolicyOptions, PolitenessPolicy, Rest, RestInput, SchedulerOptions, backoffMs, checkPolicy, equalJitterMs, policyFor, realClock, restAfter };
+import { A as validatorsToSend, C as backoffMs, D as policyFor, E as nextValidatorMemo, O as realClock, S as ValidatorVerdict, T as equalJitterMs, _ as VALIDATOR_MEMO, a as HostScheduler, b as ValidatorMemo, c as HostStopped, d as Paced, f as PolicyOptions, g as SchedulerOptions, h as RestInput, i as HostResult, k as restAfter, l as MAX_STOP_MS, m as Rest, n as DEFAULT_POLICY, o as HostStateStore, p as PolitenessPolicy, r as HostCounts, s as HostStop, t as Clock, u as MemoryHostStateStore, v as ValidatorAnswer, w as checkPolicy, x as ValidatorReport, y as ValidatorLimits } from "./politeness-yI3xBEx9.mjs";
+export { Clock, DEFAULT_POLICY, HostCounts, HostResult, HostScheduler, HostStateStore, HostStop, HostStopped, MAX_STOP_MS, MemoryHostStateStore, Paced, PolicyOptions, PolitenessPolicy, Rest, RestInput, SchedulerOptions, VALIDATOR_MEMO, ValidatorAnswer, ValidatorLimits, ValidatorMemo, ValidatorReport, ValidatorVerdict, backoffMs, checkPolicy, equalJitterMs, nextValidatorMemo, policyFor, realClock, restAfter, validatorsToSend };

package/build/politeness.mjs

@@ -40,13 +40,104 @@ var HostStopped = class extends Error {
 		this.until = until;
 	}
 };
+/** These are the limits of the validator memo of the crawler. */
+const VALIDATOR_MEMO = {
+	misses: 3,
+	probeEvery: 20,
+	probeAgeMs: 6048e5
+};
+const EMPTY_MEMO = {
+	skipEtag: false,
+	etag: "unknown",
+	lastModified: "unknown",
+	etagMisses: 0,
+	dateMisses: 0,
+	sinceProbe: 0,
+	changedAt: 0
+};
+/**
+* This function tells which validators the next conditional request to a host sends. It sends
+* each validator of the stored copy, except when the memo stopped the ETag and the copy has a
+* `Last-Modified` date. Then it sends only `If-Modified-Since`, and the ETag too for a probe.
+* A probe comes after `probeEvery` requests or after `probeAgeMs`.
+*/
+const validatorsToSend = (memo, stored, now) => {
+	if (memo?.skipEtag !== true || !stored.etag || !stored.lastModified) return stored;
+	return {
+		etag: memo.sinceProbe >= VALIDATOR_MEMO.probeEvery || now - memo.changedAt >= VALIDATOR_MEMO.probeAgeMs,
+		lastModified: true
+	};
+};
+/**
+* This function adds one conditional request to the memo of a host, and gives the new memo.
+* These are the rules, with the limits of `VALIDATOR_MEMO`:
+*
+* 1. A 304 to a request with `If-None-Match` marks the ETag `works`. After a probe, the crawler sends the ETag again.
+* 2. After `misses` stored bodies in sequence with `If-None-Match`, the ETag fails. When the request also sent
+*    `If-Modified-Since`, the crawler stops the ETag for the host.
+* 3. A probe that gets a 200 keeps the ETag stopped, and the count to the next probe starts again.
+* 4. After `misses` stored bodies in sequence with only `If-Modified-Since`, that validator fails too. Then the
+*    crawler sends the two validators again, because they cost nothing. It tries the date alone again after `probeAgeMs`.
+* 5. A changed body tells nothing about a validator.
+*/
+const nextValidatorMemo = (memo, report, now) => {
+	const next = { ...memo ?? EMPTY_MEMO };
+	if (report.etag) {
+		const probe = next.skipEtag;
+		if (report.answer === "not-modified") {
+			next.etag = "works";
+			next.etagMisses = 0;
+			if (probe) {
+				next.skipEtag = false;
+				next.sinceProbe = 0;
+				next.changedAt = now;
+			}
+		} else if (probe) {
+			if (report.answer === "same") next.etagMisses += 1;
+			next.sinceProbe = 0;
+			next.changedAt = now;
+		} else if (report.answer === "same") {
+			next.etagMisses += 1;
+			if (next.etagMisses >= VALIDATOR_MEMO.misses) {
+				next.etag = "fails";
+				const dateLeft = next.lastModified !== "fails" || now - next.changedAt >= VALIDATOR_MEMO.probeAgeMs;
+				if (report.lastModified && dateLeft) {
+					next.skipEtag = true;
+					next.dateMisses = 0;
+					next.sinceProbe = 0;
+					next.changedAt = now;
+				}
+			}
+		}
+	} else if (report.lastModified) {
+		if (next.skipEtag) next.sinceProbe += 1;
+		if (report.answer === "not-modified") {
+			next.lastModified = "works";
+			next.dateMisses = 0;
+		} else if (report.answer === "same") {
+			next.dateMisses += 1;
+			if (next.dateMisses >= VALIDATOR_MEMO.misses) {
+				next.lastModified = "fails";
+				if (next.skipEtag) {
+					next.skipEtag = false;
+					next.sinceProbe = 0;
+					next.changedAt = now;
+				}
+			}
+		}
+	}
+	return next;
+};
 /**
 * This store keeps the state of each host in memory, for one process. It removes the state of an
 * idle host (no rest left, no lease, no counts and no stop) when it holds more than `maxHosts` hosts.
+* It keeps the validator memos of the `maxHosts` hosts that it used last.
 */
 var MemoryHostStateStore = class {
 	maxHosts;
 	states = /* @__PURE__ */ new Map();
+	/** The validator memos, the memo used last at the end. */
+	memos = /* @__PURE__ */ new Map();
 	sweepAt;
 	constructor(maxHosts = 1e4) {
 		this.maxHosts = maxHosts;
@@ -126,6 +217,16 @@ var MemoryHostStateStore = class {
 			if (state !== void 0) clear(state);
 		}
 	}
+	validators(host) {
+		return this.memos.get(host);
+	}
+	noteValidators(host, report, now) {
+		const memo = nextValidatorMemo(this.memos.get(host), report, now);
+		this.memos.delete(host);
+		this.memos.set(host, memo);
+		if (this.memos.size > this.maxHosts) this.memos.delete(this.memos.keys().next().value);
+		return memo;
+	}
 };
 const aborted = (signal) => signal.reason instanceof Error ? signal.reason : new DOMException("The wait was aborted", "AbortError");
 var HostScheduler = class {
@@ -367,6 +468,6 @@ const policyFor = (options, host) => {
 	return Object.assign({}, DEFAULT_POLICY, base, ...matching.map(({ values }) => values));
 };
 //#endregion
-export { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, MemoryHostStateStore, backoffMs, checkPolicy, equalJitterMs, policyFor, realClock, restAfter };
+export { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, MemoryHostStateStore, VALIDATOR_MEMO, backoffMs, checkPolicy, equalJitterMs, nextValidatorMemo, policyFor, realClock, restAfter, validatorsToSend };
 
 //# sourceMappingURL=politeness.mjs.map

package/build/politeness.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"politeness.mjs","names":[],"sources":["../src/politeness.ts"],"sourcesContent":["/**\n * The scheduler controls the pace of each host. The speed of a polite crawler is the number of\n * hosts in progress divided by the delay for each host. Thus the scheduler keeps one request in\n * progress for each host, and it waits the delay of the host between two requests. It uses its\n * concurrency across many hosts, as the back queues of the Mercator crawler do.\n *\n * The caller decides the delay after each request, and `run` gets it from the task. Only the caller\n * knows the documented rate of the source, the crawl delay of the site and the duration of the\n * request. A wait holds no global slot, so a host that waits does not block a different host.\n *\n * The state of each host (its next time, its counts and its stop) is in a `HostStateStore`. The\n * default store is in memory. A store that processes share keeps one pace for each host across them.\n */\n\nexport interface Clock {\n  readonly now: () => number;\n  /** This function waits `ms` milliseconds. When the signal aborts, the wait ends early. */\n  readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;\n}\n\n/** A timer of Node.js holds a maximum of 2^31 − 1 milliseconds. A longer timer fires at once. */\nconst MAX_TIMER_MS = 2 ** 31 - 1;\n\nexport const realClock: Clock = {\n  now: () => Date.now(),\n  sleep: (ms, signal) =>\n    new Promise<void>((resolve) => {\n      if (signal?.aborted === true) {\n        resolve();\n        return;\n      }\n      // a long wait goes in parts, because a timer longer than the maximum fires at once\n      let left = Number.isFinite(ms) ? Math.max(0, ms) : 0;\n      let timer: ReturnType<typeof setTimeout> | undefined;\n      const done = (): void => {\n        if (timer !== undefined) clearTimeout(timer);\n        signal?.removeEventListener(\"abort\", done);\n        resolve();\n      };\n      const step = (): void => {\n        if (left <= 0) {\n          done();\n          return;\n        }\n        const part = Math.min(left, MAX_TIMER_MS);\n        left -= part;\n        timer = setTimeout(step, part);\n      };\n      signal?.addEventListener(\"abort\", done, { once: true });\n      step();\n    }),\n};\n\n/** The crawler does not ask a host again until the stop ends, after a refusal (a challenge, 403 or 402). */\nexport class HostStopped extends Error {\n  constructor(\n    readonly host: string,\n    readonly reason: string,\n    /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */\n    readonly until: number = Number.POSITIVE_INFINITY,\n  ) {\n    super(`${host} stopped: ${reason}`);\n  }\n}\n\nexport interface Paced<T> {\n  readonly value: T;\n  /** This value is the time that the host rests before its next request. */\n  readonly delayMs: number;\n}\n\n/** A stop keeps the crawler away from a host until a time. */\nexport interface HostStop {\n  /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */\n  readonly until: number;\n  readonly reason: string;\n}\n\n/** These are the counts of a host. Each count gives the results of one kind in sequence. */\nexport interface HostCounts {\n  /** This value counts errors and requests with no answer. At the limit of the policy, the host stops. */\n  readonly failures: number;\n  /** This value counts the requests to wait (429, 502, 503, 504). It sets the backoff and does not stop the host. */\n  readonly waits: number;\n}\n\n/**\n * This value tells how one result changes the counts of a host:\n *\n * - `success` sets the two counts to zero.\n * - `failure` adds one to `failures`.\n * - `wait` adds one to `waits`.\n * - `neutral` changes nothing. A client error (a 400) is neutral, because a different request can succeed.\n */\nexport type HostResult = \"success\" | \"failure\" | \"wait\" | \"neutral\";\n\n/**\n * A host state store keeps the state of each host for the scheduler. Its methods are synchronous,\n * and each method is one atomic step on the state of one host. Thus a store can be a table of\n * SQLite that processes share. Times are milliseconds since the epoch.\n *\n * The state of a host has five parts:\n *\n * - the next time: the earliest time of the next request\n * - the lease: the end of the slot of the request in progress, when there is one\n * - the counts (`HostCounts`)\n * - the stop: an end time and a reason, when there is one\n */\nexport interface HostStateStore {\n  /**\n   * This method asks for the slot of a host at the time `now`. When the host has no rest left\n   * and no lease, the store gives the slot to the caller until `now + leaseMs` and gives 0.\n   * Otherwise it changes nothing and gives the time to wait in milliseconds: the time to the next\n   * time, or to the end of the lease. A store that processes share gives a short wait (one second\n   * or less) during a lease, because the holder usually releases it early. The caller asks again after the wait.\n   */\n  reserve(host: string, now: number, leaseMs: number): number;\n  /** This method changes the counts of a host for one result, and it gives the new counts. */\n  record(host: string, result: HostResult): HostCounts;\n  /**\n   * This method ends the lease of the caller. It sets the next time to `nextAt` when `nextAt` is\n   * later than the stored next time. Only the holder of the lease uses it.\n   */\n  release(host: string, nextAt: number): void;\n  /** This method sets the next time to `until` when `until` is later. It does not touch the lease. */\n  defer(host: string, until: number): void;\n  /** This method stops a host until a time. When the host has a stop that ends later, that stop stays. */\n  stop(host: string, until: number, reason: string): void;\n  /** This method gives the stop of a host at the time `now`. A stop that ended gives no value. */\n  stopped(host: string, now: number): HostStop | undefined;\n  /**\n   * This method removes the stops and the counts of one host, or of each host when there is no\n   * host. It keeps the next times and the leases, so a reset cannot make a host go faster than its pace.\n   */\n  reset(host?: string): void;\n}\n\ninterface MemoryState {\n  nextAt: number;\n  leasedUntil: number;\n  failures: number;\n  waits: number;\n  stop: HostStop | undefined;\n}\n\n/**\n * This store keeps the state of each host in memory, for one process. It removes the state of an\n * idle host (no rest left, no lease, no counts and no stop) when it holds more than `maxHosts` hosts.\n */\nexport class MemoryHostStateStore implements HostStateStore {\n  private readonly states = new Map<string, MemoryState>();\n  private sweepAt: number;\n\n  constructor(private readonly maxHosts: number = 10_000) {\n    this.sweepAt = maxHosts;\n  }\n\n  /** This value is the number of hosts that the store holds. */\n  get size(): number {\n    return this.states.size;\n  }\n\n  private state(host: string): MemoryState {\n    let state = this.states.get(host);\n    if (state === undefined) {\n      state = { nextAt: 0, leasedUntil: 0, failures: 0, waits: 0, stop: undefined };\n      this.states.set(host, state);\n    }\n    return state;\n  }\n\n  private sweep(now: number): void {\n    for (const [host, state] of this.states) {\n      const idle =\n        state.nextAt <= now &&\n        state.leasedUntil <= now &&\n        state.failures === 0 &&\n        state.waits === 0 &&\n        (state.stop === undefined || state.stop.until <= now);\n      if (idle) this.states.delete(host);\n    }\n    // the next sweep comes after a tenth more hosts, so a store of busy hosts does not sweep at each request\n    this.sweepAt = Math.max(this.maxHosts, Math.ceil(this.states.size * 1.1));\n  }\n\n  reserve(host: string, now: number, leaseMs: number): number {\n    if (this.states.size > this.sweepAt) this.sweep(now);\n    const state = this.state(host);\n    if (state.leasedUntil > now) return state.leasedUntil - now;\n    if (state.nextAt > now) return state.nextAt - now;\n    state.leasedUntil = now + leaseMs;\n    return 0;\n  }\n\n  record(host: string, result: HostResult): HostCounts {\n    const state = this.state(host);\n    if (result === \"success\") {\n      state.failures = 0;\n      state.waits = 0;\n    } else if (result === \"failure\") state.failures += 1;\n    else if (result === \"wait\") state.waits += 1;\n    return { failures: state.failures, waits: state.waits };\n  }\n\n  release(host: string, nextAt: number): void {\n    const state = this.state(host);\n    state.leasedUntil = 0;\n    if (nextAt > state.nextAt) state.nextAt = nextAt;\n  }\n\n  defer(host: string, until: number): void {\n    const state = this.state(host);\n    if (until > state.nextAt) state.nextAt = until;\n  }\n\n  stop(host: string, until: number, reason: string): void {\n    const state = this.state(host);\n    if (state.stop === undefined || state.stop.until < until) state.stop = { until, reason };\n  }\n\n  stopped(host: string, now: number): HostStop | undefined {\n    const stop = this.states.get(host)?.stop;\n    return stop !== undefined && stop.until > now ? stop : undefined;\n  }\n\n  reset(host?: string): void {\n    const clear = (state: MemoryState): void => {\n      state.failures = 0;\n      state.waits = 0;\n      state.stop = undefined;\n    };\n    if (host === undefined) for (const state of this.states.values()) clear(state);\n    else {\n      const state = this.states.get(host);\n      if (state !== undefined) clear(state);\n    }\n  }\n}\n\n/** These options change the scheduler. */\nexport interface SchedulerOptions {\n  /** This store keeps the state of each host. The default is a `MemoryHostStateStore`. */\n  readonly store?: HostStateStore | undefined;\n  /** This value is the longest slot of one request. A slot that a crashed process holds ends after it. The default is 5 minutes. */\n  readonly leaseMs?: number | undefined;\n  /** A host rests this long after a task that throws an error, or a task that gives no valid delay. The default is 1 second. */\n  readonly fallbackDelayMs?: number | undefined;\n}\n\nconst aborted = (signal: AbortSignal): Error =>\n  signal.reason instanceof Error\n    ? signal.reason\n    : new DOMException(\"The wait was aborted\", \"AbortError\");\n\nexport class HostScheduler {\n  /** This store keeps the state of each host. */\n  readonly store: HostStateStore;\n  private readonly chains = new Map<string, Promise<unknown>>();\n  private active = 0;\n  private readonly queue: Array<() => void> = [];\n  private readonly leaseMs: number;\n  private readonly fallbackDelayMs: number;\n\n  constructor(\n    private readonly clock: Clock,\n    private readonly maxActive: number,\n    options: SchedulerOptions = {},\n  ) {\n    if (!Number.isInteger(maxActive) || maxActive < 1) {\n      throw new RangeError(`maxActive must be an integer of 1 or more, not ${String(maxActive)}`);\n    }\n    this.store = options.store ?? new MemoryHostStateStore();\n    this.leaseMs = options.leaseMs ?? 300_000;\n    this.fallbackDelayMs = options.fallbackDelayMs ?? 1000;\n  }\n\n  private acquire(signal: AbortSignal | undefined): Promise<void> {\n    if (this.active < this.maxActive) {\n      this.active += 1;\n      return Promise.resolve();\n    }\n    return new Promise<void>((resolve, reject) => {\n      const grant = (): void => {\n        signal?.removeEventListener(\"abort\", cancel);\n        resolve();\n      };\n      const cancel = (): void => {\n        const at = this.queue.indexOf(grant);\n        if (at !== -1) this.queue.splice(at, 1);\n        reject(aborted(signal!));\n      };\n      this.queue.push(grant);\n      signal?.addEventListener(\"abort\", cancel, { once: true });\n    });\n  }\n\n  private release(): void {\n    // the slot goes to the next waiter directly, so a new request cannot take it in between\n    const next = this.queue.shift();\n    if (next !== undefined) next();\n    else this.active -= 1;\n  }\n\n  private async sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {\n    if (signal === undefined) {\n      await this.clock.sleep(ms);\n      return;\n    }\n    if (signal.aborted) throw aborted(signal);\n    await new Promise<void>((resolve, reject) => {\n      const cancel = (): void => reject(aborted(signal));\n      signal.addEventListener(\"abort\", cancel, { once: true });\n      this.clock.sleep(ms, signal).then(\n        () => {\n          signal.removeEventListener(\"abort\", cancel);\n          resolve();\n        },\n        (error: unknown) => {\n          signal.removeEventListener(\"abort\", cancel);\n          reject(error);\n        },\n      );\n    });\n  }\n\n  /**\n   * This method starts one request on a host, in sequence, after the rest of the host. When the\n   * signal aborts during the wait, the promise rejects with the reason of the signal.\n   */\n  run<T>(host: string, task: () => Promise<Paced<T>>, signal?: AbortSignal): Promise<T> {\n    const previous = this.chains.get(host) ?? Promise.resolve();\n    const result = previous.then(async () => {\n      // after a wait, the time is at least the end of the wait, also on a clock that a test holds still\n      let waited = Number.NEGATIVE_INFINITY;\n      for (;;) {\n        if (signal?.aborted === true) throw aborted(signal);\n        const now = Math.max(this.clock.now(), waited);\n        const stop = this.store.stopped(host, now);\n        if (stop !== undefined) throw new HostStopped(host, stop.reason, stop.until);\n        const wait = this.store.reserve(host, now, this.leaseMs);\n        if (!(wait > 0)) break;\n        await this.sleep(wait, signal);\n        waited = now + wait;\n      }\n      try {\n        await this.acquire(signal);\n      } catch (error) {\n        this.store.release(host, this.clock.now());\n        throw error;\n      }\n      let nextAt = this.clock.now() + this.fallbackDelayMs;\n      try {\n        const { value, delayMs } = await task();\n        const rest = Number.isFinite(delayMs) && delayMs >= 0 ? delayMs : this.fallbackDelayMs;\n        nextAt = this.clock.now() + rest;\n        return value;\n      } catch (error) {\n        nextAt = this.clock.now() + this.fallbackDelayMs;\n        throw error;\n      } finally {\n        this.store.release(host, nextAt);\n        this.release();\n      }\n    });\n    const tail = result.then(\n      () => undefined,\n      () => undefined,\n    );\n    this.chains.set(host, tail);\n    // a host with no queued request keeps no chain\n    void tail.then(() => {\n      if (this.chains.get(host) === tail) this.chains.delete(host);\n    });\n    return result;\n  }\n\n  /** The server asked for a pause (Retry-After), or a backoff chose one. */\n  park(host: string, untilMs: number): void {\n    this.store.defer(host, untilMs);\n  }\n\n  /** This method stops a host until `untilMs`. With no time, the stop has no end. */\n  stop(host: string, reason: string, untilMs: number = Number.POSITIVE_INFINITY): void {\n    this.store.stop(host, untilMs, reason);\n  }\n\n  stoppedReason(host: string): string | undefined {\n    return this.store.stopped(host, this.clock.now())?.reason;\n  }\n\n  /** This method gives the stop of a host, with its end, when the host is stopped. */\n  stopOf(host: string): HostStop | undefined {\n    return this.store.stopped(host, this.clock.now());\n  }\n\n  /** This value counts the failures of a host in sequence, for the backoff. A success sets it to zero. */\n  failed(host: string): number {\n    return this.store.record(host, \"failure\").failures;\n  }\n\n  /** This value counts the requests to wait (429, 503) of a host in sequence. They do not stop the host. */\n  waited(host: string): number {\n    return this.store.record(host, \"wait\").waits;\n  }\n\n  succeeded(host: string): void {\n    this.store.record(host, \"success\");\n  }\n\n  /** This method removes the stops and the counts of one host, or of each host. The pace stays. */\n  reset(host?: string): void {\n    this.store.reset(host);\n  }\n}\n\n/**\n * This function gives an exponential backoff with full jitter, from the analysis of AWS. The wait\n * is random, up to base × 2^(try − 1), with a maximum. The jitter keeps retries apart.\n */\nexport const backoffMs = (\n  attempt: number,\n  baseMs: number,\n  capMs: number,\n  random: () => number,\n): number => Math.floor(random() * Math.min(capMs, baseMs * 2 ** Math.max(0, attempt - 1)));\n\n/**\n * This function gives an exponential backoff with equal jitter. The wait is half of\n * base × 2^(try − 1), with a maximum, and a random part up to the other half. Thus each backoff\n * is at least half of its step, and it is not lost under the rest of the host.\n */\nexport const equalJitterMs = (\n  attempt: number,\n  baseMs: number,\n  capMs: number,\n  random: () => number,\n): number => {\n  const step = Math.min(capMs, baseMs * 2 ** Math.max(0, attempt - 1));\n  return Math.floor(step / 2 + random() * (step / 2));\n};\n\n/** These are the facts of one request that set the rest of its host. */\nexport interface RestInput {\n  /** This value is the kind of the request: a web page, a documented API, or a render in the browser. */\n  readonly kind: \"page\" | \"api\" | \"render\";\n  /** This value is the duration of the request, in milliseconds. */\n  readonly elapsedMs: number;\n  /** This value is the `crawl-delay` of robots.txt, in seconds. */\n  readonly crawlDelaySeconds?: number | undefined;\n  /** This value is the documented pace of the source. It replaces the interval of the policy for the kind. */\n  readonly minIntervalMs?: number | undefined;\n  /** This value is the wait that the server asked for (`Retry-After`). */\n  readonly retryAfterMs?: number | undefined;\n}\n\n/** The rest of a host after a request, and the reason for its length. */\nexport interface Rest {\n  readonly ms: number;\n  readonly reason: \"pace\" | \"slow\" | \"retry-after\" | \"crawl-delay\";\n}\n\n/**\n * This function gives the rest of a host after one request, as the crawler calculates it. These\n * are the parts:\n *\n * - The interval of the kind (3 seconds for a page), or the documented pace of the source.\n * - The `crawl-delay` of robots.txt, up to `crawlDelayCapMs`, when it is longer.\n * - After a page or a render, `elapsedFactor` times the duration of the request, up to `elapsedCapMs`.\n * - The `Retry-After` of the server, when it is longer.\n *\n * The backoff after an error or a request to wait comes on top, with jitter. The function is\n * pure, so a simulation and a study can use the pace of the crawler.\n */\nexport const restAfter = (policy: PolitenessPolicy, input: RestInput): Rest => {\n  const interval =\n    input.minIntervalMs ??\n    (input.kind === \"page\"\n      ? policy.pageIntervalMs\n      : input.kind === \"api\"\n        ? policy.apiIntervalMs\n        : policy.renderIntervalMs);\n  const crawlDelay = Math.min(\n    policy.crawlDelayCapMs,\n    Math.max(0, (input.crawlDelaySeconds ?? 0) * 1000),\n  );\n  let ms = Math.max(interval, crawlDelay);\n  let reason: Rest[\"reason\"] = crawlDelay > interval ? \"crawl-delay\" : \"pace\";\n  if (input.kind !== \"api\") {\n    const slow = Math.min(policy.elapsedCapMs, policy.elapsedFactor * Math.max(0, input.elapsedMs));\n    if (slow > ms) {\n      ms = slow;\n      reason = \"slow\";\n    }\n  }\n  if (input.retryAfterMs !== undefined && input.retryAfterMs > ms) {\n    ms = input.retryAfterMs;\n    reason = \"retry-after\";\n  }\n  return { ms, reason };\n};\n\n/**\n * The politeness policy holds each limit of the crawler. Each value has a default, and a host can\n * have values of its own. The times are in milliseconds.\n */\nexport interface PolitenessPolicy {\n  /** This value is the shortest rest of a host between two page requests. The default is 3 seconds. */\n  readonly pageIntervalMs: number;\n  /** This value is the shortest rest of a host between two API requests. The default is 1 second. */\n  readonly apiIntervalMs: number;\n  /** This value is the shortest rest of a host after a render. The default is 5 seconds. */\n  readonly renderIntervalMs: number;\n  /** After a page, the host rests this number of times the duration of the request. The default is 5. */\n  readonly elapsedFactor: number;\n  /** This value is the maximum of the rest that comes from the duration. The default is 30 seconds. */\n  readonly elapsedCapMs: number;\n  /** This value is the maximum `crawl-delay` that the crawler obeys. The default is 60 seconds. */\n  readonly crawlDelayCapMs: number;\n  /** This value is the maximum number of redirects for one fetch. The default is 5. */\n  readonly maxRedirects: number;\n  /** This value is the maximum size of a body after decompression. The default is 5 MiB. */\n  readonly maxBytes: number;\n  /** This value is the deadline of one request, the body included. The default is 90 seconds. */\n  readonly requestTimeoutMs: number;\n  /** This value is the deadline of one render. The default is 20 seconds. */\n  readonly renderTimeoutMs: number;\n  /** After this number of failures in sequence, the host stops. The default is 5. */\n  readonly maxFailures: number;\n  /** This value is the stop after too many failures. The default is 1 hour. */\n  readonly failureStopMs: number;\n  /** This value is the stop after a 401 or a 403. The default is 1 hour. */\n  readonly blockedStopMs: number;\n  /** This value is the stop after a bot challenge. The default is 24 hours. */\n  readonly challengeStopMs: number;\n  /** This value is the stop after a 402. The default is 24 hours. */\n  readonly paymentStopMs: number;\n  /** The crawler obeys a `Retry-After` up to this value. A longer wait stops the host for that time. The default is 1 hour. */\n  readonly maxRetryAfterMs: number;\n  /** This value is the first step of the backoff after an error or a request to wait. The default is 5 seconds. */\n  readonly backoffBaseMs: number;\n  /** This value is the maximum step of the backoff. The default is 300 seconds. */\n  readonly backoffCapMs: number;\n  /** The crawler keeps a robots.txt for this time. The default is 24 hours. */\n  readonly robotsTtlMs: number;\n  /** The crawler keeps an unreachable robots.txt verdict for this time first, with a backoff. The default is 5 minutes. */\n  readonly robotsRetryMs: number;\n  /** This value is the maximum time for an unreachable robots.txt verdict. The default is 1 hour. */\n  readonly robotsRetryCapMs: number;\n  /** A stored robots.txt younger than this age replaces a robots.txt that the crawler cannot read. The default is 30 days. */\n  readonly robotsStaleMs: number;\n  /** This value is the maximum freshness that the crawler gives a response with no freshness of its own. The default is 24 hours. */\n  readonly heuristicFreshnessCapMs: number;\n}\n\n/**\n * This value is the longest stop that a `Retry-After` can give: 30 days. A wait longer than\n * `maxRetryAfterMs` stops the host for the time that the server asks, up to this value.\n */\nexport const MAX_STOP_MS: number = 30 * 24 * 60 * 60 * 1000;\n\nexport const DEFAULT_POLICY: PolitenessPolicy = {\n  pageIntervalMs: 3000,\n  apiIntervalMs: 1000,\n  renderIntervalMs: 5000,\n  elapsedFactor: 5,\n  elapsedCapMs: 30_000,\n  crawlDelayCapMs: 60_000,\n  maxRedirects: 5,\n  maxBytes: 5 * 1024 * 1024,\n  requestTimeoutMs: 90_000,\n  renderTimeoutMs: 20_000,\n  maxFailures: 5,\n  failureStopMs: 60 * 60 * 1000,\n  blockedStopMs: 60 * 60 * 1000,\n  challengeStopMs: 24 * 60 * 60 * 1000,\n  paymentStopMs: 24 * 60 * 60 * 1000,\n  maxRetryAfterMs: 60 * 60 * 1000,\n  backoffBaseMs: 5000,\n  backoffCapMs: 300_000,\n  robotsTtlMs: 24 * 60 * 60 * 1000,\n  robotsRetryMs: 5 * 60 * 1000,\n  robotsRetryCapMs: 60 * 60 * 1000,\n  robotsStaleMs: 30 * 24 * 60 * 60 * 1000,\n  heuristicFreshnessCapMs: 24 * 60 * 60 * 1000,\n};\n\n/**\n * These options change the politeness policy. The values at the top apply to each host. The\n * `hosts` map gives values for one host (`api.example.org`) or for the hosts under a name\n * (`*.example.org`). The most specific entry wins.\n */\nexport interface PolicyOptions extends Partial<PolitenessPolicy> {\n  readonly hosts?: Readonly<Record<string, Partial<PolitenessPolicy>>> | undefined;\n}\n\nconst INTEGER_KEYS: ReadonlySet<string> = new Set([\"maxRedirects\", \"maxFailures\"]);\n\nconst checked = (values: Partial<PolitenessPolicy>, where: string): Partial<PolitenessPolicy> => {\n  for (const [key, value] of Object.entries(values)) {\n    if (!(key in DEFAULT_POLICY))\n      throw new RangeError(`${where}: ${key} is not a politeness setting`);\n    const good =\n      typeof value === \"number\" &&\n      value >= 0 &&\n      (Number.isFinite(value) || key.endsWith(\"StopMs\")) &&\n      (!INTEGER_KEYS.has(key) || Number.isInteger(value));\n    if (!good)\n      throw new RangeError(`${where}: ${key} must be a number of 0 or more, not ${String(value)}`);\n  }\n  return values;\n};\n\n/** This function examines the options of a policy. It throws a `RangeError` for a value that is not a valid number. */\nexport const checkPolicy = (options: PolicyOptions | undefined): void => {\n  if (options === undefined) return;\n  const { hosts, ...base } = options;\n  checked(base, \"politeness\");\n  for (const [host, values] of Object.entries(hosts ?? {}))\n    checked(values, `politeness.hosts[\"${host}\"]`);\n};\n\n/** This function gives the policy of one host: the defaults, then the general options, then the entries for the host. */\nexport const policyFor = (options: PolicyOptions | undefined, host: string): PolitenessPolicy => {\n  if (options === undefined) return DEFAULT_POLICY;\n  const { hosts, ...base } = options;\n  const name = host.toLowerCase().replace(/\\.$/, \"\");\n  const matching = Object.entries(hosts ?? {})\n    .map(([key, values]) => ({ key: key.toLowerCase(), values }))\n    .filter(({ key }) => key === name || (key.startsWith(\"*.\") && name.endsWith(key.slice(1))))\n    // an exact name wins, and a longer pattern wins over a shorter one\n    .sort(\n      (a, b) => (a.key === name ? 1 : 0) - (b.key === name ? 1 : 0) || a.key.length - b.key.length,\n    );\n  return Object.assign(\n    {},\n    DEFAULT_POLICY,\n    base,\n    ...matching.map(({ values }) => values),\n  ) as PolitenessPolicy;\n};\n"],"mappings":";;AAqBA,MAAM,eAAe,KAAK,KAAK;AAE/B,MAAa,YAAmB;CAC9B,WAAW,KAAK,IAAI;CACpB,QAAQ,IAAI,WACV,IAAI,SAAe,YAAY;EAC7B,IAAI,QAAQ,YAAY,MAAM;GAC5B,QAAQ;GACR;EACF;EAEA,IAAI,OAAO,OAAO,SAAS,EAAE,IAAI,KAAK,IAAI,GAAG,EAAE,IAAI;EACnD,IAAI;EACJ,MAAM,aAAmB;GACvB,IAAI,UAAU,KAAA,GAAW,aAAa,KAAK;GAC3C,QAAQ,oBAAoB,SAAS,IAAI;GACzC,QAAQ;EACV;EACA,MAAM,aAAmB;GACvB,IAAI,QAAQ,GAAG;IACb,KAAK;IACL;GACF;GACA,MAAM,OAAO,KAAK,IAAI,MAAM,YAAY;GACxC,QAAQ;GACR,QAAQ,WAAW,MAAM,IAAI;EAC/B;EACA,QAAQ,iBAAiB,SAAS,MAAM,EAAE,MAAM,KAAK,CAAC;EACtD,KAAK;CACP,CAAC;AACL;;AAGA,IAAa,cAAb,cAAiC,MAAM;CAE1B;CACA;CAEA;CAJX,YACE,MACA,QAEA,QAAyB,OAAO,mBAChC;EACA,MAAM,GAAG,KAAK,YAAY,QAAQ;EALzB,KAAA,OAAA;EACA,KAAA,SAAA;EAEA,KAAA,QAAA;CAGX;AACF;;;;;AAsFA,IAAa,uBAAb,MAA4D;CAI7B;CAH7B,yBAA0B,IAAI,IAAyB;CACvD;CAEA,YAAY,WAAoC,KAAQ;EAA3B,KAAA,WAAA;EAC3B,KAAK,UAAU;CACjB;;CAGA,IAAI,OAAe;EACjB,OAAO,KAAK,OAAO;CACrB;CAEA,MAAc,MAA2B;EACvC,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI;EAChC,IAAI,UAAU,KAAA,GAAW;GACvB,QAAQ;IAAE,QAAQ;IAAG,aAAa;IAAG,UAAU;IAAG,OAAO;IAAG,MAAM,KAAA;GAAU;GAC5E,KAAK,OAAO,IAAI,MAAM,KAAK;EAC7B;EACA,OAAO;CACT;CAEA,MAAc,KAAmB;EAC/B,KAAK,MAAM,CAAC,MAAM,UAAU,KAAK,QAO/B,IALE,MAAM,UAAU,OAChB,MAAM,eAAe,OACrB,MAAM,aAAa,KACnB,MAAM,UAAU,MACf,MAAM,SAAS,KAAA,KAAa,MAAM,KAAK,SAAS,MACzC,KAAK,OAAO,OAAO,IAAI;EAGnC,KAAK,UAAU,KAAK,IAAI,KAAK,UAAU,KAAK,KAAK,KAAK,OAAO,OAAO,GAAG,CAAC;CAC1E;CAEA,QAAQ,MAAc,KAAa,SAAyB;EAC1D,IAAI,KAAK,OAAO,OAAO,KAAK,SAAS,KAAK,MAAM,GAAG;EACnD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,MAAM,cAAc,KAAK,OAAO,MAAM,cAAc;EACxD,IAAI,MAAM,SAAS,KAAK,OAAO,MAAM,SAAS;EAC9C,MAAM,cAAc,MAAM;EAC1B,OAAO;CACT;CAEA,OAAO,MAAc,QAAgC;EACnD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,WAAW,WAAW;GACxB,MAAM,WAAW;GACjB,MAAM,QAAQ;EAChB,OAAO,IAAI,WAAW,WAAW,MAAM,YAAY;OAC9C,IAAI,WAAW,QAAQ,MAAM,SAAS;EAC3C,OAAO;GAAE,UAAU,MAAM;GAAU,OAAO,MAAM;EAAM;CACxD;CAEA,QAAQ,MAAc,QAAsB;EAC1C,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,MAAM,cAAc;EACpB,IAAI,SAAS,MAAM,QAAQ,MAAM,SAAS;CAC5C;CAEA,MAAM,MAAc,OAAqB;EACvC,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,QAAQ,MAAM,QAAQ,MAAM,SAAS;CAC3C;CAEA,KAAK,MAAc,OAAe,QAAsB;EACtD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,MAAM,SAAS,KAAA,KAAa,MAAM,KAAK,QAAQ,OAAO,MAAM,OAAO;GAAE;GAAO;EAAO;CACzF;CAEA,QAAQ,MAAc,KAAmC;EACvD,MAAM,OAAO,KAAK,OAAO,IAAI,IAAI,CAAC,EAAE;EACpC,OAAO,SAAS,KAAA,KAAa,KAAK,QAAQ,MAAM,OAAO,KAAA;CACzD;CAEA,MAAM,MAAqB;EACzB,MAAM,SAAS,UAA6B;GAC1C,MAAM,WAAW;GACjB,MAAM,QAAQ;GACd,MAAM,OAAO,KAAA;EACf;EACA,IAAI,SAAS,KAAA,GAAW,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GAAG,MAAM,KAAK;OACxE;GACH,MAAM,QAAQ,KAAK,OAAO,IAAI,IAAI;GAClC,IAAI,UAAU,KAAA,GAAW,MAAM,KAAK;EACtC;CACF;AACF;AAYA,MAAM,WAAW,WACf,OAAO,kBAAkB,QACrB,OAAO,SACP,IAAI,aAAa,wBAAwB,YAAY;AAE3D,IAAa,gBAAb,MAA2B;CAUN;CACA;;CATnB;CACA,yBAA0B,IAAI,IAA8B;CAC5D,SAAiB;CACjB,QAA4C,CAAC;CAC7C;CACA;CAEA,YACE,OACA,WACA,UAA4B,CAAC,GAC7B;EAHiB,KAAA,QAAA;EACA,KAAA,YAAA;EAGjB,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,MAAM,IAAI,WAAW,kDAAkD,OAAO,SAAS,GAAG;EAE5F,KAAK,QAAQ,QAAQ,SAAS,IAAI,qBAAqB;EACvD,KAAK,UAAU,QAAQ,WAAW;EAClC,KAAK,kBAAkB,QAAQ,mBAAmB;CACpD;CAEA,QAAgB,QAAgD;EAC9D,IAAI,KAAK,SAAS,KAAK,WAAW;GAChC,KAAK,UAAU;GACf,OAAO,QAAQ,QAAQ;EACzB;EACA,OAAO,IAAI,SAAe,SAAS,WAAW;GAC5C,MAAM,cAAoB;IACxB,QAAQ,oBAAoB,SAAS,MAAM;IAC3C,QAAQ;GACV;GACA,MAAM,eAAqB;IACzB,MAAM,KAAK,KAAK,MAAM,QAAQ,KAAK;IACnC,IAAI,OAAO,IAAI,KAAK,MAAM,OAAO,IAAI,CAAC;IACtC,OAAO,QAAQ,MAAO,CAAC;GACzB;GACA,KAAK,MAAM,KAAK,KAAK;GACrB,QAAQ,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;EAC1D,CAAC;CACH;CAEA,UAAwB;EAEtB,MAAM,OAAO,KAAK,MAAM,MAAM;EAC9B,IAAI,SAAS,KAAA,GAAW,KAAK;OACxB,KAAK,UAAU;CACtB;CAEA,MAAc,MAAM,IAAY,QAAgD;EAC9E,IAAI,WAAW,KAAA,GAAW;GACxB,MAAM,KAAK,MAAM,MAAM,EAAE;GACzB;EACF;EACA,IAAI,OAAO,SAAS,MAAM,QAAQ,MAAM;EACxC,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,eAAqB,OAAO,QAAQ,MAAM,CAAC;GACjD,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;GACvD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,CAAC,WACrB;IACJ,OAAO,oBAAoB,SAAS,MAAM;IAC1C,QAAQ;GACV,IACC,UAAmB;IAClB,OAAO,oBAAoB,SAAS,MAAM;IAC1C,OAAO,KAAK;GACd,CACF;EACF,CAAC;CACH;;;;;CAMA,IAAO,MAAc,MAA+B,QAAkC;EAEpF,MAAM,UADW,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ,QAAQ,EAAA,CAClC,KAAK,YAAY;GAEvC,IAAI,SAAS,OAAO;GACpB,SAAS;IACP,IAAI,QAAQ,YAAY,MAAM,MAAM,QAAQ,MAAM;IAClD,MAAM,MAAM,KAAK,IAAI,KAAK,MAAM,IAAI,GAAG,MAAM;IAC7C,MAAM,OAAO,KAAK,MAAM,QAAQ,MAAM,GAAG;IACzC,IAAI,SAAS,KAAA,GAAW,MAAM,IAAI,YAAY,MAAM,KAAK,QAAQ,KAAK,KAAK;IAC3E,MAAM,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,KAAK,OAAO;IACvD,IAAI,EAAE,OAAO,IAAI;IACjB,MAAM,KAAK,MAAM,MAAM,MAAM;IAC7B,SAAS,MAAM;GACjB;GACA,IAAI;IACF,MAAM,KAAK,QAAQ,MAAM;GAC3B,SAAS,OAAO;IACd,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC;IACzC,MAAM;GACR;GACA,IAAI,SAAS,KAAK,MAAM,IAAI,IAAI,KAAK;GACrC,IAAI;IACF,MAAM,EAAE,OAAO,YAAY,MAAM,KAAK;IACtC,MAAM,OAAO,OAAO,SAAS,OAAO,KAAK,WAAW,IAAI,UAAU,KAAK;IACvE,SAAS,KAAK,MAAM,IAAI,IAAI;IAC5B,OAAO;GACT,SAAS,OAAO;IACd,SAAS,KAAK,MAAM,IAAI,IAAI,KAAK;IACjC,MAAM;GACR,UAAU;IACR,KAAK,MAAM,QAAQ,MAAM,MAAM;IAC/B,KAAK,QAAQ;GACf;EACF,CAAC;EACD,MAAM,OAAO,OAAO,WACZ,KAAA,SACA,KAAA,CACR;EACA,KAAK,OAAO,IAAI,MAAM,IAAI;EAE1B,KAAU,WAAW;GACnB,IAAI,KAAK,OAAO,IAAI,IAAI,MAAM,MAAM,KAAK,OAAO,OAAO,IAAI;EAC7D,CAAC;EACD,OAAO;CACT;;CAGA,KAAK,MAAc,SAAuB;EACxC,KAAK,MAAM,MAAM,MAAM,OAAO;CAChC;;CAGA,KAAK,MAAc,QAAgB,UAAkB,OAAO,mBAAyB;EACnF,KAAK,MAAM,KAAK,MAAM,SAAS,MAAM;CACvC;CAEA,cAAc,MAAkC;EAC9C,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,EAAE;CACrD;;CAGA,OAAO,MAAoC;EACzC,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC;CAClD;;CAGA,OAAO,MAAsB;EAC3B,OAAO,KAAK,MAAM,OAAO,MAAM,SAAS,CAAC,CAAC;CAC5C;;CAGA,OAAO,MAAsB;EAC3B,OAAO,KAAK,MAAM,OAAO,MAAM,MAAM,CAAC,CAAC;CACzC;CAEA,UAAU,MAAoB;EAC5B,KAAK,MAAM,OAAO,MAAM,SAAS;CACnC;;CAGA,MAAM,MAAqB;EACzB,KAAK,MAAM,MAAM,IAAI;CACvB;AACF;;;;;AAMA,MAAa,aACX,SACA,QACA,OACA,WACW,KAAK,MAAM,OAAO,IAAI,KAAK,IAAI,OAAO,SAAS,KAAK,KAAK,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC;;;;;;AAO1F,MAAa,iBACX,SACA,QACA,OACA,WACW;CACX,MAAM,OAAO,KAAK,IAAI,OAAO,SAAS,KAAK,KAAK,IAAI,GAAG,UAAU,CAAC,CAAC;CACnE,OAAO,KAAK,MAAM,OAAO,IAAI,OAAO,KAAK,OAAO,EAAE;AACpD;;;;;;;;;;;;;AAkCA,MAAa,aAAa,QAA0B,UAA2B;CAC7E,MAAM,WACJ,MAAM,kBACL,MAAM,SAAS,SACZ,OAAO,iBACP,MAAM,SAAS,QACb,OAAO,gBACP,OAAO;CACf,MAAM,aAAa,KAAK,IACtB,OAAO,iBACP,KAAK,IAAI,IAAI,MAAM,qBAAqB,KAAK,GAAI,CACnD;CACA,IAAI,KAAK,KAAK,IAAI,UAAU,UAAU;CACtC,IAAI,SAAyB,aAAa,WAAW,gBAAgB;CACrE,IAAI,MAAM,SAAS,OAAO;EACxB,MAAM,OAAO,KAAK,IAAI,OAAO,cAAc,OAAO,gBAAgB,KAAK,IAAI,GAAG,MAAM,SAAS,CAAC;EAC9F,IAAI,OAAO,IAAI;GACb,KAAK;GACL,SAAS;EACX;CACF;CACA,IAAI,MAAM,iBAAiB,KAAA,KAAa,MAAM,eAAe,IAAI;EAC/D,KAAK,MAAM;EACX,SAAS;CACX;CACA,OAAO;EAAE;EAAI;CAAO;AACtB;;;;;AA2DA,MAAa,cAAsB;AAEnC,MAAa,iBAAmC;CAC9C,gBAAgB;CAChB,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,cAAc;CACd,iBAAiB;CACjB,cAAc;CACd,UAAU;CACV,kBAAkB;CAClB,iBAAiB;CACjB,aAAa;CACb,eAAe;CACf,eAAe;CACf,iBAAiB;CACjB,eAAe;CACf,iBAAiB;CACjB,eAAe;CACf,cAAc;CACd,aAAa;CACb,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,yBAAyB;AAC3B;AAWA,MAAM,+BAAoC,IAAI,IAAI,CAAC,gBAAgB,aAAa,CAAC;AAEjF,MAAM,WAAW,QAAmC,UAA6C;CAC/F,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,MAAM,GAAG;EACjD,IAAI,EAAE,OAAO,iBACX,MAAM,IAAI,WAAW,GAAG,MAAM,IAAI,IAAI,6BAA6B;EAMrE,IAAI,EAJF,OAAO,UAAU,YACjB,SAAS,MACR,OAAO,SAAS,KAAK,KAAK,IAAI,SAAS,QAAQ,OAC/C,CAAC,aAAa,IAAI,GAAG,KAAK,OAAO,UAAU,KAAK,KAEjD,MAAM,IAAI,WAAW,GAAG,MAAM,IAAI,IAAI,sCAAsC,OAAO,KAAK,GAAG;CAC/F;CACA,OAAO;AACT;;AAGA,MAAa,eAAe,YAA6C;CACvE,IAAI,YAAY,KAAA,GAAW;CAC3B,MAAM,EAAE,OAAO,GAAG,SAAS;CAC3B,QAAQ,MAAM,YAAY;CAC1B,KAAK,MAAM,CAAC,MAAM,WAAW,OAAO,QAAQ,SAAS,CAAC,CAAC,GACrD,QAAQ,QAAQ,qBAAqB,KAAK,GAAG;AACjD;;AAGA,MAAa,aAAa,SAAoC,SAAmC;CAC/F,IAAI,YAAY,KAAA,GAAW,OAAO;CAClC,MAAM,EAAE,OAAO,GAAG,SAAS;CAC3B,MAAM,OAAO,KAAK,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;CACjD,MAAM,WAAW,OAAO,QAAQ,SAAS,CAAC,CAAC,CAAC,CACzC,KAAK,CAAC,KAAK,aAAa;EAAE,KAAK,IAAI,YAAY;EAAG;CAAO,EAAE,CAAC,CAC5D,QAAQ,EAAE,UAAU,QAAQ,QAAS,IAAI,WAAW,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAE1F,MACE,GAAG,OAAO,EAAE,QAAQ,OAAO,IAAI,MAAM,EAAE,QAAQ,OAAO,IAAI,MAAM,EAAE,IAAI,SAAS,EAAE,IAAI,MACxF;CACF,OAAO,OAAO,OACZ,CAAC,GACD,gBACA,MACA,GAAG,SAAS,KAAK,EAAE,aAAa,MAAM,CACxC;AACF"}
+{"version":3,"file":"politeness.mjs","names":[],"sources":["../src/politeness.ts"],"sourcesContent":["/**\n * The scheduler controls the pace of each host. The speed of a polite crawler is the number of\n * hosts in progress divided by the delay for each host. Thus the scheduler keeps one request in\n * progress for each host, and it waits the delay of the host between two requests. It uses its\n * concurrency across many hosts, as the back queues of the Mercator crawler do.\n *\n * The caller decides the delay after each request, and `run` gets it from the task. Only the caller\n * knows the documented rate of the source, the crawl delay of the site and the duration of the\n * request. A wait holds no global slot, so a host that waits does not block a different host.\n *\n * The state of each host (its next time, its counts and its stop) is in a `HostStateStore`. The\n * default store is in memory. A store that processes share keeps one pace for each host across them.\n */\n\nexport interface Clock {\n  readonly now: () => number;\n  /** This function waits `ms` milliseconds. When the signal aborts, the wait ends early. */\n  readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;\n}\n\n/** A timer of Node.js holds a maximum of 2^31 − 1 milliseconds. A longer timer fires at once. */\nconst MAX_TIMER_MS = 2 ** 31 - 1;\n\nexport const realClock: Clock = {\n  now: () => Date.now(),\n  sleep: (ms, signal) =>\n    new Promise<void>((resolve) => {\n      if (signal?.aborted === true) {\n        resolve();\n        return;\n      }\n      // a long wait goes in parts, because a timer longer than the maximum fires at once\n      let left = Number.isFinite(ms) ? Math.max(0, ms) : 0;\n      let timer: ReturnType<typeof setTimeout> | undefined;\n      const done = (): void => {\n        if (timer !== undefined) clearTimeout(timer);\n        signal?.removeEventListener(\"abort\", done);\n        resolve();\n      };\n      const step = (): void => {\n        if (left <= 0) {\n          done();\n          return;\n        }\n        const part = Math.min(left, MAX_TIMER_MS);\n        left -= part;\n        timer = setTimeout(step, part);\n      };\n      signal?.addEventListener(\"abort\", done, { once: true });\n      step();\n    }),\n};\n\n/** The crawler does not ask a host again until the stop ends, after a refusal (a challenge, 403 or 402). */\nexport class HostStopped extends Error {\n  constructor(\n    readonly host: string,\n    readonly reason: string,\n    /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */\n    readonly until: number = Number.POSITIVE_INFINITY,\n  ) {\n    super(`${host} stopped: ${reason}`);\n  }\n}\n\nexport interface Paced<T> {\n  readonly value: T;\n  /** This value is the time that the host rests before its next request. */\n  readonly delayMs: number;\n}\n\n/** A stop keeps the crawler away from a host until a time. */\nexport interface HostStop {\n  /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */\n  readonly until: number;\n  readonly reason: string;\n}\n\n/** These are the counts of a host. Each count gives the results of one kind in sequence. */\nexport interface HostCounts {\n  /** This value counts errors and requests with no answer. At the limit of the policy, the host stops. */\n  readonly failures: number;\n  /** This value counts the requests to wait (429, 502, 503, 504). It sets the backoff and does not stop the host. */\n  readonly waits: number;\n}\n\n/**\n * This value tells how one result changes the counts of a host:\n *\n * - `success` sets the two counts to zero.\n * - `failure` adds one to `failures`.\n * - `wait` adds one to `waits`.\n * - `neutral` changes nothing. A client error (a 400) is neutral, because a different request can succeed.\n */\nexport type HostResult = \"success\" | \"failure\" | \"wait\" | \"neutral\";\n\n/**\n * A host state store keeps the state of each host for the scheduler. Its methods are synchronous,\n * and each method is one atomic step on the state of one host. Thus a store can be a table of\n * SQLite that processes share. Times are milliseconds since the epoch.\n *\n * The state of a host has five parts:\n *\n * - the next time: the earliest time of the next request\n * - the lease: the end of the slot of the request in progress, when there is one\n * - the counts (`HostCounts`)\n * - the stop: an end time and a reason, when there is one\n */\nexport interface HostStateStore {\n  /**\n   * This method asks for the slot of a host at the time `now`. When the host has no rest left\n   * and no lease, the store gives the slot to the caller until `now + leaseMs` and gives 0.\n   * Otherwise it changes nothing and gives the time to wait in milliseconds: the time to the next\n   * time, or to the end of the lease. A store that processes share gives a short wait (one second\n   * or less) during a lease, because the holder usually releases it early. The caller asks again after the wait.\n   */\n  reserve(host: string, now: number, leaseMs: number): number;\n  /** This method changes the counts of a host for one result, and it gives the new counts. */\n  record(host: string, result: HostResult): HostCounts;\n  /**\n   * This method ends the lease of the caller. It sets the next time to `nextAt` when `nextAt` is\n   * later than the stored next time. Only the holder of the lease uses it.\n   */\n  release(host: string, nextAt: number): void;\n  /** This method sets the next time to `until` when `until` is later. It does not touch the lease. */\n  defer(host: string, until: number): void;\n  /** This method stops a host until a time. When the host has a stop that ends later, that stop stays. */\n  stop(host: string, until: number, reason: string): void;\n  /** This method gives the stop of a host at the time `now`. A stop that ended gives no value. */\n  stopped(host: string, now: number): HostStop | undefined;\n  /**\n   * This method removes the stops and the counts of one host, or of each host when there is no\n   * host. It keeps the next times and the leases, so a reset cannot make a host go faster than its pace.\n   */\n  reset(host?: string): void;\n  /**\n   * This method gives the validator memo of a host, when the store has one. A store without this\n   * method and `noteValidators` has no memo. The crawler then sends each validator of a stored copy.\n   */\n  validators?(host: string): ValidatorMemo | undefined;\n  /**\n   * This method adds one conditional request to the memo of a host, with the rules of\n   * `nextValidatorMemo`, in one atomic step. It gives the new memo.\n   */\n  noteValidators?(host: string, report: ValidatorReport, now: number): ValidatorMemo;\n}\n\n/** This value tells the result that the crawler learned for one validator of a host. */\nexport type ValidatorVerdict = \"unknown\" | \"works\" | \"fails\";\n\n/**\n * This value tells what one conditional request got:\n *\n * - `not-modified`: a 304.\n * - `same`: a 200 with the body that the store has. Thus the validator did not match.\n * - `changed`: a 200 with a different body. This answer tells nothing about the validator.\n */\nexport type ValidatorAnswer = \"not-modified\" | \"same\" | \"changed\";\n\n/** This report tells the validators that one conditional request sent, and the answer that it got. */\nexport interface ValidatorReport {\n  /** This value is true when the request sent `If-None-Match`. */\n  readonly etag: boolean;\n  /** This value is true when the request sent `If-Modified-Since`. */\n  readonly lastModified: boolean;\n  readonly answer: ValidatorAnswer;\n}\n\n/**\n * A validator memo holds what the crawler learned about the validators of one host. A server\n * examines `If-None-Match` first, and it ignores `If-Modified-Since` when the request has the two\n * (RFC 9110, section 13.1.3). Thus a host with unstable ETags does not answer 304 to a request with\n * the two validators at any time. Its stable `Last-Modified` date does not help. `nextValidatorMemo`\n * gives the rules.\n */\nexport interface ValidatorMemo {\n  /** This value is true when the crawler leaves out `If-None-Match` for the host and sends only `If-Modified-Since`. */\n  readonly skipEtag: boolean;\n  /** This value tells if a request with `If-None-Match` got a 304 (`works`) or the stored body again (`fails`). */\n  readonly etag: ValidatorVerdict;\n  /** This value tells the same for a request with only `If-Modified-Since`. */\n  readonly lastModified: ValidatorVerdict;\n  /** This value counts the stored bodies in sequence that came back to a request with `If-None-Match`. */\n  readonly etagMisses: number;\n  /** This value counts the stored bodies in sequence that came back to a request with only `If-Modified-Since`. */\n  readonly dateMisses: number;\n  /** This value counts the requests with only `If-Modified-Since` since the crawler stopped the ETag, or since the last probe. */\n  readonly sinceProbe: number;\n  /** This time (milliseconds since the epoch) is the last time that the crawler stopped or started the ETag, or probed it. */\n  readonly changedAt: number;\n}\n\n/** These are the limits of the validator memo. */\nexport interface ValidatorLimits {\n  /** After this number of stored bodies in sequence, a validator fails. The value is 3. */\n  readonly misses: number;\n  /** After this number of requests with only `If-Modified-Since`, the next request sends the ETag again (a probe). The value is 20. */\n  readonly probeEvery: number;\n  /** After this time with no probe, the next request is a probe. The value is 7 days. */\n  readonly probeAgeMs: number;\n}\n\n/** These are the limits of the validator memo of the crawler. */\nexport const VALIDATOR_MEMO: ValidatorLimits = {\n  misses: 3,\n  probeEvery: 20,\n  probeAgeMs: 7 * 24 * 60 * 60 * 1000,\n};\n\nconst EMPTY_MEMO: ValidatorMemo = {\n  skipEtag: false,\n  etag: \"unknown\",\n  lastModified: \"unknown\",\n  etagMisses: 0,\n  dateMisses: 0,\n  sinceProbe: 0,\n  changedAt: 0,\n};\n\n/**\n * This function tells which validators the next conditional request to a host sends. It sends\n * each validator of the stored copy, except when the memo stopped the ETag and the copy has a\n * `Last-Modified` date. Then it sends only `If-Modified-Since`, and the ETag too for a probe.\n * A probe comes after `probeEvery` requests or after `probeAgeMs`.\n */\nexport const validatorsToSend = (\n  memo: ValidatorMemo | undefined,\n  stored: { readonly etag: boolean; readonly lastModified: boolean },\n  now: number,\n): { readonly etag: boolean; readonly lastModified: boolean } => {\n  if (memo?.skipEtag !== true || !stored.etag || !stored.lastModified) return stored;\n  const probe =\n    memo.sinceProbe >= VALIDATOR_MEMO.probeEvery ||\n    now - memo.changedAt >= VALIDATOR_MEMO.probeAgeMs;\n  return { etag: probe, lastModified: true };\n};\n\n/**\n * This function adds one conditional request to the memo of a host, and gives the new memo.\n * These are the rules, with the limits of `VALIDATOR_MEMO`:\n *\n * 1. A 304 to a request with `If-None-Match` marks the ETag `works`. After a probe, the crawler sends the ETag again.\n * 2. After `misses` stored bodies in sequence with `If-None-Match`, the ETag fails. When the request also sent\n *    `If-Modified-Since`, the crawler stops the ETag for the host.\n * 3. A probe that gets a 200 keeps the ETag stopped, and the count to the next probe starts again.\n * 4. After `misses` stored bodies in sequence with only `If-Modified-Since`, that validator fails too. Then the\n *    crawler sends the two validators again, because they cost nothing. It tries the date alone again after `probeAgeMs`.\n * 5. A changed body tells nothing about a validator.\n */\nexport const nextValidatorMemo = (\n  memo: ValidatorMemo | undefined,\n  report: ValidatorReport,\n  now: number,\n): ValidatorMemo => {\n  const next: { -readonly [K in keyof ValidatorMemo]: ValidatorMemo[K] } = {\n    ...(memo ?? EMPTY_MEMO),\n  };\n  if (report.etag) {\n    const probe = next.skipEtag;\n    if (report.answer === \"not-modified\") {\n      next.etag = \"works\";\n      next.etagMisses = 0;\n      if (probe) {\n        next.skipEtag = false;\n        next.sinceProbe = 0;\n        next.changedAt = now;\n      }\n    } else if (probe) {\n      if (report.answer === \"same\") next.etagMisses += 1;\n      next.sinceProbe = 0;\n      next.changedAt = now;\n    } else if (report.answer === \"same\") {\n      next.etagMisses += 1;\n      if (next.etagMisses >= VALIDATOR_MEMO.misses) {\n        next.etag = \"fails\";\n        // a date that failed gets a new try only after probeAgeMs, so the memo does not swing at each request\n        const dateLeft =\n          next.lastModified !== \"fails\" || now - next.changedAt >= VALIDATOR_MEMO.probeAgeMs;\n        if (report.lastModified && dateLeft) {\n          next.skipEtag = true;\n          next.dateMisses = 0;\n          next.sinceProbe = 0;\n          next.changedAt = now;\n        }\n      }\n    }\n  } else if (report.lastModified) {\n    if (next.skipEtag) next.sinceProbe += 1;\n    if (report.answer === \"not-modified\") {\n      next.lastModified = \"works\";\n      next.dateMisses = 0;\n    } else if (report.answer === \"same\") {\n      next.dateMisses += 1;\n      if (next.dateMisses >= VALIDATOR_MEMO.misses) {\n        next.lastModified = \"fails\";\n        if (next.skipEtag) {\n          next.skipEtag = false;\n          next.sinceProbe = 0;\n          next.changedAt = now;\n        }\n      }\n    }\n  }\n  return next;\n};\n\ninterface MemoryState {\n  nextAt: number;\n  leasedUntil: number;\n  failures: number;\n  waits: number;\n  stop: HostStop | undefined;\n}\n\n/**\n * This store keeps the state of each host in memory, for one process. It removes the state of an\n * idle host (no rest left, no lease, no counts and no stop) when it holds more than `maxHosts` hosts.\n * It keeps the validator memos of the `maxHosts` hosts that it used last.\n */\nexport class MemoryHostStateStore implements HostStateStore {\n  private readonly states = new Map<string, MemoryState>();\n  /** The validator memos, the memo used last at the end. */\n  private readonly memos = new Map<string, ValidatorMemo>();\n  private sweepAt: number;\n\n  constructor(private readonly maxHosts: number = 10_000) {\n    this.sweepAt = maxHosts;\n  }\n\n  /** This value is the number of hosts that the store holds. */\n  get size(): number {\n    return this.states.size;\n  }\n\n  private state(host: string): MemoryState {\n    let state = this.states.get(host);\n    if (state === undefined) {\n      state = { nextAt: 0, leasedUntil: 0, failures: 0, waits: 0, stop: undefined };\n      this.states.set(host, state);\n    }\n    return state;\n  }\n\n  private sweep(now: number): void {\n    for (const [host, state] of this.states) {\n      const idle =\n        state.nextAt <= now &&\n        state.leasedUntil <= now &&\n        state.failures === 0 &&\n        state.waits === 0 &&\n        (state.stop === undefined || state.stop.until <= now);\n      if (idle) this.states.delete(host);\n    }\n    // the next sweep comes after a tenth more hosts, so a store of busy hosts does not sweep at each request\n    this.sweepAt = Math.max(this.maxHosts, Math.ceil(this.states.size * 1.1));\n  }\n\n  reserve(host: string, now: number, leaseMs: number): number {\n    if (this.states.size > this.sweepAt) this.sweep(now);\n    const state = this.state(host);\n    if (state.leasedUntil > now) return state.leasedUntil - now;\n    if (state.nextAt > now) return state.nextAt - now;\n    state.leasedUntil = now + leaseMs;\n    return 0;\n  }\n\n  record(host: string, result: HostResult): HostCounts {\n    const state = this.state(host);\n    if (result === \"success\") {\n      state.failures = 0;\n      state.waits = 0;\n    } else if (result === \"failure\") state.failures += 1;\n    else if (result === \"wait\") state.waits += 1;\n    return { failures: state.failures, waits: state.waits };\n  }\n\n  release(host: string, nextAt: number): void {\n    const state = this.state(host);\n    state.leasedUntil = 0;\n    if (nextAt > state.nextAt) state.nextAt = nextAt;\n  }\n\n  defer(host: string, until: number): void {\n    const state = this.state(host);\n    if (until > state.nextAt) state.nextAt = until;\n  }\n\n  stop(host: string, until: number, reason: string): void {\n    const state = this.state(host);\n    if (state.stop === undefined || state.stop.until < until) state.stop = { until, reason };\n  }\n\n  stopped(host: string, now: number): HostStop | undefined {\n    const stop = this.states.get(host)?.stop;\n    return stop !== undefined && stop.until > now ? stop : undefined;\n  }\n\n  reset(host?: string): void {\n    const clear = (state: MemoryState): void => {\n      state.failures = 0;\n      state.waits = 0;\n      state.stop = undefined;\n    };\n    if (host === undefined) for (const state of this.states.values()) clear(state);\n    else {\n      const state = this.states.get(host);\n      if (state !== undefined) clear(state);\n    }\n  }\n\n  validators(host: string): ValidatorMemo | undefined {\n    return this.memos.get(host);\n  }\n\n  noteValidators(host: string, report: ValidatorReport, now: number): ValidatorMemo {\n    const memo = nextValidatorMemo(this.memos.get(host), report, now);\n    // the memo of each change goes to the end, so the first memo is the memo used longest ago\n    this.memos.delete(host);\n    this.memos.set(host, memo);\n    if (this.memos.size > this.maxHosts) this.memos.delete(this.memos.keys().next().value!);\n    return memo;\n  }\n}\n\n/** These options change the scheduler. */\nexport interface SchedulerOptions {\n  /** This store keeps the state of each host. The default is a `MemoryHostStateStore`. */\n  readonly store?: HostStateStore | undefined;\n  /** This value is the longest slot of one request. A slot that a crashed process holds ends after it. The default is 5 minutes. */\n  readonly leaseMs?: number | undefined;\n  /** A host rests this long after a task that throws an error, or a task that gives no valid delay. The default is 1 second. */\n  readonly fallbackDelayMs?: number | undefined;\n}\n\nconst aborted = (signal: AbortSignal): Error =>\n  signal.reason instanceof Error\n    ? signal.reason\n    : new DOMException(\"The wait was aborted\", \"AbortError\");\n\nexport class HostScheduler {\n  /** This store keeps the state of each host. */\n  readonly store: HostStateStore;\n  private readonly chains = new Map<string, Promise<unknown>>();\n  private active = 0;\n  private readonly queue: Array<() => void> = [];\n  private readonly leaseMs: number;\n  private readonly fallbackDelayMs: number;\n\n  constructor(\n    private readonly clock: Clock,\n    private readonly maxActive: number,\n    options: SchedulerOptions = {},\n  ) {\n    if (!Number.isInteger(maxActive) || maxActive < 1) {\n      throw new RangeError(`maxActive must be an integer of 1 or more, not ${String(maxActive)}`);\n    }\n    this.store = options.store ?? new MemoryHostStateStore();\n    this.leaseMs = options.leaseMs ?? 300_000;\n    this.fallbackDelayMs = options.fallbackDelayMs ?? 1000;\n  }\n\n  private acquire(signal: AbortSignal | undefined): Promise<void> {\n    if (this.active < this.maxActive) {\n      this.active += 1;\n      return Promise.resolve();\n    }\n    return new Promise<void>((resolve, reject) => {\n      const grant = (): void => {\n        signal?.removeEventListener(\"abort\", cancel);\n        resolve();\n      };\n      const cancel = (): void => {\n        const at = this.queue.indexOf(grant);\n        if (at !== -1) this.queue.splice(at, 1);\n        reject(aborted(signal!));\n      };\n      this.queue.push(grant);\n      signal?.addEventListener(\"abort\", cancel, { once: true });\n    });\n  }\n\n  private release(): void {\n    // the slot goes to the next waiter directly, so a new request cannot take it in between\n    const next = this.queue.shift();\n    if (next !== undefined) next();\n    else this.active -= 1;\n  }\n\n  private async sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {\n    if (signal === undefined) {\n      await this.clock.sleep(ms);\n      return;\n    }\n    if (signal.aborted) throw aborted(signal);\n    await new Promise<void>((resolve, reject) => {\n      const cancel = (): void => reject(aborted(signal));\n      signal.addEventListener(\"abort\", cancel, { once: true });\n      this.clock.sleep(ms, signal).then(\n        () => {\n          signal.removeEventListener(\"abort\", cancel);\n          resolve();\n        },\n        (error: unknown) => {\n          signal.removeEventListener(\"abort\", cancel);\n          reject(error);\n        },\n      );\n    });\n  }\n\n  /**\n   * This method starts one request on a host, in sequence, after the rest of the host. When the\n   * signal aborts during the wait, the promise rejects with the reason of the signal.\n   */\n  run<T>(host: string, task: () => Promise<Paced<T>>, signal?: AbortSignal): Promise<T> {\n    const previous = this.chains.get(host) ?? Promise.resolve();\n    const result = previous.then(async () => {\n      // after a wait, the time is at least the end of the wait, also on a clock that a test holds still\n      let waited = Number.NEGATIVE_INFINITY;\n      for (;;) {\n        if (signal?.aborted === true) throw aborted(signal);\n        const now = Math.max(this.clock.now(), waited);\n        const stop = this.store.stopped(host, now);\n        if (stop !== undefined) throw new HostStopped(host, stop.reason, stop.until);\n        const wait = this.store.reserve(host, now, this.leaseMs);\n        if (!(wait > 0)) break;\n        await this.sleep(wait, signal);\n        waited = now + wait;\n      }\n      try {\n        await this.acquire(signal);\n      } catch (error) {\n        this.store.release(host, this.clock.now());\n        throw error;\n      }\n      let nextAt = this.clock.now() + this.fallbackDelayMs;\n      try {\n        const { value, delayMs } = await task();\n        const rest = Number.isFinite(delayMs) && delayMs >= 0 ? delayMs : this.fallbackDelayMs;\n        nextAt = this.clock.now() + rest;\n        return value;\n      } catch (error) {\n        nextAt = this.clock.now() + this.fallbackDelayMs;\n        throw error;\n      } finally {\n        this.store.release(host, nextAt);\n        this.release();\n      }\n    });\n    const tail = result.then(\n      () => undefined,\n      () => undefined,\n    );\n    this.chains.set(host, tail);\n    // a host with no queued request keeps no chain\n    void tail.then(() => {\n      if (this.chains.get(host) === tail) this.chains.delete(host);\n    });\n    return result;\n  }\n\n  /** The server asked for a pause (Retry-After), or a backoff chose one. */\n  park(host: string, untilMs: number): void {\n    this.store.defer(host, untilMs);\n  }\n\n  /** This method stops a host until `untilMs`. With no time, the stop has no end. */\n  stop(host: string, reason: string, untilMs: number = Number.POSITIVE_INFINITY): void {\n    this.store.stop(host, untilMs, reason);\n  }\n\n  stoppedReason(host: string): string | undefined {\n    return this.store.stopped(host, this.clock.now())?.reason;\n  }\n\n  /** This method gives the stop of a host, with its end, when the host is stopped. */\n  stopOf(host: string): HostStop | undefined {\n    return this.store.stopped(host, this.clock.now());\n  }\n\n  /** This value counts the failures of a host in sequence, for the backoff. A success sets it to zero. */\n  failed(host: string): number {\n    return this.store.record(host, \"failure\").failures;\n  }\n\n  /** This value counts the requests to wait (429, 503) of a host in sequence. They do not stop the host. */\n  waited(host: string): number {\n    return this.store.record(host, \"wait\").waits;\n  }\n\n  succeeded(host: string): void {\n    this.store.record(host, \"success\");\n  }\n\n  /** This method removes the stops and the counts of one host, or of each host. The pace stays. */\n  reset(host?: string): void {\n    this.store.reset(host);\n  }\n}\n\n/**\n * This function gives an exponential backoff with full jitter, from the analysis of AWS. The wait\n * is random, up to base × 2^(try − 1), with a maximum. The jitter keeps retries apart.\n */\nexport const backoffMs = (\n  attempt: number,\n  baseMs: number,\n  capMs: number,\n  random: () => number,\n): number => Math.floor(random() * Math.min(capMs, baseMs * 2 ** Math.max(0, attempt - 1)));\n\n/**\n * This function gives an exponential backoff with equal jitter. The wait is half of\n * base × 2^(try − 1), with a maximum, and a random part up to the other half. Thus each backoff\n * is at least half of its step, and it is not lost under the rest of the host.\n */\nexport const equalJitterMs = (\n  attempt: number,\n  baseMs: number,\n  capMs: number,\n  random: () => number,\n): number => {\n  const step = Math.min(capMs, baseMs * 2 ** Math.max(0, attempt - 1));\n  return Math.floor(step / 2 + random() * (step / 2));\n};\n\n/** These are the facts of one request that set the rest of its host. */\nexport interface RestInput {\n  /** This value is the kind of the request: a web page, a documented API, or a render in the browser. */\n  readonly kind: \"page\" | \"api\" | \"render\";\n  /** This value is the duration of the request, in milliseconds. */\n  readonly elapsedMs: number;\n  /** This value is the `crawl-delay` of robots.txt, in seconds. */\n  readonly crawlDelaySeconds?: number | undefined;\n  /** This value is the documented pace of the source. It replaces the interval of the policy for the kind. */\n  readonly minIntervalMs?: number | undefined;\n  /** This value is the wait that the server asked for (`Retry-After`). */\n  readonly retryAfterMs?: number | undefined;\n}\n\n/** The rest of a host after a request, and the reason for its length. */\nexport interface Rest {\n  readonly ms: number;\n  readonly reason: \"pace\" | \"slow\" | \"retry-after\" | \"crawl-delay\";\n}\n\n/**\n * This function gives the rest of a host after one request, as the crawler calculates it. These\n * are the parts:\n *\n * - The interval of the kind (3 seconds for a page), or the documented pace of the source.\n * - The `crawl-delay` of robots.txt, up to `crawlDelayCapMs`, when it is longer.\n * - After a page or a render, `elapsedFactor` times the duration of the request, up to `elapsedCapMs`.\n * - The `Retry-After` of the server, when it is longer.\n *\n * The backoff after an error or a request to wait comes on top, with jitter. The function is\n * pure, so a simulation and a study can use the pace of the crawler.\n */\nexport const restAfter = (policy: PolitenessPolicy, input: RestInput): Rest => {\n  const interval =\n    input.minIntervalMs ??\n    (input.kind === \"page\"\n      ? policy.pageIntervalMs\n      : input.kind === \"api\"\n        ? policy.apiIntervalMs\n        : policy.renderIntervalMs);\n  const crawlDelay = Math.min(\n    policy.crawlDelayCapMs,\n    Math.max(0, (input.crawlDelaySeconds ?? 0) * 1000),\n  );\n  let ms = Math.max(interval, crawlDelay);\n  let reason: Rest[\"reason\"] = crawlDelay > interval ? \"crawl-delay\" : \"pace\";\n  if (input.kind !== \"api\") {\n    const slow = Math.min(policy.elapsedCapMs, policy.elapsedFactor * Math.max(0, input.elapsedMs));\n    if (slow > ms) {\n      ms = slow;\n      reason = \"slow\";\n    }\n  }\n  if (input.retryAfterMs !== undefined && input.retryAfterMs > ms) {\n    ms = input.retryAfterMs;\n    reason = \"retry-after\";\n  }\n  return { ms, reason };\n};\n\n/**\n * The politeness policy holds each limit of the crawler. Each value has a default, and a host can\n * have values of its own. The times are in milliseconds.\n */\nexport interface PolitenessPolicy {\n  /** This value is the shortest rest of a host between two page requests. The default is 3 seconds. */\n  readonly pageIntervalMs: number;\n  /** This value is the shortest rest of a host between two API requests. The default is 1 second. */\n  readonly apiIntervalMs: number;\n  /** This value is the shortest rest of a host after a render. The default is 5 seconds. */\n  readonly renderIntervalMs: number;\n  /** After a page, the host rests this number of times the duration of the request. The default is 5. */\n  readonly elapsedFactor: number;\n  /** This value is the maximum of the rest that comes from the duration. The default is 30 seconds. */\n  readonly elapsedCapMs: number;\n  /** This value is the maximum `crawl-delay` that the crawler obeys. The default is 60 seconds. */\n  readonly crawlDelayCapMs: number;\n  /** This value is the maximum number of redirects for one fetch. The default is 5. */\n  readonly maxRedirects: number;\n  /** This value is the maximum size of a body after decompression. The default is 5 MiB. */\n  readonly maxBytes: number;\n  /** This value is the deadline of one request, the body included. The default is 90 seconds. */\n  readonly requestTimeoutMs: number;\n  /** This value is the deadline of one render. The default is 20 seconds. */\n  readonly renderTimeoutMs: number;\n  /** After this number of failures in sequence, the host stops. The default is 5. */\n  readonly maxFailures: number;\n  /** This value is the stop after too many failures. The default is 1 hour. */\n  readonly failureStopMs: number;\n  /** This value is the stop after a 401 or a 403. The default is 1 hour. */\n  readonly blockedStopMs: number;\n  /** This value is the stop after a bot challenge. The default is 24 hours. */\n  readonly challengeStopMs: number;\n  /** This value is the stop after a 402. The default is 24 hours. */\n  readonly paymentStopMs: number;\n  /** The crawler obeys a `Retry-After` up to this value. A longer wait stops the host for that time. The default is 1 hour. */\n  readonly maxRetryAfterMs: number;\n  /** This value is the first step of the backoff after an error or a request to wait. The default is 5 seconds. */\n  readonly backoffBaseMs: number;\n  /** This value is the maximum step of the backoff. The default is 300 seconds. */\n  readonly backoffCapMs: number;\n  /** The crawler keeps a robots.txt for this time. The default is 24 hours. */\n  readonly robotsTtlMs: number;\n  /** The crawler keeps an unreachable robots.txt verdict for this time first, with a backoff. The default is 5 minutes. */\n  readonly robotsRetryMs: number;\n  /** This value is the maximum time for an unreachable robots.txt verdict. The default is 1 hour. */\n  readonly robotsRetryCapMs: number;\n  /** A stored robots.txt younger than this age replaces a robots.txt that the crawler cannot read. The default is 30 days. */\n  readonly robotsStaleMs: number;\n  /** This value is the maximum freshness that the crawler gives a response with no freshness of its own. The default is 24 hours. */\n  readonly heuristicFreshnessCapMs: number;\n}\n\n/**\n * This value is the longest stop that a `Retry-After` can give: 30 days. A wait longer than\n * `maxRetryAfterMs` stops the host for the time that the server asks, up to this value.\n */\nexport const MAX_STOP_MS: number = 30 * 24 * 60 * 60 * 1000;\n\nexport const DEFAULT_POLICY: PolitenessPolicy = {\n  pageIntervalMs: 3000,\n  apiIntervalMs: 1000,\n  renderIntervalMs: 5000,\n  elapsedFactor: 5,\n  elapsedCapMs: 30_000,\n  crawlDelayCapMs: 60_000,\n  maxRedirects: 5,\n  maxBytes: 5 * 1024 * 1024,\n  requestTimeoutMs: 90_000,\n  renderTimeoutMs: 20_000,\n  maxFailures: 5,\n  failureStopMs: 60 * 60 * 1000,\n  blockedStopMs: 60 * 60 * 1000,\n  challengeStopMs: 24 * 60 * 60 * 1000,\n  paymentStopMs: 24 * 60 * 60 * 1000,\n  maxRetryAfterMs: 60 * 60 * 1000,\n  backoffBaseMs: 5000,\n  backoffCapMs: 300_000,\n  robotsTtlMs: 24 * 60 * 60 * 1000,\n  robotsRetryMs: 5 * 60 * 1000,\n  robotsRetryCapMs: 60 * 60 * 1000,\n  robotsStaleMs: 30 * 24 * 60 * 60 * 1000,\n  heuristicFreshnessCapMs: 24 * 60 * 60 * 1000,\n};\n\n/**\n * These options change the politeness policy. The values at the top apply to each host. The\n * `hosts` map gives values for one host (`api.example.org`) or for the hosts under a name\n * (`*.example.org`). The most specific entry wins.\n */\nexport interface PolicyOptions extends Partial<PolitenessPolicy> {\n  readonly hosts?: Readonly<Record<string, Partial<PolitenessPolicy>>> | undefined;\n}\n\nconst INTEGER_KEYS: ReadonlySet<string> = new Set([\"maxRedirects\", \"maxFailures\"]);\n\nconst checked = (values: Partial<PolitenessPolicy>, where: string): Partial<PolitenessPolicy> => {\n  for (const [key, value] of Object.entries(values)) {\n    if (!(key in DEFAULT_POLICY))\n      throw new RangeError(`${where}: ${key} is not a politeness setting`);\n    const good =\n      typeof value === \"number\" &&\n      value >= 0 &&\n      (Number.isFinite(value) || key.endsWith(\"StopMs\")) &&\n      (!INTEGER_KEYS.has(key) || Number.isInteger(value));\n    if (!good)\n      throw new RangeError(`${where}: ${key} must be a number of 0 or more, not ${String(value)}`);\n  }\n  return values;\n};\n\n/** This function examines the options of a policy. It throws a `RangeError` for a value that is not a valid number. */\nexport const checkPolicy = (options: PolicyOptions | undefined): void => {\n  if (options === undefined) return;\n  const { hosts, ...base } = options;\n  checked(base, \"politeness\");\n  for (const [host, values] of Object.entries(hosts ?? {}))\n    checked(values, `politeness.hosts[\"${host}\"]`);\n};\n\n/** This function gives the policy of one host: the defaults, then the general options, then the entries for the host. */\nexport const policyFor = (options: PolicyOptions | undefined, host: string): PolitenessPolicy => {\n  if (options === undefined) return DEFAULT_POLICY;\n  const { hosts, ...base } = options;\n  const name = host.toLowerCase().replace(/\\.$/, \"\");\n  const matching = Object.entries(hosts ?? {})\n    .map(([key, values]) => ({ key: key.toLowerCase(), values }))\n    .filter(({ key }) => key === name || (key.startsWith(\"*.\") && name.endsWith(key.slice(1))))\n    // an exact name wins, and a longer pattern wins over a shorter one\n    .sort(\n      (a, b) => (a.key === name ? 1 : 0) - (b.key === name ? 1 : 0) || a.key.length - b.key.length,\n    );\n  return Object.assign(\n    {},\n    DEFAULT_POLICY,\n    base,\n    ...matching.map(({ values }) => values),\n  ) as PolitenessPolicy;\n};\n"],"mappings":";;AAqBA,MAAM,eAAe,KAAK,KAAK;AAE/B,MAAa,YAAmB;CAC9B,WAAW,KAAK,IAAI;CACpB,QAAQ,IAAI,WACV,IAAI,SAAe,YAAY;EAC7B,IAAI,QAAQ,YAAY,MAAM;GAC5B,QAAQ;GACR;EACF;EAEA,IAAI,OAAO,OAAO,SAAS,EAAE,IAAI,KAAK,IAAI,GAAG,EAAE,IAAI;EACnD,IAAI;EACJ,MAAM,aAAmB;GACvB,IAAI,UAAU,KAAA,GAAW,aAAa,KAAK;GAC3C,QAAQ,oBAAoB,SAAS,IAAI;GACzC,QAAQ;EACV;EACA,MAAM,aAAmB;GACvB,IAAI,QAAQ,GAAG;IACb,KAAK;IACL;GACF;GACA,MAAM,OAAO,KAAK,IAAI,MAAM,YAAY;GACxC,QAAQ;GACR,QAAQ,WAAW,MAAM,IAAI;EAC/B;EACA,QAAQ,iBAAiB,SAAS,MAAM,EAAE,MAAM,KAAK,CAAC;EACtD,KAAK;CACP,CAAC;AACL;;AAGA,IAAa,cAAb,cAAiC,MAAM;CAE1B;CACA;CAEA;CAJX,YACE,MACA,QAEA,QAAyB,OAAO,mBAChC;EACA,MAAM,GAAG,KAAK,YAAY,QAAQ;EALzB,KAAA,OAAA;EACA,KAAA,SAAA;EAEA,KAAA,QAAA;CAGX;AACF;;AA4IA,MAAa,iBAAkC;CAC7C,QAAQ;CACR,YAAY;CACZ,YAAY;AACd;AAEA,MAAM,aAA4B;CAChC,UAAU;CACV,MAAM;CACN,cAAc;CACd,YAAY;CACZ,YAAY;CACZ,YAAY;CACZ,WAAW;AACb;;;;;;;AAQA,MAAa,oBACX,MACA,QACA,QAC+D;CAC/D,IAAI,MAAM,aAAa,QAAQ,CAAC,OAAO,QAAQ,CAAC,OAAO,cAAc,OAAO;CAI5E,OAAO;EAAE,MAFP,KAAK,cAAc,eAAe,cAClC,MAAM,KAAK,aAAa,eAAe;EACnB,cAAc;CAAK;AAC3C;;;;;;;;;;;;;AAcA,MAAa,qBACX,MACA,QACA,QACkB;CAClB,MAAM,OAAmE,EACvE,GAAI,QAAQ,WACd;CACA,IAAI,OAAO,MAAM;EACf,MAAM,QAAQ,KAAK;EACnB,IAAI,OAAO,WAAW,gBAAgB;GACpC,KAAK,OAAO;GACZ,KAAK,aAAa;GAClB,IAAI,OAAO;IACT,KAAK,WAAW;IAChB,KAAK,aAAa;IAClB,KAAK,YAAY;GACnB;EACF,OAAO,IAAI,OAAO;GAChB,IAAI,OAAO,WAAW,QAAQ,KAAK,cAAc;GACjD,KAAK,aAAa;GAClB,KAAK,YAAY;EACnB,OAAO,IAAI,OAAO,WAAW,QAAQ;GACnC,KAAK,cAAc;GACnB,IAAI,KAAK,cAAc,eAAe,QAAQ;IAC5C,KAAK,OAAO;IAEZ,MAAM,WACJ,KAAK,iBAAiB,WAAW,MAAM,KAAK,aAAa,eAAe;IAC1E,IAAI,OAAO,gBAAgB,UAAU;KACnC,KAAK,WAAW;KAChB,KAAK,aAAa;KAClB,KAAK,aAAa;KAClB,KAAK,YAAY;IACnB;GACF;EACF;CACF,OAAO,IAAI,OAAO,cAAc;EAC9B,IAAI,KAAK,UAAU,KAAK,cAAc;EACtC,IAAI,OAAO,WAAW,gBAAgB;GACpC,KAAK,eAAe;GACpB,KAAK,aAAa;EACpB,OAAO,IAAI,OAAO,WAAW,QAAQ;GACnC,KAAK,cAAc;GACnB,IAAI,KAAK,cAAc,eAAe,QAAQ;IAC5C,KAAK,eAAe;IACpB,IAAI,KAAK,UAAU;KACjB,KAAK,WAAW;KAChB,KAAK,aAAa;KAClB,KAAK,YAAY;IACnB;GACF;EACF;CACF;CACA,OAAO;AACT;;;;;;AAeA,IAAa,uBAAb,MAA4D;CAM7B;CAL7B,yBAA0B,IAAI,IAAyB;;CAEvD,wBAAyB,IAAI,IAA2B;CACxD;CAEA,YAAY,WAAoC,KAAQ;EAA3B,KAAA,WAAA;EAC3B,KAAK,UAAU;CACjB;;CAGA,IAAI,OAAe;EACjB,OAAO,KAAK,OAAO;CACrB;CAEA,MAAc,MAA2B;EACvC,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI;EAChC,IAAI,UAAU,KAAA,GAAW;GACvB,QAAQ;IAAE,QAAQ;IAAG,aAAa;IAAG,UAAU;IAAG,OAAO;IAAG,MAAM,KAAA;GAAU;GAC5E,KAAK,OAAO,IAAI,MAAM,KAAK;EAC7B;EACA,OAAO;CACT;CAEA,MAAc,KAAmB;EAC/B,KAAK,MAAM,CAAC,MAAM,UAAU,KAAK,QAO/B,IALE,MAAM,UAAU,OAChB,MAAM,eAAe,OACrB,MAAM,aAAa,KACnB,MAAM,UAAU,MACf,MAAM,SAAS,KAAA,KAAa,MAAM,KAAK,SAAS,MACzC,KAAK,OAAO,OAAO,IAAI;EAGnC,KAAK,UAAU,KAAK,IAAI,KAAK,UAAU,KAAK,KAAK,KAAK,OAAO,OAAO,GAAG,CAAC;CAC1E;CAEA,QAAQ,MAAc,KAAa,SAAyB;EAC1D,IAAI,KAAK,OAAO,OAAO,KAAK,SAAS,KAAK,MAAM,GAAG;EACnD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,MAAM,cAAc,KAAK,OAAO,MAAM,cAAc;EACxD,IAAI,MAAM,SAAS,KAAK,OAAO,MAAM,SAAS;EAC9C,MAAM,cAAc,MAAM;EAC1B,OAAO;CACT;CAEA,OAAO,MAAc,QAAgC;EACnD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,WAAW,WAAW;GACxB,MAAM,WAAW;GACjB,MAAM,QAAQ;EAChB,OAAO,IAAI,WAAW,WAAW,MAAM,YAAY;OAC9C,IAAI,WAAW,QAAQ,MAAM,SAAS;EAC3C,OAAO;GAAE,UAAU,MAAM;GAAU,OAAO,MAAM;EAAM;CACxD;CAEA,QAAQ,MAAc,QAAsB;EAC1C,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,MAAM,cAAc;EACpB,IAAI,SAAS,MAAM,QAAQ,MAAM,SAAS;CAC5C;CAEA,MAAM,MAAc,OAAqB;EACvC,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,QAAQ,MAAM,QAAQ,MAAM,SAAS;CAC3C;CAEA,KAAK,MAAc,OAAe,QAAsB;EACtD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,MAAM,SAAS,KAAA,KAAa,MAAM,KAAK,QAAQ,OAAO,MAAM,OAAO;GAAE;GAAO;EAAO;CACzF;CAEA,QAAQ,MAAc,KAAmC;EACvD,MAAM,OAAO,KAAK,OAAO,IAAI,IAAI,CAAC,EAAE;EACpC,OAAO,SAAS,KAAA,KAAa,KAAK,QAAQ,MAAM,OAAO,KAAA;CACzD;CAEA,MAAM,MAAqB;EACzB,MAAM,SAAS,UAA6B;GAC1C,MAAM,WAAW;GACjB,MAAM,QAAQ;GACd,MAAM,OAAO,KAAA;EACf;EACA,IAAI,SAAS,KAAA,GAAW,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GAAG,MAAM,KAAK;OACxE;GACH,MAAM,QAAQ,KAAK,OAAO,IAAI,IAAI;GAClC,IAAI,UAAU,KAAA,GAAW,MAAM,KAAK;EACtC;CACF;CAEA,WAAW,MAAyC;EAClD,OAAO,KAAK,MAAM,IAAI,IAAI;CAC5B;CAEA,eAAe,MAAc,QAAyB,KAA4B;EAChF,MAAM,OAAO,kBAAkB,KAAK,MAAM,IAAI,IAAI,GAAG,QAAQ,GAAG;EAEhE,KAAK,MAAM,OAAO,IAAI;EACtB,KAAK,MAAM,IAAI,MAAM,IAAI;EACzB,IAAI,KAAK,MAAM,OAAO,KAAK,UAAU,KAAK,MAAM,OAAO,KAAK,MAAM,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAM;EACtF,OAAO;CACT;AACF;AAYA,MAAM,WAAW,WACf,OAAO,kBAAkB,QACrB,OAAO,SACP,IAAI,aAAa,wBAAwB,YAAY;AAE3D,IAAa,gBAAb,MAA2B;CAUN;CACA;;CATnB;CACA,yBAA0B,IAAI,IAA8B;CAC5D,SAAiB;CACjB,QAA4C,CAAC;CAC7C;CACA;CAEA,YACE,OACA,WACA,UAA4B,CAAC,GAC7B;EAHiB,KAAA,QAAA;EACA,KAAA,YAAA;EAGjB,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,MAAM,IAAI,WAAW,kDAAkD,OAAO,SAAS,GAAG;EAE5F,KAAK,QAAQ,QAAQ,SAAS,IAAI,qBAAqB;EACvD,KAAK,UAAU,QAAQ,WAAW;EAClC,KAAK,kBAAkB,QAAQ,mBAAmB;CACpD;CAEA,QAAgB,QAAgD;EAC9D,IAAI,KAAK,SAAS,KAAK,WAAW;GAChC,KAAK,UAAU;GACf,OAAO,QAAQ,QAAQ;EACzB;EACA,OAAO,IAAI,SAAe,SAAS,WAAW;GAC5C,MAAM,cAAoB;IACxB,QAAQ,oBAAoB,SAAS,MAAM;IAC3C,QAAQ;GACV;GACA,MAAM,eAAqB;IACzB,MAAM,KAAK,KAAK,MAAM,QAAQ,KAAK;IACnC,IAAI,OAAO,IAAI,KAAK,MAAM,OAAO,IAAI,CAAC;IACtC,OAAO,QAAQ,MAAO,CAAC;GACzB;GACA,KAAK,MAAM,KAAK,KAAK;GACrB,QAAQ,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;EAC1D,CAAC;CACH;CAEA,UAAwB;EAEtB,MAAM,OAAO,KAAK,MAAM,MAAM;EAC9B,IAAI,SAAS,KAAA,GAAW,KAAK;OACxB,KAAK,UAAU;CACtB;CAEA,MAAc,MAAM,IAAY,QAAgD;EAC9E,IAAI,WAAW,KAAA,GAAW;GACxB,MAAM,KAAK,MAAM,MAAM,EAAE;GACzB;EACF;EACA,IAAI,OAAO,SAAS,MAAM,QAAQ,MAAM;EACxC,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,eAAqB,OAAO,QAAQ,MAAM,CAAC;GACjD,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;GACvD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,CAAC,WACrB;IACJ,OAAO,oBAAoB,SAAS,MAAM;IAC1C,QAAQ;GACV,IACC,UAAmB;IAClB,OAAO,oBAAoB,SAAS,MAAM;IAC1C,OAAO,KAAK;GACd,CACF;EACF,CAAC;CACH;;;;;CAMA,IAAO,MAAc,MAA+B,QAAkC;EAEpF,MAAM,UADW,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ,QAAQ,EAAA,CAClC,KAAK,YAAY;GAEvC,IAAI,SAAS,OAAO;GACpB,SAAS;IACP,IAAI,QAAQ,YAAY,MAAM,MAAM,QAAQ,MAAM;IAClD,MAAM,MAAM,KAAK,IAAI,KAAK,MAAM,IAAI,GAAG,MAAM;IAC7C,MAAM,OAAO,KAAK,MAAM,QAAQ,MAAM,GAAG;IACzC,IAAI,SAAS,KAAA,GAAW,MAAM,IAAI,YAAY,MAAM,KAAK,QAAQ,KAAK,KAAK;IAC3E,MAAM,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,KAAK,OAAO;IACvD,IAAI,EAAE,OAAO,IAAI;IACjB,MAAM,KAAK,MAAM,MAAM,MAAM;IAC7B,SAAS,MAAM;GACjB;GACA,IAAI;IACF,MAAM,KAAK,QAAQ,MAAM;GAC3B,SAAS,OAAO;IACd,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC;IACzC,MAAM;GACR;GACA,IAAI,SAAS,KAAK,MAAM,IAAI,IAAI,KAAK;GACrC,IAAI;IACF,MAAM,EAAE,OAAO,YAAY,MAAM,KAAK;IACtC,MAAM,OAAO,OAAO,SAAS,OAAO,KAAK,WAAW,IAAI,UAAU,KAAK;IACvE,SAAS,KAAK,MAAM,IAAI,IAAI;IAC5B,OAAO;GACT,SAAS,OAAO;IACd,SAAS,KAAK,MAAM,IAAI,IAAI,KAAK;IACjC,MAAM;GACR,UAAU;IACR,KAAK,MAAM,QAAQ,MAAM,MAAM;IAC/B,KAAK,QAAQ;GACf;EACF,CAAC;EACD,MAAM,OAAO,OAAO,WACZ,KAAA,SACA,KAAA,CACR;EACA,KAAK,OAAO,IAAI,MAAM,IAAI;EAE1B,KAAU,WAAW;GACnB,IAAI,KAAK,OAAO,IAAI,IAAI,MAAM,MAAM,KAAK,OAAO,OAAO,IAAI;EAC7D,CAAC;EACD,OAAO;CACT;;CAGA,KAAK,MAAc,SAAuB;EACxC,KAAK,MAAM,MAAM,MAAM,OAAO;CAChC;;CAGA,KAAK,MAAc,QAAgB,UAAkB,OAAO,mBAAyB;EACnF,KAAK,MAAM,KAAK,MAAM,SAAS,MAAM;CACvC;CAEA,cAAc,MAAkC;EAC9C,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,EAAE;CACrD;;CAGA,OAAO,MAAoC;EACzC,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC;CAClD;;CAGA,OAAO,MAAsB;EAC3B,OAAO,KAAK,MAAM,OAAO,MAAM,SAAS,CAAC,CAAC;CAC5C;;CAGA,OAAO,MAAsB;EAC3B,OAAO,KAAK,MAAM,OAAO,MAAM,MAAM,CAAC,CAAC;CACzC;CAEA,UAAU,MAAoB;EAC5B,KAAK,MAAM,OAAO,MAAM,SAAS;CACnC;;CAGA,MAAM,MAAqB;EACzB,KAAK,MAAM,MAAM,IAAI;CACvB;AACF;;;;;AAMA,MAAa,aACX,SACA,QACA,OACA,WACW,KAAK,MAAM,OAAO,IAAI,KAAK,IAAI,OAAO,SAAS,KAAK,KAAK,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC;;;;;;AAO1F,MAAa,iBACX,SACA,QACA,OACA,WACW;CACX,MAAM,OAAO,KAAK,IAAI,OAAO,SAAS,KAAK,KAAK,IAAI,GAAG,UAAU,CAAC,CAAC;CACnE,OAAO,KAAK,MAAM,OAAO,IAAI,OAAO,KAAK,OAAO,EAAE;AACpD;;;;;;;;;;;;;AAkCA,MAAa,aAAa,QAA0B,UAA2B;CAC7E,MAAM,WACJ,MAAM,kBACL,MAAM,SAAS,SACZ,OAAO,iBACP,MAAM,SAAS,QACb,OAAO,gBACP,OAAO;CACf,MAAM,aAAa,KAAK,IACtB,OAAO,iBACP,KAAK,IAAI,IAAI,MAAM,qBAAqB,KAAK,GAAI,CACnD;CACA,IAAI,KAAK,KAAK,IAAI,UAAU,UAAU;CACtC,IAAI,SAAyB,aAAa,WAAW,gBAAgB;CACrE,IAAI,MAAM,SAAS,OAAO;EACxB,MAAM,OAAO,KAAK,IAAI,OAAO,cAAc,OAAO,gBAAgB,KAAK,IAAI,GAAG,MAAM,SAAS,CAAC;EAC9F,IAAI,OAAO,IAAI;GACb,KAAK;GACL,SAAS;EACX;CACF;CACA,IAAI,MAAM,iBAAiB,KAAA,KAAa,MAAM,eAAe,IAAI;EAC/D,KAAK,MAAM;EACX,SAAS;CACX;CACA,OAAO;EAAE;EAAI;CAAO;AACtB;;;;;AA2DA,MAAa,cAAsB;AAEnC,MAAa,iBAAmC;CAC9C,gBAAgB;CAChB,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,cAAc;CACd,iBAAiB;CACjB,cAAc;CACd,UAAU;CACV,kBAAkB;CAClB,iBAAiB;CACjB,aAAa;CACb,eAAe;CACf,eAAe;CACf,iBAAiB;CACjB,eAAe;CACf,iBAAiB;CACjB,eAAe;CACf,cAAc;CACd,aAAa;CACb,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,yBAAyB;AAC3B;AAWA,MAAM,+BAAoC,IAAI,IAAI,CAAC,gBAAgB,aAAa,CAAC;AAEjF,MAAM,WAAW,QAAmC,UAA6C;CAC/F,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,MAAM,GAAG;EACjD,IAAI,EAAE,OAAO,iBACX,MAAM,IAAI,WAAW,GAAG,MAAM,IAAI,IAAI,6BAA6B;EAMrE,IAAI,EAJF,OAAO,UAAU,YACjB,SAAS,MACR,OAAO,SAAS,KAAK,KAAK,IAAI,SAAS,QAAQ,OAC/C,CAAC,aAAa,IAAI,GAAG,KAAK,OAAO,UAAU,KAAK,KAEjD,MAAM,IAAI,WAAW,GAAG,MAAM,IAAI,IAAI,sCAAsC,OAAO,KAAK,GAAG;CAC/F;CACA,OAAO;AACT;;AAGA,MAAa,eAAe,YAA6C;CACvE,IAAI,YAAY,KAAA,GAAW;CAC3B,MAAM,EAAE,OAAO,GAAG,SAAS;CAC3B,QAAQ,MAAM,YAAY;CAC1B,KAAK,MAAM,CAAC,MAAM,WAAW,OAAO,QAAQ,SAAS,CAAC,CAAC,GACrD,QAAQ,QAAQ,qBAAqB,KAAK,GAAG;AACjD;;AAGA,MAAa,aAAa,SAAoC,SAAmC;CAC/F,IAAI,YAAY,KAAA,GAAW,OAAO;CAClC,MAAM,EAAE,OAAO,GAAG,SAAS;CAC3B,MAAM,OAAO,KAAK,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;CACjD,MAAM,WAAW,OAAO,QAAQ,SAAS,CAAC,CAAC,CAAC,CACzC,KAAK,CAAC,KAAK,aAAa;EAAE,KAAK,IAAI,YAAY;EAAG;CAAO,EAAE,CAAC,CAC5D,QAAQ,EAAE,UAAU,QAAQ,QAAS,IAAI,WAAW,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAE1F,MACE,GAAG,OAAO,EAAE,QAAQ,OAAO,IAAI,MAAM,EAAE,QAAQ,OAAO,IAAI,MAAM,EAAE,IAAI,SAAS,EAAE,IAAI,MACxF;CACF,OAAO,OAAO,OACZ,CAAC,GACD,gBACA,MACA,GAAG,SAAS,KAAK,EAAE,aAAa,MAAM,CACxC;AACF"}

package/build/{browser-gJ_h4AhB.mjs → render-C1CUraWF.mjs}

@@ -1,17 +1,3 @@
-import { basename, dirname, join } from "node:path";
-import { existsSync } from "node:fs";
-//#region \0rolldown/runtime.js
-var __defProp = Object.defineProperty;
-var __exportAll = (all, no_symbols) => {
-	let target = {};
-	for (var name in all) __defProp(target, name, {
-		get: all[name],
-		enumerable: true
-	});
-	if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
-	return target;
-};
-//#endregion
 //#region src/render.ts
 /**
 * This error tells that the renderer did not render the page, and that the page or the site is
@@ -132,6 +118,8 @@ const withFrames = ({ html, frames }) => {
 	return end === -1 ? `${html}${sections}` : `${html.slice(0, end)}${sections}${html.slice(end)}`;
 };
 const messageOf = (error) => error instanceof Error ? error.message : String(error);
+/** This function gives the headers with their names in lowercase, as the route sends them. */
+const lowerKeys = (headers) => Object.fromEntries(Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value]));
 const sleep = (ms) => new Promise((resolve) => {
 	setTimeout(resolve, Math.max(0, ms)).unref();
 });
@@ -192,7 +180,7 @@ const playwrightRenderer = (options = {}) => {
 		}, () => void 0);
 		running.close().catch(() => void 0);
 	};
-	const renderPage = async (url, { userAgent, timeoutMs, headers, allowRequest, maxRedirects = MAX_REDIRECTS, signal }) => {
+	const renderPage = async (url, { userAgent, timeoutMs, headers, allowRequest, maxRedirects = MAX_REDIRECTS, signal, sign }) => {
 		if (signal?.aborted === true) throw signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("the render was cancelled");
 		const deadline = Date.now() + timeoutMs;
 		const left = () => Math.max(1, deadline - Date.now());
@@ -264,6 +252,17 @@ const playwrightRenderer = (options = {}) => {
 				const crossOrigin = next.origin !== new URL(url).origin;
 				headers = Object.fromEntries(Object.entries(headers).filter(([name]) => name !== "cookie" && !(crossOrigin && name === "authorization") && !(get && (name === "content-type" || name === "content-length"))));
 				if (get) method = "GET";
+				if (sign !== void 0) {
+					const signed = sign(next.href);
+					if (signed === void 0) return {
+						refusal: `the signer did not sign the request for ${next.href}`,
+						url: next.href
+					};
+					headers = {
+						...headers,
+						...lowerKeys(signed)
+					};
+				}
 				const body = get ? void 0 : request.postDataBuffer() ?? void 0;
 				response = await context.request.fetch(next.href, {
 					method,
@@ -300,9 +299,22 @@ const playwrightRenderer = (options = {}) => {
 					await route.abort("blockedbyclient");
 					return;
 				}
-				const sent = Object.keys(extra).length === 0 ? void 0 : {
+				const signed = sign === void 0 ? {} : sign(request.url());
+				if (signed === void 0) {
+					if (main) refused = {
+						message: `the signer did not sign the request for ${request.url()}`,
+						url: request.url()
+					};
+					await route.abort("blockedbyclient");
+					return;
+				}
+				const added = {
+					...extra,
+					...lowerKeys(signed)
+				};
+				const sent = Object.keys(added).length === 0 ? void 0 : {
 					...Object.fromEntries(Object.entries(await request.allHeaders()).filter(([name]) => !name.startsWith(":") && name !== "host")),
-					...extra
+					...added
 				};
 				if (allowRequest === void 0) {
 					await route.continue(sent === void 0 ? void 0 : { headers: sent });
@@ -451,61 +463,6 @@ const playwrightRenderer = (options = {}) => {
 	};
 };
 //#endregion
-//#region src/browser.ts
-var browser_exports = /* @__PURE__ */ __exportAll({
-	NO_BROWSER: () => NO_BROWSER,
-	browserMissing: () => browserMissing,
-	checkedRenderer: () => checkedRenderer
-});
-/** The text of the note when `playwright-core` is installed, but its browser is not. */
-const NO_BROWSER = "rendering needs a browser: npx playwright-core install chromium-headless-shell";
-/**
-* This function tells if the default renderer can start a browser. It gives no value when it can,
-* and the reason when it cannot. The command line and the MCP server use it one time at start.
-* Then a missing browser costs no wait and no error capture for each page that needs JavaScript.
-*
-* The check does not start the browser. It finds the full Chromium build or the headless shell of
-* the same revision on the disk. When the check cannot find the answer, it gives no value, and the
-* renderer then tries.
-*/
-const browserMissing = async (load = () => import("playwright-core")) => {
-	let chromium;
-	try {
-		({chromium} = await load());
-	} catch {
-		return NO_PLAYWRIGHT;
-	}
-	try {
-		const executable = chromium.executablePath();
-		if (existsSync(executable)) return void 0;
-		let folder = dirname(executable);
-		while (dirname(folder) !== folder && !/^chromium-\d+$/.test(basename(folder))) folder = dirname(folder);
-		const revision = /^chromium-(\d+)$/.exec(basename(folder))?.[1];
-		if (revision === void 0) return void 0;
-		return existsSync(join(dirname(folder), `chromium_headless_shell-${revision}`)) ? void 0 : NO_BROWSER;
-	} catch {
-		return;
-	}
-};
-/**
-* This function gives a renderer that checks one time, before its first page, that the browser is
-* on the disk (`browserMissing`). With no browser, each page gets a `RenderRefused` with the reason
-* `browser` at once, and the renderer opens nothing. `createRuntime` puts its default renderer in
-* it, so a library with no browser costs no wait and no error capture.
-*/
-const checkedRenderer = (renderer, check = () => browserMissing()) => {
-	let missing;
-	return {
-		async render(url, options) {
-			missing ??= check().catch(() => void 0);
-			const reason = await missing;
-			if (reason !== void 0) throw new RenderRefused("browser", reason);
-			return renderer.render(url, options);
-		},
-		close: () => renderer.close()
-	};
-};
-//#endregion
-export { RenderRefused as a, NO_PLAYWRIGHT as i, browser_exports as n, playwrightRenderer as o, checkedRenderer as r, __exportAll as s, browserMissing as t };
+export { RenderRefused as n, playwrightRenderer as r, NO_PLAYWRIGHT as t };
 
-//# sourceMappingURL=browser-gJ_h4AhB.mjs.map
+//# sourceMappingURL=render-C1CUraWF.mjs.map