polite-crawler 0.1.0 → 0.1.1

package/build/{open-CZqwY7GH.d.mts → open-eHcriTds.d.mts}

@@ -1,6 +1,6 @@
 import { n as Outcome } from "./classify-CrfyXcwo.mjs";
 import { r as Signals, t as AiInput } from "./signals-BHpy2xb2.mjs";
-import { f as PolicyOptions, o as HostStateStore, t as Clock } from "./politeness-C5bZ1OKM.mjs";
+import { f as PolicyOptions, o as HostStateStore, t as Clock } from "./politeness-yI3xBEx9.mjs";
 import { a as Robots } from "./robots-R-qaEBOb.mjs";
 import { Dispatcher } from "undici";
 import { LookupAddress, LookupAllOptions } from "node:dns";
@@ -222,6 +222,30 @@ interface VersionText extends DocVersion {
   readonly full_text: string | undefined;
   readonly meta: Readonly<Record<string, unknown>>;
 }
+/**
+ * This value chooses the ranking of a search:
+ *
+ * - `exact` (the default): BM25 ranks each document that matches.
+ * - `fast`: the form of the query that needs one part leaves out the common parts. A common part
+ *   is in more than `FAST_RANK_SHARE` of the documents. The rarest part stays when each part is common.
+ *
+ * The documents that hold each part come first in the two modes, with the same rank. In the mode
+ * `fast`, the rest of the results leaves out a document that holds only common parts. The common
+ * parts also do not add to the score of the rest. Thus `fast` is faster for a long question, and
+ * its results can be different. The README gives the measured difference.
+ */
+type RankMode = "exact" | "fast";
+/** In the mode `fast`, a part is common when more than this share of the documents hold it. */
+declare const FAST_RANK_SHARE = .2;
+/** These are the options of a search of the index. */
+interface FindIndexOptions {
+  /** This value is the maximum number of results. The index gives 1000 or fewer. */
+  readonly limit: number;
+  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
+  readonly kind?: string | null | undefined;
+  /** This value chooses the ranking. The default is `exact`. */
+  readonly rank?: RankMode | undefined;
+}
 interface Found {
   readonly doc: string;
   readonly kind: string;
@@ -308,11 +332,10 @@ interface ReadingIndex {
   docsOfCapture(capture: string): Doc[];
   /** This method tells if a capture gave a version of a document. */
   hasVersion(capture: string): boolean;
-  find(q: string, options: {
-    readonly limit: number;
-    readonly kind?: string | null;
-  }): Found[];
-  rankWithin(q: string, docs: readonly string[]): Array<{
+  find(q: string, options: FindIndexOptions): Found[];
+  rankWithin(q: string, docs: readonly string[], options?: {
+    readonly rank?: RankMode | undefined;
+  }): Array<{
     readonly doc: string;
     readonly score: number;
   }>;
@@ -407,21 +430,27 @@ declare class DocIndex implements ReadingIndex {
    * This method searches the documents with BM25, the best first. The syntax of the query is the
    * syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it
    * adds the documents that match one part. A document that asked not to be AI input is not in
-   * the results, so a search cannot show its text.
+   * the results, so a search cannot show its text. The option `rank` chooses the ranking (`RankMode`).
    */
-  find(q: string, options: {
-    readonly limit: number;
-    readonly kind?: string | null;
-  }): Found[];
+  find(q: string, options: FindIndexOptions): Found[];
   /**
    * This method scores a set of documents against a query, with the best first. It leaves out a
    * document that matches no term, and a document that asked not to be AI input. It ranks the
    * results of a source again, because the order of the source is only as good as its search engine.
+   * In the mode `fast`, it also leaves out a document that holds only common parts (`RankMode`).
    */
-  rankWithin(q: string, docs: readonly string[]): Array<{
+  rankWithin(q: string, docs: readonly string[], options?: {
+    readonly rank?: RankMode | undefined;
+  }): Array<{
     readonly doc: string;
     readonly score: number;
   }>;
+  /**
+   * This method gives the form of a query that needs one part, without the common parts (the mode
+   * `fast` of `RankMode`). It counts the documents that hold each part. When each part is common,
+   * it keeps the rarest part.
+   */
+  private rareParts;
   /** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */
   latest(url: string, variant?: Variant): Latest | undefined;
   private putLatest;
@@ -797,6 +826,8 @@ declare class Crawler {
   private policyOf;
   private emit;
   private track;
+  /** This method gives the validator memo of a host. The memo is a hint, so an error of its store gives no memo. */
+  private memoOf;
   private stopHost;
   private record;
   private skip;
@@ -1228,11 +1259,13 @@ declare const readDocument: (runtime: CrawlRuntime, id: string, options?: ReadOp
 declare const verifyQuote: (runtime: CrawlRuntime, id: string, quote: string, options?: VerifyOptions) => Promise<VerifyResult>;
 /**
  * This function searches the stored documents. The syntax of the query is the syntax of
- * `parseQuery`. A held-back document is not in the results.
+ * `parseQuery`. A held-back document is not in the results. The option `rank` chooses the
+ * ranking (`RankMode`), and the default is `exact`.
  */
 declare const findDocuments: (runtime: CrawlRuntime, q: string, options?: {
   readonly limit?: number | undefined;
   readonly kind?: string | null | undefined;
+  readonly rank?: RankMode | undefined;
 }) => Promise<Found[]>;
 /** The counts of a rebuild. */
 interface ReindexResult {
@@ -1413,6 +1446,11 @@ interface FindOptions {
   readonly limit?: number | undefined;
   /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
   readonly kind?: string | null | undefined;
+  /**
+   * This value chooses the ranking. The default `exact` ranks each match with BM25. `fast` leaves
+   * the common words out of the second part of the search, so a long question is faster (`RankMode`).
+   */
+  readonly rank?: RankMode | undefined;
 }
 /** A polite crawler fetches each source one time, keeps it, and answers the next questions from the disk. */
 interface PoliteCrawler {
@@ -1457,5 +1495,5 @@ interface PoliteCrawler {
  */
 declare const openCrawler: (options: OpenOptions) => PoliteCrawler;
 //#endregion
-export { FetchOutcome as $, findDocuments as A, CaptureFilter as B, ReadResult as C, RenderOptions as Ct, catchUpIndex as D, playwrightRenderer as Dt, VerifyResult as E, Renderer as Et, readDocument as F, CAPTURE_SCHEMA as G, CaptureReader as H, recordOverride as I, CaptureWriter as J, CacheEntry as K, reindex as L, isHeldBack as M, pageDocId as N, fetchAndIndex as O, BlobStore as Ot, pageExtractor as P, DEFAULT_SECRET_PARAMS as Q, renderAndIndex as R, ReadOptions as S, RefusedReason as St, VerifyOptions as T, Rendered as Tt, DamagedLine as U, CaptureLog as V, ScanOptions as W, Crawler as X, CrawlEvent as Y, CrawlerOptions as Z, Extractors as _, ReadingIndex as _t, PoliteCrawler as a, RecordInput as at, PageOptions as b, NO_PLAYWRIGHT as bt, createRuntime as c, EgressRefused as ct, isMailbox as d, DocIndex as dt, FetchRequest as et, openCrawler as f, DocInput as ft, Extractor as g, Latest as gt, CrawlRuntime as h, Found as ht, OpenOptions as i, CaptureRecord as it, heldBackMeta as j, fetchPage as k, FileBlobStore as kt, defaultUserAgent as l, egressLookup as lt, CatchUpResult as m, ExtractedDoc as mt, FetchOptions as n, RenderRequest as nt, RuntimeOptions as o, makeRecord as ot, CapturePayload as p, DocVersion as pt, CacheIndex as q, FindOptions as r, renderedKey as rt, StoreRuntime as s, EgressOptions as st, Duration as t, Fetched as tt, durationMs as u, Doc as ut, HELD_BACK as v, Variant as vt, ReindexResult as w, RenderRefused as wt, PageResult as x, PlaywrightRenderer as xt, Indexed as y, VersionText as yt, verifyQuote as z };
-//# sourceMappingURL=open-CZqwY7GH.d.mts.map
+export { FetchOutcome as $, findDocuments as A, playwrightRenderer as At, CaptureFilter as B, ReadResult as C, NO_PLAYWRIGHT as Ct, catchUpIndex as D, RenderRefused as Dt, VerifyResult as E, RenderOptions as Et, readDocument as F, CAPTURE_SCHEMA as G, CaptureReader as H, recordOverride as I, CaptureWriter as J, CacheEntry as K, reindex as L, isHeldBack as M, FileBlobStore as Mt, pageDocId as N, fetchAndIndex as O, Rendered as Ot, pageExtractor as P, DEFAULT_SECRET_PARAMS as Q, renderAndIndex as R, ReadOptions as S, VersionText as St, VerifyOptions as T, RefusedReason as Tt, DamagedLine as U, CaptureLog as V, ScanOptions as W, Crawler as X, CrawlEvent as Y, CrawlerOptions as Z, Extractors as _, Found as _t, PoliteCrawler as a, RecordInput as at, PageOptions as b, ReadingIndex as bt, createRuntime as c, EgressRefused as ct, isMailbox as d, DocIndex as dt, FetchRequest as et, openCrawler as f, DocInput as ft, Extractor as g, FindIndexOptions as gt, CrawlRuntime as h, FAST_RANK_SHARE as ht, OpenOptions as i, CaptureRecord as it, heldBackMeta as j, BlobStore as jt, fetchPage as k, Renderer as kt, defaultUserAgent as l, egressLookup as lt, CatchUpResult as m, ExtractedDoc as mt, FetchOptions as n, RenderRequest as nt, RuntimeOptions as o, makeRecord as ot, CapturePayload as p, DocVersion as pt, CacheIndex as q, FindOptions as r, renderedKey as rt, StoreRuntime as s, EgressOptions as st, Duration as t, Fetched as tt, durationMs as u, Doc as ut, HELD_BACK as v, Latest as vt, ReindexResult as w, PlaywrightRenderer as wt, PageResult as x, Variant as xt, Indexed as y, RankMode as yt, verifyQuote as z };
+//# sourceMappingURL=open-eHcriTds.d.mts.map

package/build/{politeness-C5bZ1OKM.d.mts → politeness-yI3xBEx9.d.mts}

@@ -92,14 +92,105 @@ interface HostStateStore {
    * host. It keeps the next times and the leases, so a reset cannot make a host go faster than its pace.
    */
   reset(host?: string): void;
+  /**
+   * This method gives the validator memo of a host, when the store has one. A store without this
+   * method and `noteValidators` has no memo. The crawler then sends each validator of a stored copy.
+   */
+  validators?(host: string): ValidatorMemo | undefined;
+  /**
+   * This method adds one conditional request to the memo of a host, with the rules of
+   * `nextValidatorMemo`, in one atomic step. It gives the new memo.
+   */
+  noteValidators?(host: string, report: ValidatorReport, now: number): ValidatorMemo;
 }
+/** This value tells the result that the crawler learned for one validator of a host. */
+type ValidatorVerdict = "unknown" | "works" | "fails";
+/**
+ * This value tells what one conditional request got:
+ *
+ * - `not-modified`: a 304.
+ * - `same`: a 200 with the body that the store has. Thus the validator did not match.
+ * - `changed`: a 200 with a different body. This answer tells nothing about the validator.
+ */
+type ValidatorAnswer = "not-modified" | "same" | "changed";
+/** This report tells the validators that one conditional request sent, and the answer that it got. */
+interface ValidatorReport {
+  /** This value is true when the request sent `If-None-Match`. */
+  readonly etag: boolean;
+  /** This value is true when the request sent `If-Modified-Since`. */
+  readonly lastModified: boolean;
+  readonly answer: ValidatorAnswer;
+}
+/**
+ * A validator memo holds what the crawler learned about the validators of one host. A server
+ * examines `If-None-Match` first, and it ignores `If-Modified-Since` when the request has the two
+ * (RFC 9110, section 13.1.3). Thus a host with unstable ETags does not answer 304 to a request with
+ * the two validators at any time. Its stable `Last-Modified` date does not help. `nextValidatorMemo`
+ * gives the rules.
+ */
+interface ValidatorMemo {
+  /** This value is true when the crawler leaves out `If-None-Match` for the host and sends only `If-Modified-Since`. */
+  readonly skipEtag: boolean;
+  /** This value tells if a request with `If-None-Match` got a 304 (`works`) or the stored body again (`fails`). */
+  readonly etag: ValidatorVerdict;
+  /** This value tells the same for a request with only `If-Modified-Since`. */
+  readonly lastModified: ValidatorVerdict;
+  /** This value counts the stored bodies in sequence that came back to a request with `If-None-Match`. */
+  readonly etagMisses: number;
+  /** This value counts the stored bodies in sequence that came back to a request with only `If-Modified-Since`. */
+  readonly dateMisses: number;
+  /** This value counts the requests with only `If-Modified-Since` since the crawler stopped the ETag, or since the last probe. */
+  readonly sinceProbe: number;
+  /** This time (milliseconds since the epoch) is the last time that the crawler stopped or started the ETag, or probed it. */
+  readonly changedAt: number;
+}
+/** These are the limits of the validator memo. */
+interface ValidatorLimits {
+  /** After this number of stored bodies in sequence, a validator fails. The value is 3. */
+  readonly misses: number;
+  /** After this number of requests with only `If-Modified-Since`, the next request sends the ETag again (a probe). The value is 20. */
+  readonly probeEvery: number;
+  /** After this time with no probe, the next request is a probe. The value is 7 days. */
+  readonly probeAgeMs: number;
+}
+/** These are the limits of the validator memo of the crawler. */
+declare const VALIDATOR_MEMO: ValidatorLimits;
+/**
+ * This function tells which validators the next conditional request to a host sends. It sends
+ * each validator of the stored copy, except when the memo stopped the ETag and the copy has a
+ * `Last-Modified` date. Then it sends only `If-Modified-Since`, and the ETag too for a probe.
+ * A probe comes after `probeEvery` requests or after `probeAgeMs`.
+ */
+declare const validatorsToSend: (memo: ValidatorMemo | undefined, stored: {
+  readonly etag: boolean;
+  readonly lastModified: boolean;
+}, now: number) => {
+  readonly etag: boolean;
+  readonly lastModified: boolean;
+};
+/**
+ * This function adds one conditional request to the memo of a host, and gives the new memo.
+ * These are the rules, with the limits of `VALIDATOR_MEMO`:
+ *
+ * 1. A 304 to a request with `If-None-Match` marks the ETag `works`. After a probe, the crawler sends the ETag again.
+ * 2. After `misses` stored bodies in sequence with `If-None-Match`, the ETag fails. When the request also sent
+ *    `If-Modified-Since`, the crawler stops the ETag for the host.
+ * 3. A probe that gets a 200 keeps the ETag stopped, and the count to the next probe starts again.
+ * 4. After `misses` stored bodies in sequence with only `If-Modified-Since`, that validator fails too. Then the
+ *    crawler sends the two validators again, because they cost nothing. It tries the date alone again after `probeAgeMs`.
+ * 5. A changed body tells nothing about a validator.
+ */
+declare const nextValidatorMemo: (memo: ValidatorMemo | undefined, report: ValidatorReport, now: number) => ValidatorMemo;
 /**
  * This store keeps the state of each host in memory, for one process. It removes the state of an
  * idle host (no rest left, no lease, no counts and no stop) when it holds more than `maxHosts` hosts.
+ * It keeps the validator memos of the `maxHosts` hosts that it used last.
  */
 declare class MemoryHostStateStore implements HostStateStore {
   private readonly maxHosts;
   private readonly states;
+  /** The validator memos, the memo used last at the end. */
+  private readonly memos;
   private sweepAt;
   constructor(maxHosts?: number);
   /** This value is the number of hosts that the store holds. */
@@ -113,6 +204,8 @@ declare class MemoryHostStateStore implements HostStateStore {
   stop(host: string, until: number, reason: string): void;
   stopped(host: string, now: number): HostStop | undefined;
   reset(host?: string): void;
+  validators(host: string): ValidatorMemo | undefined;
+  noteValidators(host: string, report: ValidatorReport, now: number): ValidatorMemo;
 }
 /** These options change the scheduler. */
 interface SchedulerOptions {
@@ -270,5 +363,5 @@ declare const checkPolicy: (options: PolicyOptions | undefined) => void;
 /** This function gives the policy of one host: the defaults, then the general options, then the entries for the host. */
 declare const policyFor: (options: PolicyOptions | undefined, host: string) => PolitenessPolicy;
 //#endregion
-export { restAfter as S, backoffMs as _, HostScheduler as a, policyFor as b, HostStopped as c, Paced as d, PolicyOptions as f, SchedulerOptions as g, RestInput as h, HostResult as i, MAX_STOP_MS as l, Rest as m, DEFAULT_POLICY as n, HostStateStore as o, PolitenessPolicy as p, HostCounts as r, HostStop as s, Clock as t, MemoryHostStateStore as u, checkPolicy as v, realClock as x, equalJitterMs as y };
-//# sourceMappingURL=politeness-C5bZ1OKM.d.mts.map
+export { validatorsToSend as A, backoffMs as C, policyFor as D, nextValidatorMemo as E, realClock as O, ValidatorVerdict as S, equalJitterMs as T, VALIDATOR_MEMO as _, HostScheduler as a, ValidatorMemo as b, HostStopped as c, Paced as d, PolicyOptions as f, SchedulerOptions as g, RestInput as h, HostResult as i, restAfter as k, MAX_STOP_MS as l, Rest as m, DEFAULT_POLICY as n, HostStateStore as o, PolitenessPolicy as p, HostCounts as r, HostStop as s, Clock as t, MemoryHostStateStore as u, ValidatorAnswer as v, checkPolicy as w, ValidatorReport as x, ValidatorLimits as y };
+//# sourceMappingURL=politeness-yI3xBEx9.d.mts.map

package/build/politeness.d.mts

@@ -1,2 +1,2 @@
-import { S as restAfter, _ as backoffMs, a as HostScheduler, b as policyFor, c as HostStopped, d as Paced, f as PolicyOptions, g as SchedulerOptions, h as RestInput, i as HostResult, l as MAX_STOP_MS, m as Rest, n as DEFAULT_POLICY, o as HostStateStore, p as PolitenessPolicy, r as HostCounts, s as HostStop, t as Clock, u as MemoryHostStateStore, v as checkPolicy, x as realClock, y as equalJitterMs } from "./politeness-C5bZ1OKM.mjs";
-export { Clock, DEFAULT_POLICY, HostCounts, HostResult, HostScheduler, HostStateStore, HostStop, HostStopped, MAX_STOP_MS, MemoryHostStateStore, Paced, PolicyOptions, PolitenessPolicy, Rest, RestInput, SchedulerOptions, backoffMs, checkPolicy, equalJitterMs, policyFor, realClock, restAfter };
+import { A as validatorsToSend, C as backoffMs, D as policyFor, E as nextValidatorMemo, O as realClock, S as ValidatorVerdict, T as equalJitterMs, _ as VALIDATOR_MEMO, a as HostScheduler, b as ValidatorMemo, c as HostStopped, d as Paced, f as PolicyOptions, g as SchedulerOptions, h as RestInput, i as HostResult, k as restAfter, l as MAX_STOP_MS, m as Rest, n as DEFAULT_POLICY, o as HostStateStore, p as PolitenessPolicy, r as HostCounts, s as HostStop, t as Clock, u as MemoryHostStateStore, v as ValidatorAnswer, w as checkPolicy, x as ValidatorReport, y as ValidatorLimits } from "./politeness-yI3xBEx9.mjs";
+export { Clock, DEFAULT_POLICY, HostCounts, HostResult, HostScheduler, HostStateStore, HostStop, HostStopped, MAX_STOP_MS, MemoryHostStateStore, Paced, PolicyOptions, PolitenessPolicy, Rest, RestInput, SchedulerOptions, VALIDATOR_MEMO, ValidatorAnswer, ValidatorLimits, ValidatorMemo, ValidatorReport, ValidatorVerdict, backoffMs, checkPolicy, equalJitterMs, nextValidatorMemo, policyFor, realClock, restAfter, validatorsToSend };

package/build/politeness.mjs

@@ -40,13 +40,104 @@ var HostStopped = class extends Error {
 		this.until = until;
 	}
 };
+/** These are the limits of the validator memo of the crawler. */
+const VALIDATOR_MEMO = {
+	misses: 3,
+	probeEvery: 20,
+	probeAgeMs: 6048e5
+};
+const EMPTY_MEMO = {
+	skipEtag: false,
+	etag: "unknown",
+	lastModified: "unknown",
+	etagMisses: 0,
+	dateMisses: 0,
+	sinceProbe: 0,
+	changedAt: 0
+};
+/**
+* This function tells which validators the next conditional request to a host sends. It sends
+* each validator of the stored copy, except when the memo stopped the ETag and the copy has a
+* `Last-Modified` date. Then it sends only `If-Modified-Since`, and the ETag too for a probe.
+* A probe comes after `probeEvery` requests or after `probeAgeMs`.
+*/
+const validatorsToSend = (memo, stored, now) => {
+	if (memo?.skipEtag !== true || !stored.etag || !stored.lastModified) return stored;
+	return {
+		etag: memo.sinceProbe >= VALIDATOR_MEMO.probeEvery || now - memo.changedAt >= VALIDATOR_MEMO.probeAgeMs,
+		lastModified: true
+	};
+};
+/**
+* This function adds one conditional request to the memo of a host, and gives the new memo.
+* These are the rules, with the limits of `VALIDATOR_MEMO`:
+*
+* 1. A 304 to a request with `If-None-Match` marks the ETag `works`. After a probe, the crawler sends the ETag again.
+* 2. After `misses` stored bodies in sequence with `If-None-Match`, the ETag fails. When the request also sent
+*    `If-Modified-Since`, the crawler stops the ETag for the host.
+* 3. A probe that gets a 200 keeps the ETag stopped, and the count to the next probe starts again.
+* 4. After `misses` stored bodies in sequence with only `If-Modified-Since`, that validator fails too. Then the
+*    crawler sends the two validators again, because they cost nothing. It tries the date alone again after `probeAgeMs`.
+* 5. A changed body tells nothing about a validator.
+*/
+const nextValidatorMemo = (memo, report, now) => {
+	const next = { ...memo ?? EMPTY_MEMO };
+	if (report.etag) {
+		const probe = next.skipEtag;
+		if (report.answer === "not-modified") {
+			next.etag = "works";
+			next.etagMisses = 0;
+			if (probe) {
+				next.skipEtag = false;
+				next.sinceProbe = 0;
+				next.changedAt = now;
+			}
+		} else if (probe) {
+			if (report.answer === "same") next.etagMisses += 1;
+			next.sinceProbe = 0;
+			next.changedAt = now;
+		} else if (report.answer === "same") {
+			next.etagMisses += 1;
+			if (next.etagMisses >= VALIDATOR_MEMO.misses) {
+				next.etag = "fails";
+				const dateLeft = next.lastModified !== "fails" || now - next.changedAt >= VALIDATOR_MEMO.probeAgeMs;
+				if (report.lastModified && dateLeft) {
+					next.skipEtag = true;
+					next.dateMisses = 0;
+					next.sinceProbe = 0;
+					next.changedAt = now;
+				}
+			}
+		}
+	} else if (report.lastModified) {
+		if (next.skipEtag) next.sinceProbe += 1;
+		if (report.answer === "not-modified") {
+			next.lastModified = "works";
+			next.dateMisses = 0;
+		} else if (report.answer === "same") {
+			next.dateMisses += 1;
+			if (next.dateMisses >= VALIDATOR_MEMO.misses) {
+				next.lastModified = "fails";
+				if (next.skipEtag) {
+					next.skipEtag = false;
+					next.sinceProbe = 0;
+					next.changedAt = now;
+				}
+			}
+		}
+	}
+	return next;
+};
 /**
 * This store keeps the state of each host in memory, for one process. It removes the state of an
 * idle host (no rest left, no lease, no counts and no stop) when it holds more than `maxHosts` hosts.
+* It keeps the validator memos of the `maxHosts` hosts that it used last.
 */
 var MemoryHostStateStore = class {
 	maxHosts;
 	states = /* @__PURE__ */ new Map();
+	/** The validator memos, the memo used last at the end. */
+	memos = /* @__PURE__ */ new Map();
 	sweepAt;
 	constructor(maxHosts = 1e4) {
 		this.maxHosts = maxHosts;
@@ -126,6 +217,16 @@ var MemoryHostStateStore = class {
 			if (state !== void 0) clear(state);
 		}
 	}
+	validators(host) {
+		return this.memos.get(host);
+	}
+	noteValidators(host, report, now) {
+		const memo = nextValidatorMemo(this.memos.get(host), report, now);
+		this.memos.delete(host);
+		this.memos.set(host, memo);
+		if (this.memos.size > this.maxHosts) this.memos.delete(this.memos.keys().next().value);
+		return memo;
+	}
 };
 const aborted = (signal) => signal.reason instanceof Error ? signal.reason : new DOMException("The wait was aborted", "AbortError");
 var HostScheduler = class {
@@ -367,6 +468,6 @@ const policyFor = (options, host) => {
 	return Object.assign({}, DEFAULT_POLICY, base, ...matching.map(({ values }) => values));
 };
 //#endregion
-export { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, MemoryHostStateStore, backoffMs, checkPolicy, equalJitterMs, policyFor, realClock, restAfter };
+export { DEFAULT_POLICY, HostScheduler, HostStopped, MAX_STOP_MS, MemoryHostStateStore, VALIDATOR_MEMO, backoffMs, checkPolicy, equalJitterMs, nextValidatorMemo, policyFor, realClock, restAfter, validatorsToSend };
 
 //# sourceMappingURL=politeness.mjs.map

package/build/politeness.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"politeness.mjs","names":[],"sources":["../src/politeness.ts"],"sourcesContent":["/**\n * The scheduler controls the pace of each host. The speed of a polite crawler is the number of\n * hosts in progress divided by the delay for each host. Thus the scheduler keeps one request in\n * progress for each host, and it waits the delay of the host between two requests. It uses its\n * concurrency across many hosts, as the back queues of the Mercator crawler do.\n *\n * The caller decides the delay after each request, and `run` gets it from the task. Only the caller\n * knows the documented rate of the source, the crawl delay of the site and the duration of the\n * request. A wait holds no global slot, so a host that waits does not block a different host.\n *\n * The state of each host (its next time, its counts and its stop) is in a `HostStateStore`. The\n * default store is in memory. A store that processes share keeps one pace for each host across them.\n */\n\nexport interface Clock {\n  readonly now: () => number;\n  /** This function waits `ms` milliseconds. When the signal aborts, the wait ends early. */\n  readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;\n}\n\n/** A timer of Node.js holds a maximum of 2^31 − 1 milliseconds. A longer timer fires at once. */\nconst MAX_TIMER_MS = 2 ** 31 - 1;\n\nexport const realClock: Clock = {\n  now: () => Date.now(),\n  sleep: (ms, signal) =>\n    new Promise<void>((resolve) => {\n      if (signal?.aborted === true) {\n        resolve();\n        return;\n      }\n      // a long wait goes in parts, because a timer longer than the maximum fires at once\n      let left = Number.isFinite(ms) ? Math.max(0, ms) : 0;\n      let timer: ReturnType<typeof setTimeout> | undefined;\n      const done = (): void => {\n        if (timer !== undefined) clearTimeout(timer);\n        signal?.removeEventListener(\"abort\", done);\n        resolve();\n      };\n      const step = (): void => {\n        if (left <= 0) {\n          done();\n          return;\n        }\n        const part = Math.min(left, MAX_TIMER_MS);\n        left -= part;\n        timer = setTimeout(step, part);\n      };\n      signal?.addEventListener(\"abort\", done, { once: true });\n      step();\n    }),\n};\n\n/** The crawler does not ask a host again until the stop ends, after a refusal (a challenge, 403 or 402). */\nexport class HostStopped extends Error {\n  constructor(\n    readonly host: string,\n    readonly reason: string,\n    /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */\n    readonly until: number = Number.POSITIVE_INFINITY,\n  ) {\n    super(`${host} stopped: ${reason}`);\n  }\n}\n\nexport interface Paced<T> {\n  readonly value: T;\n  /** This value is the time that the host rests before its next request. */\n  readonly delayMs: number;\n}\n\n/** A stop keeps the crawler away from a host until a time. */\nexport interface HostStop {\n  /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */\n  readonly until: number;\n  readonly reason: string;\n}\n\n/** These are the counts of a host. Each count gives the results of one kind in sequence. */\nexport interface HostCounts {\n  /** This value counts errors and requests with no answer. At the limit of the policy, the host stops. */\n  readonly failures: number;\n  /** This value counts the requests to wait (429, 502, 503, 504). It sets the backoff and does not stop the host. */\n  readonly waits: number;\n}\n\n/**\n * This value tells how one result changes the counts of a host:\n *\n * - `success` sets the two counts to zero.\n * - `failure` adds one to `failures`.\n * - `wait` adds one to `waits`.\n * - `neutral` changes nothing. A client error (a 400) is neutral, because a different request can succeed.\n */\nexport type HostResult = \"success\" | \"failure\" | \"wait\" | \"neutral\";\n\n/**\n * A host state store keeps the state of each host for the scheduler. Its methods are synchronous,\n * and each method is one atomic step on the state of one host. Thus a store can be a table of\n * SQLite that processes share. Times are milliseconds since the epoch.\n *\n * The state of a host has five parts:\n *\n * - the next time: the earliest time of the next request\n * - the lease: the end of the slot of the request in progress, when there is one\n * - the counts (`HostCounts`)\n * - the stop: an end time and a reason, when there is one\n */\nexport interface HostStateStore {\n  /**\n   * This method asks for the slot of a host at the time `now`. When the host has no rest left\n   * and no lease, the store gives the slot to the caller until `now + leaseMs` and gives 0.\n   * Otherwise it changes nothing and gives the time to wait in milliseconds: the time to the next\n   * time, or to the end of the lease. A store that processes share gives a short wait (one second\n   * or less) during a lease, because the holder usually releases it early. The caller asks again after the wait.\n   */\n  reserve(host: string, now: number, leaseMs: number): number;\n  /** This method changes the counts of a host for one result, and it gives the new counts. */\n  record(host: string, result: HostResult): HostCounts;\n  /**\n   * This method ends the lease of the caller. It sets the next time to `nextAt` when `nextAt` is\n   * later than the stored next time. Only the holder of the lease uses it.\n   */\n  release(host: string, nextAt: number): void;\n  /** This method sets the next time to `until` when `until` is later. It does not touch the lease. */\n  defer(host: string, until: number): void;\n  /** This method stops a host until a time. When the host has a stop that ends later, that stop stays. */\n  stop(host: string, until: number, reason: string): void;\n  /** This method gives the stop of a host at the time `now`. A stop that ended gives no value. */\n  stopped(host: string, now: number): HostStop | undefined;\n  /**\n   * This method removes the stops and the counts of one host, or of each host when there is no\n   * host. It keeps the next times and the leases, so a reset cannot make a host go faster than its pace.\n   */\n  reset(host?: string): void;\n}\n\ninterface MemoryState {\n  nextAt: number;\n  leasedUntil: number;\n  failures: number;\n  waits: number;\n  stop: HostStop | undefined;\n}\n\n/**\n * This store keeps the state of each host in memory, for one process. It removes the state of an\n * idle host (no rest left, no lease, no counts and no stop) when it holds more than `maxHosts` hosts.\n */\nexport class MemoryHostStateStore implements HostStateStore {\n  private readonly states = new Map<string, MemoryState>();\n  private sweepAt: number;\n\n  constructor(private readonly maxHosts: number = 10_000) {\n    this.sweepAt = maxHosts;\n  }\n\n  /** This value is the number of hosts that the store holds. */\n  get size(): number {\n    return this.states.size;\n  }\n\n  private state(host: string): MemoryState {\n    let state = this.states.get(host);\n    if (state === undefined) {\n      state = { nextAt: 0, leasedUntil: 0, failures: 0, waits: 0, stop: undefined };\n      this.states.set(host, state);\n    }\n    return state;\n  }\n\n  private sweep(now: number): void {\n    for (const [host, state] of this.states) {\n      const idle =\n        state.nextAt <= now &&\n        state.leasedUntil <= now &&\n        state.failures === 0 &&\n        state.waits === 0 &&\n        (state.stop === undefined || state.stop.until <= now);\n      if (idle) this.states.delete(host);\n    }\n    // the next sweep comes after a tenth more hosts, so a store of busy hosts does not sweep at each request\n    this.sweepAt = Math.max(this.maxHosts, Math.ceil(this.states.size * 1.1));\n  }\n\n  reserve(host: string, now: number, leaseMs: number): number {\n    if (this.states.size > this.sweepAt) this.sweep(now);\n    const state = this.state(host);\n    if (state.leasedUntil > now) return state.leasedUntil - now;\n    if (state.nextAt > now) return state.nextAt - now;\n    state.leasedUntil = now + leaseMs;\n    return 0;\n  }\n\n  record(host: string, result: HostResult): HostCounts {\n    const state = this.state(host);\n    if (result === \"success\") {\n      state.failures = 0;\n      state.waits = 0;\n    } else if (result === \"failure\") state.failures += 1;\n    else if (result === \"wait\") state.waits += 1;\n    return { failures: state.failures, waits: state.waits };\n  }\n\n  release(host: string, nextAt: number): void {\n    const state = this.state(host);\n    state.leasedUntil = 0;\n    if (nextAt > state.nextAt) state.nextAt = nextAt;\n  }\n\n  defer(host: string, until: number): void {\n    const state = this.state(host);\n    if (until > state.nextAt) state.nextAt = until;\n  }\n\n  stop(host: string, until: number, reason: string): void {\n    const state = this.state(host);\n    if (state.stop === undefined || state.stop.until < until) state.stop = { until, reason };\n  }\n\n  stopped(host: string, now: number): HostStop | undefined {\n    const stop = this.states.get(host)?.stop;\n    return stop !== undefined && stop.until > now ? stop : undefined;\n  }\n\n  reset(host?: string): void {\n    const clear = (state: MemoryState): void => {\n      state.failures = 0;\n      state.waits = 0;\n      state.stop = undefined;\n    };\n    if (host === undefined) for (const state of this.states.values()) clear(state);\n    else {\n      const state = this.states.get(host);\n      if (state !== undefined) clear(state);\n    }\n  }\n}\n\n/** These options change the scheduler. */\nexport interface SchedulerOptions {\n  /** This store keeps the state of each host. The default is a `MemoryHostStateStore`. */\n  readonly store?: HostStateStore | undefined;\n  /** This value is the longest slot of one request. A slot that a crashed process holds ends after it. The default is 5 minutes. */\n  readonly leaseMs?: number | undefined;\n  /** A host rests this long after a task that throws an error, or a task that gives no valid delay. The default is 1 second. */\n  readonly fallbackDelayMs?: number | undefined;\n}\n\nconst aborted = (signal: AbortSignal): Error =>\n  signal.reason instanceof Error\n    ? signal.reason\n    : new DOMException(\"The wait was aborted\", \"AbortError\");\n\nexport class HostScheduler {\n  /** This store keeps the state of each host. */\n  readonly store: HostStateStore;\n  private readonly chains = new Map<string, Promise<unknown>>();\n  private active = 0;\n  private readonly queue: Array<() => void> = [];\n  private readonly leaseMs: number;\n  private readonly fallbackDelayMs: number;\n\n  constructor(\n    private readonly clock: Clock,\n    private readonly maxActive: number,\n    options: SchedulerOptions = {},\n  ) {\n    if (!Number.isInteger(maxActive) || maxActive < 1) {\n      throw new RangeError(`maxActive must be an integer of 1 or more, not ${String(maxActive)}`);\n    }\n    this.store = options.store ?? new MemoryHostStateStore();\n    this.leaseMs = options.leaseMs ?? 300_000;\n    this.fallbackDelayMs = options.fallbackDelayMs ?? 1000;\n  }\n\n  private acquire(signal: AbortSignal | undefined): Promise<void> {\n    if (this.active < this.maxActive) {\n      this.active += 1;\n      return Promise.resolve();\n    }\n    return new Promise<void>((resolve, reject) => {\n      const grant = (): void => {\n        signal?.removeEventListener(\"abort\", cancel);\n        resolve();\n      };\n      const cancel = (): void => {\n        const at = this.queue.indexOf(grant);\n        if (at !== -1) this.queue.splice(at, 1);\n        reject(aborted(signal!));\n      };\n      this.queue.push(grant);\n      signal?.addEventListener(\"abort\", cancel, { once: true });\n    });\n  }\n\n  private release(): void {\n    // the slot goes to the next waiter directly, so a new request cannot take it in between\n    const next = this.queue.shift();\n    if (next !== undefined) next();\n    else this.active -= 1;\n  }\n\n  private async sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {\n    if (signal === undefined) {\n      await this.clock.sleep(ms);\n      return;\n    }\n    if (signal.aborted) throw aborted(signal);\n    await new Promise<void>((resolve, reject) => {\n      const cancel = (): void => reject(aborted(signal));\n      signal.addEventListener(\"abort\", cancel, { once: true });\n      this.clock.sleep(ms, signal).then(\n        () => {\n          signal.removeEventListener(\"abort\", cancel);\n          resolve();\n        },\n        (error: unknown) => {\n          signal.removeEventListener(\"abort\", cancel);\n          reject(error);\n        },\n      );\n    });\n  }\n\n  /**\n   * This method starts one request on a host, in sequence, after the rest of the host. When the\n   * signal aborts during the wait, the promise rejects with the reason of the signal.\n   */\n  run<T>(host: string, task: () => Promise<Paced<T>>, signal?: AbortSignal): Promise<T> {\n    const previous = this.chains.get(host) ?? Promise.resolve();\n    const result = previous.then(async () => {\n      // after a wait, the time is at least the end of the wait, also on a clock that a test holds still\n      let waited = Number.NEGATIVE_INFINITY;\n      for (;;) {\n        if (signal?.aborted === true) throw aborted(signal);\n        const now = Math.max(this.clock.now(), waited);\n        const stop = this.store.stopped(host, now);\n        if (stop !== undefined) throw new HostStopped(host, stop.reason, stop.until);\n        const wait = this.store.reserve(host, now, this.leaseMs);\n        if (!(wait > 0)) break;\n        await this.sleep(wait, signal);\n        waited = now + wait;\n      }\n      try {\n        await this.acquire(signal);\n      } catch (error) {\n        this.store.release(host, this.clock.now());\n        throw error;\n      }\n      let nextAt = this.clock.now() + this.fallbackDelayMs;\n      try {\n        const { value, delayMs } = await task();\n        const rest = Number.isFinite(delayMs) && delayMs >= 0 ? delayMs : this.fallbackDelayMs;\n        nextAt = this.clock.now() + rest;\n        return value;\n      } catch (error) {\n        nextAt = this.clock.now() + this.fallbackDelayMs;\n        throw error;\n      } finally {\n        this.store.release(host, nextAt);\n        this.release();\n      }\n    });\n    const tail = result.then(\n      () => undefined,\n      () => undefined,\n    );\n    this.chains.set(host, tail);\n    // a host with no queued request keeps no chain\n    void tail.then(() => {\n      if (this.chains.get(host) === tail) this.chains.delete(host);\n    });\n    return result;\n  }\n\n  /** The server asked for a pause (Retry-After), or a backoff chose one. */\n  park(host: string, untilMs: number): void {\n    this.store.defer(host, untilMs);\n  }\n\n  /** This method stops a host until `untilMs`. With no time, the stop has no end. */\n  stop(host: string, reason: string, untilMs: number = Number.POSITIVE_INFINITY): void {\n    this.store.stop(host, untilMs, reason);\n  }\n\n  stoppedReason(host: string): string | undefined {\n    return this.store.stopped(host, this.clock.now())?.reason;\n  }\n\n  /** This method gives the stop of a host, with its end, when the host is stopped. */\n  stopOf(host: string): HostStop | undefined {\n    return this.store.stopped(host, this.clock.now());\n  }\n\n  /** This value counts the failures of a host in sequence, for the backoff. A success sets it to zero. */\n  failed(host: string): number {\n    return this.store.record(host, \"failure\").failures;\n  }\n\n  /** This value counts the requests to wait (429, 503) of a host in sequence. They do not stop the host. */\n  waited(host: string): number {\n    return this.store.record(host, \"wait\").waits;\n  }\n\n  succeeded(host: string): void {\n    this.store.record(host, \"success\");\n  }\n\n  /** This method removes the stops and the counts of one host, or of each host. The pace stays. */\n  reset(host?: string): void {\n    this.store.reset(host);\n  }\n}\n\n/**\n * This function gives an exponential backoff with full jitter, from the analysis of AWS. The wait\n * is random, up to base × 2^(try − 1), with a maximum. The jitter keeps retries apart.\n */\nexport const backoffMs = (\n  attempt: number,\n  baseMs: number,\n  capMs: number,\n  random: () => number,\n): number => Math.floor(random() * Math.min(capMs, baseMs * 2 ** Math.max(0, attempt - 1)));\n\n/**\n * This function gives an exponential backoff with equal jitter. The wait is half of\n * base × 2^(try − 1), with a maximum, and a random part up to the other half. Thus each backoff\n * is at least half of its step, and it is not lost under the rest of the host.\n */\nexport const equalJitterMs = (\n  attempt: number,\n  baseMs: number,\n  capMs: number,\n  random: () => number,\n): number => {\n  const step = Math.min(capMs, baseMs * 2 ** Math.max(0, attempt - 1));\n  return Math.floor(step / 2 + random() * (step / 2));\n};\n\n/** These are the facts of one request that set the rest of its host. */\nexport interface RestInput {\n  /** This value is the kind of the request: a web page, a documented API, or a render in the browser. */\n  readonly kind: \"page\" | \"api\" | \"render\";\n  /** This value is the duration of the request, in milliseconds. */\n  readonly elapsedMs: number;\n  /** This value is the `crawl-delay` of robots.txt, in seconds. */\n  readonly crawlDelaySeconds?: number | undefined;\n  /** This value is the documented pace of the source. It replaces the interval of the policy for the kind. */\n  readonly minIntervalMs?: number | undefined;\n  /** This value is the wait that the server asked for (`Retry-After`). */\n  readonly retryAfterMs?: number | undefined;\n}\n\n/** The rest of a host after a request, and the reason for its length. */\nexport interface Rest {\n  readonly ms: number;\n  readonly reason: \"pace\" | \"slow\" | \"retry-after\" | \"crawl-delay\";\n}\n\n/**\n * This function gives the rest of a host after one request, as the crawler calculates it. These\n * are the parts:\n *\n * - The interval of the kind (3 seconds for a page), or the documented pace of the source.\n * - The `crawl-delay` of robots.txt, up to `crawlDelayCapMs`, when it is longer.\n * - After a page or a render, `elapsedFactor` times the duration of the request, up to `elapsedCapMs`.\n * - The `Retry-After` of the server, when it is longer.\n *\n * The backoff after an error or a request to wait comes on top, with jitter. The function is\n * pure, so a simulation and a study can use the pace of the crawler.\n */\nexport const restAfter = (policy: PolitenessPolicy, input: RestInput): Rest => {\n  const interval =\n    input.minIntervalMs ??\n    (input.kind === \"page\"\n      ? policy.pageIntervalMs\n      : input.kind === \"api\"\n        ? policy.apiIntervalMs\n        : policy.renderIntervalMs);\n  const crawlDelay = Math.min(\n    policy.crawlDelayCapMs,\n    Math.max(0, (input.crawlDelaySeconds ?? 0) * 1000),\n  );\n  let ms = Math.max(interval, crawlDelay);\n  let reason: Rest[\"reason\"] = crawlDelay > interval ? \"crawl-delay\" : \"pace\";\n  if (input.kind !== \"api\") {\n    const slow = Math.min(policy.elapsedCapMs, policy.elapsedFactor * Math.max(0, input.elapsedMs));\n    if (slow > ms) {\n      ms = slow;\n      reason = \"slow\";\n    }\n  }\n  if (input.retryAfterMs !== undefined && input.retryAfterMs > ms) {\n    ms = input.retryAfterMs;\n    reason = \"retry-after\";\n  }\n  return { ms, reason };\n};\n\n/**\n * The politeness policy holds each limit of the crawler. Each value has a default, and a host can\n * have values of its own. The times are in milliseconds.\n */\nexport interface PolitenessPolicy {\n  /** This value is the shortest rest of a host between two page requests. The default is 3 seconds. */\n  readonly pageIntervalMs: number;\n  /** This value is the shortest rest of a host between two API requests. The default is 1 second. */\n  readonly apiIntervalMs: number;\n  /** This value is the shortest rest of a host after a render. The default is 5 seconds. */\n  readonly renderIntervalMs: number;\n  /** After a page, the host rests this number of times the duration of the request. The default is 5. */\n  readonly elapsedFactor: number;\n  /** This value is the maximum of the rest that comes from the duration. The default is 30 seconds. */\n  readonly elapsedCapMs: number;\n  /** This value is the maximum `crawl-delay` that the crawler obeys. The default is 60 seconds. */\n  readonly crawlDelayCapMs: number;\n  /** This value is the maximum number of redirects for one fetch. The default is 5. */\n  readonly maxRedirects: number;\n  /** This value is the maximum size of a body after decompression. The default is 5 MiB. */\n  readonly maxBytes: number;\n  /** This value is the deadline of one request, the body included. The default is 90 seconds. */\n  readonly requestTimeoutMs: number;\n  /** This value is the deadline of one render. The default is 20 seconds. */\n  readonly renderTimeoutMs: number;\n  /** After this number of failures in sequence, the host stops. The default is 5. */\n  readonly maxFailures: number;\n  /** This value is the stop after too many failures. The default is 1 hour. */\n  readonly failureStopMs: number;\n  /** This value is the stop after a 401 or a 403. The default is 1 hour. */\n  readonly blockedStopMs: number;\n  /** This value is the stop after a bot challenge. The default is 24 hours. */\n  readonly challengeStopMs: number;\n  /** This value is the stop after a 402. The default is 24 hours. */\n  readonly paymentStopMs: number;\n  /** The crawler obeys a `Retry-After` up to this value. A longer wait stops the host for that time. The default is 1 hour. */\n  readonly maxRetryAfterMs: number;\n  /** This value is the first step of the backoff after an error or a request to wait. The default is 5 seconds. */\n  readonly backoffBaseMs: number;\n  /** This value is the maximum step of the backoff. The default is 300 seconds. */\n  readonly backoffCapMs: number;\n  /** The crawler keeps a robots.txt for this time. The default is 24 hours. */\n  readonly robotsTtlMs: number;\n  /** The crawler keeps an unreachable robots.txt verdict for this time first, with a backoff. The default is 5 minutes. */\n  readonly robotsRetryMs: number;\n  /** This value is the maximum time for an unreachable robots.txt verdict. The default is 1 hour. */\n  readonly robotsRetryCapMs: number;\n  /** A stored robots.txt younger than this age replaces a robots.txt that the crawler cannot read. The default is 30 days. */\n  readonly robotsStaleMs: number;\n  /** This value is the maximum freshness that the crawler gives a response with no freshness of its own. The default is 24 hours. */\n  readonly heuristicFreshnessCapMs: number;\n}\n\n/**\n * This value is the longest stop that a `Retry-After` can give: 30 days. A wait longer than\n * `maxRetryAfterMs` stops the host for the time that the server asks, up to this value.\n */\nexport const MAX_STOP_MS: number = 30 * 24 * 60 * 60 * 1000;\n\nexport const DEFAULT_POLICY: PolitenessPolicy = {\n  pageIntervalMs: 3000,\n  apiIntervalMs: 1000,\n  renderIntervalMs: 5000,\n  elapsedFactor: 5,\n  elapsedCapMs: 30_000,\n  crawlDelayCapMs: 60_000,\n  maxRedirects: 5,\n  maxBytes: 5 * 1024 * 1024,\n  requestTimeoutMs: 90_000,\n  renderTimeoutMs: 20_000,\n  maxFailures: 5,\n  failureStopMs: 60 * 60 * 1000,\n  blockedStopMs: 60 * 60 * 1000,\n  challengeStopMs: 24 * 60 * 60 * 1000,\n  paymentStopMs: 24 * 60 * 60 * 1000,\n  maxRetryAfterMs: 60 * 60 * 1000,\n  backoffBaseMs: 5000,\n  backoffCapMs: 300_000,\n  robotsTtlMs: 24 * 60 * 60 * 1000,\n  robotsRetryMs: 5 * 60 * 1000,\n  robotsRetryCapMs: 60 * 60 * 1000,\n  robotsStaleMs: 30 * 24 * 60 * 60 * 1000,\n  heuristicFreshnessCapMs: 24 * 60 * 60 * 1000,\n};\n\n/**\n * These options change the politeness policy. The values at the top apply to each host. The\n * `hosts` map gives values for one host (`api.example.org`) or for the hosts under a name\n * (`*.example.org`). The most specific entry wins.\n */\nexport interface PolicyOptions extends Partial<PolitenessPolicy> {\n  readonly hosts?: Readonly<Record<string, Partial<PolitenessPolicy>>> | undefined;\n}\n\nconst INTEGER_KEYS: ReadonlySet<string> = new Set([\"maxRedirects\", \"maxFailures\"]);\n\nconst checked = (values: Partial<PolitenessPolicy>, where: string): Partial<PolitenessPolicy> => {\n  for (const [key, value] of Object.entries(values)) {\n    if (!(key in DEFAULT_POLICY))\n      throw new RangeError(`${where}: ${key} is not a politeness setting`);\n    const good =\n      typeof value === \"number\" &&\n      value >= 0 &&\n      (Number.isFinite(value) || key.endsWith(\"StopMs\")) &&\n      (!INTEGER_KEYS.has(key) || Number.isInteger(value));\n    if (!good)\n      throw new RangeError(`${where}: ${key} must be a number of 0 or more, not ${String(value)}`);\n  }\n  return values;\n};\n\n/** This function examines the options of a policy. It throws a `RangeError` for a value that is not a valid number. */\nexport const checkPolicy = (options: PolicyOptions | undefined): void => {\n  if (options === undefined) return;\n  const { hosts, ...base } = options;\n  checked(base, \"politeness\");\n  for (const [host, values] of Object.entries(hosts ?? {}))\n    checked(values, `politeness.hosts[\"${host}\"]`);\n};\n\n/** This function gives the policy of one host: the defaults, then the general options, then the entries for the host. */\nexport const policyFor = (options: PolicyOptions | undefined, host: string): PolitenessPolicy => {\n  if (options === undefined) return DEFAULT_POLICY;\n  const { hosts, ...base } = options;\n  const name = host.toLowerCase().replace(/\\.$/, \"\");\n  const matching = Object.entries(hosts ?? {})\n    .map(([key, values]) => ({ key: key.toLowerCase(), values }))\n    .filter(({ key }) => key === name || (key.startsWith(\"*.\") && name.endsWith(key.slice(1))))\n    // an exact name wins, and a longer pattern wins over a shorter one\n    .sort(\n      (a, b) => (a.key === name ? 1 : 0) - (b.key === name ? 1 : 0) || a.key.length - b.key.length,\n    );\n  return Object.assign(\n    {},\n    DEFAULT_POLICY,\n    base,\n    ...matching.map(({ values }) => values),\n  ) as PolitenessPolicy;\n};\n"],"mappings":";;AAqBA,MAAM,eAAe,KAAK,KAAK;AAE/B,MAAa,YAAmB;CAC9B,WAAW,KAAK,IAAI;CACpB,QAAQ,IAAI,WACV,IAAI,SAAe,YAAY;EAC7B,IAAI,QAAQ,YAAY,MAAM;GAC5B,QAAQ;GACR;EACF;EAEA,IAAI,OAAO,OAAO,SAAS,EAAE,IAAI,KAAK,IAAI,GAAG,EAAE,IAAI;EACnD,IAAI;EACJ,MAAM,aAAmB;GACvB,IAAI,UAAU,KAAA,GAAW,aAAa,KAAK;GAC3C,QAAQ,oBAAoB,SAAS,IAAI;GACzC,QAAQ;EACV;EACA,MAAM,aAAmB;GACvB,IAAI,QAAQ,GAAG;IACb,KAAK;IACL;GACF;GACA,MAAM,OAAO,KAAK,IAAI,MAAM,YAAY;GACxC,QAAQ;GACR,QAAQ,WAAW,MAAM,IAAI;EAC/B;EACA,QAAQ,iBAAiB,SAAS,MAAM,EAAE,MAAM,KAAK,CAAC;EACtD,KAAK;CACP,CAAC;AACL;;AAGA,IAAa,cAAb,cAAiC,MAAM;CAE1B;CACA;CAEA;CAJX,YACE,MACA,QAEA,QAAyB,OAAO,mBAChC;EACA,MAAM,GAAG,KAAK,YAAY,QAAQ;EALzB,KAAA,OAAA;EACA,KAAA,SAAA;EAEA,KAAA,QAAA;CAGX;AACF;;;;;AAsFA,IAAa,uBAAb,MAA4D;CAI7B;CAH7B,yBAA0B,IAAI,IAAyB;CACvD;CAEA,YAAY,WAAoC,KAAQ;EAA3B,KAAA,WAAA;EAC3B,KAAK,UAAU;CACjB;;CAGA,IAAI,OAAe;EACjB,OAAO,KAAK,OAAO;CACrB;CAEA,MAAc,MAA2B;EACvC,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI;EAChC,IAAI,UAAU,KAAA,GAAW;GACvB,QAAQ;IAAE,QAAQ;IAAG,aAAa;IAAG,UAAU;IAAG,OAAO;IAAG,MAAM,KAAA;GAAU;GAC5E,KAAK,OAAO,IAAI,MAAM,KAAK;EAC7B;EACA,OAAO;CACT;CAEA,MAAc,KAAmB;EAC/B,KAAK,MAAM,CAAC,MAAM,UAAU,KAAK,QAO/B,IALE,MAAM,UAAU,OAChB,MAAM,eAAe,OACrB,MAAM,aAAa,KACnB,MAAM,UAAU,MACf,MAAM,SAAS,KAAA,KAAa,MAAM,KAAK,SAAS,MACzC,KAAK,OAAO,OAAO,IAAI;EAGnC,KAAK,UAAU,KAAK,IAAI,KAAK,UAAU,KAAK,KAAK,KAAK,OAAO,OAAO,GAAG,CAAC;CAC1E;CAEA,QAAQ,MAAc,KAAa,SAAyB;EAC1D,IAAI,KAAK,OAAO,OAAO,KAAK,SAAS,KAAK,MAAM,GAAG;EACnD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,MAAM,cAAc,KAAK,OAAO,MAAM,cAAc;EACxD,IAAI,MAAM,SAAS,KAAK,OAAO,MAAM,SAAS;EAC9C,MAAM,cAAc,MAAM;EAC1B,OAAO;CACT;CAEA,OAAO,MAAc,QAAgC;EACnD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,WAAW,WAAW;GACxB,MAAM,WAAW;GACjB,MAAM,QAAQ;EAChB,OAAO,IAAI,WAAW,WAAW,MAAM,YAAY;OAC9C,IAAI,WAAW,QAAQ,MAAM,SAAS;EAC3C,OAAO;GAAE,UAAU,MAAM;GAAU,OAAO,MAAM;EAAM;CACxD;CAEA,QAAQ,MAAc,QAAsB;EAC1C,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,MAAM,cAAc;EACpB,IAAI,SAAS,MAAM,QAAQ,MAAM,SAAS;CAC5C;CAEA,MAAM,MAAc,OAAqB;EACvC,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,QAAQ,MAAM,QAAQ,MAAM,SAAS;CAC3C;CAEA,KAAK,MAAc,OAAe,QAAsB;EACtD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,MAAM,SAAS,KAAA,KAAa,MAAM,KAAK,QAAQ,OAAO,MAAM,OAAO;GAAE;GAAO;EAAO;CACzF;CAEA,QAAQ,MAAc,KAAmC;EACvD,MAAM,OAAO,KAAK,OAAO,IAAI,IAAI,CAAC,EAAE;EACpC,OAAO,SAAS,KAAA,KAAa,KAAK,QAAQ,MAAM,OAAO,KAAA;CACzD;CAEA,MAAM,MAAqB;EACzB,MAAM,SAAS,UAA6B;GAC1C,MAAM,WAAW;GACjB,MAAM,QAAQ;GACd,MAAM,OAAO,KAAA;EACf;EACA,IAAI,SAAS,KAAA,GAAW,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GAAG,MAAM,KAAK;OACxE;GACH,MAAM,QAAQ,KAAK,OAAO,IAAI,IAAI;GAClC,IAAI,UAAU,KAAA,GAAW,MAAM,KAAK;EACtC;CACF;AACF;AAYA,MAAM,WAAW,WACf,OAAO,kBAAkB,QACrB,OAAO,SACP,IAAI,aAAa,wBAAwB,YAAY;AAE3D,IAAa,gBAAb,MAA2B;CAUN;CACA;;CATnB;CACA,yBAA0B,IAAI,IAA8B;CAC5D,SAAiB;CACjB,QAA4C,CAAC;CAC7C;CACA;CAEA,YACE,OACA,WACA,UAA4B,CAAC,GAC7B;EAHiB,KAAA,QAAA;EACA,KAAA,YAAA;EAGjB,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,MAAM,IAAI,WAAW,kDAAkD,OAAO,SAAS,GAAG;EAE5F,KAAK,QAAQ,QAAQ,SAAS,IAAI,qBAAqB;EACvD,KAAK,UAAU,QAAQ,WAAW;EAClC,KAAK,kBAAkB,QAAQ,mBAAmB;CACpD;CAEA,QAAgB,QAAgD;EAC9D,IAAI,KAAK,SAAS,KAAK,WAAW;GAChC,KAAK,UAAU;GACf,OAAO,QAAQ,QAAQ;EACzB;EACA,OAAO,IAAI,SAAe,SAAS,WAAW;GAC5C,MAAM,cAAoB;IACxB,QAAQ,oBAAoB,SAAS,MAAM;IAC3C,QAAQ;GACV;GACA,MAAM,eAAqB;IACzB,MAAM,KAAK,KAAK,MAAM,QAAQ,KAAK;IACnC,IAAI,OAAO,IAAI,KAAK,MAAM,OAAO,IAAI,CAAC;IACtC,OAAO,QAAQ,MAAO,CAAC;GACzB;GACA,KAAK,MAAM,KAAK,KAAK;GACrB,QAAQ,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;EAC1D,CAAC;CACH;CAEA,UAAwB;EAEtB,MAAM,OAAO,KAAK,MAAM,MAAM;EAC9B,IAAI,SAAS,KAAA,GAAW,KAAK;OACxB,KAAK,UAAU;CACtB;CAEA,MAAc,MAAM,IAAY,QAAgD;EAC9E,IAAI,WAAW,KAAA,GAAW;GACxB,MAAM,KAAK,MAAM,MAAM,EAAE;GACzB;EACF;EACA,IAAI,OAAO,SAAS,MAAM,QAAQ,MAAM;EACxC,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,eAAqB,OAAO,QAAQ,MAAM,CAAC;GACjD,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;GACvD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,CAAC,WACrB;IACJ,OAAO,oBAAoB,SAAS,MAAM;IAC1C,QAAQ;GACV,IACC,UAAmB;IAClB,OAAO,oBAAoB,SAAS,MAAM;IAC1C,OAAO,KAAK;GACd,CACF;EACF,CAAC;CACH;;;;;CAMA,IAAO,MAAc,MAA+B,QAAkC;EAEpF,MAAM,UADW,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ,QAAQ,EAAA,CAClC,KAAK,YAAY;GAEvC,IAAI,SAAS,OAAO;GACpB,SAAS;IACP,IAAI,QAAQ,YAAY,MAAM,MAAM,QAAQ,MAAM;IAClD,MAAM,MAAM,KAAK,IAAI,KAAK,MAAM,IAAI,GAAG,MAAM;IAC7C,MAAM,OAAO,KAAK,MAAM,QAAQ,MAAM,GAAG;IACzC,IAAI,SAAS,KAAA,GAAW,MAAM,IAAI,YAAY,MAAM,KAAK,QAAQ,KAAK,KAAK;IAC3E,MAAM,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,KAAK,OAAO;IACvD,IAAI,EAAE,OAAO,IAAI;IACjB,MAAM,KAAK,MAAM,MAAM,MAAM;IAC7B,SAAS,MAAM;GACjB;GACA,IAAI;IACF,MAAM,KAAK,QAAQ,MAAM;GAC3B,SAAS,OAAO;IACd,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC;IACzC,MAAM;GACR;GACA,IAAI,SAAS,KAAK,MAAM,IAAI,IAAI,KAAK;GACrC,IAAI;IACF,MAAM,EAAE,OAAO,YAAY,MAAM,KAAK;IACtC,MAAM,OAAO,OAAO,SAAS,OAAO,KAAK,WAAW,IAAI,UAAU,KAAK;IACvE,SAAS,KAAK,MAAM,IAAI,IAAI;IAC5B,OAAO;GACT,SAAS,OAAO;IACd,SAAS,KAAK,MAAM,IAAI,IAAI,KAAK;IACjC,MAAM;GACR,UAAU;IACR,KAAK,MAAM,QAAQ,MAAM,MAAM;IAC/B,KAAK,QAAQ;GACf;EACF,CAAC;EACD,MAAM,OAAO,OAAO,WACZ,KAAA,SACA,KAAA,CACR;EACA,KAAK,OAAO,IAAI,MAAM,IAAI;EAE1B,KAAU,WAAW;GACnB,IAAI,KAAK,OAAO,IAAI,IAAI,MAAM,MAAM,KAAK,OAAO,OAAO,IAAI;EAC7D,CAAC;EACD,OAAO;CACT;;CAGA,KAAK,MAAc,SAAuB;EACxC,KAAK,MAAM,MAAM,MAAM,OAAO;CAChC;;CAGA,KAAK,MAAc,QAAgB,UAAkB,OAAO,mBAAyB;EACnF,KAAK,MAAM,KAAK,MAAM,SAAS,MAAM;CACvC;CAEA,cAAc,MAAkC;EAC9C,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,EAAE;CACrD;;CAGA,OAAO,MAAoC;EACzC,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC;CAClD;;CAGA,OAAO,MAAsB;EAC3B,OAAO,KAAK,MAAM,OAAO,MAAM,SAAS,CAAC,CAAC;CAC5C;;CAGA,OAAO,MAAsB;EAC3B,OAAO,KAAK,MAAM,OAAO,MAAM,MAAM,CAAC,CAAC;CACzC;CAEA,UAAU,MAAoB;EAC5B,KAAK,MAAM,OAAO,MAAM,SAAS;CACnC;;CAGA,MAAM,MAAqB;EACzB,KAAK,MAAM,MAAM,IAAI;CACvB;AACF;;;;;AAMA,MAAa,aACX,SACA,QACA,OACA,WACW,KAAK,MAAM,OAAO,IAAI,KAAK,IAAI,OAAO,SAAS,KAAK,KAAK,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC;;;;;;AAO1F,MAAa,iBACX,SACA,QACA,OACA,WACW;CACX,MAAM,OAAO,KAAK,IAAI,OAAO,SAAS,KAAK,KAAK,IAAI,GAAG,UAAU,CAAC,CAAC;CACnE,OAAO,KAAK,MAAM,OAAO,IAAI,OAAO,KAAK,OAAO,EAAE;AACpD;;;;;;;;;;;;;AAkCA,MAAa,aAAa,QAA0B,UAA2B;CAC7E,MAAM,WACJ,MAAM,kBACL,MAAM,SAAS,SACZ,OAAO,iBACP,MAAM,SAAS,QACb,OAAO,gBACP,OAAO;CACf,MAAM,aAAa,KAAK,IACtB,OAAO,iBACP,KAAK,IAAI,IAAI,MAAM,qBAAqB,KAAK,GAAI,CACnD;CACA,IAAI,KAAK,KAAK,IAAI,UAAU,UAAU;CACtC,IAAI,SAAyB,aAAa,WAAW,gBAAgB;CACrE,IAAI,MAAM,SAAS,OAAO;EACxB,MAAM,OAAO,KAAK,IAAI,OAAO,cAAc,OAAO,gBAAgB,KAAK,IAAI,GAAG,MAAM,SAAS,CAAC;EAC9F,IAAI,OAAO,IAAI;GACb,KAAK;GACL,SAAS;EACX;CACF;CACA,IAAI,MAAM,iBAAiB,KAAA,KAAa,MAAM,eAAe,IAAI;EAC/D,KAAK,MAAM;EACX,SAAS;CACX;CACA,OAAO;EAAE;EAAI;CAAO;AACtB;;;;;AA2DA,MAAa,cAAsB;AAEnC,MAAa,iBAAmC;CAC9C,gBAAgB;CAChB,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,cAAc;CACd,iBAAiB;CACjB,cAAc;CACd,UAAU;CACV,kBAAkB;CAClB,iBAAiB;CACjB,aAAa;CACb,eAAe;CACf,eAAe;CACf,iBAAiB;CACjB,eAAe;CACf,iBAAiB;CACjB,eAAe;CACf,cAAc;CACd,aAAa;CACb,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,yBAAyB;AAC3B;AAWA,MAAM,+BAAoC,IAAI,IAAI,CAAC,gBAAgB,aAAa,CAAC;AAEjF,MAAM,WAAW,QAAmC,UAA6C;CAC/F,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,MAAM,GAAG;EACjD,IAAI,EAAE,OAAO,iBACX,MAAM,IAAI,WAAW,GAAG,MAAM,IAAI,IAAI,6BAA6B;EAMrE,IAAI,EAJF,OAAO,UAAU,YACjB,SAAS,MACR,OAAO,SAAS,KAAK,KAAK,IAAI,SAAS,QAAQ,OAC/C,CAAC,aAAa,IAAI,GAAG,KAAK,OAAO,UAAU,KAAK,KAEjD,MAAM,IAAI,WAAW,GAAG,MAAM,IAAI,IAAI,sCAAsC,OAAO,KAAK,GAAG;CAC/F;CACA,OAAO;AACT;;AAGA,MAAa,eAAe,YAA6C;CACvE,IAAI,YAAY,KAAA,GAAW;CAC3B,MAAM,EAAE,OAAO,GAAG,SAAS;CAC3B,QAAQ,MAAM,YAAY;CAC1B,KAAK,MAAM,CAAC,MAAM,WAAW,OAAO,QAAQ,SAAS,CAAC,CAAC,GACrD,QAAQ,QAAQ,qBAAqB,KAAK,GAAG;AACjD;;AAGA,MAAa,aAAa,SAAoC,SAAmC;CAC/F,IAAI,YAAY,KAAA,GAAW,OAAO;CAClC,MAAM,EAAE,OAAO,GAAG,SAAS;CAC3B,MAAM,OAAO,KAAK,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;CACjD,MAAM,WAAW,OAAO,QAAQ,SAAS,CAAC,CAAC,CAAC,CACzC,KAAK,CAAC,KAAK,aAAa;EAAE,KAAK,IAAI,YAAY;EAAG;CAAO,EAAE,CAAC,CAC5D,QAAQ,EAAE,UAAU,QAAQ,QAAS,IAAI,WAAW,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAE1F,MACE,GAAG,OAAO,EAAE,QAAQ,OAAO,IAAI,MAAM,EAAE,QAAQ,OAAO,IAAI,MAAM,EAAE,IAAI,SAAS,EAAE,IAAI,MACxF;CACF,OAAO,OAAO,OACZ,CAAC,GACD,gBACA,MACA,GAAG,SAAS,KAAK,EAAE,aAAa,MAAM,CACxC;AACF"}
+{"version":3,"file":"politeness.mjs","names":[],"sources":["../src/politeness.ts"],"sourcesContent":["/**\n * The scheduler controls the pace of each host. The speed of a polite crawler is the number of\n * hosts in progress divided by the delay for each host. Thus the scheduler keeps one request in\n * progress for each host, and it waits the delay of the host between two requests. It uses its\n * concurrency across many hosts, as the back queues of the Mercator crawler do.\n *\n * The caller decides the delay after each request, and `run` gets it from the task. Only the caller\n * knows the documented rate of the source, the crawl delay of the site and the duration of the\n * request. A wait holds no global slot, so a host that waits does not block a different host.\n *\n * The state of each host (its next time, its counts and its stop) is in a `HostStateStore`. The\n * default store is in memory. A store that processes share keeps one pace for each host across them.\n */\n\nexport interface Clock {\n  readonly now: () => number;\n  /** This function waits `ms` milliseconds. When the signal aborts, the wait ends early. */\n  readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;\n}\n\n/** A timer of Node.js holds a maximum of 2^31 − 1 milliseconds. A longer timer fires at once. */\nconst MAX_TIMER_MS = 2 ** 31 - 1;\n\nexport const realClock: Clock = {\n  now: () => Date.now(),\n  sleep: (ms, signal) =>\n    new Promise<void>((resolve) => {\n      if (signal?.aborted === true) {\n        resolve();\n        return;\n      }\n      // a long wait goes in parts, because a timer longer than the maximum fires at once\n      let left = Number.isFinite(ms) ? Math.max(0, ms) : 0;\n      let timer: ReturnType<typeof setTimeout> | undefined;\n      const done = (): void => {\n        if (timer !== undefined) clearTimeout(timer);\n        signal?.removeEventListener(\"abort\", done);\n        resolve();\n      };\n      const step = (): void => {\n        if (left <= 0) {\n          done();\n          return;\n        }\n        const part = Math.min(left, MAX_TIMER_MS);\n        left -= part;\n        timer = setTimeout(step, part);\n      };\n      signal?.addEventListener(\"abort\", done, { once: true });\n      step();\n    }),\n};\n\n/** The crawler does not ask a host again until the stop ends, after a refusal (a challenge, 403 or 402). */\nexport class HostStopped extends Error {\n  constructor(\n    readonly host: string,\n    readonly reason: string,\n    /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */\n    readonly until: number = Number.POSITIVE_INFINITY,\n  ) {\n    super(`${host} stopped: ${reason}`);\n  }\n}\n\nexport interface Paced<T> {\n  readonly value: T;\n  /** This value is the time that the host rests before its next request. */\n  readonly delayMs: number;\n}\n\n/** A stop keeps the crawler away from a host until a time. */\nexport interface HostStop {\n  /** This value is the end of the stop, in milliseconds since the epoch. `Infinity` is a stop with no end. */\n  readonly until: number;\n  readonly reason: string;\n}\n\n/** These are the counts of a host. Each count gives the results of one kind in sequence. */\nexport interface HostCounts {\n  /** This value counts errors and requests with no answer. At the limit of the policy, the host stops. */\n  readonly failures: number;\n  /** This value counts the requests to wait (429, 502, 503, 504). It sets the backoff and does not stop the host. */\n  readonly waits: number;\n}\n\n/**\n * This value tells how one result changes the counts of a host:\n *\n * - `success` sets the two counts to zero.\n * - `failure` adds one to `failures`.\n * - `wait` adds one to `waits`.\n * - `neutral` changes nothing. A client error (a 400) is neutral, because a different request can succeed.\n */\nexport type HostResult = \"success\" | \"failure\" | \"wait\" | \"neutral\";\n\n/**\n * A host state store keeps the state of each host for the scheduler. Its methods are synchronous,\n * and each method is one atomic step on the state of one host. Thus a store can be a table of\n * SQLite that processes share. Times are milliseconds since the epoch.\n *\n * The state of a host has five parts:\n *\n * - the next time: the earliest time of the next request\n * - the lease: the end of the slot of the request in progress, when there is one\n * - the counts (`HostCounts`)\n * - the stop: an end time and a reason, when there is one\n */\nexport interface HostStateStore {\n  /**\n   * This method asks for the slot of a host at the time `now`. When the host has no rest left\n   * and no lease, the store gives the slot to the caller until `now + leaseMs` and gives 0.\n   * Otherwise it changes nothing and gives the time to wait in milliseconds: the time to the next\n   * time, or to the end of the lease. A store that processes share gives a short wait (one second\n   * or less) during a lease, because the holder usually releases it early. The caller asks again after the wait.\n   */\n  reserve(host: string, now: number, leaseMs: number): number;\n  /** This method changes the counts of a host for one result, and it gives the new counts. */\n  record(host: string, result: HostResult): HostCounts;\n  /**\n   * This method ends the lease of the caller. It sets the next time to `nextAt` when `nextAt` is\n   * later than the stored next time. Only the holder of the lease uses it.\n   */\n  release(host: string, nextAt: number): void;\n  /** This method sets the next time to `until` when `until` is later. It does not touch the lease. */\n  defer(host: string, until: number): void;\n  /** This method stops a host until a time. When the host has a stop that ends later, that stop stays. */\n  stop(host: string, until: number, reason: string): void;\n  /** This method gives the stop of a host at the time `now`. A stop that ended gives no value. */\n  stopped(host: string, now: number): HostStop | undefined;\n  /**\n   * This method removes the stops and the counts of one host, or of each host when there is no\n   * host. It keeps the next times and the leases, so a reset cannot make a host go faster than its pace.\n   */\n  reset(host?: string): void;\n  /**\n   * This method gives the validator memo of a host, when the store has one. A store without this\n   * method and `noteValidators` has no memo. The crawler then sends each validator of a stored copy.\n   */\n  validators?(host: string): ValidatorMemo | undefined;\n  /**\n   * This method adds one conditional request to the memo of a host, with the rules of\n   * `nextValidatorMemo`, in one atomic step. It gives the new memo.\n   */\n  noteValidators?(host: string, report: ValidatorReport, now: number): ValidatorMemo;\n}\n\n/** This value tells the result that the crawler learned for one validator of a host. */\nexport type ValidatorVerdict = \"unknown\" | \"works\" | \"fails\";\n\n/**\n * This value tells what one conditional request got:\n *\n * - `not-modified`: a 304.\n * - `same`: a 200 with the body that the store has. Thus the validator did not match.\n * - `changed`: a 200 with a different body. This answer tells nothing about the validator.\n */\nexport type ValidatorAnswer = \"not-modified\" | \"same\" | \"changed\";\n\n/** This report tells the validators that one conditional request sent, and the answer that it got. */\nexport interface ValidatorReport {\n  /** This value is true when the request sent `If-None-Match`. */\n  readonly etag: boolean;\n  /** This value is true when the request sent `If-Modified-Since`. */\n  readonly lastModified: boolean;\n  readonly answer: ValidatorAnswer;\n}\n\n/**\n * A validator memo holds what the crawler learned about the validators of one host. A server\n * examines `If-None-Match` first, and it ignores `If-Modified-Since` when the request has the two\n * (RFC 9110, section 13.1.3). Thus a host with unstable ETags does not answer 304 to a request with\n * the two validators at any time. Its stable `Last-Modified` date does not help. `nextValidatorMemo`\n * gives the rules.\n */\nexport interface ValidatorMemo {\n  /** This value is true when the crawler leaves out `If-None-Match` for the host and sends only `If-Modified-Since`. */\n  readonly skipEtag: boolean;\n  /** This value tells if a request with `If-None-Match` got a 304 (`works`) or the stored body again (`fails`). */\n  readonly etag: ValidatorVerdict;\n  /** This value tells the same for a request with only `If-Modified-Since`. */\n  readonly lastModified: ValidatorVerdict;\n  /** This value counts the stored bodies in sequence that came back to a request with `If-None-Match`. */\n  readonly etagMisses: number;\n  /** This value counts the stored bodies in sequence that came back to a request with only `If-Modified-Since`. */\n  readonly dateMisses: number;\n  /** This value counts the requests with only `If-Modified-Since` since the crawler stopped the ETag, or since the last probe. */\n  readonly sinceProbe: number;\n  /** This time (milliseconds since the epoch) is the last time that the crawler stopped or started the ETag, or probed it. */\n  readonly changedAt: number;\n}\n\n/** These are the limits of the validator memo. */\nexport interface ValidatorLimits {\n  /** After this number of stored bodies in sequence, a validator fails. The value is 3. */\n  readonly misses: number;\n  /** After this number of requests with only `If-Modified-Since`, the next request sends the ETag again (a probe). The value is 20. */\n  readonly probeEvery: number;\n  /** After this time with no probe, the next request is a probe. The value is 7 days. */\n  readonly probeAgeMs: number;\n}\n\n/** These are the limits of the validator memo of the crawler. */\nexport const VALIDATOR_MEMO: ValidatorLimits = {\n  misses: 3,\n  probeEvery: 20,\n  probeAgeMs: 7 * 24 * 60 * 60 * 1000,\n};\n\nconst EMPTY_MEMO: ValidatorMemo = {\n  skipEtag: false,\n  etag: \"unknown\",\n  lastModified: \"unknown\",\n  etagMisses: 0,\n  dateMisses: 0,\n  sinceProbe: 0,\n  changedAt: 0,\n};\n\n/**\n * This function tells which validators the next conditional request to a host sends. It sends\n * each validator of the stored copy, except when the memo stopped the ETag and the copy has a\n * `Last-Modified` date. Then it sends only `If-Modified-Since`, and the ETag too for a probe.\n * A probe comes after `probeEvery` requests or after `probeAgeMs`.\n */\nexport const validatorsToSend = (\n  memo: ValidatorMemo | undefined,\n  stored: { readonly etag: boolean; readonly lastModified: boolean },\n  now: number,\n): { readonly etag: boolean; readonly lastModified: boolean } => {\n  if (memo?.skipEtag !== true || !stored.etag || !stored.lastModified) return stored;\n  const probe =\n    memo.sinceProbe >= VALIDATOR_MEMO.probeEvery ||\n    now - memo.changedAt >= VALIDATOR_MEMO.probeAgeMs;\n  return { etag: probe, lastModified: true };\n};\n\n/**\n * This function adds one conditional request to the memo of a host, and gives the new memo.\n * These are the rules, with the limits of `VALIDATOR_MEMO`:\n *\n * 1. A 304 to a request with `If-None-Match` marks the ETag `works`. After a probe, the crawler sends the ETag again.\n * 2. After `misses` stored bodies in sequence with `If-None-Match`, the ETag fails. When the request also sent\n *    `If-Modified-Since`, the crawler stops the ETag for the host.\n * 3. A probe that gets a 200 keeps the ETag stopped, and the count to the next probe starts again.\n * 4. After `misses` stored bodies in sequence with only `If-Modified-Since`, that validator fails too. Then the\n *    crawler sends the two validators again, because they cost nothing. It tries the date alone again after `probeAgeMs`.\n * 5. A changed body tells nothing about a validator.\n */\nexport const nextValidatorMemo = (\n  memo: ValidatorMemo | undefined,\n  report: ValidatorReport,\n  now: number,\n): ValidatorMemo => {\n  const next: { -readonly [K in keyof ValidatorMemo]: ValidatorMemo[K] } = {\n    ...(memo ?? EMPTY_MEMO),\n  };\n  if (report.etag) {\n    const probe = next.skipEtag;\n    if (report.answer === \"not-modified\") {\n      next.etag = \"works\";\n      next.etagMisses = 0;\n      if (probe) {\n        next.skipEtag = false;\n        next.sinceProbe = 0;\n        next.changedAt = now;\n      }\n    } else if (probe) {\n      if (report.answer === \"same\") next.etagMisses += 1;\n      next.sinceProbe = 0;\n      next.changedAt = now;\n    } else if (report.answer === \"same\") {\n      next.etagMisses += 1;\n      if (next.etagMisses >= VALIDATOR_MEMO.misses) {\n        next.etag = \"fails\";\n        // a date that failed gets a new try only after probeAgeMs, so the memo does not swing at each request\n        const dateLeft =\n          next.lastModified !== \"fails\" || now - next.changedAt >= VALIDATOR_MEMO.probeAgeMs;\n        if (report.lastModified && dateLeft) {\n          next.skipEtag = true;\n          next.dateMisses = 0;\n          next.sinceProbe = 0;\n          next.changedAt = now;\n        }\n      }\n    }\n  } else if (report.lastModified) {\n    if (next.skipEtag) next.sinceProbe += 1;\n    if (report.answer === \"not-modified\") {\n      next.lastModified = \"works\";\n      next.dateMisses = 0;\n    } else if (report.answer === \"same\") {\n      next.dateMisses += 1;\n      if (next.dateMisses >= VALIDATOR_MEMO.misses) {\n        next.lastModified = \"fails\";\n        if (next.skipEtag) {\n          next.skipEtag = false;\n          next.sinceProbe = 0;\n          next.changedAt = now;\n        }\n      }\n    }\n  }\n  return next;\n};\n\ninterface MemoryState {\n  nextAt: number;\n  leasedUntil: number;\n  failures: number;\n  waits: number;\n  stop: HostStop | undefined;\n}\n\n/**\n * This store keeps the state of each host in memory, for one process. It removes the state of an\n * idle host (no rest left, no lease, no counts and no stop) when it holds more than `maxHosts` hosts.\n * It keeps the validator memos of the `maxHosts` hosts that it used last.\n */\nexport class MemoryHostStateStore implements HostStateStore {\n  private readonly states = new Map<string, MemoryState>();\n  /** The validator memos, the memo used last at the end. */\n  private readonly memos = new Map<string, ValidatorMemo>();\n  private sweepAt: number;\n\n  constructor(private readonly maxHosts: number = 10_000) {\n    this.sweepAt = maxHosts;\n  }\n\n  /** This value is the number of hosts that the store holds. */\n  get size(): number {\n    return this.states.size;\n  }\n\n  private state(host: string): MemoryState {\n    let state = this.states.get(host);\n    if (state === undefined) {\n      state = { nextAt: 0, leasedUntil: 0, failures: 0, waits: 0, stop: undefined };\n      this.states.set(host, state);\n    }\n    return state;\n  }\n\n  private sweep(now: number): void {\n    for (const [host, state] of this.states) {\n      const idle =\n        state.nextAt <= now &&\n        state.leasedUntil <= now &&\n        state.failures === 0 &&\n        state.waits === 0 &&\n        (state.stop === undefined || state.stop.until <= now);\n      if (idle) this.states.delete(host);\n    }\n    // the next sweep comes after a tenth more hosts, so a store of busy hosts does not sweep at each request\n    this.sweepAt = Math.max(this.maxHosts, Math.ceil(this.states.size * 1.1));\n  }\n\n  reserve(host: string, now: number, leaseMs: number): number {\n    if (this.states.size > this.sweepAt) this.sweep(now);\n    const state = this.state(host);\n    if (state.leasedUntil > now) return state.leasedUntil - now;\n    if (state.nextAt > now) return state.nextAt - now;\n    state.leasedUntil = now + leaseMs;\n    return 0;\n  }\n\n  record(host: string, result: HostResult): HostCounts {\n    const state = this.state(host);\n    if (result === \"success\") {\n      state.failures = 0;\n      state.waits = 0;\n    } else if (result === \"failure\") state.failures += 1;\n    else if (result === \"wait\") state.waits += 1;\n    return { failures: state.failures, waits: state.waits };\n  }\n\n  release(host: string, nextAt: number): void {\n    const state = this.state(host);\n    state.leasedUntil = 0;\n    if (nextAt > state.nextAt) state.nextAt = nextAt;\n  }\n\n  defer(host: string, until: number): void {\n    const state = this.state(host);\n    if (until > state.nextAt) state.nextAt = until;\n  }\n\n  stop(host: string, until: number, reason: string): void {\n    const state = this.state(host);\n    if (state.stop === undefined || state.stop.until < until) state.stop = { until, reason };\n  }\n\n  stopped(host: string, now: number): HostStop | undefined {\n    const stop = this.states.get(host)?.stop;\n    return stop !== undefined && stop.until > now ? stop : undefined;\n  }\n\n  reset(host?: string): void {\n    const clear = (state: MemoryState): void => {\n      state.failures = 0;\n      state.waits = 0;\n      state.stop = undefined;\n    };\n    if (host === undefined) for (const state of this.states.values()) clear(state);\n    else {\n      const state = this.states.get(host);\n      if (state !== undefined) clear(state);\n    }\n  }\n\n  validators(host: string): ValidatorMemo | undefined {\n    return this.memos.get(host);\n  }\n\n  noteValidators(host: string, report: ValidatorReport, now: number): ValidatorMemo {\n    const memo = nextValidatorMemo(this.memos.get(host), report, now);\n    // the memo of each change goes to the end, so the first memo is the memo used longest ago\n    this.memos.delete(host);\n    this.memos.set(host, memo);\n    if (this.memos.size > this.maxHosts) this.memos.delete(this.memos.keys().next().value!);\n    return memo;\n  }\n}\n\n/** These options change the scheduler. */\nexport interface SchedulerOptions {\n  /** This store keeps the state of each host. The default is a `MemoryHostStateStore`. */\n  readonly store?: HostStateStore | undefined;\n  /** This value is the longest slot of one request. A slot that a crashed process holds ends after it. The default is 5 minutes. */\n  readonly leaseMs?: number | undefined;\n  /** A host rests this long after a task that throws an error, or a task that gives no valid delay. The default is 1 second. */\n  readonly fallbackDelayMs?: number | undefined;\n}\n\nconst aborted = (signal: AbortSignal): Error =>\n  signal.reason instanceof Error\n    ? signal.reason\n    : new DOMException(\"The wait was aborted\", \"AbortError\");\n\nexport class HostScheduler {\n  /** This store keeps the state of each host. */\n  readonly store: HostStateStore;\n  private readonly chains = new Map<string, Promise<unknown>>();\n  private active = 0;\n  private readonly queue: Array<() => void> = [];\n  private readonly leaseMs: number;\n  private readonly fallbackDelayMs: number;\n\n  constructor(\n    private readonly clock: Clock,\n    private readonly maxActive: number,\n    options: SchedulerOptions = {},\n  ) {\n    if (!Number.isInteger(maxActive) || maxActive < 1) {\n      throw new RangeError(`maxActive must be an integer of 1 or more, not ${String(maxActive)}`);\n    }\n    this.store = options.store ?? new MemoryHostStateStore();\n    this.leaseMs = options.leaseMs ?? 300_000;\n    this.fallbackDelayMs = options.fallbackDelayMs ?? 1000;\n  }\n\n  private acquire(signal: AbortSignal | undefined): Promise<void> {\n    if (this.active < this.maxActive) {\n      this.active += 1;\n      return Promise.resolve();\n    }\n    return new Promise<void>((resolve, reject) => {\n      const grant = (): void => {\n        signal?.removeEventListener(\"abort\", cancel);\n        resolve();\n      };\n      const cancel = (): void => {\n        const at = this.queue.indexOf(grant);\n        if (at !== -1) this.queue.splice(at, 1);\n        reject(aborted(signal!));\n      };\n      this.queue.push(grant);\n      signal?.addEventListener(\"abort\", cancel, { once: true });\n    });\n  }\n\n  private release(): void {\n    // the slot goes to the next waiter directly, so a new request cannot take it in between\n    const next = this.queue.shift();\n    if (next !== undefined) next();\n    else this.active -= 1;\n  }\n\n  private async sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {\n    if (signal === undefined) {\n      await this.clock.sleep(ms);\n      return;\n    }\n    if (signal.aborted) throw aborted(signal);\n    await new Promise<void>((resolve, reject) => {\n      const cancel = (): void => reject(aborted(signal));\n      signal.addEventListener(\"abort\", cancel, { once: true });\n      this.clock.sleep(ms, signal).then(\n        () => {\n          signal.removeEventListener(\"abort\", cancel);\n          resolve();\n        },\n        (error: unknown) => {\n          signal.removeEventListener(\"abort\", cancel);\n          reject(error);\n        },\n      );\n    });\n  }\n\n  /**\n   * This method starts one request on a host, in sequence, after the rest of the host. When the\n   * signal aborts during the wait, the promise rejects with the reason of the signal.\n   */\n  run<T>(host: string, task: () => Promise<Paced<T>>, signal?: AbortSignal): Promise<T> {\n    const previous = this.chains.get(host) ?? Promise.resolve();\n    const result = previous.then(async () => {\n      // after a wait, the time is at least the end of the wait, also on a clock that a test holds still\n      let waited = Number.NEGATIVE_INFINITY;\n      for (;;) {\n        if (signal?.aborted === true) throw aborted(signal);\n        const now = Math.max(this.clock.now(), waited);\n        const stop = this.store.stopped(host, now);\n        if (stop !== undefined) throw new HostStopped(host, stop.reason, stop.until);\n        const wait = this.store.reserve(host, now, this.leaseMs);\n        if (!(wait > 0)) break;\n        await this.sleep(wait, signal);\n        waited = now + wait;\n      }\n      try {\n        await this.acquire(signal);\n      } catch (error) {\n        this.store.release(host, this.clock.now());\n        throw error;\n      }\n      let nextAt = this.clock.now() + this.fallbackDelayMs;\n      try {\n        const { value, delayMs } = await task();\n        const rest = Number.isFinite(delayMs) && delayMs >= 0 ? delayMs : this.fallbackDelayMs;\n        nextAt = this.clock.now() + rest;\n        return value;\n      } catch (error) {\n        nextAt = this.clock.now() + this.fallbackDelayMs;\n        throw error;\n      } finally {\n        this.store.release(host, nextAt);\n        this.release();\n      }\n    });\n    const tail = result.then(\n      () => undefined,\n      () => undefined,\n    );\n    this.chains.set(host, tail);\n    // a host with no queued request keeps no chain\n    void tail.then(() => {\n      if (this.chains.get(host) === tail) this.chains.delete(host);\n    });\n    return result;\n  }\n\n  /** The server asked for a pause (Retry-After), or a backoff chose one. */\n  park(host: string, untilMs: number): void {\n    this.store.defer(host, untilMs);\n  }\n\n  /** This method stops a host until `untilMs`. With no time, the stop has no end. */\n  stop(host: string, reason: string, untilMs: number = Number.POSITIVE_INFINITY): void {\n    this.store.stop(host, untilMs, reason);\n  }\n\n  stoppedReason(host: string): string | undefined {\n    return this.store.stopped(host, this.clock.now())?.reason;\n  }\n\n  /** This method gives the stop of a host, with its end, when the host is stopped. */\n  stopOf(host: string): HostStop | undefined {\n    return this.store.stopped(host, this.clock.now());\n  }\n\n  /** This value counts the failures of a host in sequence, for the backoff. A success sets it to zero. */\n  failed(host: string): number {\n    return this.store.record(host, \"failure\").failures;\n  }\n\n  /** This value counts the requests to wait (429, 503) of a host in sequence. They do not stop the host. */\n  waited(host: string): number {\n    return this.store.record(host, \"wait\").waits;\n  }\n\n  succeeded(host: string): void {\n    this.store.record(host, \"success\");\n  }\n\n  /** This method removes the stops and the counts of one host, or of each host. The pace stays. */\n  reset(host?: string): void {\n    this.store.reset(host);\n  }\n}\n\n/**\n * This function gives an exponential backoff with full jitter, from the analysis of AWS. The wait\n * is random, up to base × 2^(try − 1), with a maximum. The jitter keeps retries apart.\n */\nexport const backoffMs = (\n  attempt: number,\n  baseMs: number,\n  capMs: number,\n  random: () => number,\n): number => Math.floor(random() * Math.min(capMs, baseMs * 2 ** Math.max(0, attempt - 1)));\n\n/**\n * This function gives an exponential backoff with equal jitter. The wait is half of\n * base × 2^(try − 1), with a maximum, and a random part up to the other half. Thus each backoff\n * is at least half of its step, and it is not lost under the rest of the host.\n */\nexport const equalJitterMs = (\n  attempt: number,\n  baseMs: number,\n  capMs: number,\n  random: () => number,\n): number => {\n  const step = Math.min(capMs, baseMs * 2 ** Math.max(0, attempt - 1));\n  return Math.floor(step / 2 + random() * (step / 2));\n};\n\n/** These are the facts of one request that set the rest of its host. */\nexport interface RestInput {\n  /** This value is the kind of the request: a web page, a documented API, or a render in the browser. */\n  readonly kind: \"page\" | \"api\" | \"render\";\n  /** This value is the duration of the request, in milliseconds. */\n  readonly elapsedMs: number;\n  /** This value is the `crawl-delay` of robots.txt, in seconds. */\n  readonly crawlDelaySeconds?: number | undefined;\n  /** This value is the documented pace of the source. It replaces the interval of the policy for the kind. */\n  readonly minIntervalMs?: number | undefined;\n  /** This value is the wait that the server asked for (`Retry-After`). */\n  readonly retryAfterMs?: number | undefined;\n}\n\n/** The rest of a host after a request, and the reason for its length. */\nexport interface Rest {\n  readonly ms: number;\n  readonly reason: \"pace\" | \"slow\" | \"retry-after\" | \"crawl-delay\";\n}\n\n/**\n * This function gives the rest of a host after one request, as the crawler calculates it. These\n * are the parts:\n *\n * - The interval of the kind (3 seconds for a page), or the documented pace of the source.\n * - The `crawl-delay` of robots.txt, up to `crawlDelayCapMs`, when it is longer.\n * - After a page or a render, `elapsedFactor` times the duration of the request, up to `elapsedCapMs`.\n * - The `Retry-After` of the server, when it is longer.\n *\n * The backoff after an error or a request to wait comes on top, with jitter. The function is\n * pure, so a simulation and a study can use the pace of the crawler.\n */\nexport const restAfter = (policy: PolitenessPolicy, input: RestInput): Rest => {\n  const interval =\n    input.minIntervalMs ??\n    (input.kind === \"page\"\n      ? policy.pageIntervalMs\n      : input.kind === \"api\"\n        ? policy.apiIntervalMs\n        : policy.renderIntervalMs);\n  const crawlDelay = Math.min(\n    policy.crawlDelayCapMs,\n    Math.max(0, (input.crawlDelaySeconds ?? 0) * 1000),\n  );\n  let ms = Math.max(interval, crawlDelay);\n  let reason: Rest[\"reason\"] = crawlDelay > interval ? \"crawl-delay\" : \"pace\";\n  if (input.kind !== \"api\") {\n    const slow = Math.min(policy.elapsedCapMs, policy.elapsedFactor * Math.max(0, input.elapsedMs));\n    if (slow > ms) {\n      ms = slow;\n      reason = \"slow\";\n    }\n  }\n  if (input.retryAfterMs !== undefined && input.retryAfterMs > ms) {\n    ms = input.retryAfterMs;\n    reason = \"retry-after\";\n  }\n  return { ms, reason };\n};\n\n/**\n * The politeness policy holds each limit of the crawler. Each value has a default, and a host can\n * have values of its own. The times are in milliseconds.\n */\nexport interface PolitenessPolicy {\n  /** This value is the shortest rest of a host between two page requests. The default is 3 seconds. */\n  readonly pageIntervalMs: number;\n  /** This value is the shortest rest of a host between two API requests. The default is 1 second. */\n  readonly apiIntervalMs: number;\n  /** This value is the shortest rest of a host after a render. The default is 5 seconds. */\n  readonly renderIntervalMs: number;\n  /** After a page, the host rests this number of times the duration of the request. The default is 5. */\n  readonly elapsedFactor: number;\n  /** This value is the maximum of the rest that comes from the duration. The default is 30 seconds. */\n  readonly elapsedCapMs: number;\n  /** This value is the maximum `crawl-delay` that the crawler obeys. The default is 60 seconds. */\n  readonly crawlDelayCapMs: number;\n  /** This value is the maximum number of redirects for one fetch. The default is 5. */\n  readonly maxRedirects: number;\n  /** This value is the maximum size of a body after decompression. The default is 5 MiB. */\n  readonly maxBytes: number;\n  /** This value is the deadline of one request, the body included. The default is 90 seconds. */\n  readonly requestTimeoutMs: number;\n  /** This value is the deadline of one render. The default is 20 seconds. */\n  readonly renderTimeoutMs: number;\n  /** After this number of failures in sequence, the host stops. The default is 5. */\n  readonly maxFailures: number;\n  /** This value is the stop after too many failures. The default is 1 hour. */\n  readonly failureStopMs: number;\n  /** This value is the stop after a 401 or a 403. The default is 1 hour. */\n  readonly blockedStopMs: number;\n  /** This value is the stop after a bot challenge. The default is 24 hours. */\n  readonly challengeStopMs: number;\n  /** This value is the stop after a 402. The default is 24 hours. */\n  readonly paymentStopMs: number;\n  /** The crawler obeys a `Retry-After` up to this value. A longer wait stops the host for that time. The default is 1 hour. */\n  readonly maxRetryAfterMs: number;\n  /** This value is the first step of the backoff after an error or a request to wait. The default is 5 seconds. */\n  readonly backoffBaseMs: number;\n  /** This value is the maximum step of the backoff. The default is 300 seconds. */\n  readonly backoffCapMs: number;\n  /** The crawler keeps a robots.txt for this time. The default is 24 hours. */\n  readonly robotsTtlMs: number;\n  /** The crawler keeps an unreachable robots.txt verdict for this time first, with a backoff. The default is 5 minutes. */\n  readonly robotsRetryMs: number;\n  /** This value is the maximum time for an unreachable robots.txt verdict. The default is 1 hour. */\n  readonly robotsRetryCapMs: number;\n  /** A stored robots.txt younger than this age replaces a robots.txt that the crawler cannot read. The default is 30 days. */\n  readonly robotsStaleMs: number;\n  /** This value is the maximum freshness that the crawler gives a response with no freshness of its own. The default is 24 hours. */\n  readonly heuristicFreshnessCapMs: number;\n}\n\n/**\n * This value is the longest stop that a `Retry-After` can give: 30 days. A wait longer than\n * `maxRetryAfterMs` stops the host for the time that the server asks, up to this value.\n */\nexport const MAX_STOP_MS: number = 30 * 24 * 60 * 60 * 1000;\n\nexport const DEFAULT_POLICY: PolitenessPolicy = {\n  pageIntervalMs: 3000,\n  apiIntervalMs: 1000,\n  renderIntervalMs: 5000,\n  elapsedFactor: 5,\n  elapsedCapMs: 30_000,\n  crawlDelayCapMs: 60_000,\n  maxRedirects: 5,\n  maxBytes: 5 * 1024 * 1024,\n  requestTimeoutMs: 90_000,\n  renderTimeoutMs: 20_000,\n  maxFailures: 5,\n  failureStopMs: 60 * 60 * 1000,\n  blockedStopMs: 60 * 60 * 1000,\n  challengeStopMs: 24 * 60 * 60 * 1000,\n  paymentStopMs: 24 * 60 * 60 * 1000,\n  maxRetryAfterMs: 60 * 60 * 1000,\n  backoffBaseMs: 5000,\n  backoffCapMs: 300_000,\n  robotsTtlMs: 24 * 60 * 60 * 1000,\n  robotsRetryMs: 5 * 60 * 1000,\n  robotsRetryCapMs: 60 * 60 * 1000,\n  robotsStaleMs: 30 * 24 * 60 * 60 * 1000,\n  heuristicFreshnessCapMs: 24 * 60 * 60 * 1000,\n};\n\n/**\n * These options change the politeness policy. The values at the top apply to each host. The\n * `hosts` map gives values for one host (`api.example.org`) or for the hosts under a name\n * (`*.example.org`). The most specific entry wins.\n */\nexport interface PolicyOptions extends Partial<PolitenessPolicy> {\n  readonly hosts?: Readonly<Record<string, Partial<PolitenessPolicy>>> | undefined;\n}\n\nconst INTEGER_KEYS: ReadonlySet<string> = new Set([\"maxRedirects\", \"maxFailures\"]);\n\nconst checked = (values: Partial<PolitenessPolicy>, where: string): Partial<PolitenessPolicy> => {\n  for (const [key, value] of Object.entries(values)) {\n    if (!(key in DEFAULT_POLICY))\n      throw new RangeError(`${where}: ${key} is not a politeness setting`);\n    const good =\n      typeof value === \"number\" &&\n      value >= 0 &&\n      (Number.isFinite(value) || key.endsWith(\"StopMs\")) &&\n      (!INTEGER_KEYS.has(key) || Number.isInteger(value));\n    if (!good)\n      throw new RangeError(`${where}: ${key} must be a number of 0 or more, not ${String(value)}`);\n  }\n  return values;\n};\n\n/** This function examines the options of a policy. It throws a `RangeError` for a value that is not a valid number. */\nexport const checkPolicy = (options: PolicyOptions | undefined): void => {\n  if (options === undefined) return;\n  const { hosts, ...base } = options;\n  checked(base, \"politeness\");\n  for (const [host, values] of Object.entries(hosts ?? {}))\n    checked(values, `politeness.hosts[\"${host}\"]`);\n};\n\n/** This function gives the policy of one host: the defaults, then the general options, then the entries for the host. */\nexport const policyFor = (options: PolicyOptions | undefined, host: string): PolitenessPolicy => {\n  if (options === undefined) return DEFAULT_POLICY;\n  const { hosts, ...base } = options;\n  const name = host.toLowerCase().replace(/\\.$/, \"\");\n  const matching = Object.entries(hosts ?? {})\n    .map(([key, values]) => ({ key: key.toLowerCase(), values }))\n    .filter(({ key }) => key === name || (key.startsWith(\"*.\") && name.endsWith(key.slice(1))))\n    // an exact name wins, and a longer pattern wins over a shorter one\n    .sort(\n      (a, b) => (a.key === name ? 1 : 0) - (b.key === name ? 1 : 0) || a.key.length - b.key.length,\n    );\n  return Object.assign(\n    {},\n    DEFAULT_POLICY,\n    base,\n    ...matching.map(({ values }) => values),\n  ) as PolitenessPolicy;\n};\n"],"mappings":";;AAqBA,MAAM,eAAe,KAAK,KAAK;AAE/B,MAAa,YAAmB;CAC9B,WAAW,KAAK,IAAI;CACpB,QAAQ,IAAI,WACV,IAAI,SAAe,YAAY;EAC7B,IAAI,QAAQ,YAAY,MAAM;GAC5B,QAAQ;GACR;EACF;EAEA,IAAI,OAAO,OAAO,SAAS,EAAE,IAAI,KAAK,IAAI,GAAG,EAAE,IAAI;EACnD,IAAI;EACJ,MAAM,aAAmB;GACvB,IAAI,UAAU,KAAA,GAAW,aAAa,KAAK;GAC3C,QAAQ,oBAAoB,SAAS,IAAI;GACzC,QAAQ;EACV;EACA,MAAM,aAAmB;GACvB,IAAI,QAAQ,GAAG;IACb,KAAK;IACL;GACF;GACA,MAAM,OAAO,KAAK,IAAI,MAAM,YAAY;GACxC,QAAQ;GACR,QAAQ,WAAW,MAAM,IAAI;EAC/B;EACA,QAAQ,iBAAiB,SAAS,MAAM,EAAE,MAAM,KAAK,CAAC;EACtD,KAAK;CACP,CAAC;AACL;;AAGA,IAAa,cAAb,cAAiC,MAAM;CAE1B;CACA;CAEA;CAJX,YACE,MACA,QAEA,QAAyB,OAAO,mBAChC;EACA,MAAM,GAAG,KAAK,YAAY,QAAQ;EALzB,KAAA,OAAA;EACA,KAAA,SAAA;EAEA,KAAA,QAAA;CAGX;AACF;;AA4IA,MAAa,iBAAkC;CAC7C,QAAQ;CACR,YAAY;CACZ,YAAY;AACd;AAEA,MAAM,aAA4B;CAChC,UAAU;CACV,MAAM;CACN,cAAc;CACd,YAAY;CACZ,YAAY;CACZ,YAAY;CACZ,WAAW;AACb;;;;;;;AAQA,MAAa,oBACX,MACA,QACA,QAC+D;CAC/D,IAAI,MAAM,aAAa,QAAQ,CAAC,OAAO,QAAQ,CAAC,OAAO,cAAc,OAAO;CAI5E,OAAO;EAAE,MAFP,KAAK,cAAc,eAAe,cAClC,MAAM,KAAK,aAAa,eAAe;EACnB,cAAc;CAAK;AAC3C;;;;;;;;;;;;;AAcA,MAAa,qBACX,MACA,QACA,QACkB;CAClB,MAAM,OAAmE,EACvE,GAAI,QAAQ,WACd;CACA,IAAI,OAAO,MAAM;EACf,MAAM,QAAQ,KAAK;EACnB,IAAI,OAAO,WAAW,gBAAgB;GACpC,KAAK,OAAO;GACZ,KAAK,aAAa;GAClB,IAAI,OAAO;IACT,KAAK,WAAW;IAChB,KAAK,aAAa;IAClB,KAAK,YAAY;GACnB;EACF,OAAO,IAAI,OAAO;GAChB,IAAI,OAAO,WAAW,QAAQ,KAAK,cAAc;GACjD,KAAK,aAAa;GAClB,KAAK,YAAY;EACnB,OAAO,IAAI,OAAO,WAAW,QAAQ;GACnC,KAAK,cAAc;GACnB,IAAI,KAAK,cAAc,eAAe,QAAQ;IAC5C,KAAK,OAAO;IAEZ,MAAM,WACJ,KAAK,iBAAiB,WAAW,MAAM,KAAK,aAAa,eAAe;IAC1E,IAAI,OAAO,gBAAgB,UAAU;KACnC,KAAK,WAAW;KAChB,KAAK,aAAa;KAClB,KAAK,aAAa;KAClB,KAAK,YAAY;IACnB;GACF;EACF;CACF,OAAO,IAAI,OAAO,cAAc;EAC9B,IAAI,KAAK,UAAU,KAAK,cAAc;EACtC,IAAI,OAAO,WAAW,gBAAgB;GACpC,KAAK,eAAe;GACpB,KAAK,aAAa;EACpB,OAAO,IAAI,OAAO,WAAW,QAAQ;GACnC,KAAK,cAAc;GACnB,IAAI,KAAK,cAAc,eAAe,QAAQ;IAC5C,KAAK,eAAe;IACpB,IAAI,KAAK,UAAU;KACjB,KAAK,WAAW;KAChB,KAAK,aAAa;KAClB,KAAK,YAAY;IACnB;GACF;EACF;CACF;CACA,OAAO;AACT;;;;;;AAeA,IAAa,uBAAb,MAA4D;CAM7B;CAL7B,yBAA0B,IAAI,IAAyB;;CAEvD,wBAAyB,IAAI,IAA2B;CACxD;CAEA,YAAY,WAAoC,KAAQ;EAA3B,KAAA,WAAA;EAC3B,KAAK,UAAU;CACjB;;CAGA,IAAI,OAAe;EACjB,OAAO,KAAK,OAAO;CACrB;CAEA,MAAc,MAA2B;EACvC,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI;EAChC,IAAI,UAAU,KAAA,GAAW;GACvB,QAAQ;IAAE,QAAQ;IAAG,aAAa;IAAG,UAAU;IAAG,OAAO;IAAG,MAAM,KAAA;GAAU;GAC5E,KAAK,OAAO,IAAI,MAAM,KAAK;EAC7B;EACA,OAAO;CACT;CAEA,MAAc,KAAmB;EAC/B,KAAK,MAAM,CAAC,MAAM,UAAU,KAAK,QAO/B,IALE,MAAM,UAAU,OAChB,MAAM,eAAe,OACrB,MAAM,aAAa,KACnB,MAAM,UAAU,MACf,MAAM,SAAS,KAAA,KAAa,MAAM,KAAK,SAAS,MACzC,KAAK,OAAO,OAAO,IAAI;EAGnC,KAAK,UAAU,KAAK,IAAI,KAAK,UAAU,KAAK,KAAK,KAAK,OAAO,OAAO,GAAG,CAAC;CAC1E;CAEA,QAAQ,MAAc,KAAa,SAAyB;EAC1D,IAAI,KAAK,OAAO,OAAO,KAAK,SAAS,KAAK,MAAM,GAAG;EACnD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,MAAM,cAAc,KAAK,OAAO,MAAM,cAAc;EACxD,IAAI,MAAM,SAAS,KAAK,OAAO,MAAM,SAAS;EAC9C,MAAM,cAAc,MAAM;EAC1B,OAAO;CACT;CAEA,OAAO,MAAc,QAAgC;EACnD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,WAAW,WAAW;GACxB,MAAM,WAAW;GACjB,MAAM,QAAQ;EAChB,OAAO,IAAI,WAAW,WAAW,MAAM,YAAY;OAC9C,IAAI,WAAW,QAAQ,MAAM,SAAS;EAC3C,OAAO;GAAE,UAAU,MAAM;GAAU,OAAO,MAAM;EAAM;CACxD;CAEA,QAAQ,MAAc,QAAsB;EAC1C,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,MAAM,cAAc;EACpB,IAAI,SAAS,MAAM,QAAQ,MAAM,SAAS;CAC5C;CAEA,MAAM,MAAc,OAAqB;EACvC,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,QAAQ,MAAM,QAAQ,MAAM,SAAS;CAC3C;CAEA,KAAK,MAAc,OAAe,QAAsB;EACtD,MAAM,QAAQ,KAAK,MAAM,IAAI;EAC7B,IAAI,MAAM,SAAS,KAAA,KAAa,MAAM,KAAK,QAAQ,OAAO,MAAM,OAAO;GAAE;GAAO;EAAO;CACzF;CAEA,QAAQ,MAAc,KAAmC;EACvD,MAAM,OAAO,KAAK,OAAO,IAAI,IAAI,CAAC,EAAE;EACpC,OAAO,SAAS,KAAA,KAAa,KAAK,QAAQ,MAAM,OAAO,KAAA;CACzD;CAEA,MAAM,MAAqB;EACzB,MAAM,SAAS,UAA6B;GAC1C,MAAM,WAAW;GACjB,MAAM,QAAQ;GACd,MAAM,OAAO,KAAA;EACf;EACA,IAAI,SAAS,KAAA,GAAW,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GAAG,MAAM,KAAK;OACxE;GACH,MAAM,QAAQ,KAAK,OAAO,IAAI,IAAI;GAClC,IAAI,UAAU,KAAA,GAAW,MAAM,KAAK;EACtC;CACF;CAEA,WAAW,MAAyC;EAClD,OAAO,KAAK,MAAM,IAAI,IAAI;CAC5B;CAEA,eAAe,MAAc,QAAyB,KAA4B;EAChF,MAAM,OAAO,kBAAkB,KAAK,MAAM,IAAI,IAAI,GAAG,QAAQ,GAAG;EAEhE,KAAK,MAAM,OAAO,IAAI;EACtB,KAAK,MAAM,IAAI,MAAM,IAAI;EACzB,IAAI,KAAK,MAAM,OAAO,KAAK,UAAU,KAAK,MAAM,OAAO,KAAK,MAAM,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAM;EACtF,OAAO;CACT;AACF;AAYA,MAAM,WAAW,WACf,OAAO,kBAAkB,QACrB,OAAO,SACP,IAAI,aAAa,wBAAwB,YAAY;AAE3D,IAAa,gBAAb,MAA2B;CAUN;CACA;;CATnB;CACA,yBAA0B,IAAI,IAA8B;CAC5D,SAAiB;CACjB,QAA4C,CAAC;CAC7C;CACA;CAEA,YACE,OACA,WACA,UAA4B,CAAC,GAC7B;EAHiB,KAAA,QAAA;EACA,KAAA,YAAA;EAGjB,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,MAAM,IAAI,WAAW,kDAAkD,OAAO,SAAS,GAAG;EAE5F,KAAK,QAAQ,QAAQ,SAAS,IAAI,qBAAqB;EACvD,KAAK,UAAU,QAAQ,WAAW;EAClC,KAAK,kBAAkB,QAAQ,mBAAmB;CACpD;CAEA,QAAgB,QAAgD;EAC9D,IAAI,KAAK,SAAS,KAAK,WAAW;GAChC,KAAK,UAAU;GACf,OAAO,QAAQ,QAAQ;EACzB;EACA,OAAO,IAAI,SAAe,SAAS,WAAW;GAC5C,MAAM,cAAoB;IACxB,QAAQ,oBAAoB,SAAS,MAAM;IAC3C,QAAQ;GACV;GACA,MAAM,eAAqB;IACzB,MAAM,KAAK,KAAK,MAAM,QAAQ,KAAK;IACnC,IAAI,OAAO,IAAI,KAAK,MAAM,OAAO,IAAI,CAAC;IACtC,OAAO,QAAQ,MAAO,CAAC;GACzB;GACA,KAAK,MAAM,KAAK,KAAK;GACrB,QAAQ,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;EAC1D,CAAC;CACH;CAEA,UAAwB;EAEtB,MAAM,OAAO,KAAK,MAAM,MAAM;EAC9B,IAAI,SAAS,KAAA,GAAW,KAAK;OACxB,KAAK,UAAU;CACtB;CAEA,MAAc,MAAM,IAAY,QAAgD;EAC9E,IAAI,WAAW,KAAA,GAAW;GACxB,MAAM,KAAK,MAAM,MAAM,EAAE;GACzB;EACF;EACA,IAAI,OAAO,SAAS,MAAM,QAAQ,MAAM;EACxC,MAAM,IAAI,SAAe,SAAS,WAAW;GAC3C,MAAM,eAAqB,OAAO,QAAQ,MAAM,CAAC;GACjD,OAAO,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;GACvD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,CAAC,WACrB;IACJ,OAAO,oBAAoB,SAAS,MAAM;IAC1C,QAAQ;GACV,IACC,UAAmB;IAClB,OAAO,oBAAoB,SAAS,MAAM;IAC1C,OAAO,KAAK;GACd,CACF;EACF,CAAC;CACH;;;;;CAMA,IAAO,MAAc,MAA+B,QAAkC;EAEpF,MAAM,UADW,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ,QAAQ,EAAA,CAClC,KAAK,YAAY;GAEvC,IAAI,SAAS,OAAO;GACpB,SAAS;IACP,IAAI,QAAQ,YAAY,MAAM,MAAM,QAAQ,MAAM;IAClD,MAAM,MAAM,KAAK,IAAI,KAAK,MAAM,IAAI,GAAG,MAAM;IAC7C,MAAM,OAAO,KAAK,MAAM,QAAQ,MAAM,GAAG;IACzC,IAAI,SAAS,KAAA,GAAW,MAAM,IAAI,YAAY,MAAM,KAAK,QAAQ,KAAK,KAAK;IAC3E,MAAM,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,KAAK,OAAO;IACvD,IAAI,EAAE,OAAO,IAAI;IACjB,MAAM,KAAK,MAAM,MAAM,MAAM;IAC7B,SAAS,MAAM;GACjB;GACA,IAAI;IACF,MAAM,KAAK,QAAQ,MAAM;GAC3B,SAAS,OAAO;IACd,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC;IACzC,MAAM;GACR;GACA,IAAI,SAAS,KAAK,MAAM,IAAI,IAAI,KAAK;GACrC,IAAI;IACF,MAAM,EAAE,OAAO,YAAY,MAAM,KAAK;IACtC,MAAM,OAAO,OAAO,SAAS,OAAO,KAAK,WAAW,IAAI,UAAU,KAAK;IACvE,SAAS,KAAK,MAAM,IAAI,IAAI;IAC5B,OAAO;GACT,SAAS,OAAO;IACd,SAAS,KAAK,MAAM,IAAI,IAAI,KAAK;IACjC,MAAM;GACR,UAAU;IACR,KAAK,MAAM,QAAQ,MAAM,MAAM;IAC/B,KAAK,QAAQ;GACf;EACF,CAAC;EACD,MAAM,OAAO,OAAO,WACZ,KAAA,SACA,KAAA,CACR;EACA,KAAK,OAAO,IAAI,MAAM,IAAI;EAE1B,KAAU,WAAW;GACnB,IAAI,KAAK,OAAO,IAAI,IAAI,MAAM,MAAM,KAAK,OAAO,OAAO,IAAI;EAC7D,CAAC;EACD,OAAO;CACT;;CAGA,KAAK,MAAc,SAAuB;EACxC,KAAK,MAAM,MAAM,MAAM,OAAO;CAChC;;CAGA,KAAK,MAAc,QAAgB,UAAkB,OAAO,mBAAyB;EACnF,KAAK,MAAM,KAAK,MAAM,SAAS,MAAM;CACvC;CAEA,cAAc,MAAkC;EAC9C,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,EAAE;CACrD;;CAGA,OAAO,MAAoC;EACzC,OAAO,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,IAAI,CAAC;CAClD;;CAGA,OAAO,MAAsB;EAC3B,OAAO,KAAK,MAAM,OAAO,MAAM,SAAS,CAAC,CAAC;CAC5C;;CAGA,OAAO,MAAsB;EAC3B,OAAO,KAAK,MAAM,OAAO,MAAM,MAAM,CAAC,CAAC;CACzC;CAEA,UAAU,MAAoB;EAC5B,KAAK,MAAM,OAAO,MAAM,SAAS;CACnC;;CAGA,MAAM,MAAqB;EACzB,KAAK,MAAM,MAAM,IAAI;CACvB;AACF;;;;;AAMA,MAAa,aACX,SACA,QACA,OACA,WACW,KAAK,MAAM,OAAO,IAAI,KAAK,IAAI,OAAO,SAAS,KAAK,KAAK,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC;;;;;;AAO1F,MAAa,iBACX,SACA,QACA,OACA,WACW;CACX,MAAM,OAAO,KAAK,IAAI,OAAO,SAAS,KAAK,KAAK,IAAI,GAAG,UAAU,CAAC,CAAC;CACnE,OAAO,KAAK,MAAM,OAAO,IAAI,OAAO,KAAK,OAAO,EAAE;AACpD;;;;;;;;;;;;;AAkCA,MAAa,aAAa,QAA0B,UAA2B;CAC7E,MAAM,WACJ,MAAM,kBACL,MAAM,SAAS,SACZ,OAAO,iBACP,MAAM,SAAS,QACb,OAAO,gBACP,OAAO;CACf,MAAM,aAAa,KAAK,IACtB,OAAO,iBACP,KAAK,IAAI,IAAI,MAAM,qBAAqB,KAAK,GAAI,CACnD;CACA,IAAI,KAAK,KAAK,IAAI,UAAU,UAAU;CACtC,IAAI,SAAyB,aAAa,WAAW,gBAAgB;CACrE,IAAI,MAAM,SAAS,OAAO;EACxB,MAAM,OAAO,KAAK,IAAI,OAAO,cAAc,OAAO,gBAAgB,KAAK,IAAI,GAAG,MAAM,SAAS,CAAC;EAC9F,IAAI,OAAO,IAAI;GACb,KAAK;GACL,SAAS;EACX;CACF;CACA,IAAI,MAAM,iBAAiB,KAAA,KAAa,MAAM,eAAe,IAAI;EAC/D,KAAK,MAAM;EACX,SAAS;CACX;CACA,OAAO;EAAE;EAAI;CAAO;AACtB;;;;;AA2DA,MAAa,cAAsB;AAEnC,MAAa,iBAAmC;CAC9C,gBAAgB;CAChB,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,cAAc;CACd,iBAAiB;CACjB,cAAc;CACd,UAAU;CACV,kBAAkB;CAClB,iBAAiB;CACjB,aAAa;CACb,eAAe;CACf,eAAe;CACf,iBAAiB;CACjB,eAAe;CACf,iBAAiB;CACjB,eAAe;CACf,cAAc;CACd,aAAa;CACb,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,yBAAyB;AAC3B;AAWA,MAAM,+BAAoC,IAAI,IAAI,CAAC,gBAAgB,aAAa,CAAC;AAEjF,MAAM,WAAW,QAAmC,UAA6C;CAC/F,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,MAAM,GAAG;EACjD,IAAI,EAAE,OAAO,iBACX,MAAM,IAAI,WAAW,GAAG,MAAM,IAAI,IAAI,6BAA6B;EAMrE,IAAI,EAJF,OAAO,UAAU,YACjB,SAAS,MACR,OAAO,SAAS,KAAK,KAAK,IAAI,SAAS,QAAQ,OAC/C,CAAC,aAAa,IAAI,GAAG,KAAK,OAAO,UAAU,KAAK,KAEjD,MAAM,IAAI,WAAW,GAAG,MAAM,IAAI,IAAI,sCAAsC,OAAO,KAAK,GAAG;CAC/F;CACA,OAAO;AACT;;AAGA,MAAa,eAAe,YAA6C;CACvE,IAAI,YAAY,KAAA,GAAW;CAC3B,MAAM,EAAE,OAAO,GAAG,SAAS;CAC3B,QAAQ,MAAM,YAAY;CAC1B,KAAK,MAAM,CAAC,MAAM,WAAW,OAAO,QAAQ,SAAS,CAAC,CAAC,GACrD,QAAQ,QAAQ,qBAAqB,KAAK,GAAG;AACjD;;AAGA,MAAa,aAAa,SAAoC,SAAmC;CAC/F,IAAI,YAAY,KAAA,GAAW,OAAO;CAClC,MAAM,EAAE,OAAO,GAAG,SAAS;CAC3B,MAAM,OAAO,KAAK,YAAY,CAAC,CAAC,QAAQ,OAAO,EAAE;CACjD,MAAM,WAAW,OAAO,QAAQ,SAAS,CAAC,CAAC,CAAC,CACzC,KAAK,CAAC,KAAK,aAAa;EAAE,KAAK,IAAI,YAAY;EAAG;CAAO,EAAE,CAAC,CAC5D,QAAQ,EAAE,UAAU,QAAQ,QAAS,IAAI,WAAW,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAE1F,MACE,GAAG,OAAO,EAAE,QAAQ,OAAO,IAAI,MAAM,EAAE,QAAQ,OAAO,IAAI,MAAM,EAAE,IAAI,SAAS,EAAE,IAAI,MACxF;CACF,OAAO,OAAO,OACZ,CAAC,GACD,gBACA,MACA,GAAG,SAAS,KAAK,EAAE,aAAa,MAAM,CACxC;AACF"}

package/build/{version-rVx8_y1_.mjs → version-no12qgos.mjs}

@@ -1,6 +1,6 @@
 //#region src/version.ts
 /** This is the version of the package. A test makes sure that it agrees with `package.json`. */
-const VERSION = "0.1.0";
+const VERSION = "0.1.1";
 /** This is the product token that robots.txt rules match when an application gives no token. */
 const DEFAULT_TOKEN = "PoliteCrawler";
 /** This page tells a site owner what the crawler is and how to control it. */
@@ -8,4 +8,4 @@ const BOT_PAGE = "https://mark1russell7.github.io/crawler/bot/";
 //#endregion
 export { DEFAULT_TOKEN as n, VERSION as r, BOT_PAGE as t };
 
-//# sourceMappingURL=version-rVx8_y1_.mjs.map
+//# sourceMappingURL=version-no12qgos.mjs.map

package/build/{version-rVx8_y1_.mjs.map → version-no12qgos.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"version-rVx8_y1_.mjs","names":[],"sources":["../src/version.ts"],"sourcesContent":["/** This is the version of the package. A test makes sure that it agrees with `package.json`. */\nexport const VERSION = \"0.1.0\";\n\n/** This is the product token that robots.txt rules match when an application gives no token. */\nexport const DEFAULT_TOKEN = \"PoliteCrawler\";\n\n/** This page tells a site owner what the crawler is and how to control it. */\nexport const BOT_PAGE = \"https://mark1russell7.github.io/crawler/bot/\";\n"],"mappings":";;AACA,MAAa,UAAU;;AAGvB,MAAa,gBAAgB;;AAG7B,MAAa,WAAW"}
+{"version":3,"file":"version-no12qgos.mjs","names":[],"sources":["../src/version.ts"],"sourcesContent":["/** This is the version of the package. A test makes sure that it agrees with `package.json`. */\nexport const VERSION = \"0.1.1\";\n\n/** This is the product token that robots.txt rules match when an application gives no token. */\nexport const DEFAULT_TOKEN = \"PoliteCrawler\";\n\n/** This page tells a site owner what the crawler is and how to control it. */\nexport const BOT_PAGE = \"https://mark1russell7.github.io/crawler/bot/\";\n"],"mappings":";;AACA,MAAa,UAAU;;AAGvB,MAAa,gBAAgB;;AAG7B,MAAa,WAAW"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "polite-crawler",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "mcpName": "io.github.mark1russell7/polite-crawler",
   "description": "A polite web crawler for research and AI agents. It fetches a source once, keeps it, and reads it many times: robots.txt obeyed, one request per site, full-text search, an MCP server.",
   "keywords": [

package/record.schema.json

@@ -105,6 +105,11 @@
         },
         "etag": { "$ref": "#/$defs/nullableString" },
         "last_modified": { "$ref": "#/$defs/nullableString" },
+        "validators": {
+          "description": "The validators that a conditional request sent. The validator memo of the host can leave out if-none-match.",
+          "type": "array",
+          "items": { "enum": ["if-none-match", "if-modified-since"] }
+        },
         "revisit_of": {
           "description": "The capture whose body a 304 confirmed.",
           "type": "string"