polite-crawler 0.1.1 → 0.1.2

package/build/open-Ca1lcWR3.d.mts

@@ -0,0 +1,2875 @@
+import { n as Outcome } from "./classify-CrfyXcwo.mjs";
+import { t as ScopeOptions } from "./urls-BTDSgDLC.mjs";
+import { r as Signals, t as AiInput } from "./signals-BHpy2xb2.mjs";
+import { f as PolicyOptions, o as HostStateStore, t as Clock } from "./politeness-yI3xBEx9.mjs";
+import { a as Robots } from "./robots-R-qaEBOb.mjs";
+import { JsonWebKey, KeyObject } from "node:crypto";
+import { Dispatcher } from "undici";
+import { LookupAddress, LookupAllOptions } from "node:dns";
+import { LookupFunction } from "node:net";
+//#region src/blobs.d.ts
+/**
+ * A blob store keeps the bytes of each response, and the hash of the bytes is the key. The crawler
+ * needs only these two operations, so an application can give its own store.
+ */
+interface BlobStore {
+  put(bytes: Uint8Array): Promise<{
+    readonly hash: string;
+  }>;
+  get(hash: string): Promise<Uint8Array | undefined>;
+}
+/** These options change a file blob store. */
+interface FileBlobStoreOptions {
+  /** When this value is true (the default), a blob is on the disk (`fsync`) before `put` finishes. */
+  readonly sync?: boolean | undefined;
+}
+/**
+ * This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash
+ * of the bytes. These are its rules:
+ *
+ * - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is
+ *   written again.
+ * - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.
+ *   A crash thus cannot leave half a blob under its name.
+ * - Two writers can put the same bytes at the same time. When the rename of one fails because the
+ *   other was first, the put succeeds if the stored file has the same hash.
+ * - The method `get` examines the hash of the bytes. A damaged file gets a new name
+ *   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.
+ */
+declare class FileBlobStore implements BlobStore {
+  readonly dir: string;
+  private readonly writing;
+  private readonly sync;
+  constructor(dir: string, options?: FileBlobStoreOptions);
+  put(bytes: Uint8Array): Promise<{
+    readonly hash: string;
+    readonly existed: boolean;
+  }>;
+  /** This method tells if a file holds exactly the bytes of a hash. */
+  private static holds;
+  private write;
+  get(hash: string): Promise<Uint8Array | undefined>;
+}
+//#endregion
+//#region src/render.d.ts
+/**
+ * The renderer opens the headless browser only for a page that shows almost nothing without
+ * JavaScript. The settings:
+ *
+ * - The default `chromium-headless-shell` of Playwright, the lighter build.
+ * - One browser for many pages, and a new context for each page. Thus no cookie or storage goes
+ *   from one site to the next. A context has its own HTTP cache, so routing each request costs
+ *   no cache across sites.
+ * - Each request of the page goes through a route, the first navigation too. The route refuses
+ *   images, fonts and media by their resource type. It asks `allowRequest` about each other
+ *   request, and it sends `headers` with it.
+ * - Playwright routes only the first request of a redirect chain. Thus with `allowRequest`, the
+ *   route sends each request itself, follows each redirect itself and asks about each step. The
+ *   browser then gets each body whole, not as a stream.
+ * - No download, no popup, no service worker and no dialog.
+ * - A wait until the visible text stops growing, with a limit. Playwright discourages
+ *   `networkidle`. The text includes open shadow roots and the frames of the same origin.
+ * - One deadline for the whole render: the navigation, the wait and the reading of the page.
+ * - After a crash or a disconnection, the next page launches a new browser.
+ * - The browser closes after two idle minutes. An idle Chromium holds hundreds of megabytes, so
+ *   it stays open only while pages need it. The open browser keeps the process alive, so a
+ *   script uses `close()` when it is done.
+ */
+interface Rendered {
+  readonly html: string;
+  readonly finalUrl: string;
+  readonly status: number | null;
+  readonly headers: Readonly<Record<string, string>>;
+}
+/** These are the options of one render. */
+interface RenderOptions {
+  readonly userAgent: string;
+  /** This value is the deadline of the whole render, in milliseconds. */
+  readonly timeoutMs: number;
+  /** These headers go with each request of the page, for example the `From` header. */
+  readonly headers?: Readonly<Record<string, string>> | undefined;
+  /**
+   * This function decides each request that the page makes, the first navigation included. When
+   * it gives false, the browser does not send the request. The crawler uses it to apply the
+   * egress policy and robots.txt to the requests of the page.
+   */
+  readonly allowRequest?: ((url: string) => Promise<boolean> | boolean) | undefined;
+  /** This value is the maximum number of redirects of one request of the page. The default is 5, as for a plain fetch. */
+  readonly maxRedirects?: number | undefined;
+  /** When this signal aborts, the renderer stops the page: it closes the context, and the render rejects. */
+  readonly signal?: AbortSignal | undefined;
+  /**
+   * This function gives the signature headers of one request of the page (Web Bot Auth). The route
+   * adds them to the request and to each step of its redirects. When the function gives undefined,
+   * the browser does not send the request.
+   */
+  readonly sign?: ((url: string) => Readonly<Record<string, string>> | undefined) | undefined;
+}
+interface Renderer {
+  render(url: string, options: RenderOptions): Promise<Rendered>;
+  close(): Promise<void>;
+}
+/** The reason of a `RenderRefused`. */
+type RefusedReason = "refused" | "download" | "scheme" | "browser";
+/**
+ * This error tells that the renderer did not render the page, and that the page or the site is
+ * not at fault. The reason is one of these:
+ *
+ * - `refused`: `allowRequest` refused the first navigation, a step of its redirects, or a later
+ *   navigation of the page that left it on an error page. `url` names the refused URL.
+ * - `download`: the URL gives a file, not a page.
+ * - `scheme`: the URL is not an http or https URL.
+ * - `browser`: the renderer has no browser (`playwright-core` or its browser is not installed). The
+ *   crawler then renders no more pages, and gives each one the note at once.
+ *
+ * A crawler does not count this error as a failure of the host.
+ */
+declare class RenderRefused extends Error {
+  readonly reason: RefusedReason;
+  /** This URL is the request that `allowRequest` or the limit of redirects refused, when the reason is `refused`. */
+  readonly url: string | undefined;
+  constructor(reason: RefusedReason, message: string, url?: string);
+}
+/** The text of the error when the optional package `playwright-core` is not installed. */
+declare const NO_PLAYWRIGHT = "rendering needs the package playwright-core and a browser: npm install playwright-core, then npx playwright-core install chromium-headless-shell. For the MCP server with npx, start it with npx -y -p polite-crawler -p playwright-core polite-crawler mcp";
+interface PlaywrightRenderer extends Renderer {
+  /** This function tells if a browser is open. */
+  readonly open: () => boolean;
+}
+declare const playwrightRenderer: (options?: {
+  readonly idleMs?: number;
+  /**
+   * This function starts the browser: it gives a `Browser` of `playwright-core`. The default is
+   * the headless shell of Playwright. The type names no type of `playwright-core`, so a program
+   * with no `playwright-core` can check its types.
+   */
+  readonly launch?: (() => Promise<object>) | undefined;
+}) => PlaywrightRenderer;
+//#endregion
+//#region src/record.d.ts
+/**
+ * A capture record is one line of the capture log. Each fetch decision makes one record, and a
+ * fetch that the crawler declines makes one too.
+ *
+ * The record id comes from a hash of its content, so the same decision always gets the same id.
+ * A citation names this id.
+ *
+ * The id has 48 bits: the source and the first 12 hexadecimal digits of the hash. Approximately
+ * 1 million records give a chance of a collision of 0.2 percent. The package keeps this length on purpose,
+ * because the ids of the earlier capture logs and their citations must stay the same. A later
+ * version can make the ids of new records longer behind a record version. It must still read a
+ * 12-digit id.
+ */
+interface CaptureRecord {
+  readonly id: string;
+  /** The source of the record. The crawler writes `fetch`, and an application can add its own sources. */
+  readonly source: string;
+  readonly kind: string;
+  readonly external_id?: string | undefined;
+  readonly occurred_at?: string | undefined;
+  readonly received_at: string;
+  readonly payload: unknown;
+  readonly content_hash: string;
+  /** This label comes from the application. The crawler writes it as it is and does not read it. */
+  readonly scope?: string | undefined;
+  /**
+   * These labels come from the application, for example the lineage of a request. The crawler writes
+   * them as they are and does not read them. Like `scope`, they are not part of the hash.
+   */
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+}
+/** These are the fields of a new record. The function `makeRecord` calculates the id and the hash. */
+interface RecordInput {
+  readonly source: string;
+  readonly kind: string;
+  readonly payload: unknown;
+  readonly received_at: string;
+  readonly external_id?: string | undefined;
+  readonly occurred_at?: string | undefined;
+  readonly scope?: string | undefined;
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+}
+/**
+ * This function makes a capture record. The hash covers the source, the kind, the external id
+ * and the payload. The id is the source and the first 12 digits of the hash. The times, `scope`
+ * and `labels` are not part of the hash.
+ */
+declare const makeRecord: (input: RecordInput) => CaptureRecord;
+//#endregion
+//#region src/log.d.ts
+/** A scan filter selects the records of one source, one kind, or the records after one time. */
+interface CaptureFilter {
+  readonly source?: string | undefined;
+  readonly kind?: string | undefined;
+  /** This value is an ISO 8601 time. The scan gives the records received at this time or later. */
+  readonly since?: string | undefined;
+}
+/** A damaged line is a line of the log that is not a record: a torn append, or bytes that a crash left. */
+interface DamagedLine {
+  /** This value is the path of the file. */
+  readonly path: string;
+  /** This value is the number of the line in the file, from 1. */
+  readonly line: number;
+}
+/** These options change a scan. */
+interface ScanOptions {
+  /** The scan skips a damaged line, and it gives the line to this function. */
+  readonly onDamaged?: ((damaged: DamagedLine) => void) | undefined;
+}
+/** The read side of the capture log is what `reindex` replays. */
+interface CaptureReader {
+  scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;
+}
+/** These options change a capture log. */
+interface CaptureLogOptions {
+  /**
+   * When this value is true (the default), the log writes each batch of appends to the disk
+   * (`fdatasync`) before the appends finish. Thus a power cut does not lose a record that the
+   * crawler reported.
+   */
+  readonly sync?: boolean | undefined;
+}
+/**
+ * The capture log keeps each fetch decision as one line of JSON in
+ * `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.
+ *
+ * These are the rules of the log:
+ *
+ * - The log appends a batch of records with one write, and it writes the batch to the disk before
+ *   the appends finish.
+ * - An append always starts on a new line. After a crash leaves a partial line, the next append
+ *   starts after a line break. The partial line stays as a damaged line, and the record is good.
+ * - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.
+ * - The log does not remove or change a line.
+ *
+ * Two processes can append to one log, because each batch is one write to a file in append mode.
+ */
+declare class CaptureLog implements CaptureReader {
+  readonly dir: string;
+  private readonly handles;
+  private readonly pending;
+  private draining;
+  private readonly sync;
+  constructor(dir: string, options?: CaptureLogOptions);
+  private pathOf;
+  private handleFor;
+  /** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */
+  private static needsBreak;
+  append(record: CaptureRecord): Promise<void>;
+  /** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */
+  private kick;
+  /** This method writes the waiting records in batches: one write and one sync for each file. */
+  private drain;
+  /** This method waits until each append in progress is on the disk. */
+  flush(): Promise<void>;
+  /** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */
+  private filesOf;
+  /** This method reads the records of one source in the sequence of the files. */
+  private readSource;
+  /**
+   * This method reads the records that agree with the filter. It first waits for the appends that
+   * are in progress. It reads each file as a stream, so its memory does not grow with the log.
+   *
+   * The records of one source come in the sequence of the files. The records of two or more sources
+   * merge in the sequence of `received_at`.
+   */
+  scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;
+  /** This method writes the appends in progress, then closes the files. A later append opens them again. */
+  close(): Promise<void>;
+}
+//#endregion
+//#region src/runtime.d.ts
+/**
+ * An extractor changes one captured body into the documents that it holds. It has no side effects,
+ * so `reindex` can replay each capture through it and build the index again with no request. It can
+ * answer later, because the crawler reads a PDF asynchronously.
+ *
+ * An extractor that throws does not stop a fetch, a render or a rebuild. The runtime records the
+ * error and gives no document for that capture.
+ */
+type Extractor = (body: Uint8Array, capture: {
+  readonly url: string;
+  readonly contentType: string | null;
+}) => ReadonlyArray<ExtractedDoc> | Promise<ReadonlyArray<ExtractedDoc>>;
+/** This function gives the document id of a web page. Each page has one id. */
+declare const pageDocId: (url: string) => string;
+/**
+ * This extractor is the built-in recipe for a web page, plain or rendered. It keeps the visible
+ * text, the JSON-LD (whole objects, up to a limit) and the links. It decodes the body in the
+ * encoding that the page declares, with `decodeBody`. It does not throw: a page that it cannot read
+ * gives a document that names the error. When the reader cut the text, the document keeps the whole
+ * text in `full_text`, so `verify` finds a quote past the cut.
+ */
+declare const pageExtractor: Extractor;
+declare class Extractors {
+  private readonly byRecipe;
+  private readonly replaceable;
+  register(recipe: string, extractor: Extractor): void;
+  get(recipe: string): Extractor | undefined;
+  /** This method gives the names of the recipes that have an extractor. */
+  recipes(): string[];
+}
+interface CrawlRuntime {
+  readonly crawler: Crawler;
+  /** This index keeps the documents and the cache entries. `DocIndex` is the SQLite index. */
+  readonly index: ReadingIndex;
+  readonly blobs: BlobStore;
+  /** This reader is the read side of the capture log. The function `reindex` replays it. */
+  readonly captures: CaptureReader;
+  /** This log is the write side of the capture log. */
+  readonly log: CaptureLog;
+  readonly extractors: Extractors;
+  readonly now: () => string;
+}
+/** This is the payload of a capture record, as the crawler writes it. */
+interface CapturePayload {
+  readonly url: string;
+  readonly recipe: string;
+  readonly mode: "api" | "page";
+  readonly status?: number;
+  readonly outcome?: string;
+  readonly blob?: string | null;
+  readonly content_type?: string | null;
+  readonly etag?: string | null;
+  readonly last_modified?: string | null;
+  readonly fetched_at: string;
+  readonly signals?: {
+    readonly ai_input?: AiInput;
+  };
+  /** This value is true when the headless browser made the capture, not a plain request. */
+  readonly rendered?: boolean;
+  /** This value is true when the body is only the first part of the response. */
+  readonly truncated?: boolean;
+  /** This value is true when the server cut a compressed body. The capture is not a copy to keep. */
+  readonly cut?: boolean;
+  /** This value is true when an API answered with a web page that the request did not accept. The capture is not a copy to keep. */
+  readonly unasked?: boolean;
+  /** This time (ISO 8601) is the end of the freshness that the server gave. Captures before version 0.2 do not have it. */
+  readonly fresh_until?: string | null;
+  /** This text holds the request headers that the response varies on. Captures before version 0.2 do not have it. */
+  readonly vary_key?: string | null;
+  /** This URL is the target of a redirect, with its secrets as `REDACTED`. */
+  readonly location?: string | null;
+  readonly note?: string;
+  /** This text is the byte range of a range request (`bytes=0-99`). The capture is a part of the resource, not its stored copy. */
+  readonly range?: string;
+}
+/** The note of a document that is held back. */
+declare const HELD_BACK = "This page asked not to be used as AI input, so its text is held back. Open the URL to read it, or read again with userProvided when a person gave this URL directly (the crawler logs that).";
+/**
+ * This function tells if a document is held back: the page asked not to be AI input, and no person
+ * gave its URL directly. The only way to read a held-back document is `userProvided`.
+ */
+declare const isHeldBack: (doc: {
+  readonly ai_input: AiInput;
+}, userProvided?: boolean) => boolean;
+/** This function gives the metadata that a held-back document shows: only the structural fields, which hold no text of the page. */
+declare const heldBackMeta: (meta: Readonly<Record<string, unknown>>) => Record<string, unknown>;
+/** The result of `fetchAndIndex` and `renderAndIndex`. */
+interface Indexed {
+  readonly fetched: Fetched;
+  readonly docs: readonly Doc[];
+  /** This text tells why the capture gave no document: the error of the extractor, or a bot check. */
+  readonly error?: string | undefined;
+  /** This value names the bot check or the firewall that sent its page in place of the content. */
+  readonly blocked?: string | undefined;
+}
+/**
+ * This function fetches one URL and adds its documents to the index. Each recipe uses it. A fetch
+ * from the cache gives the documents that the index has for that capture. Thus a repeated
+ * question sends no request and reads nothing again. An extractor that throws gives no document and an
+ * `error`, and the fetch still succeeds.
+ */
+declare const fetchAndIndex: (runtime: CrawlRuntime, request: FetchRequest) => Promise<Indexed>;
+/** This function renders a page in the headless browser and adds what the page shows to the index. */
+declare const renderAndIndex: (runtime: CrawlRuntime, request: RenderRequest) => Promise<Indexed>;
+interface PageResult {
+  readonly url: string;
+  readonly doc: string | null;
+  readonly outcome: string;
+  readonly status: number;
+  /** This value is the title of the page. A held-back page shows no title. */
+  readonly title: string | null;
+  readonly chars: number;
+  readonly rendered: boolean;
+  /** These are the schema.org types that the JSON-LD of the page declares. */
+  readonly structured: readonly string[];
+  readonly links: number;
+  /** These ids are the captures behind the current text, the newest first, 10 or fewer. */
+  readonly captures: readonly string[];
+  /** This id is the capture of the text that the document shows. A citation names it. */
+  readonly capture: string | null;
+  /** This value is the number of captures of the document. */
+  readonly capture_count: number;
+  /** This time (ISO 8601) is the fetch time of the text that the document shows. */
+  readonly fetched_at: string | null;
+  /** This value is true when the stored copy answered, and the crawler sent no request. */
+  readonly from_cache: boolean;
+  /** This value is true when the page asked not to be AI input. Its title and text are then held back. */
+  readonly held_back: boolean;
+  readonly note: string | null;
+  /**
+   * This value tells that the document is an archived copy, not a fresh fetch: the archive, the date
+   * of the copy and its URL there. The outcome and the status stay those of the live fetch.
+   */
+  readonly archived?: Archived | undefined;
+}
+/** These options control one page. */
+interface PageOptions {
+  readonly render: "auto" | "never" | "always";
+  readonly maxAgeMs?: number | "auto" | undefined;
+  readonly onBehalfOf?: readonly string[] | undefined;
+  readonly job?: string | undefined;
+  /** When this signal aborts, the crawler stops the fetch and the render. */
+  readonly signal?: AbortSignal | undefined;
+  readonly scope?: string | undefined;
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+  /** This time (milliseconds since the epoch) is the end of a budget. No request of the page starts at or after it (`FetchRequest.deadline`). */
+  readonly deadline?: number | undefined;
+  /** The value `archive` gives a copy from a public web archive when the site refuses the page, or is down. The default is `none`. */
+  readonly fallback?: "archive" | "none" | undefined;
+  /** The value true also gives an archived copy of a page that the current robots.txt of its site disallows. The default is false. */
+  readonly archiveDespiteRobots?: boolean | undefined;
+}
+/**
+ * This function reads one web page. It first sends a plain request. It opens the browser only when
+ * the page shows almost nothing without JavaScript, or when the caller asks for it. It renders
+ * only an HTML page: not a PDF, a different file or a bot check. A page that sends the reader on
+ * with a `<meta http-equiv="refresh">` gets one more plain fetch, of its target.
+ */
+declare const fetchPage: (runtime: CrawlRuntime, url: string, options: PageOptions) => Promise<PageResult>;
+/** This result is a part of one stored document. */
+interface ReadResult {
+  readonly doc: string;
+  readonly kind: string;
+  /** This value is the title of the document. A held-back document shows no title. */
+  readonly title: string;
+  readonly url: string | null;
+  /** This value is the metadata of the document. A held-back document shows only the structural fields. */
+  readonly meta: Readonly<Record<string, unknown>>;
+  readonly from: number;
+  readonly to: number;
+  readonly total: number;
+  readonly text: string;
+  readonly captures: readonly string[];
+  /** This id is the capture of the text that the result reads. */
+  readonly capture: string;
+  /** This time (ISO 8601) is the fetch time of the text that the result reads. */
+  readonly fetched_at: string;
+  readonly ai_input: AiInput;
+  /** This value is true when the page asked not to be AI input. The text is then empty. */
+  readonly held_back: boolean;
+  readonly note: string | null;
+  /** This time (ISO 8601) is the `at` of the read. The result has it only when the read gave `at`. */
+  readonly at?: string | undefined;
+  /**
+   * This value is true when `at` is before the first capture of the document. Then no version
+   * answers: the text, the title, `capture` and `fetched_at` are empty, and the note tells why.
+   */
+  readonly before_first?: boolean | undefined;
+  /** This value tells that the text is an archived copy, with the archive and the date of the copy. */
+  readonly archived?: Archived | undefined;
+}
+/** This result tells if a quote is in a stored document, and where. */
+interface VerifyResult {
+  readonly doc: string;
+  /** This value is true when the quote is in the text. A held-back document gives false. */
+  readonly found: boolean;
+  /** These values are the positions of the quote in the text of `capture`. */
+  readonly start: number | null;
+  readonly end: number | null;
+  readonly excerpt: string | null;
+  readonly url: string | null;
+  readonly captures: readonly string[];
+  /** This id is the capture whose text holds the quote, or the capture that the result examined. */
+  readonly capture: string | null;
+  /** This hash is the blob of that capture. */
+  readonly blob: string | null;
+  /** This time (ISO 8601) is the fetch time of that capture. */
+  readonly fetched_at: string | null;
+  /** This value is true when only an earlier version of the document holds the quote. */
+  readonly stale: boolean;
+  /** This value is true when the quote is in the whole text of a page that the reader cut. The positions are then in that whole text. */
+  readonly in_full_text: boolean;
+  /** This value is false when the examined text is not complete (a PDF that the reader stopped). A quote that is not found can then be in the file. */
+  readonly complete: boolean;
+  /** This value is the number of places where the quote is. */
+  readonly count: number;
+  readonly held_back: boolean;
+  readonly note: string | null;
+  /** This time (ISO 8601) is the `at` of the verification. The result has it only when the verification gave `at`. */
+  readonly at?: string | undefined;
+  /** This value is true when `at` is before the first capture of the document. Then no version holds the quote, and the note tells why. */
+  readonly before_first?: boolean | undefined;
+  /**
+   * This value tells that the examined text is an archived copy, with the archive and the date of
+   * the copy. A citation of it names the archive and that date, not a date of a fresh fetch.
+   */
+  readonly archived?: Archived | undefined;
+}
+/** These options control a read. */
+interface ReadOptions {
+  /** This value is the first character to read. The default is 0. */
+  readonly from?: number | undefined;
+  /** This value is the number of characters to read. The default is 4000. */
+  readonly length?: number | undefined;
+  /** This value is true when a person gave this URL directly. It is the only way to read a held-back document. The crawler logs it. */
+  readonly userProvided?: boolean | undefined;
+  /** This id reads the version of one capture, not the current version. */
+  readonly capture?: string | undefined;
+  /**
+   * This time reads the version that was current at that time: an ISO 8601 text or a `Date`. A
+   * date with no time is the start of that day in UTC. Give `capture` or `at`, not the two.
+   */
+  readonly at?: string | Date | undefined;
+}
+/** These options control a verification. */
+interface VerifyOptions {
+  /** This id examines the version of one capture only. A citation of that capture thus stays true after a new fetch. */
+  readonly capture?: string | undefined;
+  /** This value is true when a person gave this URL directly. It is the only way to verify a quote in a held-back document. */
+  readonly userProvided?: boolean | undefined;
+  /** This time examines only the version that was current at that time (`ReadOptions.at`). Give `capture` or `at`, not the two. */
+  readonly at?: string | Date | undefined;
+}
+/**
+ * This function gives the ISO 8601 form of a time. A text that is not a time throws a `RangeError`.
+ * A date with no time is the start of that day in UTC.
+ */
+declare const isoTime: (name: string, value: string | Date) => string;
+/**
+ * This function gives the version of a document that was current at a time (ISO 8601). It uses
+ * `ReadingIndex.versionAt` when the index has it. Else it takes the newest version at or before
+ * the time.
+ */
+declare const versionAtTime: (runtime: CrawlRuntime, id: string, at: string) => VersionAt;
+/**
+ * This function writes an `override` record to the capture log. A person gave the URL of a
+ * held-back document directly, and the hold stops for one action (`read` or `verify`). The record
+ * takes the scope and the labels of `marks`, or else those of the document, as its captures do.
+ */
+declare const recordOverride: (runtime: CrawlRuntime, doc: {
+  readonly doc: string;
+  readonly url: string | null;
+  readonly scope?: string | undefined;
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+}, action: string, marks?: {
+  readonly scope?: string | undefined;
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+}) => Promise<void>;
+/**
+ * This function reads a range of characters of one stored document. It applies the rule for a
+ * held-back document: no title, no text, and only the structural metadata, unless `userProvided`.
+ * The positions do not cut a character in two.
+ */
+declare const readDocument: (runtime: CrawlRuntime, id: string, options?: ReadOptions) => Promise<ReadResult>;
+/**
+ * This function finds a quote in a stored document, with `locate`. It tells which capture holds the
+ * quote, with its blob, its fetch time and the positions:
+ *
+ * - With `capture`, it examines the version of that capture only.
+ * - With no capture, it examines the current text first. When the quote is not there, it examines
+ *   the earlier texts, the newest first. A match there has `stale: true`.
+ * - When the reader cut a text, it also examines the whole text.
+ *
+ * A held-back document gives `found: false` and no positions, so a quote cannot test its text,
+ * unless `userProvided`. The same rule applies to each version. A version that asked not to be AI
+ * input stays held back after a later capture drops the signal. The crawler logs an override when
+ * `userProvided` opens a held-back version.
+ */
+declare const verifyQuote: (runtime: CrawlRuntime, id: string, quote: string, options?: VerifyOptions) => Promise<VerifyResult>;
+/**
+ * This function searches the stored documents. The syntax of the query is the syntax of
+ * `parseQuery`. A held-back document is not in the results. The option `rank` chooses the
+ * ranking (`RankMode`), and the default is `exact`.
+ */
+declare const findDocuments: (runtime: CrawlRuntime, q: string, options?: {
+  readonly limit?: number | undefined;
+  readonly kind?: string | null | undefined;
+  readonly rank?: RankMode | undefined;
+}) => Promise<Found[]>;
+/** The counts of a rebuild. */
+interface ReindexResult {
+  /** This value is the number of captures that the rebuild read again. */
+  readonly captures: number;
+  readonly docs: number;
+  readonly urls: number;
+  /** This value is the number of captures whose extractor threw. */
+  readonly failed: number;
+  /** This value is the number of captures whose blob is missing or damaged. */
+  readonly missing: number;
+  /** This value is the number of damaged lines of the log that the rebuild skipped. */
+  readonly damaged: number;
+  /** This value is the number of documents of recipes with no extractor, kept from the old index. */
+  readonly carried: number;
+  /** These recipes have captures in the log and no extractor in this runtime. */
+  readonly unknown_recipes: readonly string[];
+  /** These are the first errors of the extractors. */
+  readonly errors: ReadonlyArray<{
+    readonly capture: string;
+    readonly error: string;
+  }>;
+}
+/**
+ * This function builds the reading index again from the capture log and the blobs. The index is a
+ * projection, and this function is the proof that its deletion loses nothing. These are its rules:
+ *
+ * - It builds a new generation of the tables and commits it in batches. The old index answers
+ *   until the end, and a failed rebuild leaves it as it was.
+ * - It reads the log as a stream, so its memory does not grow with the log.
+ * - It counts and skips a damaged line, a missing blob and an extractor that throws.
+ * - The documents of a recipe that has no extractor in this runtime stay, from the old index.
+ * - A capture that a different writer adds during the rebuild goes into the new index.
+ */
+declare const reindex: (runtime: CrawlRuntime, options?: {
+  readonly batch?: number | undefined;
+}) => Promise<ReindexResult>;
+/** The result of `catchUpIndex`. */
+interface CatchUpResult extends ReindexResult {
+  /**
+   * The value `rebuild` tells that the index was stale, and the function built it again from the
+   * whole log. The value `tail` tells that it read only the captures that the index missed.
+   */
+  readonly mode: "rebuild" | "tail";
+}
+/**
+ * This function brings the reading index up to the capture log, with no request. `openCrawler`
+ * uses it at open. An application that uses `createRuntime` uses it the same way:
+ *
+ * - A stale index (a new schema, or a deleted file) is built again with `reindex`.
+ * - An index that is behind the log gets the captures that it missed. A process can stop between
+ *   the log and the index, and SQLite can lose its last transactions after a power cut. The
+ *   function reads the log from a short time before the newest cache entry.
+ *
+ * It gives the counts, or undefined when the index missed nothing.
+ */
+declare const catchUpIndex: (runtime: CrawlRuntime) => Promise<CatchUpResult | undefined>;
+//#endregion
+//#region src/fallback.d.ts
+/**
+ * The archived fallback. When a site refuses a page, or is down, the crawler can give a copy from a
+ * public web archive: the Wayback Machine, then Common Crawl. The crawler does not show the copy as a fresh fetch:
+ *
+ * - It is a document of its own (`archive:<url>`), of the kind `archived`. Its metadata and each
+ *   result that shows it have `archived`: the archive, the date of the copy and its URL there.
+ * - The result keeps the outcome of the live fetch (`blocked`, for example), and its note names the
+ *   archive and the date.
+ *
+ * The fallback is opt-in (`fallback: "archive"`). The archives are third-party APIs with their own
+ * politeness, so each request to them is a fetch of the same crawler. Their robots.txt, the pace of
+ * each host, the egress policy and the identity of the crawler apply, and the capture log records
+ * each request.
+ *
+ * By default, the crawler gives no archived copy of a page that the current robots.txt of its site
+ * disallows for this agent. Only a stop, an error or a dead host gets the fallback.
+ * `archiveDespiteRobots` widens it. The usage signals of the copy apply as for a live page.
+ */
+/** The public web archives of the fallback. */
+type WebArchive = "wayback" | "commoncrawl";
+/** An archived copy: the archive, the time when the archive captured the page, and where the copy is. */
+interface Archived {
+  readonly source: WebArchive;
+  /** This time (ISO 8601) is when the archive captured the page. It is not the time of the fetch from the archive. */
+  readonly timestamp: string;
+  /** This URL is the copy in the archive: a Wayback Machine page, or the Common Crawl WARC file. */
+  readonly url: string;
+  /** This URL is the page that the archive captured. */
+  readonly original: string;
+  /** This id is the `WARC-Record-ID` of a Common Crawl copy in its WARC file. */
+  readonly record?: string | undefined;
+}
+/** This function gives the document id of the archived copy of a page. */
+declare const archivedDocId: (url: string) => string;
+/**
+ * This extractor reads the captures of the recipe `archive-copy`. It has no side effects, so
+ * `reindex` replays them. It reads two forms:
+ *
+ * - A Wayback Machine copy in the raw form (`/web/<time>id_/<url>`). The body is the body of the page.
+ * - A Common Crawl WARC record (a gzip member). It reads the HTTP headers and the body from the record.
+ *
+ * It reads the body with the page reader, and gives a document of the kind `archived` with
+ * `archived` in its metadata. The usage signals of the HTTP headers in a WARC record apply too.
+ */
+declare const archiveExtractor: Extractor;
+//#endregion
+//#region src/docs.d.ts
+/** This is one document, as an extractor gives it. */
+interface ExtractedDoc {
+  readonly doc: string;
+  readonly kind: string;
+  readonly title: string;
+  /** This text is the readable text. For a paper, it is the title and the abstract. */
+  readonly text: string;
+  readonly url: string | null;
+  readonly meta: Readonly<Record<string, unknown>>;
+  /** This rule chooses between the texts of two recipes. The default is `richest`: the longer text. */
+  readonly merge?: "latest" | "richest" | undefined;
+  /** When the extractor cut the text, this text is the whole text. The function `verify` looks for a quote here too. */
+  readonly full_text?: string | undefined;
+  /**
+   * The value `no` holds the document back from the AI reader, also when the capture said nothing.
+   * An extractor sets it when the body states it, for example the headers in an archived response.
+   */
+  readonly ai_input?: AiInput | undefined;
+}
+/** This is one version of a document: the document as one capture gave it. */
+interface DocInput extends ExtractedDoc {
+  /** These ids are the capture records that this version came from. The first id names the version. */
+  readonly captures: readonly string[];
+  readonly ai_input?: AiInput | undefined;
+  /** This id is the capture of the version. The default is the first id of `captures`. */
+  readonly capture?: string | undefined;
+  /** This time (ISO 8601) is the time of the fetch. The default is the time of the upsert. */
+  readonly fetched_at?: string | undefined;
+  readonly blob?: string | null | undefined;
+  /** This value is the recipe that read the capture. A version with no recipe has the recipe `""`. */
+  readonly recipe?: string | undefined;
+  /** This value is true when the headless browser made the capture. */
+  readonly rendered?: boolean | undefined;
+  /** This label comes from the capture. The index writes it as it is and does not read it. */
+  readonly scope?: string | undefined;
+  /** These labels come from the capture. The index writes them as they are and does not read them. */
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+}
+/** This is the current version of a document. */
+interface Doc {
+  readonly doc: string;
+  readonly kind: string;
+  readonly title: string;
+  readonly text: string;
+  readonly url: string | null;
+  readonly meta: Readonly<Record<string, unknown>>;
+  /** These ids are the captures behind the current text, the newest first, `MAX_CAPTURES` or fewer. */
+  readonly captures: readonly string[];
+  /** This value is the number of versions of the document. */
+  readonly capture_count: number;
+  /** This id is the capture of the current text. */
+  readonly capture: string;
+  readonly blob: string | null;
+  /** This time (ISO 8601) is the fetch time of the current text. */
+  readonly fetched_at: string;
+  readonly recipe: string;
+  readonly rendered: boolean;
+  readonly ai_input: AiInput;
+  /** This value is true when the extractor cut the text, and the index keeps the whole text for `verify`. */
+  readonly cut: boolean;
+  readonly scope?: string | undefined;
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+  readonly updated_at: string;
+}
+/** This is one version of a document, with no text. */
+interface DocVersion {
+  readonly doc: string;
+  readonly capture: string;
+  readonly recipe: string;
+  readonly rendered: boolean;
+  readonly fetched_at: string;
+  readonly blob: string | null;
+  readonly kind: string;
+  readonly title: string;
+  readonly url: string | null;
+  readonly ai_input: AiInput;
+  /** This value is the length of the text of the version. */
+  readonly chars: number;
+  /** This value is true when the index keeps a whole text that is longer than the text. */
+  readonly cut: boolean;
+}
+/** This is the text of one version, with its metadata. */
+interface VersionText extends DocVersion {
+  readonly text: string;
+  /** This text is the whole text when the extractor cut `text`. */
+  readonly full_text: string | undefined;
+  readonly meta: Readonly<Record<string, unknown>>;
+}
+/** This is the version of a document that was current at a time (`ReadingIndex.versionAt`). */
+interface VersionAt {
+  /**
+   * This version was current at the time, or it is undefined when the time is before the first
+   * version. Its `ai_input` is `no` when the document was held back at that time.
+   */
+  readonly version: VersionText | undefined;
+  /** This time (ISO 8601) is the fetch time of the first version of the document, or undefined for an unknown document. */
+  readonly first: string | undefined;
+}
+/**
+ * This value chooses the ranking of a search:
+ *
+ * - `exact` (the default): BM25 ranks each document that matches.
+ * - `fast`: the form of the query that needs one part leaves out the common parts. A common part
+ *   is in more than `FAST_RANK_SHARE` of the documents. The rarest part stays when each part is common.
+ *
+ * The documents that hold each part come first in the two modes, with the same rank. In the mode
+ * `fast`, the rest of the results leaves out a document that holds only common parts. The common
+ * parts also do not add to the score of the rest. Thus `fast` is faster for a long question, and
+ * its results can be different. The README gives the measured difference.
+ */
+type RankMode = "exact" | "fast";
+/** In the mode `fast`, a part is common when more than this share of the documents hold it. */
+declare const FAST_RANK_SHARE = .2;
+/** These are the options of a search of the index. */
+interface FindIndexOptions {
+  /** This value is the maximum number of results. The index gives 1000 or fewer. */
+  readonly limit: number;
+  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
+  readonly kind?: string | null | undefined;
+  /** This value chooses the ranking. The default is `exact`. */
+  readonly rank?: RankMode | undefined;
+}
+interface Found {
+  readonly doc: string;
+  readonly kind: string;
+  readonly title: string;
+  readonly url: string | null;
+  readonly snippet: string;
+  readonly score: number;
+  readonly ai_input: AiInput;
+  /** This time (ISO 8601) is the fetch time of the current text. */
+  readonly fetched_at: string;
+  /** This id is the capture of the current text. */
+  readonly capture: string;
+  /** This value tells that the document is an archived copy (the kind `archived`), with the archive and the date of the copy. */
+  readonly archived?: Archived | undefined;
+}
+/** The variant of a cache entry: the plain response, or the page that the browser rendered. */
+type Variant = "plain" | "rendered";
+interface Latest {
+  readonly url: string;
+  readonly capture: string;
+  readonly blob: string | null;
+  readonly etag: string | null;
+  readonly last_modified: string | null;
+  readonly content_type: string | null;
+  readonly fetched_at: string;
+  readonly status: number;
+  /** This value is what the page said about AI input at the time of the fetch. */
+  readonly ai_input: AiInput;
+  /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */
+  readonly location?: string | null | undefined;
+  /** This value is true when the stored body is only the first part of the response. */
+  readonly truncated?: boolean | undefined;
+  /** This time (ISO 8601) is the end of the freshness that the server gave. */
+  readonly fresh_until?: string | null | undefined;
+  /** This text holds the request headers that the response varies on (`Vary`), with their values. */
+  readonly vary_key?: string | null | undefined;
+  /** The default is `plain`. A key that ends in `#rendered` (the form of version 0.1) is `rendered`. */
+  readonly variant?: Variant | undefined;
+}
+/** These are the counts of the index. */
+interface Counts {
+  readonly docs: number;
+  readonly urls: number;
+}
+/** A rebuild of the index writes a new generation of the tables. The old generation answers until the commit. */
+interface IndexBuild {
+  /** This method adds a version to the new generation. The build writes the versions in batches. */
+  upsert(input: DocInput, now: string): void;
+  /** This method sets a cache entry in the new generation, when it is newer than the stored entry. */
+  setLatest(entry: Latest): void;
+  /** This method reads a cache entry of the new generation. */
+  latest(url: string, variant?: Variant): Latest | undefined;
+  /** This method writes the waiting versions in one transaction. */
+  flush(): void;
+  /**
+   * This method copies the versions of each recipe that is not in `recipes` from the current
+   * generation. A runtime with no extractor for a recipe thus keeps the documents of that recipe.
+   * It gives the number of documents that it copied.
+   */
+  carry(recipes: ReadonlySet<string>): number;
+  /**
+   * This method makes the new generation current. It first copies the rows that a different writer
+   * added to the current generation since `since`. Then it removes the old generation.
+   */
+  commit(since: string): Counts;
+  /** This method removes the new generation. The current generation stays. */
+  abort(): void;
+}
+/**
+ * A reading index keeps the documents, their versions, the search and the cache entries. `DocIndex`
+ * is the SQLite index. A runtime takes any object with these methods, for example a search index
+ * of a different kind that wraps a `DocIndex`.
+ */
+interface ReadingIndex {
+  /** This value is true when the index opened empty and a rebuild did not fill it yet. */
+  readonly stale: boolean;
+  get(doc: string): Doc | undefined;
+  upsert(input: DocInput, now: string): Doc;
+  versions(doc: string, options?: {
+    readonly limit?: number;
+  }): DocVersion[];
+  versionText(doc: string, capture?: string): VersionText | undefined;
+  /**
+   * This optional method gives the version that was current at a time (ISO 8601), with the rules
+   * that choose the current version. An index without it gets the newest version at or before the
+   * time, from `versions` and `versionText`.
+   */
+  versionAt?(doc: string, at: string): VersionAt;
+  earlierTexts(doc: string, options?: {
+    readonly limit?: number;
+  }): VersionText[];
+  docsOfCapture(capture: string): Doc[];
+  /** This method tells if a capture gave a version of a document. */
+  hasVersion(capture: string): boolean;
+  find(q: string, options: FindIndexOptions): Found[];
+  rankWithin(q: string, docs: readonly string[], options?: {
+    readonly rank?: RankMode | undefined;
+  }): Array<{
+    readonly doc: string;
+    readonly score: number;
+  }>;
+  latest(url: string, variant?: Variant): Latest | undefined;
+  setLatest(entry: Latest): void;
+  /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
+  newestFetch(): string | undefined;
+  counts(): Counts;
+  beginRebuild(options?: {
+    readonly batch?: number;
+  }): IndexBuild;
+}
+/** These are the options of a reading index. */
+interface DocIndexOptions {
+  /** This value is the time in milliseconds that a write waits for the lock of a different process. The default is 10 seconds. */
+  readonly busyTimeoutMs?: number | undefined;
+}
+declare class DocIndex implements ReadingIndex {
+  readonly path: string;
+  private readonly db;
+  private readonly statements;
+  private depth;
+  private closed;
+  private empty;
+  /**
+   * This value is true when the file had a different schema, or no tables, so the index opened
+   * empty. A store with captures then needs `reindex`. The commit of a rebuild makes it false.
+   */
+  get stale(): boolean;
+  constructor(path: string, options?: DocIndexOptions);
+  /** This method makes the tables of the schema. It gives true when the index opened empty. */
+  private prepareSchema;
+  private setMeta;
+  private getMeta;
+  private stmt;
+  /**
+   * This method does a write in one transaction. A transaction in progress takes the write in,
+   * also a transaction that the caller opened on the connection.
+   */
+  private write;
+  private generation;
+  /**
+   * This method does a step on the tables of the current generation. A rebuild of a different
+   * process can remove those tables in the meantime. Then the method reads the generation again
+   * and does the step once more.
+   */
+  private live;
+  /** This method removes the tables of a generation. */
+  private dropGeneration;
+  /** This method removes the generations of builds that a stopped process left. */
+  private dropAbandoned;
+  private putText;
+  private textOf;
+  private putVersion;
+  /** This method gives the newest version of a document for one recipe and one variant, at `until` or before when it is given. */
+  private newest;
+  /**
+   * This method chooses the current version of a document with the rules of the class. With
+   * `until`, it uses only the versions that the crawler fetched at that time or before. Thus it
+   * gives the version that was current at that time. It gives undefined when no version agrees.
+   */
+  private choose;
+  /** This method chooses the current version of a document from its versions, and writes it with its search entry. */
+  private recompute;
+  private docOf;
+  private toDoc;
+  private versionTextOf;
+  get(doc: string): Doc | undefined;
+  /**
+   * This method adds one version of a document, and it gives the current document. The rules of
+   * the class choose the current version. An upsert of a known capture replaces that version.
+   */
+  upsert(input: DocInput, now: string): Doc;
+  /** This method gives the versions of a document, the newest first. */
+  versions(doc: string, options?: {
+    readonly limit?: number;
+  }): DocVersion[];
+  /**
+   * This method gives the text of one version of a document. With no capture, it gives the current
+   * version. It also gives the text of a held-back version: the caller applies `isHeldBack` to the
+   * `ai_input` of the version.
+   */
+  versionText(doc: string, capture?: string): VersionText | undefined;
+  /**
+   * This method gives the version of a document that was current at a time (ISO 8601). It applies
+   * the rules of the class to the versions that the crawler fetched at that time or before. Thus
+   * a later capture does not change the answer. It also gives a held-back version: its `ai_input`
+   * is `no` when the document was held back at that time, and the caller applies `isHeldBack`.
+   */
+  versionAt(doc: string, at: string): VersionAt;
+  /**
+   * This method gives the texts of the earlier versions of a document, the newest first. Each
+   * text comes one time, with the newest version that has it. It skips the current text. It also
+   * gives held-back texts: the caller applies `isHeldBack` to the `ai_input` of each version.
+   */
+  earlierTexts(doc: string, options?: {
+    readonly limit?: number;
+  }): VersionText[];
+  /** This method tells if a capture gave a version of a document. */
+  hasVersion(capture: string): boolean;
+  /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
+  newestFetch(): string | undefined;
+  /** This method gives the current documents that a capture gave. */
+  docsOfCapture(capture: string): Doc[];
+  /**
+   * This method searches the documents with BM25, the best first. The syntax of the query is the
+   * syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it
+   * adds the documents that match one part. A document that asked not to be AI input is not in
+   * the results, so a search cannot show its text. The option `rank` chooses the ranking (`RankMode`).
+   */
+  find(q: string, options: FindIndexOptions): Found[];
+  /**
+   * This method scores a set of documents against a query, with the best first. It leaves out a
+   * document that matches no term, and a document that asked not to be AI input. It ranks the
+   * results of a source again, because the order of the source is only as good as its search engine.
+   * In the mode `fast`, it also leaves out a document that holds only common parts (`RankMode`).
+   */
+  rankWithin(q: string, docs: readonly string[], options?: {
+    readonly rank?: RankMode | undefined;
+  }): Array<{
+    readonly doc: string;
+    readonly score: number;
+  }>;
+  /**
+   * This method gives the form of a query that needs one part, without the common parts (the mode
+   * `fast` of `RankMode`). It counts the documents that hold each part. When each part is common,
+   * it keeps the rarest part.
+   */
+  private rareParts;
+  /** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */
+  latest(url: string, variant?: Variant): Latest | undefined;
+  private putLatest;
+  /** This method keeps the last answer for a URL. An entry that is older than the stored entry changes nothing. */
+  setLatest(entry: Latest): void;
+  counts(): Counts;
+  /** This method empties the current generation. A rebuild uses `beginRebuild`, which keeps the index until its commit. */
+  reset(): void;
+  /**
+   * This method starts a rebuild into a new generation of the tables. The current generation
+   * answers each read and takes each write until the commit. A failed rebuild thus leaves the
+   * index as it was.
+   */
+  beginRebuild(options?: {
+    readonly batch?: number;
+  }): IndexBuild;
+  /** This method closes the file. A second close does nothing. */
+  close(): void;
+}
+//#endregion
+//#region src/egress.d.ts
+/**
+ * The egress policy keeps the crawler on the public internet. A page or an agent can ask for an
+ * internal address. Examples are a loopback service, a router on the LAN and the metadata
+ * service of a cloud host. The crawler refuses these addresses by default.
+ *
+ * - The crawler fetches only `http:` and `https:` URLs.
+ * - The check uses the address after DNS. The `lookup` hook of the connection checks the address
+ *   that the socket then uses, so a DNS answer that changes cannot pass the check.
+ * - The crawler examines a literal IP host directly, in each spelling that the URL parser reads.
+ * - The crawler examines each redirect hop again.
+ *
+ * A refusal is a skipped fetch with a clear note. It does not throw an error.
+ */
+/** These options change the egress policy. With no options, the crawler fetches only public addresses. */
+interface EgressOptions {
+  /** The value true lets the crawler fetch loopback, private and link-local addresses. The default is false. */
+  readonly allowPrivate?: boolean | undefined;
+  /**
+   * These hosts are permitted although their address is not public. An entry is a host name, a
+   * name pattern (`*.corp.example`), an IP address or a CIDR range (`10.0.0.0/8`).
+   */
+  readonly allowHosts?: readonly string[] | undefined;
+}
+/** The connection hook gives this error when the policy refuses each address of a host. */
+declare class EgressRefused extends Error {
+  readonly address: string;
+  readonly kind: string;
+  readonly code: string;
+  constructor(address: string, kind: string);
+}
+/** This is the form of `dns.lookup` that the egress hook uses. Tests give a resolver of their own. */
+type Resolve = (host: string, options: LookupAllOptions, callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void) => void;
+/**
+ * This function gives a `lookup` hook for the connection options of an undici `Agent`. The hook
+ * resolves the name, removes each address that the policy refuses, and gives the rest to the
+ * socket. When no address stays, the connection fails with `EgressRefused`.
+ */
+declare const egressLookup: (policy: EgressOptions, resolve?: Resolve) => LookupFunction;
+//#endregion
+//#region src/webbotauth.d.ts
+/**
+ * Signed requests: HTTP Message Signatures (RFC 9421), as the IETF Web Bot Auth profile tells
+ * (`draft-ietf-webbotauth-httpsig-protocol-00`, 1 September 2026). A signature lets a site verify that
+ * a request comes from the holder of a key that the operator publishes.
+ *
+ * - Each request gets three headers: `Signature-Agent`, `Signature-Input` and `Signature`.
+ * - `Signature-Agent` is a dictionary. Its member names the key directory of the operator.
+ * - The signature covers `@authority` and that member of `Signature-Agent`.
+ * - The parameters are `created`, `keyid` (the JWK thumbprint of the key), `alg="ed25519"`,
+ *   `expires`, a random `nonce` and `tag="web-bot-auth"`.
+ * - The key is an Ed25519 key, and `node:crypto` signs. The package adds no dependency.
+ *
+ * A signature adds an identity. It does not replace the `User-Agent` header, which stays honest.
+ */
+/** The path of the key directory on the host of the operator (the well-known URI of the draft). */
+declare const KEY_DIRECTORY_PATH = "/.well-known/http-message-signatures-directory";
+/** The media type of the key directory. The host of the directory sends it as `Content-Type`. */
+declare const KEY_DIRECTORY_TYPE = "application/http-message-signatures-directory+json";
+/**
+ * This value tells how a site finds the keys from the URL of `Signature-Agent`:
+ *
+ * - `directory` (the default): the URL is an origin, and the keys are at `KEY_DIRECTORY_PATH` on it.
+ * - `jwks_uri`: the URL is the address of a JSON Web Key Set, on any path.
+ */
+type SignatureAgentType = "directory" | "jwks_uri";
+/** A key in one of the forms that `node:crypto` reads: a `KeyObject`, PEM text, DER bytes (PKCS#8 or SPKI) or a JWK. */
+type KeyInput = KeyObject | string | Uint8Array | JsonWebKey;
+/** These options make a signer for Web Bot Auth. */
+interface WebBotAuthOptions {
+  /** This key signs. It is an Ed25519 private key: a `KeyObject`, PKCS#8 (PEM text or DER bytes) or a JWK with `d`. */
+  readonly privateKey: KeyInput;
+  /**
+   * This text is the `keyid` of each signature: the JWK thumbprint of the key (RFC 8037). The signer
+   * calculates it. A value that is not the thumbprint of the key throws an error.
+   */
+  readonly keyId?: string | undefined;
+  /**
+   * This URL tells a site where the keys are. For the type `directory`, it is an https origin, for
+   * example `https://bot.example.com`. For `jwks_uri`, it is the https URL of a JSON Web Key Set.
+   */
+  readonly signatureAgent: string;
+  /** This value tells how a site finds the keys from `signatureAgent`. The default is `directory`. */
+  readonly agentType?: SignatureAgentType | undefined;
+  /** This value is the time that a signature stays valid, in milliseconds. The default is 5 minutes, and the maximum is 24 hours. */
+  readonly lifetimeMs?: number | undefined;
+  /** This text is the label of the signature, and the key of the member of `Signature-Agent`. The default is `sig1`. */
+  readonly label?: string | undefined;
+}
+/**
+ * A request signer gives the signature headers of one request. The crawler uses it for each
+ * request, robots.txt too. `createWebBotAuthSigner` makes one. A signer of your own can sign with a
+ * key in a hardware module, for example.
+ */
+interface RequestSigner {
+  /** This text is the `keyid` of the signatures. */
+  readonly keyId: string;
+  /** This URL is the value of the member of `Signature-Agent`: where a site finds the keys. */
+  readonly signatureAgent: string;
+  /** This method gives the headers that sign a request to `url` at the time `now` (milliseconds since the epoch). */
+  sign(url: URL, now: number): Readonly<Record<string, string>>;
+}
+/** The signer that `createWebBotAuthSigner` makes. It also gives its public key for the key directory. */
+interface WebBotAuthSigner extends RequestSigner {
+  /** This JWK is the public key of the signer, with no private part. Give it to `keyDirectory`. */
+  readonly publicKey: JsonWebKey;
+}
+/** One key of a key directory: an Ed25519 public key as a JWK, with its thumbprint as `kid`. */
+interface DirectoryKey {
+  readonly kty: "OKP";
+  readonly crv: "Ed25519";
+  readonly kid: string;
+  readonly x: string;
+  readonly use: "sig";
+}
+/** A key directory: a JSON Web Key Set. The operator serves it as JSON at `KEY_DIRECTORY_PATH`, with the type `KEY_DIRECTORY_TYPE`. */
+interface KeyDirectory {
+  readonly keys: readonly DirectoryKey[];
+}
+/** The parts of a request that `verifySignature` reads. */
+interface SignedRequest {
+  readonly url: string;
+  /** The default is `GET`. */
+  readonly method?: string | undefined;
+  /** These are the headers of the request. A name can be in any case, and a header can have more than one line. */
+  readonly headers: Readonly<Record<string, string | readonly string[] | undefined>>;
+}
+/** The result of `verifySignature`. */
+interface SignatureCheck {
+  /** This value is true when a Web Bot Auth signature of the request verifies with a key of the directory. */
+  readonly verified: boolean;
+  /** This text is the label of the signature that the check examined last, or null. */
+  readonly label: string | null;
+  readonly keyId: string | null;
+  /** This URL is the member of `Signature-Agent` that the signature covers, or null. */
+  readonly agent: string | null;
+  /** This text tells why the request does not verify. It is null when it verifies. */
+  readonly reason: string | null;
+}
+/**
+ * This function makes the key directory for the operator to host: a JSON Web Key Set with each
+ * public key and its thumbprint as `kid`. It takes public keys, or private keys, and it writes only
+ * their public parts. Serve the JSON at `KEY_DIRECTORY_PATH` with the type `KEY_DIRECTORY_TYPE`.
+ */
+declare const keyDirectory: (publicKeys: readonly KeyInput[]) => KeyDirectory;
+/**
+ * This function makes a signer for Web Bot Auth with an Ed25519 key. Each signature covers
+ * `@authority` and the member of `Signature-Agent`. It has a new random nonce, and it expires after
+ * `lifetimeMs`. The signer keeps the private key in a closure: no property of the signer holds it.
+ */
+declare const createWebBotAuthSigner: (options: WebBotAuthOptions) => WebBotAuthSigner;
+/**
+ * This function verifies the Web Bot Auth signatures of a request with the keys of a directory. It
+ * applies RFC 9421 section 3.2 and the profile of the draft. The profile asks for the tag
+ * `web-bot-auth`, `created`, `expires` and `@authority`. It also asks for the member of
+ * `Signature-Agent`, and a key with the thumbprint of `keyid`. The tests use it, and an operator can check a setup with it. It sends no request.
+ */
+declare const verifySignature: (request: SignedRequest, directory: KeyDirectory, options?: {
+  readonly now?: number | undefined;
+}) => SignatureCheck;
+//#endregion
+//#region src/crawler.d.ts
+/**
+ * The crawler is a polite fetcher. It sends one request at a time to each host, and it obeys
+ * robots.txt for web pages. Its `User-Agent` header tells what it is. It asks again with the
+ * validators of the stored copy, and it stops when a site refuses. Each decision goes into the
+ * capture log, and a fetch that the crawler declines goes there too.
+ *
+ * There are two kinds of request. A documented public API (`api`) has its own terms and rate
+ * limits. A web page (`page`) obeys robots.txt. The settings come from measurements:
+ *
+ * - The `request` function of undici on one `Agent`.
+ * - A keep-alive time longer than the delay for each host. With the default of 4 seconds, each
+ *   polite request opens a new connection.
+ * - A backoff with jitter in the queue, not the retry handler of undici. That handler limits
+ *   Retry-After to 30 seconds and adds no jitter.
+ * - A maximum size for each body, after decompression. The crawler keeps the first part of a
+ *   longer body and marks it as truncated.
+ *
+ * A fetch and a render go through one admission. It examines the URL, the egress policy, the
+ * product tokens, robots.txt, the cache and the stops of the host. Only the transport is
+ * different: an HTTP request or the headless browser.
+ *
+ * The egress policy (`EgressOptions`) keeps the crawler on public addresses by default. The
+ * politeness policy (`PolitenessPolicy`) holds each limit, and a host can have its own values.
+ * The fetch and the render do not throw an error: each problem is an outcome with a note.
+ */
+/**
+ * The outcome of a fetch or a render. A `skipped` request did not go out. A render of a URL that
+ * gives a file and not a page is `not-html`.
+ */
+type FetchOutcome = Outcome | "skipped" | "not-html";
+interface FetchRequest {
+  readonly url: string;
+  /** The value `api` is a documented API with its own terms. The value `page` is a web page, and robots.txt applies. */
+  readonly mode: "api" | "page";
+  /** This recipe reads the capture again. The function `reindex` uses it to replay the capture. */
+  readonly recipe: string;
+  readonly accept?: string | undefined;
+  /**
+   * These are more request headers. The crawler does not write them into the capture. They
+   * cannot replace `User-Agent`, `From`, `Accept-Encoding` or the validators. A redirect to a
+   * different origin removes `Authorization`, `Cookie` and `Proxy-Authorization`.
+   */
+  readonly headers?: Readonly<Record<string, string>> | undefined;
+  /**
+   * These query parameters contain secrets. Each capture shows `REDACTED` in their place. The
+   * names in `DEFAULT_SECRET_PARAMS` get the same treatment with no list.
+   */
+  readonly secretParams?: readonly string[] | undefined;
+  /** This value is the documented pace of the source: the shortest time between two of its requests. */
+  readonly minIntervalMs?: number | undefined;
+  /**
+   * A stored capture younger than this age answers, and the crawler sends no request. The value
+   * `auto` uses the freshness that the server gave (RFC 9111): `Cache-Control`, `Expires`, or 10
+   * percent of the age of `Last-Modified`.
+   */
+  readonly maxAgeMs?: number | "auto" | undefined;
+  readonly maxBytes?: number | undefined;
+  /**
+   * These are the product tokens of an agent that the fetch acts for, for example `Claude-User`.
+   * The robots.txt rules for these tokens apply too.
+   */
+  readonly onBehalfOf?: readonly string[] | undefined;
+  /** This name connects the captures to the job that asked for them. */
+  readonly job?: string | undefined;
+  /** When this signal aborts, the crawler stops the fetch. The result is then `skipped` or `error`, with a note. */
+  readonly signal?: AbortSignal | undefined;
+  /** This label goes into each capture record of the request. It replaces the `scope` of the crawler. */
+  readonly scope?: string | undefined;
+  /** These labels go into each capture record of the request, for example its lineage. They are not part of the capture id. */
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+  /**
+   * This time (milliseconds since the epoch, on the clock of the crawler) is the end of a budget.
+   * The request does not start at or after it. When the host is not free before it, the result is
+   * `skipped`, and its note starts with `DEADLINE_NOTE`.
+   */
+  readonly deadline?: number | undefined;
+  /**
+   * This byte range is the part of the resource to fetch (a `Range` request). The crawler then sends
+   * no validators, keeps no stored copy for the URL, and reads `length` bytes at most. The capture
+   * records the range.
+   */
+  readonly range?: {
+    readonly offset: number;
+    readonly length: number;
+  } | undefined;
+  /** These lines of usage signals apply to the response too. A copy from a web archive gets the content signals of the robots.txt of its site. */
+  readonly signalLines?: readonly string[] | undefined;
+  /**
+   * This prefix names the response headers that hold the headers of an archived response, for
+   * example `x-archive-orig-` of the Wayback Machine. The usage signals of those headers apply too.
+   */
+  readonly originalHeaderPrefix?: string | undefined;
+}
+interface Fetched {
+  /** This URL is the URL of the capture. Its secrets show as `REDACTED`. */
+  readonly url: string;
+  readonly status: number;
+  readonly outcome: FetchOutcome;
+  /** This value is true when a stored capture answered, with no request. */
+  readonly fromCache: boolean;
+  /** This id is the capture record behind the body. A citation names it. */
+  readonly capture: string | undefined;
+  readonly blob: string | undefined;
+  readonly body: Uint8Array | undefined;
+  readonly contentType: string | undefined;
+  readonly signals: Signals;
+  readonly note: string | undefined;
+  /** This value is true when the body was longer than the maximum, so the crawler kept only its first part. */
+  readonly truncated?: boolean | undefined;
+  /** This time (ISO 8601) is the time of the fetch of the body. For a stored copy, it is the time of that copy. */
+  readonly fetchedAt?: string | undefined;
+  /** This label is the scope of the capture. */
+  readonly scope?: string | undefined;
+  /** These labels are the labels of the capture. */
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+}
+/** The crawler writes each record to a capture writer. The class `CaptureLog` is one. */
+interface CaptureWriter {
+  append(record: CaptureRecord): Promise<void>;
+  close?(): Promise<void>;
+}
+/** A cache entry is the last answer for a URL: its capture, its body and its validators. */
+interface CacheEntry {
+  /** This text is the cache key: the canonical URL, with its secrets as `REDACTED`. */
+  readonly url: string;
+  readonly capture: string;
+  readonly blob: string | null;
+  readonly etag: string | null;
+  readonly last_modified: string | null;
+  readonly content_type: string | null;
+  readonly fetched_at: string;
+  readonly status: number;
+  /** This value is what the page said about AI input at the time of the fetch. */
+  readonly ai_input: AiInput;
+  /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */
+  readonly location?: string | null | undefined;
+  /** This value is true when the stored body is only the first part of the response. */
+  readonly truncated?: boolean | undefined;
+  /** This time (ISO 8601) is the end of the freshness that the server gave. */
+  readonly fresh_until?: string | null | undefined;
+  /** This text holds the request headers that the response varies on (`Vary`), with their values. */
+  readonly vary_key?: string | null | undefined;
+  /** This value tells a plain response from a page that the browser rendered. The default is `plain`. */
+  readonly variant?: Variant | undefined;
+}
+/**
+ * The cache index holds the last answer for each URL. The crawler reads it before a request, and
+ * it writes it after an answer. The class `DocIndex` is one. An index that keeps only the
+ * necessary fields of `CacheEntry` works, and the crawler keeps the other fields in memory.
+ */
+interface CacheIndex {
+  latest(url: string, variant?: Variant): CacheEntry | undefined;
+  setLatest(entry: CacheEntry): void;
+}
+/** An event tells a listener what the crawler does. The listener cannot change it. */
+type CrawlEvent = {
+  readonly type: "request";
+  readonly url: string;
+  readonly host: string;
+  readonly transport: "http" | "browser";
+} | {
+  readonly type: "response";
+  readonly url: string;
+  readonly host: string;
+  readonly status: number;
+  readonly outcome: FetchOutcome;
+  readonly elapsedMs: number;
+} | {
+  readonly type: "wait";
+  readonly host: string;
+  readonly ms: number;
+  readonly reason: string;
+} | {
+  readonly type: "stop";
+  readonly host: string;
+  readonly until: number;
+  readonly reason: string;
+} | {
+  readonly type: "skip";
+  readonly url: string;
+  readonly note: string;
+};
+interface CrawlerOptions {
+  /** This product token is the name that robots.txt rules match. It contains only letters, `_` and `-` (RFC 9309). */
+  readonly token: string;
+  /** This text is the full `User-Agent` header. */
+  readonly userAgent: string;
+  /** This contact goes into the `From` header. Without it, the crawler sends no `From` header. */
+  readonly from?: string | undefined;
+  readonly log: CaptureWriter;
+  readonly blobs: BlobStore;
+  readonly index: CacheIndex;
+  readonly clock?: Clock | undefined;
+  readonly random?: (() => number) | undefined;
+  /** This value is the number of hosts that get requests at one time. The default is 16. */
+  readonly maxActive?: number | undefined;
+  /**
+   * This value tells if the crawler also obeys the robots.txt group of `PoliteCrawler`. That applies
+   * when the `User-Agent` links the bot page of polite-crawler (`BOT_PAGE`) and the token is different.
+   * Site owners read that page, and they write their rules for `PoliteCrawler`. The most restrictive
+   * of the groups applies, as for `onBehalfOf`. The default is true. Set false for a bot page of your own.
+   */
+  readonly obeyDefaultToken?: boolean | undefined;
+  /**
+   * This value is the number of pages that the browser renders at one time. The default is 2. Each
+   * render is a browser context of its own, which holds much memory. A render waits for a slot before
+   * the lease of its host, so the wait does not count toward the time of the host.
+   */
+  readonly maxRenders?: number | undefined;
+  /**
+   * This undici dispatcher sends the requests. With a dispatcher of your own, the crawler checks
+   * only literal addresses. Put `egressLookup` in its connection options to check the address after DNS.
+   */
+  readonly dispatcher?: Dispatcher | undefined;
+  /** This label goes into each capture record as it is. */
+  readonly scope?: string | undefined;
+  /**
+   * This function ranks the scopes of the requests, the most private scope highest. One fetch of a
+   * robots.txt can serve requests of different scopes. Its record then takes the scope with the
+   * highest rank, and the labels of each request. With no function, the first request gives the scope.
+   */
+  readonly scopeRank?: ((scope: string) => number) | undefined;
+  /** This browser renders the pages that need JavaScript. Without it, the crawler renders no page. */
+  readonly renderer?: Renderer | undefined;
+  /** This policy controls the addresses that the crawler fetches. By default, the crawler fetches only public addresses. */
+  readonly egress?: EgressOptions | undefined;
+  /** These values change the limits of the politeness policy, for each host or for some hosts. */
+  readonly politeness?: PolicyOptions | undefined;
+  /** This store keeps the pace, the counts and the stops of each host. The default keeps them in memory. */
+  readonly hostState?: HostStateStore | undefined;
+  /** This function gets an event for each request, response, wait, stop and skip. An error in it does not stop the crawler. */
+  readonly onEvent?: ((event: CrawlEvent) => void) | undefined;
+  /**
+   * This signer signs each request, robots.txt and the requests of a rendered page too. Give the
+   * options of `createWebBotAuthSigner`, or a `RequestSigner`. Without it, the crawler signs nothing.
+   * A signature adds an identity: the `User-Agent` and `From` headers stay as they are.
+   */
+  readonly signer?: RequestSigner | WebBotAuthOptions | undefined;
+}
+interface RenderRequest {
+  readonly url: string;
+  readonly recipe: string;
+  /** A stored render younger than this age answers. The value `auto` uses the freshness that the server gave. */
+  readonly maxAgeMs?: number | "auto" | undefined;
+  readonly onBehalfOf?: readonly string[] | undefined;
+  readonly job?: string | undefined;
+  /** When this signal aborts, the crawler stops the render, and the browser stops the page. */
+  readonly signal?: AbortSignal | undefined;
+  /** This label goes into the capture record of the render. It replaces the `scope` of the crawler. */
+  readonly scope?: string | undefined;
+  /** These labels go into the capture record of the render. */
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+  /** This time (milliseconds since the epoch) is the end of a budget. The render does not start at or after it (`FetchRequest.deadline`). */
+  readonly deadline?: number | undefined;
+}
+/** The note of a request starts with this text when its host was not free before its deadline, so the request did not start. */
+declare const DEADLINE_NOTE = "deadline reached";
+/**
+ * This function gives the key of the cache entry of a rendered page in the form of version 0.1.
+ * The index keeps the variant in a column of its own, and it reads this form too.
+ */
+declare const renderedKey: (url: string) => string;
+/** This value is the version of the payload of the records that the crawler writes. A record with no `schema` is version 1. */
+declare const CAPTURE_SCHEMA = 2;
+/**
+ * These query parameters hold secrets in many APIs. The crawler writes `REDACTED` in place of
+ * their values in each capture, in addition to the names of `secretParams`.
+ */
+declare const DEFAULT_SECRET_PARAMS: readonly string[];
+declare class Crawler {
+  private readonly options;
+  private readonly clock;
+  private readonly random;
+  private readonly scheduler;
+  private readonly dispatcher;
+  private readonly ownsDispatcher;
+  private readonly egress;
+  private readonly robots;
+  /** The fields of a cache entry that the index does not keep, and the raw target of a redirect. */
+  private readonly meta;
+  private readonly inFlight;
+  private readonly running;
+  private readonly closing;
+  private closed;
+  private closingDone;
+  /** This note tells why the renderer has no browser. After the first page that finds it, the crawler renders nothing. */
+  private renderOff;
+  /** The renders that run at one time (`maxRenders`). */
+  private readonly renders;
+  /** These tokens apply because the `User-Agent` links the bot page (`obeyDefaultToken`). */
+  private readonly pageTokens;
+  /** The deadlines of the requests that have one, by the signal that the scheduler gets for each of them. */
+  private readonly deadlines;
+  /** This signer signs each request. It holds the key, and the crawler writes only its key id and its agent. */
+  private readonly signer;
+  constructor(options: CrawlerOptions);
+  private iso;
+  private policyOf;
+  /**
+   * This method gives the signal of a request for the scheduler. A request with a deadline gets a
+   * signal of its own, and the clock of the scheduler reads the deadline of that signal.
+   */
+  private paced;
+  /** This method throws `DeadlineReached` when the deadline of a request came before its slot. */
+  private beforeDeadline;
+  private emit;
+  private track;
+  /** This method gives the validator memo of a host. The memo is a hint, so an error of its store gives no memo. */
+  private memoOf;
+  private stopHost;
+  private record;
+  private skip;
+  /** This method reads the cache entry of a URL, with the fields that the crawler keeps in memory. */
+  private lookup;
+  private store;
+  /** This method gives the target of a stored redirect, when the crawler can follow it. */
+  private followable;
+  /** This method answers from the cache, when the stored entry is young or fresh enough. */
+  private fromCache;
+  /**
+   * This method gives the signature headers of one request, and the facts of the capture: the key id
+   * and the agent. With no signer, it gives nothing. A signer that throws gives the note of a skip,
+   * because the crawler sends no unsigned request when it has a signer.
+   */
+  private signFor;
+  private requestHeaders;
+  /**
+   * This method records the result of a request for the host, and it gives the rest of the host.
+   * The rest comes from `restAfter`. A 429 or 503 waits, an error counts toward a stop, and a
+   * refusal stops the host for a time.
+   */
+  private settle;
+  /**
+   * This method stops the host of a URL as a challenge does: for the `challengeStopMs` of the policy.
+   * The runtime uses it when a page that answered 200 is a bot check (`meta.blocked`).
+   */
+  reportBlocked(url: string, what: string): void;
+  /**
+   * This method applies the rules that a fetch and a render share. The sequence is the URL, the
+   * options, the egress policy, the product tokens and the rules for an agent. Then come the
+   * cache, the stops of the host and robots.txt. It gives the answer when a rule decides, or the
+   * facts that the transport needs.
+   */
+  private admit;
+  /** This method fetches a URL. It follows the redirects up to the maximum, and each step is a separate decision. */
+  fetch(request_: FetchRequest): Promise<Fetched>;
+  private follow;
+  /** This method joins a request to an equal request in progress, so one URL gets one request at a time. */
+  private hop;
+  private knownRobots;
+  /**
+   * This method gives the robots.txt of the origin of a URL. A good copy stays for the TTL of the
+   * policy. An unreachable verdict stays for some minutes, with a backoff and the `Retry-After`
+   * of the server. During that time the last good copy applies, when there is one (RFC 9309 §2.4).
+   */
+  private robotsFor;
+  /**
+   * This method gives the robots.txt rules that apply to a URL, as the crawler applies them. They
+   * come from its memory, or from a fetch of `/robots.txt` with its redirects. A robots.txt that
+   * the crawler cannot read gives rules that permit nothing, and `unreachable` gives the reason.
+   */
+  readRobots(url: string, marks?: {
+    readonly scope?: string | undefined;
+    readonly labels?: Readonly<Record<string, string>> | undefined;
+  }): Promise<{
+    readonly robots: Robots;
+    readonly capture: string | undefined;
+    readonly stale?: string | undefined;
+  }>;
+  /**
+   * This method tells if robots.txt lets the crawler fetch a URL as a page. The rules of the token
+   * of the crawler and of the agents of `onBehalfOf` apply, as for a fetch. It reads robots.txt as
+   * a fetch does, so a robots.txt that the crawler cannot read permits nothing. When the signal
+   * aborts, the promise rejects, and a fetch of robots.txt in progress continues.
+   */
+  robotsAllow(url: string, options?: {
+    readonly onBehalfOf?: readonly string[] | undefined;
+    readonly scope?: string | undefined;
+    readonly labels?: Readonly<Record<string, string>> | undefined;
+    readonly signal?: AbortSignal | undefined;
+  }): Promise<{
+    readonly allowed: boolean;
+    readonly note: string | undefined;
+    /** These are the lines of usage signals (`Content-Signal`) of robots.txt that apply to the URL, when there are any. The archived fallback applies them to a copy. */
+    readonly signals?: readonly string[] | undefined;
+  }>;
+  /** This method reads the stored copy of a robots.txt, when it is younger than the limit of the policy. */
+  private storedRobots;
+  /** This method adds the scope and the labels of one request to the requests that wait for a robots.txt. The most private scope wins. */
+  private join;
+  private fetchRobots;
+  private fetchOnce;
+  /**
+   * This method gives the function that decides each request of a rendered page. It refuses an
+   * address that the egress policy refuses, and a URL of the page's host that robots.txt
+   * disallows. For a different host, it uses a robots.txt that the crawler holds, when there is one.
+   */
+  private allowRequestFor;
+  /**
+   * This method tells why the browser did not go on to a URL that the page moved to: the egress
+   * policy or robots.txt. For a different site, it uses a robots.txt that the crawler holds. It
+   * gives undefined for the page itself, or for a URL that it cannot read.
+   */
+  private movedRefusal;
+  /**
+   * This method renders a page in the headless browser and captures what the page shows. The rules
+   * of a fetch apply:
+   *
+   * - the egress policy and robots.txt, for the page, its final URL and each request of the page
+   * - one request at a time for each host
+   * - the backoff after a 429 or a 503
+   * - a capture for each decision
+   *
+   * A render costs the site more than a fetch, so the host rests longer after one. A render that
+   * the browser refuses (`RenderRefused`) does not count against the host.
+   */
+  render(request_: RenderRequest): Promise<Fetched>;
+  private renderOnce;
+  /**
+   * This method removes the stops and the failure counts of one host, or of each host. It also
+   * forgets the robots.txt verdicts that came from an unreachable robots.txt. The pace of each host
+   * stays. A process that runs for a long time uses it, for example after the network comes back.
+   */
+  resetHosts(host?: string): void;
+  /**
+   * This method gives the stop of the host of a URL, or undefined when the host is not stopped. A
+   * watch uses it, so a page of a stopped host is due again at the end of the stop.
+   */
+  hostStop(url: string): ReturnType<HostStateStore["stopped"]>;
+  /**
+   * This method closes the crawler. Work in the queue ends as `skipped` with the note
+   * `crawler closed`, and the crawler writes no capture after the close. Then it closes the
+   * browser, the log and its own dispatcher. A second close waits for the first.
+   */
+  close(): Promise<void>;
+}
+//#endregion
+//#region src/diff.d.ts
+/**
+ * A diff compares two versions of a stored document. It gives the changes of the text as a line
+ * diff, with a word diff inside each changed line. It gives the changes of the title, the URL and
+ * the metadata (the JSON-LD too) as changes at a path.
+ *
+ * The line diff is the algorithm of Myers (1986). Its time is linear in the length of the texts
+ * when the change is small. A budget of steps (`DIFF_LIMITS`) bounds the time of a large change.
+ * After the budget, the diff gives the rest of the change as one removal and one addition. Then
+ * the result says that it is not minimal. A huge text gives its first part, and the result says so.
+ *
+ * A held-back version gives no text and no title to a diff, and only its structural metadata.
+ */
+/** These are the limits of a diff. Each limit bounds the time or the size of a result. */
+declare const DIFF_LIMITS: {
+  /** The characters of each text that a diff compares. A longer text gives its first part. */
+  readonly maxChars: 1_000_000;
+  /** The lines of each text that a diff compares. A longer text gives its first lines. */
+  readonly maxLines: 50_000;
+  /** The steps of the line diff. After them, the rest of the change is one removal and one addition. */
+  readonly maxSteps: 1_000_000;
+  /** The steps of the word diffs of all the changed lines together. */
+  readonly maxWordSteps: 1_000_000;
+  /** The words of one line that a word diff compares. A longer line gets no word diff. */
+  readonly maxLineWords: 5_000;
+  /** The lines of context before and after each change. */
+  readonly context: 2;
+  /** The lines that the hunks of a result show. The summary counts each line. */
+  readonly maxHunkLines: 2_000;
+  /** The changes of the metadata that a result shows. The summary counts each change. */
+  readonly maxMetaChanges: 200;
+  /** The depth of the metadata that a diff compares. A deeper change is a change of the whole value. */
+  readonly maxMetaDepth: 16;
+  /** The characters of the JSON of each value of a change of the metadata. */
+  readonly maxValueChars: 300;
+};
+/** These are the limits of a diff, one by one. `DIFF_LIMITS` gives the defaults. */
+type DiffLimits = { readonly [K in keyof typeof DIFF_LIMITS]: number; };
+/** This is a part of a changed line: words that stay, words that the new text adds, or words that it removes. */
+interface WordPart {
+  readonly op: "same" | "added" | "removed";
+  readonly text: string;
+}
+/**
+ * This is one line of a hunk. A `same` line is context. A `changed` line pairs a removed line with
+ * an added line, and `words` gives the word diff, or null when the word budget ended.
+ */
+type DiffLine = {
+  readonly op: "same" | "added" | "removed";
+  readonly text: string;
+} | {
+  readonly op: "changed";
+  readonly before: string;
+  readonly after: string;
+  readonly words: readonly WordPart[] | null;
+};
+/** This is a part of the diff of the text: the changed lines with some context, as in a unified diff. */
+interface DiffHunk {
+  /** This value is the number (from 1) of the first line of the hunk in the old text. */
+  readonly from_line: number;
+  /** This value is the number of lines of the old text in the hunk. */
+  readonly from_count: number;
+  /** This value is the number (from 1) of the first line of the hunk in the new text. */
+  readonly to_line: number;
+  /** This value is the number of lines of the new text in the hunk. */
+  readonly to_count: number;
+  readonly lines: readonly DiffLine[];
+}
+/**
+ * This is one change of the title, the URL or the metadata. The path names the value, for
+ * example `meta.jsonld[0].openingHours`. `before` and `after` are the JSON of the values, cut to
+ * `DIFF_LIMITS.maxValueChars`.
+ */
+interface MetaChange {
+  readonly path: string;
+  readonly change: "added" | "removed" | "changed";
+  /** This text is the JSON of the old value, or null when the new version adds the value. */
+  readonly before: string | null;
+  /** This text is the JSON of the new value, or null when the new version removes the value. */
+  readonly after: string | null;
+}
+/** These are the counts of a diff. */
+interface DiffSummary {
+  /** This value counts the lines that only the new text has. */
+  readonly lines_added: number;
+  /** This value counts the lines that only the old text has. */
+  readonly lines_removed: number;
+  /** This value counts the pairs of a removed line and an added line that the result shows as one changed line. */
+  readonly lines_changed: number;
+  /** This value counts the words that the new text adds, in the added lines and in the changed lines. */
+  readonly words_added: number;
+  /** This value counts the words that the new text removes, in the removed lines and in the changed lines. */
+  readonly words_removed: number;
+  /** This value counts the changes of the title, the URL and the metadata. */
+  readonly meta_changes: number;
+}
+/** This is the input of `diffTexts`: the parts of a version that a diff compares. */
+interface DiffInput {
+  readonly text: string;
+  readonly fields: Readonly<Record<string, unknown>>;
+}
+/** This is the result of `diffTexts`. */
+interface TextDiff {
+  readonly summary: DiffSummary;
+  readonly hunks: readonly DiffHunk[];
+  readonly meta: readonly MetaChange[];
+  /** This value is true when the text and the fields are the same. */
+  readonly same: boolean;
+  /** This value is true when the line diff is minimal and each changed line has its word diff. */
+  readonly exact: boolean;
+  /** This value is true when the result leaves out a part. It can leave out the end of a long text, lines of the hunks or changes of the metadata. */
+  readonly truncated: boolean;
+  /** This value counts the steps of the line diff and of the word diffs. A test reads it: a step count does not change with the computer. */
+  readonly steps: number;
+  readonly notes: readonly string[];
+}
+/**
+ * This function compares two versions: the lines of the text, the words of each changed line, and
+ * the fields (the title, the URL and the metadata) at their paths. It writes nothing, and its time
+ * and its result stay inside the limits.
+ */
+declare const diffTexts: (before: DiffInput, after: DiffInput, given?: Partial<DiffLimits>) => TextDiff;
+/** A version of a document for a diff: a capture id, or the version that was current at a time. */
+type VersionChoice = string | {
+  readonly at: string | Date;
+};
+/** These options choose the two versions of a diff. */
+interface DiffOptions {
+  /**
+   * This version is the old side: a capture id, or `{ at }`. The default is the version before
+   * `to` with the same recipe and the same kind of capture (plain or rendered).
+   */
+  readonly from?: VersionChoice | undefined;
+  /** This version is the new side: a capture id, or `{ at }`. The default is the newest version. */
+  readonly to?: VersionChoice | undefined;
+  /** These values change the limits of the diff (`DIFF_LIMITS`). */
+  readonly limits?: Partial<DiffLimits> | undefined;
+}
+/** This is one side of a diff: the version that it compares. */
+interface DiffSide {
+  /** This id is the capture of the version. A citation names it. */
+  readonly capture: string;
+  /** This time (ISO 8601) is the fetch time of the version. */
+  readonly fetched_at: string;
+  /** This value is the length of the text of the version. */
+  readonly chars: number;
+  readonly rendered: boolean;
+  /** This value is true when the version is held back, so the diff does not show its text. */
+  readonly held_back: boolean;
+  /** This time (ISO 8601) is the `at` that chose the version, or null for a capture id or a default. */
+  readonly at: string | null;
+}
+/** The result of a diff of two versions of a stored document. */
+interface DiffResult {
+  readonly doc: string;
+  readonly url: string | null;
+  readonly from: DiffSide;
+  readonly to: DiffSide;
+  /**
+   * This value is true when the two versions have the same text, title, URL and metadata. It is
+   * null when a side is held back, because then the diff does not compare the texts.
+   */
+  readonly same: boolean | null;
+  readonly summary: DiffSummary;
+  readonly hunks: readonly DiffHunk[];
+  readonly meta: readonly MetaChange[];
+  /** This value is true when a side is held back. Then the result has no text and no title, and only the structural metadata. */
+  readonly held_back: boolean;
+  /** This value is true when the line diff is minimal and each changed line has its word diff. */
+  readonly exact: boolean;
+  /** This value is true when the result leaves out a part. The note tells which part. */
+  readonly truncated: boolean;
+  /** This value counts the steps of the diff. */
+  readonly steps: number;
+  readonly note: string | null;
+}
+/**
+ * This function compares two versions of a stored document, and writes nothing. It applies the
+ * rule for a held-back document. When the document or a version is held back, the diff compares
+ * only the structural metadata, and it shows no text and no title.
+ */
+declare const diffDocument: (runtime: CrawlRuntime, id: string, options?: DiffOptions) => Promise<DiffResult>;
+//#endregion
+//#region src/duration.d.ts
+/** A duration is a count of milliseconds, or a text, for example `"30m"`, `"24h"` or `"7d"`. */
+type Duration = number | string;
+/**
+ * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,
+ * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`
+ * is zero. The function throws an error for a text that it cannot read.
+ */
+declare const durationMs: (duration: Duration) => number;
+//#endregion
+//#region src/sitemaps.d.ts
+/**
+ * The sitemap reader reads the files that list the pages of a site. These are an XML sitemap, a
+ * sitemap index, an RSS 2.0 or RSS 1.0 feed and an Atom feed. A text sitemap with one URL on each
+ * line is a file of this kind too. The reader is small and strict, because the files come from the web:
+ *
+ * - It reads the text one time from the start to the end, and it builds no tree. Its time is
+ *   linear in the length of the text.
+ * - It expands no entity. It knows only the five entities of XML and the character references. A
+ *   DTD in the file changes nothing.
+ * - It stops at a maximum depth of elements, at a maximum number of entries and at the first
+ *   error in the markup. The entries before the stop stay.
+ *
+ * It imports no Node.js module, so a browser can use it.
+ */
+/** The kind of a file: the name of its root element, or `text` for a text sitemap. */
+type SitemapKind = "urlset" | "sitemapindex" | "rss" | "atom" | "rdf" | "text" | "unknown";
+/** The values of `<changefreq>` in the sitemap protocol. */
+type Changefreq = "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never";
+/** One entry of a file: a page, or a sitemap of an index. */
+interface SitemapEntry {
+  /** This URL is the canonical form of the URL of the entry. */
+  readonly url: string;
+  /** This time (ISO 8601) is the last change that the file gives, or null. */
+  readonly lastmod: string | null;
+  /** This value is the `<changefreq>` of a sitemap entry, or null. */
+  readonly changefreq: Changefreq | null;
+}
+/** The result of one file. */
+interface SitemapRead {
+  readonly kind: SitemapKind;
+  /** These entries are the pages of the file, or the sitemaps of an index, in the order of the file. */
+  readonly entries: readonly SitemapEntry[];
+  /** This value counts the entries with no valid http(s) URL. */
+  readonly invalid: number;
+  /** This value is false when the reader stopped before the end of the file. */
+  readonly complete: boolean;
+  /** These notes tell why the reader stopped, or what it ignored. */
+  readonly notes: readonly string[];
+}
+/** These limits stop the reader on a large or a hostile file. */
+interface SitemapLimits {
+  /** This value is the maximum number of entries. The default is 50,000, the limit of the sitemap protocol. */
+  readonly maxEntries?: number | undefined;
+  /** This value is the maximum depth of the elements. The default is 32. */
+  readonly maxDepth?: number | undefined;
+}
+/** These are the default limits of the reader, and the maximum length of one value. */
+declare const SITEMAP_LIMITS: {
+  readonly maxEntries: 50_000;
+  readonly maxDepth: 32;
+  /** A value (a URL or a date) longer than this number of characters is not valid. */
+  readonly maxValue: 4096;
+};
+/**
+ * This function changes a date of a sitemap or a feed into ISO 8601. It reads the W3C form of the
+ * sitemap protocol and Atom, and the mail form of RSS. A time with no zone is UTC. It gives null
+ * for a text that it cannot read.
+ */
+declare const feedDate: (value: string) => string | null;
+/**
+ * This function reads a sitemap, a sitemap index, an RSS or Atom feed, or a text sitemap. The
+ * root element of the file tells its kind. The entries of a feed can have URLs relative to `base`
+ * (the URL of the file). The entries of a sitemap must have absolute URLs.
+ */
+declare const readSitemap: (text: string, base: string, limits?: SitemapLimits) => SitemapRead;
+//#endregion
+//#region src/discover.d.ts
+/**
+ * Discovery finds the pages of a site from its sitemaps and its feeds. Each file is a fetch of the
+ * crawler, so each polite rule applies to it. These rules are robots.txt, one request at a time for
+ * each host, the pace, the stops, the egress policy and the capture log. These are the sources, in sequence:
+ *
+ * 1. The page at the URL of the site, for the RSS and Atom feeds that it links.
+ * 2. The feeds that the caller gives.
+ * 3. The sitemaps that the caller gives, and the `Sitemap:` lines of robots.txt. When there are
+ *    none, `/sitemap.xml`.
+ * 4. The sitemaps of each sitemap index, the newest first, to a maximum depth.
+ *
+ * Discovery keeps a URL only when it is in scope (the same site by default) and robots.txt permits
+ * it. It stops at a maximum of files, a maximum of URLs and a maximum time.
+ */
+/** These are the default limits of `discover`. */
+declare const DISCOVER_LIMITS: {
+  /** The most sitemap and feed files that one discovery reads. */
+  readonly maxFiles: 20;
+  /** The most URLs that one discovery gives. */
+  readonly maxUrls: 10_000;
+  /** The most bytes of one file, after decompression: 10 MiB. */
+  readonly maxBytes: 10_485_760;
+  /** The most levels of sitemap index under a file that discovery found first. */
+  readonly maxNesting: 3;
+  /** The longest time of one discovery: 1 minute. */
+  readonly budgetMs: 60_000;
+  /** The most origins whose robots.txt one discovery reads for the URLs that it finds. */
+  readonly maxOrigins: 10;
+  /** The most feeds that discovery reads from the links of the page. */
+  readonly maxLinkedFeeds: 5;
+};
+/** These options control a discovery. Each limit has a default in `DISCOVER_LIMITS`. */
+interface DiscoverOptions extends ScopeOptions {
+  /** This value is the most sitemap and feed files to read. The default is 20. */
+  readonly maxFiles?: number | undefined;
+  /** This value is the most URLs to give. The default is 10,000. */
+  readonly maxUrls?: number | undefined;
+  /** This value is the most bytes of one file, after decompression. The default is 10 MiB. */
+  readonly maxBytes?: number | undefined;
+  /** This value is the most levels of sitemap index to follow. The default is 3. */
+  readonly maxNesting?: number | undefined;
+  /** This value is the longest time of the discovery, in milliseconds. The default is 1 minute. */
+  readonly budgetMs?: number | undefined;
+  /** These sitemaps are read too, in addition to the `Sitemap:` lines of robots.txt. */
+  readonly sitemaps?: readonly string[] | undefined;
+  /** These RSS or Atom feeds are read too. */
+  readonly feeds?: readonly string[] | undefined;
+  /** With the default value true, discovery reads the page of the site for the feeds that it links. */
+  readonly linkedFeeds?: boolean | undefined;
+  /**
+   * A stored copy of a file younger than this age answers, with no request. The default is `24h`.
+   * The value `auto` uses the freshness that the server gave.
+   */
+  readonly maxAge?: Duration | undefined;
+  /** These agents get their robots.txt rules applied to each fetch and to each URL. */
+  readonly onBehalfOf?: readonly string[] | undefined;
+  /** When this signal aborts, the discovery stops, and it gives what it found. */
+  readonly signal?: AbortSignal | undefined;
+  /** This label goes into the capture records of the discovery. */
+  readonly scope?: string | undefined;
+  /** These labels go into the capture records of the discovery. */
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+}
+/** These options control `discoverSite`: the options of `discover`, and a deadline. */
+interface DiscoverRunOptions extends DiscoverOptions {
+  /** This time (milliseconds since the epoch) ends the discovery when it comes before the end of `budgetMs`. */
+  readonly deadline?: number | undefined;
+}
+/** One URL that a discovery found. */
+interface DiscoveredUrl {
+  /** This URL is the canonical form of the URL of the page. */
+  readonly url: string;
+  /** This time (ISO 8601) is the last change that the file gave, or null. */
+  readonly lastmod: string | null;
+  readonly changefreq: Changefreq | null;
+  /** This value tells if a sitemap or a feed listed the URL. */
+  readonly via: "sitemap" | "feed";
+  /** This URL is the file that listed the URL first. */
+  readonly source: string;
+}
+/** One sitemap or feed file that a discovery read, or tried to read. */
+interface DiscoverSource {
+  readonly url: string;
+  readonly via: "sitemap" | "feed";
+  /** This value tells how discovery found the file. */
+  readonly from: "robots.txt" | "default" | "given" | "page" | "index";
+  /** This value is the kind that the reader found, or null for a file that the crawler did not get. */
+  readonly kind: SitemapKind | null;
+  /** This value is the outcome of the fetch: `ok`, `not-modified`, `skipped`, `not-found` and others. */
+  readonly outcome: string;
+  readonly status: number;
+  /** This id is the capture of the file. */
+  readonly capture: string | null;
+  readonly fetched_at: string | null;
+  readonly from_cache: boolean;
+  /** This value is the number of entries that the file listed. */
+  readonly entries: number;
+  /** This value is the number of new URLs that discovery kept from the file. */
+  readonly urls: number;
+  /** This value is true when the file was longer than `maxBytes`, so the reader read only its first part. */
+  readonly truncated: boolean;
+  readonly note: string | null;
+}
+/** The reason that a discovery stopped. */
+type DiscoverStop = "done" | "max-files" | "max-urls" | "budget" | "cancelled" | "closed" | "robots";
+/** The result of a discovery. */
+interface DiscoverResult {
+  /** This URL is the origin of the site. */
+  readonly site: string;
+  /** These URLs are the pages that discovery found, the newest `lastmod` first. */
+  readonly urls: readonly DiscoveredUrl[];
+  /** These are the files that discovery read, in the sequence of the reads. */
+  readonly sources: readonly DiscoverSource[];
+  /** These values count the URLs that discovery did not keep, for each reason. */
+  readonly dropped: {
+    /** The URLs on a different site, and the files of an index on a different site. */
+    readonly scope: number;
+    /** The URLs that robots.txt does not permit. */
+    readonly robots: number;
+    /** The URLs that discovery found before. */
+    readonly duplicate: number;
+    /** The entries with no valid http(s) URL. */
+    readonly invalid: number;
+    /** The URLs and the files over a limit: the origins, the depth of the index, or the URLs. */
+    readonly limit: number;
+  };
+  readonly stopped: DiscoverStop;
+  /** This text tells why the discovery stopped early, or null. */
+  readonly note: string | null;
+}
+/**
+ * This function finds the pages of a site from its sitemaps and its feeds. `PoliteCrawler.discover`
+ * uses it. Each file is a fetch of the crawler with the recipe `sitemap` or `feed`, so the
+ * capture log records it, and robots.txt applies to it.
+ */
+declare const discoverSite: (runtime: CrawlRuntime, site: string, options?: DiscoverRunOptions) => Promise<DiscoverResult>;
+//#endregion
+//#region src/extract.d.ts
+/**
+ * The field check takes the values that a caller proposes for the fields of a stored page. It finds
+ * each value in the stored text or in the JSON-LD. A model or a person gives the values. The check
+ * does not make values: it only checks and locates the values that it gets.
+ *
+ * Each field gets one status:
+ *
+ * - `verified`: the stored text holds the exact value. The result gives the span (or the JSON-LD
+ *   path) and the capture.
+ * - `normalized`: the stored text holds the value under a stated normalization, for example a
+ *   different number format. The result names the normalization and gives the span of the stored text.
+ * - `ambiguous`: the value matches, but the stored text does not give one meaning. The result gives
+ *   each span, and it does not choose one.
+ * - `not-found`: the stored text does not hold the value.
+ * - `held-back`: the page asked not to be AI input. The result tells nothing about the value.
+ * - `invalid`: the value is not a value of its kind, so the check did not search for it.
+ *
+ * The time of the check is linear in the length of the text, for a fixed set of fields.
+ */
+/** The kinds of field that the check reads. `json-ld:<path>` reads a value at a path of the JSON-LD, for example `json-ld:Product.offers.price`. */
+type FieldKind = "text" | "number" | "money" | "date" | "url" | "email" | "phone" | `json-ld:${string}`;
+/** These kinds read the text of the page. The kind `json-ld:<path>` reads the JSON-LD. */
+declare const FIELD_KINDS: readonly string[];
+/** This is the value that a caller proposes for one field. */
+interface FieldClaim {
+  /** This value is the proposal. The check removes the spaces at its start and at its end. */
+  readonly value: string | number;
+  /** This value is the kind of the field. The default is `number` for a number and `text` for a text. */
+  readonly kind?: FieldKind | undefined;
+}
+/** The status of one field. The module comment of `extract.ts` and the README tell what each status means. */
+type FieldStatus = "verified" | "normalized" | "ambiguous" | "not-found" | "held-back" | "invalid";
+/** These positions locate a value in the text of a version. The end is not part of the value. */
+interface FieldSpan {
+  readonly start: number;
+  readonly end: number;
+  /** This value is `full_text` when the value is only in the whole text of a page that the reader cut. */
+  readonly in: "text" | "full_text";
+}
+/** This is one place that an ambiguous field matches. */
+interface FieldCandidate {
+  readonly span: FieldSpan | null;
+  /** This path locates a value of the JSON-LD (`jsonld[0].offers[1].price`) or a stored link (`links[3]`). */
+  readonly path: string | null;
+  /** This value is the exact stored text, or the stored value of the JSON-LD. */
+  readonly raw: string | number | boolean;
+  readonly excerpt: string;
+  /** These are the meanings that the stored text can have, for example two dates for `03/04/2026`. */
+  readonly readings: readonly string[];
+}
+/** This is the check of one field, before the runtime adds the capture. */
+interface FieldCheck {
+  readonly status: FieldStatus;
+  /** This value is the kind that the check used. */
+  readonly kind: string;
+  readonly span: FieldSpan | null;
+  readonly path: string | null;
+  /** This value is the exact stored text of the match, or the stored value of the JSON-LD. */
+  readonly raw: string | number | boolean | null;
+  /** This text is the stored text around the match. */
+  readonly excerpt: string | null;
+  /** This value names the normalization of a `normalized` match, for example `thousands-separator`. Two or more names are joined with `+`. */
+  readonly normalization: string | null;
+  /** This value is the meaning of the stored text that matched, for example `1234.5` or `2026-03-04`. */
+  readonly reading: string | null;
+  /** This value is the number of places that match. */
+  readonly count: number;
+  /** These are the places of an ambiguous field, one for each meaning, 5 or fewer. */
+  readonly candidates: readonly FieldCandidate[];
+  readonly note: string | null;
+}
+/** This is the result of one field: the check, and the capture of the text that the check read. */
+interface FieldResult extends FieldCheck {
+  readonly capture: string | null;
+  /** This time (ISO 8601) is the fetch time of that capture. */
+  readonly fetched_at: string | null;
+}
+/** These are the counts of the statuses of the fields. */
+interface ExtractSummary {
+  readonly fields: number;
+  readonly verified: number;
+  readonly normalized: number;
+  readonly ambiguous: number;
+  readonly not_found: number;
+  readonly held_back: number;
+  readonly invalid: number;
+}
+/** This is the text of a document that `checkFields` reads, with its metadata. */
+interface ExtractSource {
+  readonly text: string;
+  /** This text is the whole text of a page that the reader cut. The check reads it when the value is not in `text`. */
+  readonly full_text?: string | undefined;
+  /** This URL is the address of the page. The check resolves a relative URL against it. */
+  readonly url?: string | null | undefined;
+  /** The check reads `jsonld`, `jsonld_dropped`, `links` and `truncated` of this metadata. */
+  readonly meta?: Readonly<Record<string, unknown>> | undefined;
+}
+/** This is the result of `checkFields`. */
+interface FieldsChecked {
+  readonly fields: Readonly<Record<string, FieldCheck>>;
+  readonly summary: ExtractSummary;
+  /** This value counts the units of work of the check. It grows in proportion to the length of the text. */
+  readonly steps: number;
+}
+/** These options control a check of fields against a stored document. */
+interface ExtractOptions {
+  /** This id checks the version of one capture, not the current version. */
+  readonly capture?: string | undefined;
+  /**
+   * This time checks the version that was current at that time: an ISO 8601 text or a `Date`. A date
+   * with no time is the start of that day in UTC. Give `capture` or `at`, not the two.
+   */
+  readonly at?: string | Date | undefined;
+  /** This value is true when a person gave this URL directly. It is the only way to check a held-back document. The crawler logs it. */
+  readonly userProvided?: boolean | undefined;
+}
+/** This is the result of a check of fields against a stored document. */
+interface ExtractResult {
+  readonly doc: string;
+  readonly url: string | null;
+  readonly captures: readonly string[];
+  /** This id is the capture of the text that the check read. A held-back document gives null. */
+  readonly capture: string | null;
+  /** This hash is the blob of that capture. */
+  readonly blob: string | null;
+  /** This time (ISO 8601) is the fetch time of that capture. */
+  readonly fetched_at: string | null;
+  readonly held_back: boolean;
+  /** These are the results of the fields, with the names that the caller gave. */
+  readonly fields: Readonly<Record<string, FieldResult>>;
+  readonly summary: ExtractSummary;
+  readonly note: string | null;
+  /** This time (ISO 8601) is the `at` of the check. The result has it only when the check gave `at`. */
+  readonly at?: string | undefined;
+  /** This value is true when `at` is before the first capture of the document. Then no field is found. */
+  readonly before_first?: boolean | undefined;
+  /**
+   * This value tells that the checked text is an archived copy, with the archive and the date of the
+   * copy. The note says so too. A citation of a value names the archive and that date.
+   */
+  readonly archived?: Archived | undefined;
+}
+/**
+ * This function checks the proposed values of fields against the text and the metadata of a
+ * document. It sends no request and writes nothing. It does not make values. A field that is not
+ * valid throws a `RangeError`. The work grows in proportion to the length of the text.
+ */
+declare const checkFields: (source: ExtractSource, fields: Readonly<Record<string, FieldClaim>>) => FieldsChecked;
+/**
+ * This function checks the proposed values of fields against a stored document. It does not make
+ * values. It applies the rules of `verify`:
+ *
+ * - With no option, it checks the current version. `capture` checks the version of one capture,
+ *   and `at` checks the version that was current at a time.
+ * - A held-back document gives `held-back` for each field, with no position and no capture, unless
+ *   `userProvided`. The crawler then logs an override.
+ * - A field that is not valid throws a `RangeError`, and an unknown document throws an error.
+ */
+declare const extractDocument: (runtime: CrawlRuntime, id: string, fields: Readonly<Record<string, FieldClaim>>, options?: ExtractOptions) => Promise<ExtractResult>;
+//#endregion
+//#region src/frontier.d.ts
+/**
+ * The frontier is a small crawl of a site. It starts at the seeds. It follows the links of each page
+ * that it fetched, to a maximum of pages, a maximum depth and a maximum time.
+ *
+ * Each page is a fetch of the crawler, so each polite rule applies. These rules are robots.txt, one
+ * request at a time for each host, the pace, the stops, the egress policy and the capture log. The
+ * frontier does not start a second request to a host with a request in progress. The crawler
+ * limits the hosts in progress (`maxActive`).
+ *
+ * The state of the frontier is in memory, for one call. A later version can make it resumable.
+ */
+/** These are the default limits of `crawl`, and the limits of the memory of the frontier. */
+declare const CRAWL_LIMITS: {
+  /** The most pages that one crawl gives. */
+  readonly maxPages: 100;
+  /** The most links from a seed to a page. A seed has the depth 0. */
+  readonly maxDepth: 3;
+  /** The longest time of one crawl: 10 minutes. */
+  readonly budgetMs: 600_000;
+  /** The most hosts that get a request of the crawl at one time. */
+  readonly hostsAtOnce: 16;
+  /** The most URLs that wait in the frontier. */
+  readonly maxQueue: 10_000;
+  /** The most URLs that the frontier remembers, so it fetches none of them two times. */
+  readonly maxSeen: 100_000;
+};
+/**
+ * A pattern for the URLs of a crawl. A text that starts with `/` matches the path and the query.
+ * A different text matches the whole URL. In a text, `*` matches any characters and a `$` at the
+ * end matches the end, as in robots.txt. A regular expression tests the whole URL.
+ */
+type UrlPattern = string | RegExp;
+/** These options control a crawl. Each limit has a default in `CRAWL_LIMITS`. */
+interface CrawlOptions extends ScopeOptions {
+  /** This value is the most pages that the crawl gives. The default is 100. */
+  readonly maxPages?: number | undefined;
+  /** This value is the most links from a seed to a page. The default is 3, and 0 fetches only the seeds. */
+  readonly maxDepth?: number | undefined;
+  /** This value is the longest time of the crawl in milliseconds. No request starts after it. The default is 10 minutes. */
+  readonly budgetMs?: number | undefined;
+  /** When this list is not empty, the crawl follows only a URL that matches a pattern of it. */
+  readonly include?: readonly UrlPattern[] | undefined;
+  /** The crawl follows no URL that matches a pattern of this list. */
+  readonly exclude?: readonly UrlPattern[] | undefined;
+  /** With the value true, the crawl adds the pages of the sitemaps of each site of the seeds (`discover`). */
+  readonly useSitemaps?: boolean | undefined;
+  /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */
+  readonly render?: "auto" | "never" | "always" | undefined;
+  /** A stored copy younger than this age answers, with no request. The default is `24h`. */
+  readonly maxAge?: Duration | undefined;
+  /** These agents get their robots.txt rules applied to each fetch. */
+  readonly onBehalfOf?: readonly string[] | undefined;
+  /** When this signal aborts, the crawl stops the fetches in progress and ends. */
+  readonly signal?: AbortSignal | undefined;
+  /** This label goes into the capture records of the crawl. */
+  readonly scope?: string | undefined;
+  /** These labels go into the capture records of the crawl. */
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+}
+/** These options control `crawlSite`: the options of `crawl`, and a step before the first fetch. */
+interface CrawlRunOptions extends CrawlOptions {
+  /** This function operates before the first fetch. `openCrawler` waits there for the open of the store. */
+  readonly ready?: (() => Promise<void>) | undefined;
+}
+/** A page of a crawl: the result of its fetch, and the place of the page in the crawl. */
+interface CrawlPage extends PageResult {
+  /** This value is the number of links from a seed to the page. */
+  readonly depth: number;
+  /** This value tells if the page is a seed, a page of a sitemap, or the target of a link. */
+  readonly source: "seed" | "sitemap" | "link";
+  /** This URL is the page whose link the crawl followed, or null. */
+  readonly referrer: string | null;
+}
+/** The reason that a crawl stopped. */
+type CrawlStop = "done" | "max-pages" | "budget" | "cancelled" | "closed" | "error";
+/** The summary of a crawl. */
+interface CrawlSummary {
+  /** This value is the number of pages that the crawl gave. */
+  readonly pages: number;
+  /** This value counts the pages with a document: `ok` or `not-modified`, from the store too. */
+  readonly fetched: number;
+  /** This value counts the pages that a stored copy answered, with no request. */
+  readonly from_cache: number;
+  /**
+   * This value counts the refusals. These are the links that robots.txt does not permit, and the
+   * pages that the crawler declined (`skipped`). The pages that the site refused (`blocked`,
+   * `challenge`, `payment-required`) are refusals too.
+   */
+  readonly refused: number;
+  /** This value counts the pages with a different outcome, for example `error` or `not-found`. */
+  readonly failed: number;
+  /** This value counts the links that the crawl did not follow. `skips` gives the reasons. */
+  readonly skipped: number;
+  readonly skips: {
+    /** The links to a host that is not in scope. */
+    readonly scope: number;
+    /** The links that `include` or `exclude` left out. */
+    readonly pattern: number;
+    /** The links deeper than `maxDepth`. */
+    readonly depth: number;
+    /** The pages that asked not to be AI input: the crawl does not follow their links. */
+    readonly held_back: number;
+    /** The links over the limits of the memory of the frontier. */
+    readonly limit: number;
+    /** The URLs that a fetched page had as its final or its canonical URL. */
+    readonly duplicate: number;
+  };
+  /** This value counts the URLs that waited in the frontier at the stop. */
+  readonly queued: number;
+  /** This value counts the URLs that the sitemaps gave, with `useSitemaps`. */
+  readonly sitemap_urls: number;
+  readonly stopped: CrawlStop;
+  /** This text tells why the crawl stopped. */
+  readonly reason: string;
+  /** This value is the time of the crawl in milliseconds, on the clock of the crawler. */
+  readonly elapsed_ms: number;
+}
+/**
+ * A crawl in progress. A `for await` loop gets each page when its fetch ends. `summary` resolves
+ * when the loop ends: at the last page, at a stop or at a `break`. The crawl starts at the first
+ * step of the loop, and each crawl operates one time only.
+ */
+interface CrawlRun extends AsyncIterable<CrawlPage> {
+  readonly summary: Promise<CrawlSummary>;
+}
+/**
+ * This function crawls a site from its seeds. `PoliteCrawler.crawl` uses it. It gives each page
+ * when its fetch ends, and a summary at the end. The seeds come first, then the pages of the
+ * sitemaps (the newest first), then the links, the lower depth first.
+ */
+declare const crawlSite: (runtime: CrawlRuntime, seeds: string | readonly string[], options?: CrawlRunOptions) => CrawlRun;
+//#endregion
+//#region src/watch.d.ts
+/**
+ * A watch list keeps the pages that the crawler checks again on a schedule. `watch.run` fetches the
+ * pages that are due through the normal fetch, so each polite rule applies. It gives an event for
+ * each page: new, changed, unchanged, gone, refused or error.
+ *
+ * These are the rules of a watch:
+ *
+ * - A watch compares the title, the text and the JSON-LD of the page with the last confirmed version.
+ * - With `confirm: 2` (the default), a change is suspected when a check first sees it. The next
+ *   check confirms it when it sees the same new version. A check that sees the earlier version
+ *   again clears the suspicion, because one read can be a glitch. With `confirm: 1`, each check confirms.
+ * - `every` has a floor of 1 hour by default. A check that a stop of the host refused is due again
+ *   at the end of the stop, or later.
+ * - The list is in `watch.sqlite` in the store, so it stays after a restart. A run claims each entry
+ *   with a lease in that file, so two processes do not check one entry two times.
+ *
+ * The package has no daemon. A scheduler of the operator starts `watch.run`.
+ */
+/** This value is the version of the schema of `watch.sqlite`. */
+declare const WATCH_SCHEMA = 1;
+/** These are the defaults and the limits of the watch list. */
+declare const WATCH_LIMITS: {
+  /** The shortest `every` of a watch by default: 1 hour, so a watch cannot hammer a site. */
+  readonly floorMs: 3_600_000;
+  /** The default `every`: 24 hours. */
+  readonly everyMs: 86_400_000;
+  /** The default time of one run: 10 minutes. No request starts after it. */
+  readonly budgetMs: 600_000;
+  /** The default number of entries that one run checks. */
+  readonly maxPages: 100;
+  /** The most hosts that get a check of one run at one time. */
+  readonly hostsAtOnce: 8;
+  /** A run holds the lease of an entry until the end of its budget and this time more: 2 minutes. */
+  readonly leaseMs: 120_000;
+};
+/** The kind of a check: the page is new, changed, unchanged or gone, or the check got no version. */
+type WatchKind = "new" | "changed" | "unchanged" | "gone" | "refused" | "error";
+/** These options add a page to the watch list, or change its watch. */
+interface WatchOptions {
+  /** This value is the time between two checks. The default is `24h`, and the floor is `1h` (`watchFloor`). */
+  readonly every?: Duration | undefined;
+  /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */
+  readonly render?: "auto" | "never" | "always" | undefined;
+  /** These labels go into the capture records of each check. */
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+  /** This label goes into the capture records of each check. */
+  readonly scope?: string | undefined;
+  /** These agents get their robots.txt rules applied to each check. */
+  readonly onBehalfOf?: readonly string[] | undefined;
+  /** This value is the number of checks that must see a change before it is confirmed. The default is 2. */
+  readonly confirm?: 1 | 2 | undefined;
+}
+/** A change that one check saw, and that the next check can confirm. */
+interface WatchPending {
+  /** This value is `changed`, `gone` or `new` (a page that came back). */
+  readonly kind: "changed" | "gone" | "new";
+  /** This id is the capture of the suspected version, or null for a page that is gone. */
+  readonly capture: string | null;
+  /** This time (ISO 8601) is the time of the check that first saw the change. */
+  readonly since: string;
+}
+/** This is one entry of the watch list. */
+interface WatchEntry {
+  /** This URL is the page that the watch fetches. */
+  readonly url: string;
+  /** This value is the time between two checks in milliseconds. */
+  readonly every_ms: number;
+  readonly render: "auto" | "never" | "always";
+  readonly confirm: 1 | 2;
+  readonly scope: string | null;
+  readonly labels: Readonly<Record<string, string>> | null;
+  readonly on_behalf_of: readonly string[];
+  /** This time (ISO 8601) is the time of the first add. */
+  readonly added_at: string;
+  /** This time (ISO 8601) is the time when the entry is due. */
+  readonly next_at: string;
+  /** This time (ISO 8601) is the time of the last check, or null before the first check. */
+  readonly checked_at: string | null;
+  readonly last_kind: WatchKind | null;
+  readonly last_note: string | null;
+  /** This value counts the checks. */
+  readonly checks: number;
+  /** This value counts the confirmed changes. */
+  readonly changes: number;
+  /** This id is the document of the page. */
+  readonly doc: string | null;
+  /** This id is the capture of the last confirmed version, or null. */
+  readonly capture: string | null;
+  /** This time (ISO 8601) is the fetch time of the last confirmed version, or null. */
+  readonly fetched_at: string | null;
+  /** This value is true when the last confirmed state is that the page is gone. */
+  readonly gone: boolean;
+  /** This value is the change that the last check suspected, or null. */
+  readonly pending: WatchPending | null;
+  /** This time (ISO 8601) is the end of the lease of a run that checks the entry, or null. */
+  readonly leased_until: string | null;
+}
+/** These options control a run of the watch list. */
+interface WatchRunOptions {
+  /** This value is the longest time of the run in milliseconds. No request starts after it. The default is 10 minutes. */
+  readonly budgetMs?: number | undefined;
+  /** This value is the most entries that the run checks. The default is 100. */
+  readonly maxPages?: number | undefined;
+  /** When this signal aborts, the run stops the checks in progress and ends. */
+  readonly signal?: AbortSignal | undefined;
+}
+/** These options control `runWatch`: the options of a run, and the parts of `openCrawler`. */
+interface WatchRunnerOptions extends WatchRunOptions {
+  /** These agents get their robots.txt rules applied to each check, before the agents of the entry. */
+  readonly onBehalfOf?: readonly string[] | undefined;
+  /** This function operates before the first check. `openCrawler` waits there for the open of the store. */
+  readonly ready?: (() => Promise<void>) | undefined;
+}
+/** This is the event of one check of a run. */
+interface WatchEvent {
+  /** This URL is the page of the watch. */
+  readonly url: string;
+  readonly doc: string | null;
+  readonly kind: WatchKind;
+  /** This value is true when the check saw a change for the first time, and the next check must confirm it. */
+  readonly suspected: boolean;
+  /** This value is true when the check confirmed a change. Its version is then the last confirmed version. */
+  readonly confirmed: boolean;
+  /** This id is the capture that the check got, or null. */
+  readonly capture: string | null;
+  /** This time (ISO 8601) is the fetch time of that capture, or null. */
+  readonly fetched_at: string | null;
+  /** This id is the capture of the last confirmed version before the check, or null. */
+  readonly previous: string | null;
+  /** This time (ISO 8601) is the fetch time of that version, or null. */
+  readonly previous_fetched_at: string | null;
+  readonly outcome: string;
+  readonly status: number;
+  /** These are the counts of the diff from the last confirmed version, for a change. A held-back page gives null. */
+  readonly diff: DiffSummary | null;
+  /** This value is true when the page asked not to be AI input. */
+  readonly held_back: boolean;
+  /** This time (ISO 8601) is the time when the entry is due again. */
+  readonly next_at: string;
+  readonly note: string | null;
+}
+/** The reason that a run stopped. */
+type WatchStop = "done" | "max-pages" | "budget" | "cancelled" | "closed" | "error";
+/** The summary of a run. */
+interface WatchSummary {
+  /** This value counts the checks that gave an event. */
+  readonly checked: number;
+  readonly new: number;
+  readonly changed: number;
+  readonly unchanged: number;
+  readonly gone: number;
+  readonly refused: number;
+  readonly error: number;
+  /** This value counts the changes that wait for the next check. */
+  readonly suspected: number;
+  /** This value counts the changes that the run confirmed. */
+  readonly confirmed: number;
+  /** This value counts the entries that are due at the end of the run. */
+  readonly due: number;
+  readonly stopped: WatchStop;
+  /** This text tells why the run stopped. */
+  readonly reason: string;
+  /** This value is the time of the run in milliseconds, on the clock of the crawler. */
+  readonly elapsed_ms: number;
+}
+/** A run in progress. A `for await` loop gets each event. `summary` resolves when the loop ends. */
+interface WatchRun extends AsyncIterable<WatchEvent> {
+  readonly summary: Promise<WatchSummary>;
+}
+/** This is the watch list of `openCrawler`. Each method gives a promise, except `run`. */
+interface WatchList {
+  /** This method adds a page, or changes the options of its watch. A new entry is due at once. */
+  add(url: string, options?: WatchOptions): Promise<WatchEntry>;
+  /** This method removes a page from the list. It gives false when the list does not have the page. */
+  remove(url: string): Promise<boolean>;
+  /** This method gives the entries, the first due first. */
+  list(): Promise<WatchEntry[]>;
+  /** This method checks the entries that are due, to a budget. A `for await` loop gets each event. */
+  run(options?: WatchRunOptions): WatchRun;
+}
+/** These options open a watch store. */
+interface WatchStoreOptions {
+  /** This value is the time in milliseconds that a step waits for the lock of a different process. The default is 10 seconds. */
+  readonly busyTimeoutMs?: number | undefined;
+  /** This value is the shortest `every` in milliseconds. The default is 1 hour. */
+  readonly floorMs?: number | undefined;
+}
+interface Row {
+  url: string;
+  target: string;
+  host: string;
+  every_ms: number;
+  render: string;
+  confirm: number;
+  scope: string | null;
+  labels: string | null;
+  on_behalf_of: string | null;
+  added_at: number;
+  next_at: number;
+  lease_until: number;
+  lease_owner: string | null;
+  checks: number;
+  changes: number;
+  last_at: number | null;
+  last_kind: string | null;
+  last_note: string | null;
+  doc: string | null;
+  base_state: string | null;
+  base_sha: string | null;
+  base_capture: string | null;
+  base_fetched_at: string | null;
+  pend_state: string | null;
+  pend_sha: string | null;
+  pend_capture: string | null;
+  pend_fetched_at: string | null;
+  pend_since: number | null;
+}
+/** The state of a page after a check: what a later check compares. */
+interface State {
+  readonly state: "present" | "gone";
+  readonly sha: string | null;
+  readonly capture: string | null;
+  readonly fetched_at: string | null;
+}
+/** This is the change of an entry after a check. */
+interface Update {
+  readonly next_at: number;
+  readonly last_at: number;
+  readonly last_kind: WatchKind;
+  readonly last_note: string | null;
+  readonly changed: boolean;
+  readonly doc: string | null;
+  /** The new confirmed state, or undefined when it stays. */
+  readonly base?: State | undefined;
+  /** The new suspected state, null to clear it, or undefined when it stays. */
+  readonly pending?: (State & {
+    readonly since: number;
+  }) | null | undefined;
+}
+/**
+ * This store keeps the watch list in a SQLite file (`watch.sqlite`). Two or more processes on one
+ * store share it. Each step is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock of a
+ * different process (`busy_timeout`).
+ *
+ * The list is the decision of the operator, not a projection of the capture log. Thus it has its
+ * own file: a deletion of `hosts.sqlite` or of the index does not lose it.
+ */
+declare class WatchStore {
+  readonly path: string;
+  private readonly db;
+  private readonly statements;
+  private closed;
+  /** This value is the shortest `every` in milliseconds. */
+  readonly floorMs: number;
+  constructor(path: string, options?: WatchStoreOptions);
+  private stmt;
+  /** This method does one step in one transaction. */
+  private step;
+  private row;
+  /**
+   * This method adds a page, or changes the options of its watch and keeps its state. A new entry
+   * is due at once. A changed entry is due one `every` after its last check. A bad option throws
+   * a `RangeError`.
+   */
+  add(url: string, options: WatchOptions, now: number): WatchEntry;
+  /** This method removes a page. It gives false when the list does not have the page. */
+  remove(url: string): boolean;
+  /** This method gives the entry of a page, or undefined. */
+  get(url: string, now: number): WatchEntry | undefined;
+  /** This method gives the entries, the first due first. */
+  list(now: number): WatchEntry[];
+  /** This method counts the entries that are due at a time. */
+  due(now: number): number;
+  /**
+   * This method claims the first due entry with no lease, on a host that is not in `busy`. The
+   * lease of `owner` holds until `until`. Thus two processes do not get one entry at one time.
+   */
+  claim(now: number, owner: string, until: number, busy: readonly string[]): Row | undefined;
+  /** This method ends the lease of `owner` and changes nothing else, so the entry stays due. A closed store does nothing. */
+  release(key: string, owner: string): void;
+  /**
+   * This method writes the result of a check and ends the lease. It gives false when `owner` does
+   * not hold the lease, or when the store is closed.
+   */
+  settle(key: string, owner: string, update: Update): boolean;
+  /** This method closes the file. A second close does nothing. */
+  close(): void;
+}
+/**
+ * This function checks the entries of a watch store that are due. `PoliteCrawler.watch.run` uses
+ * it. Each check is a fetch of the page with `fetchPage`, with no stored copy (`maxAgeMs: 0`), so
+ * each polite rule applies. A check that the budget stops before its request stays due. The store
+ * can be a function that opens it: the run uses it before the first check.
+ */
+declare const runWatch: (runtime: CrawlRuntime, source: WatchStore | (() => Promise<WatchStore>), options?: WatchRunnerOptions) => WatchRun;
+//#endregion
+//#region src/open.d.ts
+/**
+ * This function gives the default `User-Agent` header. The header names the crawler, its version
+ * and the page about it, so a site owner can find out what made each request.
+ */
+declare const defaultUserAgent: (contact?: string) => string;
+/**
+ * This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,
+ * section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in
+ * the `User-Agent` only.
+ */
+declare const isMailbox: (contact: string) => boolean;
+/**
+ * These options make a runtime. A store needs a log, a blob store and an index. Give `dir`, and
+ * the runtime makes the three parts in it, or give the parts. A part that you give replaces the
+ * part in `dir`.
+ */
+interface RuntimeOptions {
+  /** This folder keeps the capture log (`captures/`), the bodies (`blobs/`), the index (`index.sqlite`) and the state of each host (`hosts.sqlite`). */
+  readonly dir?: string | undefined;
+  /** This log replaces `<dir>/captures`. */
+  readonly log?: CaptureLog | undefined;
+  /** This blob store replaces `<dir>/blobs`. */
+  readonly blobs?: BlobStore | undefined;
+  /** This index replaces `<dir>/index.sqlite`. A runtime takes any `ReadingIndex`. */
+  readonly index?: ReadingIndex | undefined;
+  /**
+   * This store keeps the pace and the stops of each host. The default with `dir` is a SQLite file
+   * (`hosts.sqlite`), so the processes on one store share the pace. With no `dir`, it is in memory.
+   */
+  readonly hostState?: HostStateStore | undefined;
+  /** This token is the name that robots.txt rules match. The default is `PoliteCrawler`. */
+  readonly token?: string | undefined;
+  /** This text is the full `User-Agent` header. The default names the crawler and its page. */
+  readonly userAgent?: string | undefined;
+  /**
+   * This address goes into the `User-Agent` header. A mailbox (`ops@example.com`) also goes into the
+   * `From` header, and a URL does not. The crawler sends no contact without it.
+   */
+  readonly contact?: string | undefined;
+  /**
+   * This browser renders the pages that need JavaScript. The value `false` turns rendering off. The
+   * default checks one time that `playwright-core` and its browser are installed. With no browser,
+   * each page that needs JavaScript gets a note at once, with no wait.
+   */
+  readonly renderer?: Renderer | false | undefined;
+  /** This value is the number of sites that the crawler fetches from at one time. The default is 16. */
+  readonly maxActive?: number | undefined;
+  /** This value is the number of pages that the browser renders at one time. The default is 2: each render holds a browser context. */
+  readonly maxRenders?: number | undefined;
+  /** This `undici` dispatcher sends the requests. Tests give a mock agent, and a proxy is possible. */
+  readonly dispatcher?: Dispatcher | undefined;
+  /** This policy controls the addresses that the crawler fetches. By default, it fetches only public addresses. */
+  readonly egress?: EgressOptions | undefined;
+  /** These values change the limits of the politeness policy, for each host or for some hosts. */
+  readonly politeness?: PolicyOptions | undefined;
+  /** This clock gives the time to the crawler and to the index. Tests give a clock that they control. */
+  readonly clock?: Clock | undefined;
+  /** This function gives the random numbers of the jitter. The default is `Math.random`. */
+  readonly random?: (() => number) | undefined;
+  /** This label goes into each capture record. */
+  readonly scope?: string | undefined;
+  /** With the value false, the crawler does not also obey the group of `PoliteCrawler` when its `User-Agent` links the bot page (`CrawlerOptions.obeyDefaultToken`). */
+  readonly obeyDefaultToken?: boolean | undefined;
+  /** This function ranks the scopes, the most private highest. A shared fetch of a robots.txt records the scope with the highest rank (`CrawlerOptions.scopeRank`). */
+  readonly scopeRank?: ((scope: string) => number) | undefined;
+  /** This function gets an event for each request, response, wait, stop and skip. */
+  readonly onEvent?: ((event: CrawlEvent) => void) | undefined;
+  /** These extractors read the captures of the recipes of an application. The recipes `page` and `archive-copy` are built in. */
+  readonly extractors?: Readonly<Record<string, Extractor>> | undefined;
+  /**
+   * This signer signs each request with Web Bot Auth (`CrawlerOptions.signer`). Give the options of
+   * `createWebBotAuthSigner`, or a `RequestSigner`. Without it, the crawler signs nothing.
+   */
+  readonly signer?: RequestSigner | WebBotAuthOptions | undefined;
+}
+/** A runtime that a store made. It owns the parts that it made, and it closes them. */
+interface StoreRuntime extends CrawlRuntime {
+  readonly hostState: HostStateStore;
+  /** This method closes the crawler, the browser, the log, the index and the host state store that the runtime made. A second close does nothing. */
+  close(): Promise<void>;
+}
+/**
+ * This function makes a runtime. It is the one place that puts the log, the blob store, the index,
+ * the host state, the crawler and the extractors together. The function `openCrawler` uses it.
+ * An application that needs the parts uses it too. The runtime and the crawler share one clock.
+ */
+declare const createRuntime: (options: RuntimeOptions) => StoreRuntime;
+/** These options open a crawler on a folder. Only `dir` is necessary. */
+interface OpenOptions extends Omit<RuntimeOptions, "dir" | "log" | "blobs" | "index"> {
+  /** This folder keeps the capture log, the response bodies, the search index and the state of each host. */
+  readonly dir: string;
+  /** These agents get their robots.txt rules applied to each fetch, for example `Claude-User`. */
+  readonly onBehalfOf?: readonly string[] | undefined;
+  /** This value is the shortest `every` of a watch. The default is `1h`, so a watch cannot hammer a site. */
+  readonly watchFloor?: Duration | undefined;
+}
+/** These options control one fetch. */
+interface FetchOptions {
+  /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */
+  readonly render?: "auto" | "never" | "always" | undefined;
+  /**
+   * A stored copy younger than this age answers, and the crawler sends no request. The default is
+   * `24h`. The value `auto` uses the freshness that the server gave. The value `0s` asks again.
+   */
+  readonly maxAge?: Duration | undefined;
+  /** These agents get their robots.txt rules applied to this fetch, in addition to the defaults. */
+  readonly onBehalfOf?: readonly string[] | undefined;
+  /** When this signal aborts, the crawler stops the fetch and the render. */
+  readonly signal?: AbortSignal | undefined;
+  /** This label goes into the capture records of this fetch. */
+  readonly scope?: string | undefined;
+  /** These labels go into the capture records of this fetch. They are not part of the capture ids. */
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+  /**
+   * The value `archive` gives a copy from a public web archive (the Wayback Machine, then Common
+   * Crawl) when the site refuses the page or is down. The result then has `archived`. The default is `none`.
+   */
+  readonly fallback?: "archive" | "none" | undefined;
+  /**
+   * The value true also gives an archived copy of a page that the current robots.txt of its site
+   * disallows for this agent. The default is false: only a stop, an error or a dead host gets a copy.
+   */
+  readonly archiveDespiteRobots?: boolean | undefined;
+}
+/** These options control a search. */
+interface FindOptions {
+  /** This value is the maximum number of results. The default is 10. */
+  readonly limit?: number | undefined;
+  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
+  readonly kind?: string | null | undefined;
+  /**
+   * This value chooses the ranking. The default `exact` ranks each match with BM25. `fast` leaves
+   * the common words out of the second part of the search, so a long question is faster (`RankMode`).
+   */
+  readonly rank?: RankMode | undefined;
+}
+/** A polite crawler fetches each source one time, keeps it, and answers the next questions from the disk. */
+interface PoliteCrawler {
+  /** The runtime gives the parts to an application that needs more control. */
+  readonly runtime: StoreRuntime;
+  /** This method fetches a page, reads it and adds it to the index. */
+  fetch(url: string, options?: FetchOptions): Promise<PageResult>;
+  /**
+   * This method searches the stored documents. It sends no request. A held-back document is not
+   * in the results. The syntax of the query is the syntax of `parseQuery`.
+   */
+  find(q: string, options?: FindOptions): Promise<Found[]>;
+  /** This method reads a range of characters of one stored document, the current version or the version of one capture. */
+  read(doc: string, options?: ReadOptions): Promise<ReadResult>;
+  /**
+   * This method finds a quote in a stored document. It ignores case, spaces, typographic
+   * punctuation and Unicode forms. It tells which capture holds the quote.
+   */
+  verify(doc: string, quote: string, options?: VerifyOptions): Promise<VerifyResult>;
+  /** This method gives the versions of a stored document, the newest first. A held-back version shows no title. */
+  versions(doc: string, options?: {
+    readonly limit?: number;
+  }): Promise<DocVersion[]>;
+  /**
+   * This method builds the index again from the capture log and the stored bodies, with no request.
+   * When the crawler rebuilt a stale index at open, the first reindex gives the counts of that
+   * rebuild, unless a fetch came before it.
+   */
+  reindex(): Promise<ReindexResult>;
+  /** This method closes the browser, the log files and the index. A second close does nothing. */
+  close(): Promise<void>;
+  /**
+   * This optional method finds the pages of a site from its sitemaps and its feeds. Each file is a
+   * polite fetch with its capture. It keeps the URLs on the site that robots.txt permits, to its
+   * limits. `openCrawler` gives it. Over a runtime of your own, use `discoverSite`.
+   */
+  discover?(site: string, options?: DiscoverOptions): Promise<DiscoverResult>;
+  /**
+   * This optional method crawls a site from its seeds. It follows the links of each page to a
+   * maximum of pages, a maximum depth and a maximum time. A `for await` loop gets each page.
+   * `summary` tells why the crawl stopped. `openCrawler` gives it. Over a runtime of your own, use
+   * `crawlSite`.
+   */
+  crawl?(seeds: string | readonly string[], options?: CrawlOptions): CrawlRun;
+  /**
+   * This optional method compares two versions of a stored document: the lines and the words of
+   * the text, and the metadata at its paths. It sends no request. `openCrawler` gives it.
+   */
+  diff?(doc: string, options?: DiffOptions): Promise<DiffResult>;
+  /** This optional list keeps the pages that the crawler checks again on a schedule. `openCrawler` gives it. */
+  readonly watch?: WatchList;
+  /**
+   * This optional method checks the proposed values of fields against a stored document, and it
+   * locates each value. It does not make values. It sends no request. `openCrawler` gives it.
+   */
+  extract?(doc: string, fields: Readonly<Record<string, FieldClaim>>, options?: ExtractOptions): Promise<ExtractResult>;
+}
+/** This crawler is the result of `openCrawler`: a `PoliteCrawler` with `discover`, `crawl`, `diff`, `watch` and `extract`. */
+interface OpenedCrawler extends PoliteCrawler {
+  /** This method finds the pages of a site from its sitemaps and its feeds. */
+  discover(site: string, options?: DiscoverOptions): Promise<DiscoverResult>;
+  /** This method crawls a site from its seeds, within its limits. A `for await` loop gets each page. */
+  crawl(seeds: string | readonly string[], options?: CrawlOptions): CrawlRun;
+  /** This method compares two versions of a stored document. It sends no request. */
+  diff(doc: string, options?: DiffOptions): Promise<DiffResult>;
+  /** This list keeps the pages that the crawler checks again on a schedule, in `watch.sqlite`. */
+  readonly watch: WatchList;
+  /**
+   * This method checks the proposed values of fields against a stored document. Each field gets
+   * `verified`, `normalized`, `ambiguous`, `not-found`, `held-back` or `invalid`.
+   */
+  extract(doc: string, fields: Readonly<Record<string, FieldClaim>>, options?: ExtractOptions): Promise<ExtractResult>;
+}
+/**
+ * This function opens a crawler on a folder. The folder keeps the capture log (`captures/`), the
+ * response bodies (`blobs/`), the search index (`index.sqlite`) and the state of each host
+ * (`hosts.sqlite`). The index is a projection: `reindex` makes it again from the log and the bodies.
+ * At open, `catchUpIndex` rebuilds an index of an earlier version, and gives an index that is
+ * behind the log the captures that it missed.
+ *
+ * Each method gives a promise, and none throws before its promise. After `close`, each method
+ * rejects.
+ */
+declare const openCrawler: (options: OpenOptions) => OpenedCrawler;
+//#endregion
+export { DiscoverOptions as $, verifyQuote as $n, SignedRequest as $t, CrawlPage as A, Extractors as An, diffTexts as At, ExtractSummary as B, catchUpIndex as Bn, FetchOutcome as Bt, WatchStop as C, WebArchive as Cn, DiffSide as Ct, runWatch as D, CatchUpResult as Dn, VersionChoice as Dt, WatchSummary as E, CapturePayload as En, TextDiff as Et, UrlPattern as F, ReadOptions as Fn, CrawlEvent as Ft, FieldKind as G, isHeldBack as Gn, DirectoryKey as Gt, FieldCandidate as H, fetchPage as Hn, Fetched as Ht, crawlSite as I, ReadResult as In, Crawler as It, FieldStatus as J, pageExtractor as Jn, KeyDirectory as Jt, FieldResult as K, isoTime as Kn, KEY_DIRECTORY_PATH as Kt, ExtractOptions as L, ReindexResult as Ln, CrawlerOptions as Lt, CrawlRunOptions as M, Indexed as Mn, CacheEntry as Mt, CrawlStop as N, PageOptions as Nn, CacheIndex as Nt, CRAWL_LIMITS as O, CrawlRuntime as On, WordPart as Ot, CrawlSummary as P, PageResult as Pn, CaptureWriter as Pt, DISCOVER_LIMITS as Q, renderAndIndex as Qn, SignatureCheck as Qt, ExtractResult as R, VerifyOptions as Rn, DEADLINE_NOTE as Rt, WatchRunnerOptions as S, Archived as Sn, DiffResult as St, WatchStoreOptions as T, archivedDocId as Tn, MetaChange as Tt, FieldCheck as U, findDocuments as Un, RenderRequest as Ut, FIELD_KINDS as V, fetchAndIndex as Vn, FetchRequest as Vt, FieldClaim as W, heldBackMeta as Wn, renderedKey as Wt, checkFields as X, recordOverride as Xn, RequestSigner as Xt, FieldsChecked as Y, readDocument as Yn, KeyInput as Yt, extractDocument as Z, reindex as Zn, SignatureAgentType as Zt, WatchList as _, RankMode as _n, BlobStore as _r, DiffHunk as _t, PoliteCrawler as a, EgressOptions as an, ScanOptions as ar, discoverSite as at, WatchRun as b, VersionAt as bn, DiffLine as bt, createRuntime as c, Doc as cn, makeRecord as cr, SitemapEntry as ct, openCrawler as d, DocVersion as dn, RefusedReason as dr, SitemapRead as dt, WebBotAuthOptions as en, versionAtTime as er, DiscoverResult as et, WATCH_LIMITS as f, ExtractedDoc as fn, RenderOptions as fr, feedDate as ft, WatchKind as g, Latest as gn, playwrightRenderer as gr, DIFF_LIMITS as gt, WatchEvent as h, Found as hn, Renderer as hr, durationMs as ht, OpenedCrawler as i, verifySignature as in, DamagedLine as ir, DiscoveredUrl as it, CrawlRun as j, HELD_BACK as jn, CAPTURE_SCHEMA as jt, CrawlOptions as k, Extractor as kn, diffDocument as kt, defaultUserAgent as l, DocIndex as ln, NO_PLAYWRIGHT as lr, SitemapKind as lt, WatchEntry as m, FindIndexOptions as mn, Rendered as mr, Duration as mt, FindOptions as n, createWebBotAuthSigner as nn, CaptureLog as nr, DiscoverSource as nt, RuntimeOptions as o, EgressRefused as on, CaptureRecord as or, Changefreq as ot, WATCH_SCHEMA as p, FAST_RANK_SHARE as pn, RenderRefused as pr, readSitemap as pt, FieldSpan as q, pageDocId as qn, KEY_DIRECTORY_TYPE as qt, OpenOptions as r, keyDirectory as rn, CaptureReader as rr, DiscoverStop as rt, StoreRuntime as s, egressLookup as sn, RecordInput as sr, SITEMAP_LIMITS as st, FetchOptions as t, WebBotAuthSigner as tn, CaptureFilter as tr, DiscoverRunOptions as tt, isMailbox as u, DocInput as un, PlaywrightRenderer as ur, SitemapLimits as ut, WatchOptions as v, ReadingIndex as vn, FileBlobStore as vr, DiffInput as vt, WatchStore as w, archiveExtractor as wn, DiffSummary as wt, WatchRunOptions as x, VersionText as xn, DiffOptions as xt, WatchPending as y, Variant as yn, DiffLimits as yt, ExtractSource as z, VerifyResult as zn, DEFAULT_SECRET_PARAMS as zt };
+//# sourceMappingURL=open-Ca1lcWR3.d.mts.map