polite-crawler 0.1.3 → 0.2.0

package/build/daemon-DP4EqjFL.d.mts

@@ -1,3772 +0,0 @@
-import { n as Outcome } from "./classify-CrfyXcwo.mjs";
-import { t as ScopeOptions } from "./urls-BTDSgDLC.mjs";
-import { r as Signals, t as AiInput } from "./signals-BHpy2xb2.mjs";
-import { f as PolicyOptions, o as HostStateStore, t as Clock } from "./politeness-yI3xBEx9.mjs";
-import { a as Robots } from "./robots-R-qaEBOb.mjs";
-import { JsonWebKey, KeyObject } from "node:crypto";
-import { Dispatcher } from "undici";
-import { LookupAddress, LookupAllOptions } from "node:dns";
-import { LookupFunction } from "node:net";
-//#region src/blobs.d.ts
-/**
- * A blob store keeps the bytes of each response, and the hash of the bytes is the key. The crawler
- * needs only these two operations, so an application can give its own store.
- */
-interface BlobStore {
-  put(bytes: Uint8Array): Promise<{
-    readonly hash: string;
-  }>;
-  get(hash: string): Promise<Uint8Array | undefined>;
-}
-/** These options change a file blob store. */
-interface FileBlobStoreOptions {
-  /** When this value is true (the default), a blob is on the disk (`fsync`) before `put` finishes. */
-  readonly sync?: boolean | undefined;
-}
-/**
- * This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash
- * of the bytes. These are its rules:
- *
- * - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is
- *   written again.
- * - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.
- *   A crash thus cannot leave half a blob under its name.
- * - Two writers can put the same bytes at the same time. When the rename of one fails because the
- *   other was first, the put succeeds if the stored file has the same hash.
- * - The method `get` examines the hash of the bytes. A damaged file gets a new name
- *   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.
- */
-declare class FileBlobStore implements BlobStore {
-  readonly dir: string;
-  private readonly writing;
-  private readonly sync;
-  constructor(dir: string, options?: FileBlobStoreOptions);
-  put(bytes: Uint8Array): Promise<{
-    readonly hash: string;
-    readonly existed: boolean;
-  }>;
-  /** This method tells if a file holds exactly the bytes of a hash. */
-  private static holds;
-  private write;
-  get(hash: string): Promise<Uint8Array | undefined>;
-}
-//#endregion
-//#region src/render.d.ts
-/**
- * The renderer opens the headless browser only for a page that shows almost nothing without
- * JavaScript. The settings:
- *
- * - The default `chromium-headless-shell` of Playwright, the lighter build.
- * - One browser for many pages, and a new context for each page. Thus no cookie or storage goes
- *   from one site to the next. A context has its own HTTP cache, so routing each request costs
- *   no cache across sites.
- * - Each request of the page goes through a route, the first navigation too. The route refuses
- *   images, fonts and media by their resource type. It asks `allowRequest` about each other
- *   request, and it sends `headers` with it.
- * - Playwright routes only the first request of a redirect chain. Thus with `allowRequest`, the
- *   route sends each request itself, follows each redirect itself and asks about each step. The
- *   browser then gets each body whole, not as a stream.
- * - No download, no popup, no service worker and no dialog.
- * - A wait until the visible text stops growing, with a limit. Playwright discourages
- *   `networkidle`. The text includes open shadow roots and the frames of the same origin.
- * - One deadline for the whole render: the navigation, the wait and the reading of the page.
- * - After a crash or a disconnection, the next page launches a new browser.
- * - The browser closes after two idle minutes. An idle Chromium holds hundreds of megabytes, so
- *   it stays open only while pages need it. The open browser keeps the process alive, so a
- *   script uses `close()` when it is done.
- */
-interface Rendered {
-  readonly html: string;
-  readonly finalUrl: string;
-  readonly status: number | null;
-  readonly headers: Readonly<Record<string, string>>;
-}
-/** These are the options of one render. */
-interface RenderOptions {
-  readonly userAgent: string;
-  /** This value is the deadline of the whole render, in milliseconds. */
-  readonly timeoutMs: number;
-  /** These headers go with each request of the page, for example the `From` header. */
-  readonly headers?: Readonly<Record<string, string>> | undefined;
-  /**
-   * This function decides each request that the page makes, the first navigation included. When
-   * it gives false, the browser does not send the request. The crawler uses it to apply the
-   * egress policy and robots.txt to the requests of the page.
-   */
-  readonly allowRequest?: ((url: string) => Promise<boolean> | boolean) | undefined;
-  /** This value is the maximum number of redirects of one request of the page. The default is 5, as for a plain fetch. */
-  readonly maxRedirects?: number | undefined;
-  /** When this signal aborts, the renderer stops the page: it closes the context, and the render rejects. */
-  readonly signal?: AbortSignal | undefined;
-  /**
-   * This function gives the signature headers of one request of the page (Web Bot Auth). The route
-   * adds them to the request and to each step of its redirects. When the function gives undefined,
-   * the browser does not send the request.
-   */
-  readonly sign?: ((url: string) => Readonly<Record<string, string>> | undefined) | undefined;
-}
-interface Renderer {
-  render(url: string, options: RenderOptions): Promise<Rendered>;
-  close(): Promise<void>;
-}
-/** The reason of a `RenderRefused`. */
-type RefusedReason = "refused" | "download" | "scheme" | "browser";
-/**
- * This error tells that the renderer did not render the page, and that the page or the site is
- * not at fault. The reason is one of these:
- *
- * - `refused`: `allowRequest` refused the first navigation, a step of its redirects, or a later
- *   navigation of the page that left it on an error page. `url` names the refused URL.
- * - `download`: the URL gives a file, not a page.
- * - `scheme`: the URL is not an http or https URL.
- * - `browser`: the renderer has no browser (`playwright-core` or its browser is not installed). The
- *   crawler then renders no more pages, and gives each one the note at once.
- *
- * A crawler does not count this error as a failure of the host.
- */
-declare class RenderRefused extends Error {
-  readonly reason: RefusedReason;
-  /** This URL is the request that `allowRequest` or the limit of redirects refused, when the reason is `refused`. */
-  readonly url: string | undefined;
-  constructor(reason: RefusedReason, message: string, url?: string);
-}
-/** The text of the error when the optional package `playwright-core` is not installed. */
-declare const NO_PLAYWRIGHT = "rendering needs the package playwright-core and a browser: npm install playwright-core, then npx playwright-core install chromium-headless-shell. For the MCP server with npx, start it with npx -y -p polite-crawler -p playwright-core polite-crawler mcp";
-interface PlaywrightRenderer extends Renderer {
-  /** This function tells if a browser is open. */
-  readonly open: () => boolean;
-}
-declare const playwrightRenderer: (options?: {
-  readonly idleMs?: number;
-  /**
-   * This function starts the browser: it gives a `Browser` of `playwright-core`. The default is
-   * the headless shell of Playwright. The type names no type of `playwright-core`, so a program
-   * with no `playwright-core` can check its types.
-   */
-  readonly launch?: (() => Promise<object>) | undefined;
-}) => PlaywrightRenderer;
-//#endregion
-//#region src/duration.d.ts
-/** A duration is a count of milliseconds, or a text, for example `"30m"`, `"24h"` or `"7d"`. */
-type Duration = number | string;
-/**
- * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,
- * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`
- * is zero. The function throws an error for a text that it cannot read.
- */
-declare const durationMs: (duration: Duration) => number;
-//#endregion
-//#region src/record.d.ts
-/**
- * A capture record is one line of the capture log. Each fetch decision makes one record, and a
- * fetch that the crawler declines makes one too.
- *
- * The record id comes from a hash of its content, so the same decision always gets the same id.
- * A citation names this id.
- *
- * The id has 48 bits: the source and the first 12 hexadecimal digits of the hash. Approximately
- * 1 million records give a chance of a collision of 0.2 percent. The package keeps this length on purpose,
- * because the ids of the earlier capture logs and their citations must stay the same. A later
- * version can make the ids of new records longer behind a record version. It must still read a
- * 12-digit id.
- */
-interface CaptureRecord {
-  readonly id: string;
-  /** The source of the record. The crawler writes `fetch`, and an application can add its own sources. */
-  readonly source: string;
-  readonly kind: string;
-  readonly external_id?: string | undefined;
-  readonly occurred_at?: string | undefined;
-  readonly received_at: string;
-  readonly payload: unknown;
-  readonly content_hash: string;
-  /** This label comes from the application. The crawler writes it as it is and does not read it. */
-  readonly scope?: string | undefined;
-  /**
-   * These labels come from the application, for example the lineage of a request. The crawler writes
-   * them as they are and does not read them. Like `scope`, they are not part of the hash.
-   */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/** These are the fields of a new record. The function `makeRecord` calculates the id and the hash. */
-interface RecordInput {
-  readonly source: string;
-  readonly kind: string;
-  readonly payload: unknown;
-  readonly received_at: string;
-  readonly external_id?: string | undefined;
-  readonly occurred_at?: string | undefined;
-  readonly scope?: string | undefined;
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/**
- * This function makes a capture record. The hash covers the source, the kind, the external id
- * and the payload. The id is the source and the first 12 digits of the hash. The times, `scope`
- * and `labels` are not part of the hash.
- */
-declare const makeRecord: (input: RecordInput) => CaptureRecord;
-//#endregion
-//#region src/log.d.ts
-/** A scan filter selects the records of one source, one kind, or the records after one time. */
-interface CaptureFilter {
-  readonly source?: string | undefined;
-  readonly kind?: string | undefined;
-  /** This value is an ISO 8601 time. The scan gives the records received at this time or later. */
-  readonly since?: string | undefined;
-}
-/** A damaged line is a line of the log that is not a record: a torn append, or bytes that a crash left. */
-interface DamagedLine {
-  /** This value is the path of the file. */
-  readonly path: string;
-  /** This value is the number of the line in the file, from 1. */
-  readonly line: number;
-}
-/** These options change a scan. */
-interface ScanOptions {
-  /** The scan skips a damaged line, and it gives the line to this function. */
-  readonly onDamaged?: ((damaged: DamagedLine) => void) | undefined;
-}
-/** The read side of the capture log is what `reindex` replays. */
-interface CaptureReader {
-  scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;
-}
-/** These options change a capture log. */
-interface CaptureLogOptions {
-  /**
-   * When this value is true (the default), the log writes each batch of appends to the disk
-   * (`fdatasync`) before the appends finish. Thus a power cut does not lose a record that the
-   * crawler reported.
-   */
-  readonly sync?: boolean | undefined;
-}
-/**
- * The capture log keeps each fetch decision as one line of JSON in
- * `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.
- *
- * These are the rules of the log:
- *
- * - The log appends a batch of records with one write, and it writes the batch to the disk before
- *   the appends finish.
- * - An append always starts on a new line. After a crash leaves a partial line, the next append
- *   starts after a line break. The partial line stays as a damaged line, and the record is good.
- * - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.
- * - The log does not remove or change a line.
- *
- * Two processes can append to one log, because each batch is one write to a file in append mode.
- */
-declare class CaptureLog implements CaptureReader {
-  readonly dir: string;
-  private readonly handles;
-  private readonly pending;
-  private draining;
-  private readonly sync;
-  constructor(dir: string, options?: CaptureLogOptions);
-  private pathOf;
-  private handleFor;
-  /** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */
-  private static needsBreak;
-  append(record: CaptureRecord): Promise<void>;
-  /** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */
-  private kick;
-  /** This method writes the waiting records in batches: one write and one sync for each file. */
-  private drain;
-  /** This method waits until each append in progress is on the disk. */
-  flush(): Promise<void>;
-  /** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */
-  private filesOf;
-  /** This method reads the records of one source in the sequence of the files. */
-  private readSource;
-  /**
-   * This method reads the records that agree with the filter. It first waits for the appends that
-   * are in progress. It reads each file as a stream, so its memory does not grow with the log.
-   *
-   * The records of one source come in the sequence of the files. The records of two or more sources
-   * merge in the sequence of `received_at`.
-   */
-  scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;
-  /** This method writes the appends in progress, then closes the files. A later append opens them again. */
-  close(): Promise<void>;
-}
-//#endregion
-//#region src/runtime.d.ts
-/**
- * An extractor changes one captured body into the documents that it holds. It has no side effects,
- * so `reindex` can replay each capture through it and build the index again with no request. It can
- * answer later, because the crawler reads a PDF asynchronously.
- *
- * An extractor that throws does not stop a fetch, a render or a rebuild. The runtime records the
- * error and gives no document for that capture.
- */
-type Extractor = (body: Uint8Array, capture: {
-  readonly url: string;
-  readonly contentType: string | null;
-}) => ReadonlyArray<ExtractedDoc> | Promise<ReadonlyArray<ExtractedDoc>>;
-/** This function gives the document id of a web page. Each page has one id. */
-declare const pageDocId: (url: string) => string;
-/**
- * This extractor is the built-in recipe for a web page, plain or rendered. It keeps the visible
- * text, the JSON-LD (whole objects, up to a limit) and the links. It decodes the body in the
- * encoding that the page declares, with `decodeBody`. It does not throw: a page that it cannot read
- * gives a document that names the error. When the reader cut the text, the document keeps the whole
- * text in `full_text`, so `verify` finds a quote past the cut.
- */
-declare const pageExtractor: Extractor;
-declare class Extractors {
-  private readonly byRecipe;
-  private readonly replaceable;
-  register(recipe: string, extractor: Extractor): void;
-  get(recipe: string): Extractor | undefined;
-  /** This method gives the names of the recipes that have an extractor. */
-  recipes(): string[];
-}
-interface CrawlRuntime {
-  readonly crawler: Crawler;
-  /** This index keeps the documents and the cache entries. `DocIndex` is the SQLite index. */
-  readonly index: ReadingIndex;
-  readonly blobs: BlobStore;
-  /** This reader is the read side of the capture log. The function `reindex` replays it. */
-  readonly captures: CaptureReader;
-  /** This log is the write side of the capture log. */
-  readonly log: CaptureLog;
-  readonly extractors: Extractors;
-  readonly now: () => string;
-}
-/** This is the payload of a capture record, as the crawler writes it. */
-interface CapturePayload {
-  readonly url: string;
-  readonly recipe: string;
-  readonly mode: "api" | "page";
-  readonly status?: number;
-  readonly outcome?: string;
-  readonly blob?: string | null;
-  readonly content_type?: string | null;
-  readonly etag?: string | null;
-  readonly last_modified?: string | null;
-  readonly fetched_at: string;
-  readonly signals?: {
-    readonly ai_input?: AiInput;
-  };
-  /** This value is true when the headless browser made the capture, not a plain request. */
-  readonly rendered?: boolean;
-  /** This value is true when the body is only the first part of the response. */
-  readonly truncated?: boolean;
-  /** This value is true when the server cut a compressed body. The capture is not a copy to keep. */
-  readonly cut?: boolean;
-  /** This value is true when an API answered with a web page that the request did not accept. The capture is not a copy to keep. */
-  readonly unasked?: boolean;
-  /** This time (ISO 8601) is the end of the freshness that the server gave. Captures before version 0.2 do not have it. */
-  readonly fresh_until?: string | null;
-  /** This text holds the request headers that the response varies on. Captures before version 0.2 do not have it. */
-  readonly vary_key?: string | null;
-  /** This URL is the target of a redirect, with its secrets as `REDACTED`. */
-  readonly location?: string | null;
-  readonly note?: string;
-  /** This text is the byte range of a range request (`bytes=0-99`). The capture is a part of the resource, not its stored copy. */
-  readonly range?: string;
-}
-/** The note of a document that is held back. */
-declare const HELD_BACK = "This page asked not to be used as AI input, so its text is held back. Open the URL to read it, or read again with userProvided when a person gave this URL directly (the crawler logs that).";
-/**
- * This function tells if a document is held back: the page asked not to be AI input, and no person
- * gave its URL directly. The only way to read a held-back document is `userProvided`.
- */
-declare const isHeldBack: (doc: {
-  readonly ai_input: AiInput;
-}, userProvided?: boolean) => boolean;
-/** This function gives the metadata that a held-back document shows: only the structural fields, which hold no text of the page. */
-declare const heldBackMeta: (meta: Readonly<Record<string, unknown>>) => Record<string, unknown>;
-/** The result of `fetchAndIndex` and `renderAndIndex`. */
-interface Indexed {
-  readonly fetched: Fetched;
-  readonly docs: readonly Doc[];
-  /** This text tells why the capture gave no document: the error of the extractor, or a bot check. */
-  readonly error?: string | undefined;
-  /** This value names the bot check or the firewall that sent its page in place of the content. */
-  readonly blocked?: string | undefined;
-}
-/**
- * This function fetches one URL and adds its documents to the index. Each recipe uses it. A fetch
- * from the cache gives the documents that the index has for that capture. Thus a repeated
- * question sends no request and reads nothing again. An extractor that throws gives no document and an
- * `error`, and the fetch still succeeds.
- */
-declare const fetchAndIndex: (runtime: CrawlRuntime, request: FetchRequest) => Promise<Indexed>;
-/** This function renders a page in the headless browser and adds what the page shows to the index. */
-declare const renderAndIndex: (runtime: CrawlRuntime, request: RenderRequest) => Promise<Indexed>;
-interface PageResult {
-  readonly url: string;
-  readonly doc: string | null;
-  readonly outcome: string;
-  readonly status: number;
-  /** This value is the title of the page. A held-back page shows no title. */
-  readonly title: string | null;
-  readonly chars: number;
-  readonly rendered: boolean;
-  /** These are the schema.org types that the JSON-LD of the page declares. */
-  readonly structured: readonly string[];
-  readonly links: number;
-  /** These ids are the captures behind the current text, the newest first, 10 or fewer. */
-  readonly captures: readonly string[];
-  /** This id is the capture of the text that the document shows. A citation names it. */
-  readonly capture: string | null;
-  /** This value is the number of captures of the document. */
-  readonly capture_count: number;
-  /** This time (ISO 8601) is the fetch time of the text that the document shows. */
-  readonly fetched_at: string | null;
-  /** This value is true when the stored copy answered, and the crawler sent no request. */
-  readonly from_cache: boolean;
-  /** This value is true when the page asked not to be AI input. Its title and text are then held back. */
-  readonly held_back: boolean;
-  readonly note: string | null;
-  /**
-   * This value tells that the document is an archived copy, not a fresh fetch: the archive, the date
-   * of the copy and its URL there. The outcome and the status stay those of the live fetch.
-   */
-  readonly archived?: Archived | undefined;
-}
-/** These options control one page. */
-interface PageOptions {
-  readonly render: "auto" | "never" | "always";
-  readonly maxAgeMs?: number | "auto" | undefined;
-  readonly onBehalfOf?: readonly string[] | undefined;
-  readonly job?: string | undefined;
-  /** When this signal aborts, the crawler stops the fetch and the render. */
-  readonly signal?: AbortSignal | undefined;
-  readonly scope?: string | undefined;
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-  /** This time (milliseconds since the epoch) is the end of a budget. No request of the page starts at or after it (`FetchRequest.deadline`). */
-  readonly deadline?: number | undefined;
-  /** The value `archive` gives a copy from a public web archive when the site refuses the page, or is down. The default is `none`. */
-  readonly fallback?: "archive" | "none" | undefined;
-  /** The value true also gives an archived copy of a page that the current robots.txt of its site disallows. The default is false. */
-  readonly archiveDespiteRobots?: boolean | undefined;
-}
-/**
- * This function reads one web page. It first sends a plain request. It opens the browser only when
- * the page shows almost nothing without JavaScript, or when the caller asks for it. It renders
- * only an HTML page: not a PDF, a different file or a bot check. A page that sends the reader on
- * with a `<meta http-equiv="refresh">` gets one more plain fetch, of its target.
- */
-declare const fetchPage: (runtime: CrawlRuntime, url: string, options: PageOptions) => Promise<PageResult>;
-/** This result is a part of one stored document. */
-interface ReadResult {
-  readonly doc: string;
-  readonly kind: string;
-  /** This value is the title of the document. A held-back document shows no title. */
-  readonly title: string;
-  readonly url: string | null;
-  /** This value is the metadata of the document. A held-back document shows only the structural fields. */
-  readonly meta: Readonly<Record<string, unknown>>;
-  readonly from: number;
-  readonly to: number;
-  readonly total: number;
-  readonly text: string;
-  readonly captures: readonly string[];
-  /** This id is the capture of the text that the result reads. */
-  readonly capture: string;
-  /** This time (ISO 8601) is the fetch time of the text that the result reads. */
-  readonly fetched_at: string;
-  readonly ai_input: AiInput;
-  /** This value is true when the page asked not to be AI input. The text is then empty. */
-  readonly held_back: boolean;
-  readonly note: string | null;
-  /** This time (ISO 8601) is the `at` of the read. The result has it only when the read gave `at`. */
-  readonly at?: string | undefined;
-  /**
-   * This value is true when `at` is before the first capture of the document. Then no version
-   * answers: the text, the title, `capture` and `fetched_at` are empty, and the note tells why.
-   */
-  readonly before_first?: boolean | undefined;
-  /** This value tells that the text is an archived copy, with the archive and the date of the copy. */
-  readonly archived?: Archived | undefined;
-}
-/** This result tells if a quote is in a stored document, and where. */
-interface VerifyResult {
-  readonly doc: string;
-  /** This value is true when the quote is in the text. A held-back document gives false. */
-  readonly found: boolean;
-  /** These values are the positions of the quote in the text of `capture`. */
-  readonly start: number | null;
-  readonly end: number | null;
-  readonly excerpt: string | null;
-  readonly url: string | null;
-  readonly captures: readonly string[];
-  /** This id is the capture whose text holds the quote, or the capture that the result examined. */
-  readonly capture: string | null;
-  /** This hash is the blob of that capture. */
-  readonly blob: string | null;
-  /** This time (ISO 8601) is the fetch time of that capture. */
-  readonly fetched_at: string | null;
-  /** This value is true when only an earlier version of the document holds the quote. */
-  readonly stale: boolean;
-  /** This value is true when the quote is in the whole text of a page that the reader cut. The positions are then in that whole text. */
-  readonly in_full_text: boolean;
-  /** This value is false when the examined text is not complete (a PDF that the reader stopped). A quote that is not found can then be in the file. */
-  readonly complete: boolean;
-  /** This value is the number of places where the quote is. */
-  readonly count: number;
-  readonly held_back: boolean;
-  readonly note: string | null;
-  /** This time (ISO 8601) is the `at` of the verification. The result has it only when the verification gave `at`. */
-  readonly at?: string | undefined;
-  /** This value is true when `at` is before the first capture of the document. Then no version holds the quote, and the note tells why. */
-  readonly before_first?: boolean | undefined;
-  /**
-   * This value tells that the examined text is an archived copy, with the archive and the date of
-   * the copy. A citation of it names the archive and that date, not a date of a fresh fetch.
-   */
-  readonly archived?: Archived | undefined;
-}
-/** These options control a read. */
-interface ReadOptions {
-  /** This value is the first character to read. The default is 0. */
-  readonly from?: number | undefined;
-  /** This value is the number of characters to read. The default is 4000. */
-  readonly length?: number | undefined;
-  /** This value is true when a person gave this URL directly. It is the only way to read a held-back document. The crawler logs it. */
-  readonly userProvided?: boolean | undefined;
-  /** This id reads the version of one capture, not the current version. */
-  readonly capture?: string | undefined;
-  /**
-   * This time reads the version that was current at that time: an ISO 8601 text or a `Date`. A
-   * date with no time is the start of that day in UTC. Give `capture` or `at`, not the two.
-   */
-  readonly at?: string | Date | undefined;
-}
-/** These options control a verification. */
-interface VerifyOptions {
-  /** This id examines the version of one capture only. A citation of that capture thus stays true after a new fetch. */
-  readonly capture?: string | undefined;
-  /** This value is true when a person gave this URL directly. It is the only way to verify a quote in a held-back document. */
-  readonly userProvided?: boolean | undefined;
-  /** This time examines only the version that was current at that time (`ReadOptions.at`). Give `capture` or `at`, not the two. */
-  readonly at?: string | Date | undefined;
-}
-/**
- * This function gives the ISO 8601 form of a time. A text that is not a time throws a `RangeError`.
- * A date with no time is the start of that day in UTC.
- */
-declare const isoTime: (name: string, value: string | Date) => string;
-/**
- * This function gives the version of a document that was current at a time (ISO 8601). It uses
- * `ReadingIndex.versionAt` when the index has it. Else it takes the newest version at or before
- * the time.
- */
-declare const versionAtTime: (runtime: CrawlRuntime, id: string, at: string) => VersionAt;
-/**
- * This function writes an `override` record to the capture log. A person gave the URL of a
- * held-back document directly, and the hold stops for one action (`read` or `verify`). The record
- * takes the scope and the labels of `marks`, or else those of the document, as its captures do.
- */
-declare const recordOverride: (runtime: CrawlRuntime, doc: {
-  readonly doc: string;
-  readonly url: string | null;
-  readonly scope?: string | undefined;
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}, action: string, marks?: {
-  readonly scope?: string | undefined;
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}) => Promise<void>;
-/**
- * This function reads a range of characters of one stored document. It applies the rule for a
- * held-back document: no title, no text, and only the structural metadata, unless `userProvided`.
- * The positions do not cut a character in two.
- */
-declare const readDocument: (runtime: CrawlRuntime, id: string, options?: ReadOptions) => Promise<ReadResult>;
-/**
- * This function finds a quote in a stored document, with `locate`. It tells which capture holds the
- * quote, with its blob, its fetch time and the positions:
- *
- * - With `capture`, it examines the version of that capture only.
- * - With no capture, it examines the current text first. When the quote is not there, it examines
- *   the earlier texts, the newest first. A match there has `stale: true`.
- * - When the reader cut a text, it also examines the whole text.
- *
- * A held-back document gives `found: false` and no positions, so a quote cannot test its text,
- * unless `userProvided`. The same rule applies to each version. A version that asked not to be AI
- * input stays held back after a later capture drops the signal. The crawler logs an override when
- * `userProvided` opens a held-back version.
- */
-declare const verifyQuote: (runtime: CrawlRuntime, id: string, quote: string, options?: VerifyOptions) => Promise<VerifyResult>;
-/**
- * This function searches the stored documents. The syntax of the query is the syntax of
- * `parseQuery`. A held-back document is not in the results. The option `rank` chooses the
- * ranking (`RankMode`), and the default is `exact`.
- */
-declare const findDocuments: (runtime: CrawlRuntime, q: string, options?: {
-  readonly limit?: number | undefined;
-  readonly kind?: string | null | undefined;
-  readonly rank?: RankMode | undefined;
-}) => Promise<Found[]>;
-/** The counts of a rebuild. */
-interface ReindexResult {
-  /** This value is the number of captures that the rebuild read again. */
-  readonly captures: number;
-  readonly docs: number;
-  readonly urls: number;
-  /** This value is the number of captures whose extractor threw. */
-  readonly failed: number;
-  /** This value is the number of captures whose blob is missing or damaged. */
-  readonly missing: number;
-  /** This value is the number of damaged lines of the log that the rebuild skipped. */
-  readonly damaged: number;
-  /** This value is the number of documents of recipes with no extractor, kept from the old index. */
-  readonly carried: number;
-  /** These recipes have captures in the log and no extractor in this runtime. */
-  readonly unknown_recipes: readonly string[];
-  /** These are the first errors of the extractors. */
-  readonly errors: ReadonlyArray<{
-    readonly capture: string;
-    readonly error: string;
-  }>;
-}
-/**
- * This function builds the reading index again from the capture log and the blobs. The index is a
- * projection, and this function is the proof that its deletion loses nothing. These are its rules:
- *
- * - It builds a new generation of the tables and commits it in batches. The old index answers
- *   until the end, and a failed rebuild leaves it as it was.
- * - It reads the log as a stream, so its memory does not grow with the log.
- * - It counts and skips a damaged line, a missing blob and an extractor that throws.
- * - The documents of a recipe that has no extractor in this runtime stay, from the old index.
- * - A capture that a different writer adds during the rebuild goes into the new index.
- */
-declare const reindex: (runtime: CrawlRuntime, options?: {
-  readonly batch?: number | undefined;
-}) => Promise<ReindexResult>;
-/** The result of `catchUpIndex`. */
-interface CatchUpResult extends ReindexResult {
-  /**
-   * The value `rebuild` tells that the index was stale, and the function built it again from the
-   * whole log. The value `tail` tells that it read only the captures that the index missed.
-   */
-  readonly mode: "rebuild" | "tail";
-}
-/**
- * This function brings the reading index up to the capture log, with no request. `openCrawler`
- * uses it at open. An application that uses `createRuntime` uses it the same way:
- *
- * - A stale index (a new schema, or a deleted file) is built again with `reindex`.
- * - An index that is behind the log gets the captures that it missed. A process can stop between
- *   the log and the index, and SQLite can lose its last transactions after a power cut. The
- *   function reads the log from a short time before the newest cache entry.
- *
- * It gives the counts, or undefined when the index missed nothing.
- */
-declare const catchUpIndex: (runtime: CrawlRuntime) => Promise<CatchUpResult | undefined>;
-//#endregion
-//#region src/fallback.d.ts
-/**
- * The archived fallback. When a site refuses a page, or is down, the crawler can give a copy from a
- * public web archive: the Wayback Machine, then Common Crawl. The crawler does not show the copy as a fresh fetch:
- *
- * - It is a document of its own (`archive:<url>`), of the kind `archived`. Its metadata and each
- *   result that shows it have `archived`: the archive, the date of the copy and its URL there.
- * - The result keeps the outcome of the live fetch (`blocked`, for example), and its note names the
- *   archive and the date.
- *
- * The fallback is opt-in (`fallback: "archive"`). The archives are third-party APIs with their own
- * politeness, so each request to them is a fetch of the same crawler. Their robots.txt, the pace of
- * each host, the egress policy and the identity of the crawler apply, and the capture log records
- * each request.
- *
- * By default, the crawler gives no archived copy of a page that the current robots.txt of its site
- * disallows for this agent. Only a stop, an error or a dead host gets the fallback.
- * `archiveDespiteRobots` widens it. The usage signals of the copy apply as for a live page.
- */
-/** The public web archives of the fallback. */
-type WebArchive = "wayback" | "commoncrawl";
-/** An archived copy: the archive, the time when the archive captured the page, and where the copy is. */
-interface Archived {
-  readonly source: WebArchive;
-  /** This time (ISO 8601) is when the archive captured the page. It is not the time of the fetch from the archive. */
-  readonly timestamp: string;
-  /** This URL is the copy in the archive: a Wayback Machine page, or the Common Crawl WARC file. */
-  readonly url: string;
-  /** This URL is the page that the archive captured. */
-  readonly original: string;
-  /** This id is the `WARC-Record-ID` of a Common Crawl copy in its WARC file. */
-  readonly record?: string | undefined;
-}
-/** This function gives the document id of the archived copy of a page. */
-declare const archivedDocId: (url: string) => string;
-/**
- * This extractor reads the captures of the recipe `archive-copy`. It has no side effects, so
- * `reindex` replays them. It reads two forms:
- *
- * - A Wayback Machine copy in the raw form (`/web/<time>id_/<url>`). The body is the body of the page.
- * - A Common Crawl WARC record (a gzip member). It reads the HTTP headers and the body from the record.
- *
- * It reads the body with the page reader, and gives a document of the kind `archived` with
- * `archived` in its metadata. The usage signals of the HTTP headers in a WARC record apply too.
- */
-declare const archiveExtractor: Extractor;
-//#endregion
-//#region src/docs.d.ts
-/** This is one document, as an extractor gives it. */
-interface ExtractedDoc {
-  readonly doc: string;
-  readonly kind: string;
-  readonly title: string;
-  /** This text is the readable text. For a paper, it is the title and the abstract. */
-  readonly text: string;
-  readonly url: string | null;
-  readonly meta: Readonly<Record<string, unknown>>;
-  /** This rule chooses between the texts of two recipes. The default is `richest`: the longer text. */
-  readonly merge?: "latest" | "richest" | undefined;
-  /** When the extractor cut the text, this text is the whole text. The function `verify` looks for a quote here too. */
-  readonly full_text?: string | undefined;
-  /**
-   * The value `no` holds the document back from the AI reader, also when the capture said nothing.
-   * An extractor sets it when the body states it, for example the headers in an archived response.
-   */
-  readonly ai_input?: AiInput | undefined;
-}
-/** This is one version of a document: the document as one capture gave it. */
-interface DocInput extends ExtractedDoc {
-  /** These ids are the capture records that this version came from. The first id names the version. */
-  readonly captures: readonly string[];
-  readonly ai_input?: AiInput | undefined;
-  /** This id is the capture of the version. The default is the first id of `captures`. */
-  readonly capture?: string | undefined;
-  /** This time (ISO 8601) is the time of the fetch. The default is the time of the upsert. */
-  readonly fetched_at?: string | undefined;
-  readonly blob?: string | null | undefined;
-  /** This value is the recipe that read the capture. A version with no recipe has the recipe `""`. */
-  readonly recipe?: string | undefined;
-  /** This value is true when the headless browser made the capture. */
-  readonly rendered?: boolean | undefined;
-  /** This label comes from the capture. The index writes it as it is and does not read it. */
-  readonly scope?: string | undefined;
-  /** These labels come from the capture. The index writes them as they are and does not read them. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/** This is the current version of a document. */
-interface Doc {
-  readonly doc: string;
-  readonly kind: string;
-  readonly title: string;
-  readonly text: string;
-  readonly url: string | null;
-  readonly meta: Readonly<Record<string, unknown>>;
-  /** These ids are the captures behind the current text, the newest first, `MAX_CAPTURES` or fewer. */
-  readonly captures: readonly string[];
-  /** This value is the number of versions of the document. */
-  readonly capture_count: number;
-  /** This id is the capture of the current text. */
-  readonly capture: string;
-  readonly blob: string | null;
-  /** This time (ISO 8601) is the fetch time of the current text. */
-  readonly fetched_at: string;
-  readonly recipe: string;
-  readonly rendered: boolean;
-  readonly ai_input: AiInput;
-  /** This value is true when the extractor cut the text, and the index keeps the whole text for `verify`. */
-  readonly cut: boolean;
-  readonly scope?: string | undefined;
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-  readonly updated_at: string;
-}
-/** This is one version of a document, with no text. */
-interface DocVersion {
-  readonly doc: string;
-  readonly capture: string;
-  readonly recipe: string;
-  readonly rendered: boolean;
-  readonly fetched_at: string;
-  readonly blob: string | null;
-  readonly kind: string;
-  readonly title: string;
-  readonly url: string | null;
-  readonly ai_input: AiInput;
-  /** This value is the length of the text of the version. */
-  readonly chars: number;
-  /** This value is true when the index keeps a whole text that is longer than the text. */
-  readonly cut: boolean;
-}
-/** This is the text of one version, with its metadata. */
-interface VersionText extends DocVersion {
-  readonly text: string;
-  /** This text is the whole text when the extractor cut `text`. */
-  readonly full_text: string | undefined;
-  readonly meta: Readonly<Record<string, unknown>>;
-}
-/** This is the version of a document that was current at a time (`ReadingIndex.versionAt`). */
-interface VersionAt {
-  /**
-   * This version was current at the time, or it is undefined when the time is before the first
-   * version. Its `ai_input` is `no` when the document was held back at that time.
-   */
-  readonly version: VersionText | undefined;
-  /** This time (ISO 8601) is the fetch time of the first version of the document, or undefined for an unknown document. */
-  readonly first: string | undefined;
-}
-/**
- * This value chooses the ranking of a search:
- *
- * - `exact` (the default): BM25 ranks each document that matches.
- * - `fast`: the form of the query that needs one part leaves out the common parts. A common part
- *   is in more than `FAST_RANK_SHARE` of the documents. The rarest part stays when each part is common.
- *
- * The documents that hold each part come first in the two modes, with the same rank. In the mode
- * `fast`, the rest of the results leaves out a document that holds only common parts. The common
- * parts also do not add to the score of the rest. Thus `fast` is faster for a long question, and
- * its results can be different. The README gives the measured difference.
- */
-type RankMode = "exact" | "fast";
-/** In the mode `fast`, a part is common when more than this share of the documents hold it. */
-declare const FAST_RANK_SHARE = .2;
-/**
- * This value chooses the ranking of a search of the crawler: a `RankMode` of the index, or
- * `hybrid`. The mode `hybrid` fuses the BM25 ranking with the ranking of the vectors of an
- * embedder (`OpenOptions.embedder`). Without an embedder, it throws an error.
- */
-type SearchRank = RankMode | "hybrid";
-/** The text of the error of hybrid search with no embedder. */
-declare const NO_EMBEDDER = "hybrid search needs an embedder: open the crawler with the option embedder (transformersEmbedder or ollamaEmbedder), or give --embedder transformers or ollama to the command line";
-/** These are the options of a search of the index. */
-interface FindIndexOptions {
-  /** This value is the maximum number of results. The index gives 1000 or fewer. */
-  readonly limit: number;
-  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
-  readonly kind?: string | null | undefined;
-  /** This value chooses the ranking. The default is `exact`. */
-  readonly rank?: RankMode | undefined;
-}
-/** This is the passage of a hit of a hybrid search: the chunk of the text that is closest to the question. */
-interface Passage {
-  /** These values are the positions of the chunk in the text of the capture of the hit, in UTF-16 units. */
-  readonly start: number;
-  readonly end: number;
-  /** This value is the cosine of the vectors of the chunk and the question, from -1 to 1. */
-  readonly similarity: number;
-}
-interface Found {
-  readonly doc: string;
-  readonly kind: string;
-  readonly title: string;
-  readonly url: string | null;
-  readonly snippet: string;
-  readonly score: number;
-  readonly ai_input: AiInput;
-  /** This time (ISO 8601) is the fetch time of the current text. */
-  readonly fetched_at: string;
-  /** This id is the capture of the current text. */
-  readonly capture: string;
-  /** This value tells that the document is an archived copy (the kind `archived`), with the archive and the date of the copy. */
-  readonly archived?: Archived | undefined;
-  /** This value is the passage that a hybrid search found. A search of the other modes does not give it. */
-  readonly passage?: Passage | undefined;
-}
-/** This is the current text of a document in short: its hash, with no text. */
-interface DocDigest {
-  readonly doc: string;
-  readonly kind: string;
-  readonly ai_input: AiInput;
-  /** This value is the SHA-256 of the current text of the document, in hex. */
-  readonly text_sha: string;
-}
-/** These options choose the documents of `ReadingIndex.digests`. */
-interface DigestOptions {
-  /** These documents, in place of a list of each document. A document that is not in the index gives nothing. */
-  readonly docs?: readonly string[] | undefined;
-  /** In a list of each document, the documents with an id after this id, in the sequence of the ids. */
-  readonly after?: string | undefined;
-  /** In a list of each document, this number of documents at most. The default is 500. */
-  readonly limit?: number | undefined;
-}
-/** The variant of a cache entry: the plain response, or the page that the browser rendered. */
-type Variant = "plain" | "rendered";
-interface Latest {
-  readonly url: string;
-  readonly capture: string;
-  readonly blob: string | null;
-  readonly etag: string | null;
-  readonly last_modified: string | null;
-  readonly content_type: string | null;
-  readonly fetched_at: string;
-  readonly status: number;
-  /** This value is what the page said about AI input at the time of the fetch. */
-  readonly ai_input: AiInput;
-  /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */
-  readonly location?: string | null | undefined;
-  /** This value is true when the stored body is only the first part of the response. */
-  readonly truncated?: boolean | undefined;
-  /** This time (ISO 8601) is the end of the freshness that the server gave. */
-  readonly fresh_until?: string | null | undefined;
-  /** This text holds the request headers that the response varies on (`Vary`), with their values. */
-  readonly vary_key?: string | null | undefined;
-  /** The default is `plain`. A key that ends in `#rendered` (the form of version 0.1) is `rendered`. */
-  readonly variant?: Variant | undefined;
-}
-/** These are the counts of the index. */
-interface Counts {
-  readonly docs: number;
-  readonly urls: number;
-}
-/** A rebuild of the index writes a new generation of the tables. The old generation answers until the commit. */
-interface IndexBuild {
-  /** This method adds a version to the new generation. The build writes the versions in batches. */
-  upsert(input: DocInput, now: string): void;
-  /** This method sets a cache entry in the new generation, when it is newer than the stored entry. */
-  setLatest(entry: Latest): void;
-  /** This method reads a cache entry of the new generation. */
-  latest(url: string, variant?: Variant): Latest | undefined;
-  /** This method writes the waiting versions in one transaction. */
-  flush(): void;
-  /**
-   * This method copies the versions of each recipe that is not in `recipes` from the current
-   * generation. A runtime with no extractor for a recipe thus keeps the documents of that recipe.
-   * It gives the number of documents that it copied.
-   */
-  carry(recipes: ReadonlySet<string>): number;
-  /**
-   * This method makes the new generation current. It first copies the rows that a different writer
-   * added to the current generation since `since`. Then it removes the old generation.
-   */
-  commit(since: string): Counts;
-  /** This method removes the new generation. The current generation stays. */
-  abort(): void;
-}
-/**
- * A reading index keeps the documents, their versions, the search and the cache entries. `DocIndex`
- * is the SQLite index. A runtime takes any object with these methods, for example a search index
- * of a different kind that wraps a `DocIndex`.
- */
-interface ReadingIndex {
-  /** This value is true when the index opened empty and a rebuild did not fill it yet. */
-  readonly stale: boolean;
-  get(doc: string): Doc | undefined;
-  upsert(input: DocInput, now: string): Doc;
-  versions(doc: string, options?: {
-    readonly limit?: number;
-  }): DocVersion[];
-  versionText(doc: string, capture?: string): VersionText | undefined;
-  /**
-   * This optional method gives the version that was current at a time (ISO 8601), with the rules
-   * that choose the current version. An index without it gets the newest version at or before the
-   * time, from `versions` and `versionText`.
-   */
-  versionAt?(doc: string, at: string): VersionAt;
-  earlierTexts(doc: string, options?: {
-    readonly limit?: number;
-  }): VersionText[];
-  docsOfCapture(capture: string): Doc[];
-  /** This method tells if a capture gave a version of a document. */
-  hasVersion(capture: string): boolean;
-  /**
-   * This optional method gives the hash of the current text of documents, with no text. Hybrid
-   * search uses it to find the documents whose text has no vectors. An index without it gives the
-   * same facts from `get`, more slowly.
-   */
-  digests?(options: DigestOptions): DocDigest[];
-  find(q: string, options: FindIndexOptions): Found[];
-  rankWithin(q: string, docs: readonly string[], options?: {
-    readonly rank?: RankMode | undefined;
-  }): Array<{
-    readonly doc: string;
-    readonly score: number;
-  }>;
-  latest(url: string, variant?: Variant): Latest | undefined;
-  setLatest(entry: Latest): void;
-  /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
-  newestFetch(): string | undefined;
-  counts(): Counts;
-  beginRebuild(options?: {
-    readonly batch?: number;
-  }): IndexBuild;
-}
-/** These are the options of a reading index. */
-interface DocIndexOptions {
-  /** This value is the time in milliseconds that a write waits for the lock of a different process. The default is 10 seconds. */
-  readonly busyTimeoutMs?: number | undefined;
-}
-declare class DocIndex implements ReadingIndex {
-  readonly path: string;
-  private readonly db;
-  private readonly statements;
-  private depth;
-  private closed;
-  private empty;
-  /**
-   * This value is true when the file had a different schema, or no tables, so the index opened
-   * empty. A store with captures then needs `reindex`. The commit of a rebuild makes it false.
-   */
-  get stale(): boolean;
-  constructor(path: string, options?: DocIndexOptions);
-  /** This method makes the tables of the schema. It gives true when the index opened empty. */
-  private prepareSchema;
-  private setMeta;
-  private getMeta;
-  private stmt;
-  /**
-   * This method does a write in one transaction. A transaction in progress takes the write in,
-   * also a transaction that the caller opened on the connection.
-   */
-  private write;
-  private generation;
-  /**
-   * This method does a step on the tables of the current generation. A rebuild of a different
-   * process can remove those tables in the meantime. Then the method reads the generation again
-   * and does the step once more.
-   */
-  private live;
-  /** This method removes the tables of a generation. */
-  private dropGeneration;
-  /** This method removes the generations of builds that a stopped process left. */
-  private dropAbandoned;
-  private putText;
-  private textOf;
-  private putVersion;
-  /** This method gives the newest version of a document for one recipe and one variant, at `until` or before when it is given. */
-  private newest;
-  /**
-   * This method chooses the current version of a document with the rules of the class. With
-   * `until`, it uses only the versions that the crawler fetched at that time or before. Thus it
-   * gives the version that was current at that time. It gives undefined when no version agrees.
-   */
-  private choose;
-  /** This method chooses the current version of a document from its versions, and writes it with its search entry. */
-  private recompute;
-  private docOf;
-  private toDoc;
-  private versionTextOf;
-  get(doc: string): Doc | undefined;
-  /**
-   * This method adds one version of a document, and it gives the current document. The rules of
-   * the class choose the current version. An upsert of a known capture replaces that version.
-   */
-  upsert(input: DocInput, now: string): Doc;
-  /** This method gives the versions of a document, the newest first. */
-  versions(doc: string, options?: {
-    readonly limit?: number;
-  }): DocVersion[];
-  /**
-   * This method gives the text of one version of a document. With no capture, it gives the current
-   * version. It also gives the text of a held-back version: the caller applies `isHeldBack` to the
-   * `ai_input` of the version.
-   */
-  versionText(doc: string, capture?: string): VersionText | undefined;
-  /**
-   * This method gives the version of a document that was current at a time (ISO 8601). It applies
-   * the rules of the class to the versions that the crawler fetched at that time or before. Thus
-   * a later capture does not change the answer. It also gives a held-back version: its `ai_input`
-   * is `no` when the document was held back at that time, and the caller applies `isHeldBack`.
-   */
-  versionAt(doc: string, at: string): VersionAt;
-  /**
-   * This method gives the texts of the earlier versions of a document, the newest first. Each
-   * text comes one time, with the newest version that has it. It skips the current text. It also
-   * gives held-back texts: the caller applies `isHeldBack` to the `ai_input` of each version.
-   */
-  earlierTexts(doc: string, options?: {
-    readonly limit?: number;
-  }): VersionText[];
-  /** This method tells if a capture gave a version of a document. */
-  hasVersion(capture: string): boolean;
-  /**
-   * This method gives the hash of the current text of documents, with no text. It gives the
-   * documents of `docs`, or each document after `after` in the sequence of the ids, to `limit`.
-   */
-  digests(options?: DigestOptions): DocDigest[];
-  /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
-  newestFetch(): string | undefined;
-  /** This method gives the current documents that a capture gave. */
-  docsOfCapture(capture: string): Doc[];
-  /**
-   * This method searches the documents with BM25, the best first. The syntax of the query is the
-   * syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it
-   * adds the documents that match one part. A document that asked not to be AI input is not in
-   * the results, so a search cannot show its text. The option `rank` chooses the ranking (`RankMode`).
-   */
-  find(q: string, options: FindIndexOptions): Found[];
-  /**
-   * This method scores a set of documents against a query, with the best first. It leaves out a
-   * document that matches no term, and a document that asked not to be AI input. It ranks the
-   * results of a source again, because the order of the source is only as good as its search engine.
-   * In the mode `fast`, it also leaves out a document that holds only common parts (`RankMode`).
-   */
-  rankWithin(q: string, docs: readonly string[], options?: {
-    readonly rank?: RankMode | undefined;
-  }): Array<{
-    readonly doc: string;
-    readonly score: number;
-  }>;
-  /**
-   * This method gives the form of a query that needs one part, without the common parts (the mode
-   * `fast` of `RankMode`). It counts the documents that hold each part. When each part is common,
-   * it keeps the rarest part.
-   */
-  private rareParts;
-  /** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */
-  latest(url: string, variant?: Variant): Latest | undefined;
-  private putLatest;
-  /** This method keeps the last answer for a URL. An entry that is older than the stored entry changes nothing. */
-  setLatest(entry: Latest): void;
-  counts(): Counts;
-  /** This method empties the current generation. A rebuild uses `beginRebuild`, which keeps the index until its commit. */
-  reset(): void;
-  /**
-   * This method starts a rebuild into a new generation of the tables. The current generation
-   * answers each read and takes each write until the commit. A failed rebuild thus leaves the
-   * index as it was.
-   */
-  beginRebuild(options?: {
-    readonly batch?: number;
-  }): IndexBuild;
-  /** This method closes the file. A second close does nothing. */
-  close(): void;
-}
-//#endregion
-//#region src/egress.d.ts
-/**
- * The egress policy keeps the crawler on the public internet. A page or an agent can ask for an
- * internal address. Examples are a loopback service, a router on the LAN and the metadata
- * service of a cloud host. The crawler refuses these addresses by default.
- *
- * - The crawler fetches only `http:` and `https:` URLs.
- * - The check uses the address after DNS. The `lookup` hook of the connection checks the address
- *   that the socket then uses, so a DNS answer that changes cannot pass the check.
- * - The crawler examines a literal IP host directly, in each spelling that the URL parser reads.
- * - The crawler examines each redirect hop again.
- *
- * A refusal is a skipped fetch with a clear note. It does not throw an error.
- */
-/** These options change the egress policy. With no options, the crawler fetches only public addresses. */
-interface EgressOptions {
-  /** The value true lets the crawler fetch loopback, private and link-local addresses. The default is false. */
-  readonly allowPrivate?: boolean | undefined;
-  /**
-   * These hosts are permitted although their address is not public. An entry is a host name, a
-   * name pattern (`*.corp.example`), an IP address or a CIDR range (`10.0.0.0/8`).
-   */
-  readonly allowHosts?: readonly string[] | undefined;
-}
-/** The connection hook gives this error when the policy refuses each address of a host. */
-declare class EgressRefused extends Error {
-  readonly address: string;
-  readonly kind: string;
-  readonly code: string;
-  constructor(address: string, kind: string);
-}
-/** This is the form of `dns.lookup` that the egress hook uses. Tests give a resolver of their own. */
-type Resolve = (host: string, options: LookupAllOptions, callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void) => void;
-/**
- * This function gives a `lookup` hook for the connection options of an undici `Agent`. The hook
- * resolves the name, removes each address that the policy refuses, and gives the rest to the
- * socket. When no address stays, the connection fails with `EgressRefused`.
- */
-declare const egressLookup: (policy: EgressOptions, resolve?: Resolve) => LookupFunction;
-//#endregion
-//#region src/webbotauth.d.ts
-/**
- * Signed requests: HTTP Message Signatures (RFC 9421), as the IETF Web Bot Auth profile tells
- * (`draft-ietf-webbotauth-httpsig-protocol-00`, 1 September 2026). A signature lets a site verify that
- * a request comes from the holder of a key that the operator publishes.
- *
- * - Each request gets three headers: `Signature-Agent`, `Signature-Input` and `Signature`.
- * - `Signature-Agent` is a dictionary. Its member names the key directory of the operator.
- * - The signature covers `@authority` and that member of `Signature-Agent`.
- * - The parameters are `created`, `keyid` (the JWK thumbprint of the key), `alg="ed25519"`,
- *   `expires`, a random `nonce` and `tag="web-bot-auth"`.
- * - The key is an Ed25519 key, and `node:crypto` signs. The package adds no dependency.
- *
- * A signature adds an identity. It does not replace the `User-Agent` header, which stays honest.
- */
-/** The path of the key directory on the host of the operator (the well-known URI of the draft). */
-declare const KEY_DIRECTORY_PATH = "/.well-known/http-message-signatures-directory";
-/** The media type of the key directory. The host of the directory sends it as `Content-Type`. */
-declare const KEY_DIRECTORY_TYPE = "application/http-message-signatures-directory+json";
-/**
- * This value tells how a site finds the keys from the URL of `Signature-Agent`:
- *
- * - `directory` (the default): the URL is an origin, and the keys are at `KEY_DIRECTORY_PATH` on it.
- * - `jwks_uri`: the URL is the address of a JSON Web Key Set, on any path.
- */
-type SignatureAgentType = "directory" | "jwks_uri";
-/** A key in one of the forms that `node:crypto` reads: a `KeyObject`, PEM text, DER bytes (PKCS#8 or SPKI) or a JWK. */
-type KeyInput = KeyObject | string | Uint8Array | JsonWebKey;
-/** These options make a signer for Web Bot Auth. */
-interface WebBotAuthOptions {
-  /** This key signs. It is an Ed25519 private key: a `KeyObject`, PKCS#8 (PEM text or DER bytes) or a JWK with `d`. */
-  readonly privateKey: KeyInput;
-  /**
-   * This text is the `keyid` of each signature: the JWK thumbprint of the key (RFC 8037). The signer
-   * calculates it. A value that is not the thumbprint of the key throws an error.
-   */
-  readonly keyId?: string | undefined;
-  /**
-   * This URL tells a site where the keys are. For the type `directory`, it is an https origin, for
-   * example `https://bot.example.com`. For `jwks_uri`, it is the https URL of a JSON Web Key Set.
-   */
-  readonly signatureAgent: string;
-  /** This value tells how a site finds the keys from `signatureAgent`. The default is `directory`. */
-  readonly agentType?: SignatureAgentType | undefined;
-  /** This value is the time that a signature stays valid, in milliseconds. The default is 5 minutes, and the maximum is 24 hours. */
-  readonly lifetimeMs?: number | undefined;
-  /** This text is the label of the signature, and the key of the member of `Signature-Agent`. The default is `sig1`. */
-  readonly label?: string | undefined;
-}
-/**
- * A request signer gives the signature headers of one request. The crawler uses it for each
- * request, robots.txt too. `createWebBotAuthSigner` makes one. A signer of your own can sign with a
- * key in a hardware module, for example.
- */
-interface RequestSigner {
-  /** This text is the `keyid` of the signatures. */
-  readonly keyId: string;
-  /** This URL is the value of the member of `Signature-Agent`: where a site finds the keys. */
-  readonly signatureAgent: string;
-  /** This method gives the headers that sign a request to `url` at the time `now` (milliseconds since the epoch). */
-  sign(url: URL, now: number): Readonly<Record<string, string>>;
-}
-/** The signer that `createWebBotAuthSigner` makes. It also gives its public key for the key directory. */
-interface WebBotAuthSigner extends RequestSigner {
-  /** This JWK is the public key of the signer, with no private part. Give it to `keyDirectory`. */
-  readonly publicKey: JsonWebKey;
-}
-/** One key of a key directory: an Ed25519 public key as a JWK, with its thumbprint as `kid`. */
-interface DirectoryKey {
-  readonly kty: "OKP";
-  readonly crv: "Ed25519";
-  readonly kid: string;
-  readonly x: string;
-  readonly use: "sig";
-}
-/** A key directory: a JSON Web Key Set. The operator serves it as JSON at `KEY_DIRECTORY_PATH`, with the type `KEY_DIRECTORY_TYPE`. */
-interface KeyDirectory {
-  readonly keys: readonly DirectoryKey[];
-}
-/** The parts of a request that `verifySignature` reads. */
-interface SignedRequest {
-  readonly url: string;
-  /** The default is `GET`. */
-  readonly method?: string | undefined;
-  /** These are the headers of the request. A name can be in any case, and a header can have more than one line. */
-  readonly headers: Readonly<Record<string, string | readonly string[] | undefined>>;
-}
-/** The result of `verifySignature`. */
-interface SignatureCheck {
-  /** This value is true when a Web Bot Auth signature of the request verifies with a key of the directory. */
-  readonly verified: boolean;
-  /** This text is the label of the signature that the check examined last, or null. */
-  readonly label: string | null;
-  readonly keyId: string | null;
-  /** This URL is the member of `Signature-Agent` that the signature covers, or null. */
-  readonly agent: string | null;
-  /** This text tells why the request does not verify. It is null when it verifies. */
-  readonly reason: string | null;
-}
-/**
- * This function makes the key directory for the operator to host: a JSON Web Key Set with each
- * public key and its thumbprint as `kid`. It takes public keys, or private keys, and it writes only
- * their public parts. Serve the JSON at `KEY_DIRECTORY_PATH` with the type `KEY_DIRECTORY_TYPE`.
- */
-declare const keyDirectory: (publicKeys: readonly KeyInput[]) => KeyDirectory;
-/**
- * This function makes a signer for Web Bot Auth with an Ed25519 key. Each signature covers
- * `@authority` and the member of `Signature-Agent`. It has a new random nonce, and it expires after
- * `lifetimeMs`. The signer keeps the private key in a closure: no property of the signer holds it.
- */
-declare const createWebBotAuthSigner: (options: WebBotAuthOptions) => WebBotAuthSigner;
-/**
- * This function verifies the Web Bot Auth signatures of a request with the keys of a directory. It
- * applies RFC 9421 section 3.2 and the profile of the draft. The profile asks for the tag
- * `web-bot-auth`, `created`, `expires` and `@authority`. It also asks for the member of
- * `Signature-Agent`, and a key with the thumbprint of `keyid`. The tests use it, and an operator can check a setup with it. It sends no request.
- */
-declare const verifySignature: (request: SignedRequest, directory: KeyDirectory, options?: {
-  readonly now?: number | undefined;
-}) => SignatureCheck;
-//#endregion
-//#region src/crawler.d.ts
-/**
- * The crawler is a polite fetcher. It sends one request at a time to each host, and it obeys
- * robots.txt for web pages. Its `User-Agent` header tells what it is. It asks again with the
- * validators of the stored copy, and it stops when a site refuses. Each decision goes into the
- * capture log, and a fetch that the crawler declines goes there too.
- *
- * There are two kinds of request. A documented public API (`api`) has its own terms and rate
- * limits. A web page (`page`) obeys robots.txt. The settings come from measurements:
- *
- * - The `request` function of undici on one `Agent`.
- * - A keep-alive time longer than the delay for each host. With the default of 4 seconds, each
- *   polite request opens a new connection.
- * - A backoff with jitter in the queue, not the retry handler of undici. That handler limits
- *   Retry-After to 30 seconds and adds no jitter.
- * - A maximum size for each body, after decompression. The crawler keeps the first part of a
- *   longer body and marks it as truncated.
- *
- * A fetch and a render go through one admission. It examines the URL, the egress policy, the
- * product tokens, robots.txt, the cache and the stops of the host. Only the transport is
- * different: an HTTP request or the headless browser.
- *
- * The egress policy (`EgressOptions`) keeps the crawler on public addresses by default. The
- * politeness policy (`PolitenessPolicy`) holds each limit, and a host can have its own values.
- * The fetch and the render do not throw an error: each problem is an outcome with a note.
- */
-/**
- * The outcome of a fetch or a render. A `skipped` request did not go out. A render of a URL that
- * gives a file and not a page is `not-html`.
- */
-type FetchOutcome = Outcome | "skipped" | "not-html";
-interface FetchRequest {
-  readonly url: string;
-  /** The value `api` is a documented API with its own terms. The value `page` is a web page, and robots.txt applies. */
-  readonly mode: "api" | "page";
-  /** This recipe reads the capture again. The function `reindex` uses it to replay the capture. */
-  readonly recipe: string;
-  readonly accept?: string | undefined;
-  /**
-   * These are more request headers. The crawler does not write them into the capture. They
-   * cannot replace `User-Agent`, `From`, `Accept-Encoding` or the validators. A redirect to a
-   * different origin removes `Authorization`, `Cookie` and `Proxy-Authorization`.
-   */
-  readonly headers?: Readonly<Record<string, string>> | undefined;
-  /**
-   * These query parameters contain secrets. Each capture shows `REDACTED` in their place. The
-   * names in `DEFAULT_SECRET_PARAMS` get the same treatment with no list.
-   */
-  readonly secretParams?: readonly string[] | undefined;
-  /** This value is the documented pace of the source: the shortest time between two of its requests. */
-  readonly minIntervalMs?: number | undefined;
-  /**
-   * A stored capture younger than this age answers, and the crawler sends no request. The value
-   * `auto` uses the freshness that the server gave (RFC 9111): `Cache-Control`, `Expires`, or 10
-   * percent of the age of `Last-Modified`.
-   */
-  readonly maxAgeMs?: number | "auto" | undefined;
-  readonly maxBytes?: number | undefined;
-  /**
-   * These are the product tokens of an agent that the fetch acts for, for example `Claude-User`.
-   * The robots.txt rules for these tokens apply too.
-   */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** This name connects the captures to the job that asked for them. */
-  readonly job?: string | undefined;
-  /** When this signal aborts, the crawler stops the fetch. The result is then `skipped` or `error`, with a note. */
-  readonly signal?: AbortSignal | undefined;
-  /** This label goes into each capture record of the request. It replaces the `scope` of the crawler. */
-  readonly scope?: string | undefined;
-  /** These labels go into each capture record of the request, for example its lineage. They are not part of the capture id. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-  /**
-   * This time (milliseconds since the epoch, on the clock of the crawler) is the end of a budget.
-   * The request does not start at or after it. When the host is not free before it, the result is
-   * `skipped`, and its note starts with `DEADLINE_NOTE`.
-   */
-  readonly deadline?: number | undefined;
-  /**
-   * This byte range is the part of the resource to fetch (a `Range` request). The crawler then sends
-   * no validators, keeps no stored copy for the URL, and reads `length` bytes at most. The capture
-   * records the range.
-   */
-  readonly range?: {
-    readonly offset: number;
-    readonly length: number;
-  } | undefined;
-  /** These lines of usage signals apply to the response too. A copy from a web archive gets the content signals of the robots.txt of its site. */
-  readonly signalLines?: readonly string[] | undefined;
-  /**
-   * This prefix names the response headers that hold the headers of an archived response, for
-   * example `x-archive-orig-` of the Wayback Machine. The usage signals of those headers apply too.
-   */
-  readonly originalHeaderPrefix?: string | undefined;
-}
-interface Fetched {
-  /** This URL is the URL of the capture. Its secrets show as `REDACTED`. */
-  readonly url: string;
-  readonly status: number;
-  readonly outcome: FetchOutcome;
-  /** This value is true when a stored capture answered, with no request. */
-  readonly fromCache: boolean;
-  /** This id is the capture record behind the body. A citation names it. */
-  readonly capture: string | undefined;
-  readonly blob: string | undefined;
-  readonly body: Uint8Array | undefined;
-  readonly contentType: string | undefined;
-  readonly signals: Signals;
-  readonly note: string | undefined;
-  /** This value is true when the body was longer than the maximum, so the crawler kept only its first part. */
-  readonly truncated?: boolean | undefined;
-  /** This time (ISO 8601) is the time of the fetch of the body. For a stored copy, it is the time of that copy. */
-  readonly fetchedAt?: string | undefined;
-  /** This label is the scope of the capture. */
-  readonly scope?: string | undefined;
-  /** These labels are the labels of the capture. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/** The crawler writes each record to a capture writer. The class `CaptureLog` is one. */
-interface CaptureWriter {
-  append(record: CaptureRecord): Promise<void>;
-  close?(): Promise<void>;
-}
-/** A cache entry is the last answer for a URL: its capture, its body and its validators. */
-interface CacheEntry {
-  /** This text is the cache key: the canonical URL, with its secrets as `REDACTED`. */
-  readonly url: string;
-  readonly capture: string;
-  readonly blob: string | null;
-  readonly etag: string | null;
-  readonly last_modified: string | null;
-  readonly content_type: string | null;
-  readonly fetched_at: string;
-  readonly status: number;
-  /** This value is what the page said about AI input at the time of the fetch. */
-  readonly ai_input: AiInput;
-  /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */
-  readonly location?: string | null | undefined;
-  /** This value is true when the stored body is only the first part of the response. */
-  readonly truncated?: boolean | undefined;
-  /** This time (ISO 8601) is the end of the freshness that the server gave. */
-  readonly fresh_until?: string | null | undefined;
-  /** This text holds the request headers that the response varies on (`Vary`), with their values. */
-  readonly vary_key?: string | null | undefined;
-  /** This value tells a plain response from a page that the browser rendered. The default is `plain`. */
-  readonly variant?: Variant | undefined;
-}
-/**
- * The cache index holds the last answer for each URL. The crawler reads it before a request, and
- * it writes it after an answer. The class `DocIndex` is one. An index that keeps only the
- * necessary fields of `CacheEntry` works, and the crawler keeps the other fields in memory.
- */
-interface CacheIndex {
-  latest(url: string, variant?: Variant): CacheEntry | undefined;
-  setLatest(entry: CacheEntry): void;
-}
-/** An event tells a listener what the crawler does. The listener cannot change it. */
-type CrawlEvent = {
-  readonly type: "request";
-  readonly url: string;
-  readonly host: string;
-  readonly transport: "http" | "browser";
-} | {
-  readonly type: "response";
-  readonly url: string;
-  readonly host: string;
-  readonly status: number;
-  readonly outcome: FetchOutcome;
-  readonly elapsedMs: number;
-} | {
-  readonly type: "wait";
-  readonly host: string;
-  readonly ms: number;
-  readonly reason: string;
-} | {
-  readonly type: "stop";
-  readonly host: string;
-  readonly until: number;
-  readonly reason: string;
-} | {
-  readonly type: "skip";
-  readonly url: string;
-  readonly note: string;
-};
-interface CrawlerOptions {
-  /** This product token is the name that robots.txt rules match. It contains only letters, `_` and `-` (RFC 9309). */
-  readonly token: string;
-  /** This text is the full `User-Agent` header. */
-  readonly userAgent: string;
-  /** This contact goes into the `From` header. Without it, the crawler sends no `From` header. */
-  readonly from?: string | undefined;
-  readonly log: CaptureWriter;
-  readonly blobs: BlobStore;
-  readonly index: CacheIndex;
-  readonly clock?: Clock | undefined;
-  readonly random?: (() => number) | undefined;
-  /** This value is the number of hosts that get requests at one time. The default is 16. */
-  readonly maxActive?: number | undefined;
-  /**
-   * This value tells if the crawler also obeys the robots.txt group of `PoliteCrawler`. That applies
-   * when the `User-Agent` links the bot page of polite-crawler (`BOT_PAGE`) and the token is different.
-   * Site owners read that page, and they write their rules for `PoliteCrawler`. The most restrictive
-   * of the groups applies, as for `onBehalfOf`. The default is true. Set false for a bot page of your own.
-   */
-  readonly obeyDefaultToken?: boolean | undefined;
-  /**
-   * This value is the number of pages that the browser renders at one time. The default is 2. Each
-   * render is a browser context of its own, which holds much memory. A render waits for a slot before
-   * the lease of its host, so the wait does not count toward the time of the host.
-   */
-  readonly maxRenders?: number | undefined;
-  /**
-   * This undici dispatcher sends the requests. With a dispatcher of your own, the crawler checks
-   * only literal addresses. Put `egressLookup` in its connection options to check the address after DNS.
-   */
-  readonly dispatcher?: Dispatcher | undefined;
-  /** This label goes into each capture record as it is. */
-  readonly scope?: string | undefined;
-  /**
-   * This function ranks the scopes of the requests, the most private scope highest. One fetch of a
-   * robots.txt can serve requests of different scopes. Its record then takes the scope with the
-   * highest rank, and the labels of each request. With no function, the first request gives the scope.
-   */
-  readonly scopeRank?: ((scope: string) => number) | undefined;
-  /** This browser renders the pages that need JavaScript. Without it, the crawler renders no page. */
-  readonly renderer?: Renderer | undefined;
-  /** This policy controls the addresses that the crawler fetches. By default, the crawler fetches only public addresses. */
-  readonly egress?: EgressOptions | undefined;
-  /** These values change the limits of the politeness policy, for each host or for some hosts. */
-  readonly politeness?: PolicyOptions | undefined;
-  /** This store keeps the pace, the counts and the stops of each host. The default keeps them in memory. */
-  readonly hostState?: HostStateStore | undefined;
-  /** This function gets an event for each request, response, wait, stop and skip. An error in it does not stop the crawler. */
-  readonly onEvent?: ((event: CrawlEvent) => void) | undefined;
-  /**
-   * This signer signs each request, robots.txt and the requests of a rendered page too. Give the
-   * options of `createWebBotAuthSigner`, or a `RequestSigner`. Without it, the crawler signs nothing.
-   * A signature adds an identity: the `User-Agent` and `From` headers stay as they are.
-   */
-  readonly signer?: RequestSigner | WebBotAuthOptions | undefined;
-}
-interface RenderRequest {
-  readonly url: string;
-  readonly recipe: string;
-  /** A stored render younger than this age answers. The value `auto` uses the freshness that the server gave. */
-  readonly maxAgeMs?: number | "auto" | undefined;
-  readonly onBehalfOf?: readonly string[] | undefined;
-  readonly job?: string | undefined;
-  /** When this signal aborts, the crawler stops the render, and the browser stops the page. */
-  readonly signal?: AbortSignal | undefined;
-  /** This label goes into the capture record of the render. It replaces the `scope` of the crawler. */
-  readonly scope?: string | undefined;
-  /** These labels go into the capture record of the render. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-  /** This time (milliseconds since the epoch) is the end of a budget. The render does not start at or after it (`FetchRequest.deadline`). */
-  readonly deadline?: number | undefined;
-}
-/** The note of a request starts with this text when its host was not free before its deadline, so the request did not start. */
-declare const DEADLINE_NOTE = "deadline reached";
-/**
- * This function gives the key of the cache entry of a rendered page in the form of version 0.1.
- * The index keeps the variant in a column of its own, and it reads this form too.
- */
-declare const renderedKey: (url: string) => string;
-/** This value is the version of the payload of the records that the crawler writes. A record with no `schema` is version 1. */
-declare const CAPTURE_SCHEMA = 2;
-/**
- * These query parameters hold secrets in many APIs. The crawler writes `REDACTED` in place of
- * their values in each capture, in addition to the names of `secretParams`.
- */
-declare const DEFAULT_SECRET_PARAMS: readonly string[];
-declare class Crawler {
-  private readonly options;
-  private readonly clock;
-  private readonly random;
-  private readonly scheduler;
-  private readonly dispatcher;
-  private readonly ownsDispatcher;
-  private readonly egress;
-  private readonly robots;
-  /** The fields of a cache entry that the index does not keep, and the raw target of a redirect. */
-  private readonly meta;
-  private readonly inFlight;
-  private readonly running;
-  private readonly closing;
-  private closed;
-  private closingDone;
-  /** This note tells why the renderer has no browser. After the first page that finds it, the crawler renders nothing. */
-  private renderOff;
-  /** The renders that run at one time (`maxRenders`). */
-  private readonly renders;
-  /** These tokens apply because the `User-Agent` links the bot page (`obeyDefaultToken`). */
-  private readonly pageTokens;
-  /** The deadlines of the requests that have one, by the signal that the scheduler gets for each of them. */
-  private readonly deadlines;
-  /** This signer signs each request. It holds the key, and the crawler writes only its key id and its agent. */
-  private readonly signer;
-  constructor(options: CrawlerOptions);
-  private iso;
-  private policyOf;
-  /**
-   * This method gives the signal of a request for the scheduler. A request with a deadline gets a
-   * signal of its own, and the clock of the scheduler reads the deadline of that signal.
-   */
-  private paced;
-  /** This method throws `DeadlineReached` when the deadline of a request came before its slot. */
-  private beforeDeadline;
-  private emit;
-  private track;
-  /** This method gives the validator memo of a host. The memo is a hint, so an error of its store gives no memo. */
-  private memoOf;
-  private stopHost;
-  private record;
-  private skip;
-  /** This method reads the cache entry of a URL, with the fields that the crawler keeps in memory. */
-  private lookup;
-  private store;
-  /** This method gives the target of a stored redirect, when the crawler can follow it. */
-  private followable;
-  /** This method answers from the cache, when the stored entry is young or fresh enough. */
-  private fromCache;
-  /**
-   * This method gives the signature headers of one request, and the facts of the capture: the key id
-   * and the agent. With no signer, it gives nothing. A signer that throws gives the note of a skip,
-   * because the crawler sends no unsigned request when it has a signer.
-   */
-  private signFor;
-  private requestHeaders;
-  /**
-   * This method records the result of a request for the host, and it gives the rest of the host.
-   * The rest comes from `restAfter`. A 429 or 503 waits, an error counts toward a stop, and a
-   * refusal stops the host for a time.
-   */
-  private settle;
-  /**
-   * This method stops the host of a URL as a challenge does: for the `challengeStopMs` of the policy.
-   * The runtime uses it when a page that answered 200 is a bot check (`meta.blocked`).
-   */
-  reportBlocked(url: string, what: string): void;
-  /**
-   * This method applies the rules that a fetch and a render share. The sequence is the URL, the
-   * options, the egress policy, the product tokens and the rules for an agent. Then come the
-   * cache, the stops of the host and robots.txt. It gives the answer when a rule decides, or the
-   * facts that the transport needs.
-   */
-  private admit;
-  /** This method fetches a URL. It follows the redirects up to the maximum, and each step is a separate decision. */
-  fetch(request_: FetchRequest): Promise<Fetched>;
-  private follow;
-  /** This method joins a request to an equal request in progress, so one URL gets one request at a time. */
-  private hop;
-  private knownRobots;
-  /**
-   * This method gives the robots.txt of the origin of a URL. A good copy stays for the TTL of the
-   * policy. An unreachable verdict stays for some minutes, with a backoff and the `Retry-After`
-   * of the server. During that time the last good copy applies, when there is one (RFC 9309 §2.4).
-   */
-  private robotsFor;
-  /**
-   * This method gives the robots.txt rules that apply to a URL, as the crawler applies them. They
-   * come from its memory, or from a fetch of `/robots.txt` with its redirects. A robots.txt that
-   * the crawler cannot read gives rules that permit nothing, and `unreachable` gives the reason.
-   */
-  readRobots(url: string, marks?: {
-    readonly scope?: string | undefined;
-    readonly labels?: Readonly<Record<string, string>> | undefined;
-  }): Promise<{
-    readonly robots: Robots;
-    readonly capture: string | undefined;
-    readonly stale?: string | undefined;
-  }>;
-  /**
-   * This method tells if robots.txt lets the crawler fetch a URL as a page. The rules of the token
-   * of the crawler and of the agents of `onBehalfOf` apply, as for a fetch. It reads robots.txt as
-   * a fetch does, so a robots.txt that the crawler cannot read permits nothing. When the signal
-   * aborts, the promise rejects, and a fetch of robots.txt in progress continues.
-   */
-  robotsAllow(url: string, options?: {
-    readonly onBehalfOf?: readonly string[] | undefined;
-    readonly scope?: string | undefined;
-    readonly labels?: Readonly<Record<string, string>> | undefined;
-    readonly signal?: AbortSignal | undefined;
-  }): Promise<{
-    readonly allowed: boolean;
-    readonly note: string | undefined;
-    /** These are the lines of usage signals (`Content-Signal`) of robots.txt that apply to the URL, when there are any. The archived fallback applies them to a copy. */
-    readonly signals?: readonly string[] | undefined;
-  }>;
-  /** This method reads the stored copy of a robots.txt, when it is younger than the limit of the policy. */
-  private storedRobots;
-  /** This method adds the scope and the labels of one request to the requests that wait for a robots.txt. The most private scope wins. */
-  private join;
-  private fetchRobots;
-  private fetchOnce;
-  /**
-   * This method gives the function that decides each request of a rendered page. It refuses an
-   * address that the egress policy refuses, and a URL of the page's host that robots.txt
-   * disallows. For a different host, it uses a robots.txt that the crawler holds, when there is one.
-   */
-  private allowRequestFor;
-  /**
-   * This method tells why the browser did not go on to a URL that the page moved to: the egress
-   * policy or robots.txt. For a different site, it uses a robots.txt that the crawler holds. It
-   * gives undefined for the page itself, or for a URL that it cannot read.
-   */
-  private movedRefusal;
-  /**
-   * This method renders a page in the headless browser and captures what the page shows. The rules
-   * of a fetch apply:
-   *
-   * - the egress policy and robots.txt, for the page, its final URL and each request of the page
-   * - one request at a time for each host
-   * - the backoff after a 429 or a 503
-   * - a capture for each decision
-   *
-   * A render costs the site more than a fetch, so the host rests longer after one. A render that
-   * the browser refuses (`RenderRefused`) does not count against the host.
-   */
-  render(request_: RenderRequest): Promise<Fetched>;
-  private renderOnce;
-  /**
-   * This method removes the stops and the failure counts of one host, or of each host. It also
-   * forgets the robots.txt verdicts that came from an unreachable robots.txt. The pace of each host
-   * stays. A process that runs for a long time uses it, for example after the network comes back.
-   */
-  resetHosts(host?: string): void;
-  /**
-   * This method gives the stop of the host of a URL, or undefined when the host is not stopped. A
-   * watch uses it, so a page of a stopped host is due again at the end of the stop.
-   */
-  hostStop(url: string): ReturnType<HostStateStore["stopped"]>;
-  /**
-   * This method closes the crawler. Work in the queue ends as `skipped` with the note
-   * `crawler closed`, and the crawler writes no capture after the close. Then it closes the
-   * browser, the log and its own dispatcher. A second close waits for the first.
-   */
-  close(): Promise<void>;
-}
-//#endregion
-//#region src/diff.d.ts
-/**
- * A diff compares two versions of a stored document. It gives the changes of the text as a line
- * diff, with a word diff inside each changed line. It gives the changes of the title, the URL and
- * the metadata (the JSON-LD too) as changes at a path.
- *
- * The line diff is the algorithm of Myers (1986). Its time is linear in the length of the texts
- * when the change is small. A budget of steps (`DIFF_LIMITS`) bounds the time of a large change.
- * After the budget, the diff gives the rest of the change as one removal and one addition. Then
- * the result says that it is not minimal. A huge text gives its first part, and the result says so.
- *
- * A held-back version gives no text and no title to a diff, and only its structural metadata.
- */
-/** These are the limits of a diff. Each limit bounds the time or the size of a result. */
-declare const DIFF_LIMITS: {
-  /** The characters of each text that a diff compares. A longer text gives its first part. */
-  readonly maxChars: 1_000_000;
-  /** The lines of each text that a diff compares. A longer text gives its first lines. */
-  readonly maxLines: 50_000;
-  /** The steps of the line diff. After them, the rest of the change is one removal and one addition. */
-  readonly maxSteps: 1_000_000;
-  /** The steps of the word diffs of all the changed lines together. */
-  readonly maxWordSteps: 1_000_000;
-  /** The words of one line that a word diff compares. A longer line gets no word diff. */
-  readonly maxLineWords: 5_000;
-  /** The lines of context before and after each change. */
-  readonly context: 2;
-  /** The lines that the hunks of a result show. The summary counts each line. */
-  readonly maxHunkLines: 2_000;
-  /** The changes of the metadata that a result shows. The summary counts each change. */
-  readonly maxMetaChanges: 200;
-  /** The depth of the metadata that a diff compares. A deeper change is a change of the whole value. */
-  readonly maxMetaDepth: 16;
-  /** The characters of the JSON of each value of a change of the metadata. */
-  readonly maxValueChars: 300;
-};
-/** These are the limits of a diff, one by one. `DIFF_LIMITS` gives the defaults. */
-type DiffLimits = { readonly [K in keyof typeof DIFF_LIMITS]: number; };
-/** This is a part of a changed line: words that stay, words that the new text adds, or words that it removes. */
-interface WordPart {
-  readonly op: "same" | "added" | "removed";
-  readonly text: string;
-}
-/**
- * This is one line of a hunk. A `same` line is context. A `changed` line pairs a removed line with
- * an added line, and `words` gives the word diff, or null when the word budget ended.
- */
-type DiffLine = {
-  readonly op: "same" | "added" | "removed";
-  readonly text: string;
-} | {
-  readonly op: "changed";
-  readonly before: string;
-  readonly after: string;
-  readonly words: readonly WordPart[] | null;
-};
-/** This is a part of the diff of the text: the changed lines with some context, as in a unified diff. */
-interface DiffHunk {
-  /** This value is the number (from 1) of the first line of the hunk in the old text. */
-  readonly from_line: number;
-  /** This value is the number of lines of the old text in the hunk. */
-  readonly from_count: number;
-  /** This value is the number (from 1) of the first line of the hunk in the new text. */
-  readonly to_line: number;
-  /** This value is the number of lines of the new text in the hunk. */
-  readonly to_count: number;
-  readonly lines: readonly DiffLine[];
-}
-/**
- * This is one change of the title, the URL or the metadata. The path names the value, for
- * example `meta.jsonld[0].openingHours`. `before` and `after` are the JSON of the values, cut to
- * `DIFF_LIMITS.maxValueChars`.
- */
-interface MetaChange {
-  readonly path: string;
-  readonly change: "added" | "removed" | "changed";
-  /** This text is the JSON of the old value, or null when the new version adds the value. */
-  readonly before: string | null;
-  /** This text is the JSON of the new value, or null when the new version removes the value. */
-  readonly after: string | null;
-}
-/** These are the counts of a diff. */
-interface DiffSummary {
-  /** This value counts the lines that only the new text has. */
-  readonly lines_added: number;
-  /** This value counts the lines that only the old text has. */
-  readonly lines_removed: number;
-  /** This value counts the pairs of a removed line and an added line that the result shows as one changed line. */
-  readonly lines_changed: number;
-  /** This value counts the words that the new text adds, in the added lines and in the changed lines. */
-  readonly words_added: number;
-  /** This value counts the words that the new text removes, in the removed lines and in the changed lines. */
-  readonly words_removed: number;
-  /** This value counts the changes of the title, the URL and the metadata. */
-  readonly meta_changes: number;
-}
-/** This is the input of `diffTexts`: the parts of a version that a diff compares. */
-interface DiffInput {
-  readonly text: string;
-  readonly fields: Readonly<Record<string, unknown>>;
-}
-/** This is the result of `diffTexts`. */
-interface TextDiff {
-  readonly summary: DiffSummary;
-  readonly hunks: readonly DiffHunk[];
-  readonly meta: readonly MetaChange[];
-  /** This value is true when the text and the fields are the same. */
-  readonly same: boolean;
-  /** This value is true when the line diff is minimal and each changed line has its word diff. */
-  readonly exact: boolean;
-  /** This value is true when the result leaves out a part. It can leave out the end of a long text, lines of the hunks or changes of the metadata. */
-  readonly truncated: boolean;
-  /** This value counts the steps of the line diff and of the word diffs. A test reads it: a step count does not change with the computer. */
-  readonly steps: number;
-  readonly notes: readonly string[];
-}
-/**
- * This function compares two versions: the lines of the text, the words of each changed line, and
- * the fields (the title, the URL and the metadata) at their paths. It writes nothing, and its time
- * and its result stay inside the limits.
- */
-declare const diffTexts: (before: DiffInput, after: DiffInput, given?: Partial<DiffLimits>) => TextDiff;
-/** A version of a document for a diff: a capture id, or the version that was current at a time. */
-type VersionChoice = string | {
-  readonly at: string | Date;
-};
-/** These options choose the two versions of a diff. */
-interface DiffOptions {
-  /**
-   * This version is the old side: a capture id, or `{ at }`. The default is the version before
-   * `to` with the same recipe and the same kind of capture (plain or rendered).
-   */
-  readonly from?: VersionChoice | undefined;
-  /** This version is the new side: a capture id, or `{ at }`. The default is the newest version. */
-  readonly to?: VersionChoice | undefined;
-  /** These values change the limits of the diff (`DIFF_LIMITS`). */
-  readonly limits?: Partial<DiffLimits> | undefined;
-}
-/** This is one side of a diff: the version that it compares. */
-interface DiffSide {
-  /** This id is the capture of the version. A citation names it. */
-  readonly capture: string;
-  /** This time (ISO 8601) is the fetch time of the version. */
-  readonly fetched_at: string;
-  /** This value is the length of the text of the version. */
-  readonly chars: number;
-  readonly rendered: boolean;
-  /** This value is true when the version is held back, so the diff does not show its text. */
-  readonly held_back: boolean;
-  /** This time (ISO 8601) is the `at` that chose the version, or null for a capture id or a default. */
-  readonly at: string | null;
-}
-/** The result of a diff of two versions of a stored document. */
-interface DiffResult {
-  readonly doc: string;
-  readonly url: string | null;
-  readonly from: DiffSide;
-  readonly to: DiffSide;
-  /**
-   * This value is true when the two versions have the same text, title, URL and metadata. It is
-   * null when a side is held back, because then the diff does not compare the texts.
-   */
-  readonly same: boolean | null;
-  readonly summary: DiffSummary;
-  readonly hunks: readonly DiffHunk[];
-  readonly meta: readonly MetaChange[];
-  /** This value is true when a side is held back. Then the result has no text and no title, and only the structural metadata. */
-  readonly held_back: boolean;
-  /** This value is true when the line diff is minimal and each changed line has its word diff. */
-  readonly exact: boolean;
-  /** This value is true when the result leaves out a part. The note tells which part. */
-  readonly truncated: boolean;
-  /** This value counts the steps of the diff. */
-  readonly steps: number;
-  readonly note: string | null;
-}
-/**
- * This function compares two versions of a stored document, and writes nothing. It applies the
- * rule for a held-back document. When the document or a version is held back, the diff compares
- * only the structural metadata, and it shows no text and no title.
- */
-declare const diffDocument: (runtime: CrawlRuntime, id: string, options?: DiffOptions) => Promise<DiffResult>;
-//#endregion
-//#region src/sitemaps.d.ts
-/**
- * The sitemap reader reads the files that list the pages of a site. These are an XML sitemap, a
- * sitemap index, an RSS 2.0 or RSS 1.0 feed and an Atom feed. A text sitemap with one URL on each
- * line is a file of this kind too. The reader is small and strict, because the files come from the web:
- *
- * - It reads the text one time from the start to the end, and it builds no tree. Its time is
- *   linear in the length of the text.
- * - It expands no entity. It knows only the five entities of XML and the character references. A
- *   DTD in the file changes nothing.
- * - It stops at a maximum depth of elements, at a maximum number of entries and at the first
- *   error in the markup. The entries before the stop stay.
- *
- * It imports no Node.js module, so a browser can use it.
- */
-/** The kind of a file: the name of its root element, or `text` for a text sitemap. */
-type SitemapKind = "urlset" | "sitemapindex" | "rss" | "atom" | "rdf" | "text" | "unknown";
-/** The values of `<changefreq>` in the sitemap protocol. */
-type Changefreq = "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never";
-/** One entry of a file: a page, or a sitemap of an index. */
-interface SitemapEntry {
-  /** This URL is the canonical form of the URL of the entry. */
-  readonly url: string;
-  /** This time (ISO 8601) is the last change that the file gives, or null. */
-  readonly lastmod: string | null;
-  /** This value is the `<changefreq>` of a sitemap entry, or null. */
-  readonly changefreq: Changefreq | null;
-}
-/** The result of one file. */
-interface SitemapRead {
-  readonly kind: SitemapKind;
-  /** These entries are the pages of the file, or the sitemaps of an index, in the order of the file. */
-  readonly entries: readonly SitemapEntry[];
-  /** This value counts the entries with no valid http(s) URL. */
-  readonly invalid: number;
-  /** This value is false when the reader stopped before the end of the file. */
-  readonly complete: boolean;
-  /** These notes tell why the reader stopped, or what it ignored. */
-  readonly notes: readonly string[];
-}
-/** These limits stop the reader on a large or a hostile file. */
-interface SitemapLimits {
-  /** This value is the maximum number of entries. The default is 50,000, the limit of the sitemap protocol. */
-  readonly maxEntries?: number | undefined;
-  /** This value is the maximum depth of the elements. The default is 32. */
-  readonly maxDepth?: number | undefined;
-}
-/** These are the default limits of the reader, and the maximum length of one value. */
-declare const SITEMAP_LIMITS: {
-  readonly maxEntries: 50_000;
-  readonly maxDepth: 32;
-  /** A value (a URL or a date) longer than this number of characters is not valid. */
-  readonly maxValue: 4096;
-};
-/**
- * This function changes a date of a sitemap or a feed into ISO 8601. It reads the W3C form of the
- * sitemap protocol and Atom, and the mail form of RSS. A time with no zone is UTC. It gives null
- * for a text that it cannot read.
- */
-declare const feedDate: (value: string) => string | null;
-/**
- * This function reads a sitemap, a sitemap index, an RSS or Atom feed, or a text sitemap. The
- * root element of the file tells its kind. The entries of a feed can have URLs relative to `base`
- * (the URL of the file). The entries of a sitemap must have absolute URLs.
- */
-declare const readSitemap: (text: string, base: string, limits?: SitemapLimits) => SitemapRead;
-//#endregion
-//#region src/discover.d.ts
-/**
- * Discovery finds the pages of a site from its sitemaps and its feeds. Each file is a fetch of the
- * crawler, so each polite rule applies to it. These rules are robots.txt, one request at a time for
- * each host, the pace, the stops, the egress policy and the capture log. These are the sources, in sequence:
- *
- * 1. The page at the URL of the site, for the RSS and Atom feeds that it links.
- * 2. The feeds that the caller gives.
- * 3. The sitemaps that the caller gives, and the `Sitemap:` lines of robots.txt. When there are
- *    none, `/sitemap.xml`.
- * 4. The sitemaps of each sitemap index, the newest first, to a maximum depth.
- *
- * Discovery keeps a URL only when it is in scope (the same site by default) and robots.txt permits
- * it. It stops at a maximum of files, a maximum of URLs and a maximum time.
- */
-/** These are the default limits of `discover`. */
-declare const DISCOVER_LIMITS: {
-  /** The most sitemap and feed files that one discovery reads. */
-  readonly maxFiles: 20;
-  /** The most URLs that one discovery gives. */
-  readonly maxUrls: 10_000;
-  /** The most bytes of one file, after decompression: 10 MiB. */
-  readonly maxBytes: 10_485_760;
-  /** The most levels of sitemap index under a file that discovery found first. */
-  readonly maxNesting: 3;
-  /** The longest time of one discovery: 1 minute. */
-  readonly budgetMs: 60_000;
-  /** The most origins whose robots.txt one discovery reads for the URLs that it finds. */
-  readonly maxOrigins: 10;
-  /** The most feeds that discovery reads from the links of the page. */
-  readonly maxLinkedFeeds: 5;
-};
-/** These options control a discovery. Each limit has a default in `DISCOVER_LIMITS`. */
-interface DiscoverOptions extends ScopeOptions {
-  /** This value is the most sitemap and feed files to read. The default is 20. */
-  readonly maxFiles?: number | undefined;
-  /** This value is the most URLs to give. The default is 10,000. */
-  readonly maxUrls?: number | undefined;
-  /** This value is the most bytes of one file, after decompression. The default is 10 MiB. */
-  readonly maxBytes?: number | undefined;
-  /** This value is the most levels of sitemap index to follow. The default is 3. */
-  readonly maxNesting?: number | undefined;
-  /** This value is the longest time of the discovery, in milliseconds. The default is 1 minute. */
-  readonly budgetMs?: number | undefined;
-  /** These sitemaps are read too, in addition to the `Sitemap:` lines of robots.txt. */
-  readonly sitemaps?: readonly string[] | undefined;
-  /** These RSS or Atom feeds are read too. */
-  readonly feeds?: readonly string[] | undefined;
-  /** With the default value true, discovery reads the page of the site for the feeds that it links. */
-  readonly linkedFeeds?: boolean | undefined;
-  /**
-   * A stored copy of a file younger than this age answers, with no request. The default is `24h`.
-   * The value `auto` uses the freshness that the server gave.
-   */
-  readonly maxAge?: Duration | undefined;
-  /** These agents get their robots.txt rules applied to each fetch and to each URL. */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** When this signal aborts, the discovery stops, and it gives what it found. */
-  readonly signal?: AbortSignal | undefined;
-  /** This label goes into the capture records of the discovery. */
-  readonly scope?: string | undefined;
-  /** These labels go into the capture records of the discovery. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/** These options control `discoverSite`: the options of `discover`, and a deadline. */
-interface DiscoverRunOptions extends DiscoverOptions {
-  /** This time (milliseconds since the epoch) ends the discovery when it comes before the end of `budgetMs`. */
-  readonly deadline?: number | undefined;
-}
-/** One URL that a discovery found. */
-interface DiscoveredUrl {
-  /** This URL is the canonical form of the URL of the page. */
-  readonly url: string;
-  /** This time (ISO 8601) is the last change that the file gave, or null. */
-  readonly lastmod: string | null;
-  readonly changefreq: Changefreq | null;
-  /** This value tells if a sitemap or a feed listed the URL. */
-  readonly via: "sitemap" | "feed";
-  /** This URL is the file that listed the URL first. */
-  readonly source: string;
-}
-/** One sitemap or feed file that a discovery read, or tried to read. */
-interface DiscoverSource {
-  readonly url: string;
-  readonly via: "sitemap" | "feed";
-  /** This value tells how discovery found the file. */
-  readonly from: "robots.txt" | "default" | "given" | "page" | "index";
-  /** This value is the kind that the reader found, or null for a file that the crawler did not get. */
-  readonly kind: SitemapKind | null;
-  /** This value is the outcome of the fetch: `ok`, `not-modified`, `skipped`, `not-found` and others. */
-  readonly outcome: string;
-  readonly status: number;
-  /** This id is the capture of the file. */
-  readonly capture: string | null;
-  readonly fetched_at: string | null;
-  readonly from_cache: boolean;
-  /** This value is the number of entries that the file listed. */
-  readonly entries: number;
-  /** This value is the number of new URLs that discovery kept from the file. */
-  readonly urls: number;
-  /** This value is true when the file was longer than `maxBytes`, so the reader read only its first part. */
-  readonly truncated: boolean;
-  readonly note: string | null;
-}
-/** The reason that a discovery stopped. */
-type DiscoverStop = "done" | "max-files" | "max-urls" | "budget" | "cancelled" | "closed" | "robots";
-/** The result of a discovery. */
-interface DiscoverResult {
-  /** This URL is the origin of the site. */
-  readonly site: string;
-  /** These URLs are the pages that discovery found, the newest `lastmod` first. */
-  readonly urls: readonly DiscoveredUrl[];
-  /** These are the files that discovery read, in the sequence of the reads. */
-  readonly sources: readonly DiscoverSource[];
-  /** These values count the URLs that discovery did not keep, for each reason. */
-  readonly dropped: {
-    /** The URLs on a different site, and the files of an index on a different site. */
-    readonly scope: number;
-    /** The URLs that robots.txt does not permit. */
-    readonly robots: number;
-    /** The URLs that discovery found before. */
-    readonly duplicate: number;
-    /** The entries with no valid http(s) URL. */
-    readonly invalid: number;
-    /** The URLs and the files over a limit: the origins, the depth of the index, or the URLs. */
-    readonly limit: number;
-  };
-  readonly stopped: DiscoverStop;
-  /** This text tells why the discovery stopped early, or null. */
-  readonly note: string | null;
-}
-/**
- * This function finds the pages of a site from its sitemaps and its feeds. `PoliteCrawler.discover`
- * uses it. Each file is a fetch of the crawler with the recipe `sitemap` or `feed`, so the
- * capture log records it, and robots.txt applies to it.
- */
-declare const discoverSite: (runtime: CrawlRuntime, site: string, options?: DiscoverRunOptions) => Promise<DiscoverResult>;
-//#endregion
-//#region src/embed.d.ts
-/**
- * The parts of hybrid search that need no store: the embedder interface, the chunks of a text, and
- * the fusion of ranked lists.
- *
- * An embedder changes texts into vectors. Two texts with a similar meaning get vectors that point
- * in a similar direction. Hybrid search ranks the stored documents two times: one time with the
- * full-text index (BM25), one time by the cosine of the vectors. Reciprocal rank fusion (RRF) then
- * makes one list of the two.
- *
- * `Embedder` is the extension point. `transformersEmbedder` and `ollamaEmbedder` are two adapters,
- * and an application can give an embedder of its own, as it can give a renderer of its own.
- */
-/** These options go to each call of `Embedder.embed`. */
-interface EmbedCallOptions {
-  /** When this signal aborts, the embedder stops and rejects with the reason of the signal. */
-  readonly signal?: AbortSignal | undefined;
-  /**
-   * This value tells what the texts are: `document` (parts of stored texts, the default) or `query`
-   * (the words of a search). Some models want a different prefix for each.
-   */
-  readonly purpose?: "document" | "query" | undefined;
-}
-/**
- * An embedder changes texts into vectors. The vector store keeps each vector with the `id` of its
- * embedder, so the vectors of two models do not mix.
- */
-interface Embedder {
-  /** This text names the model and its version. Change it when the vectors of the model change. */
-  readonly id: string;
-  /** This value is the length of each vector. The value 0 tells that the embedder learns it from its first answer. */
-  readonly dimensions: number;
-  /** This value is the largest input of the model, in tokens. The chunks of a text fit in it. The default is 256. */
-  readonly maxInputTokens?: number | undefined;
-  /** This method gives one vector for each text, in the same sequence. */
-  embed(texts: readonly string[], options: EmbedCallOptions): Promise<Float32Array[]>;
-  /** This optional method releases the model. A later call of `embed` loads it again. */
-  close?(): Promise<void>;
-}
-/** These are the limits of the chunks of a text. */
-declare const CHUNK_LIMITS: {
-  /** A chunk has this number of tokens at most, also for a model with a larger input. */
-  readonly maxTokens: number;
-  /** A chunk has this number of tokens when the embedder does not give `maxInputTokens`. */
-  readonly defaultTokens: number;
-  /** A chunk fills this share of its limit, because the count of the tokens is an estimate. */
-  readonly fill: number;
-  /** Two chunks next to each other share this share of the tokens of a chunk. */
-  readonly overlap: number;
-  /** A text gives this number of chunks at most. The rest of a longer text gets no vector. */
-  readonly maxChunks: number;
-};
-/** This is one chunk of a text: a window of paragraphs, with its position in the text. */
-interface TextChunk {
-  /** This value is the position of the chunk in the list, from 0. */
-  readonly index: number;
-  /** These values are the positions of the chunk in the text, in UTF-16 units. */
-  readonly start: number;
-  readonly end: number;
-  readonly text: string;
-}
-/** These options control the chunks of a text. */
-interface ChunkOptions {
-  /** This value is the largest number of tokens in a chunk. */
-  readonly maxTokens: number;
-  /** This value is the number of tokens that two chunks next to each other can share. The default is 10 percent of `maxTokens`. */
-  readonly overlapTokens?: number | undefined;
-  /** This value is the largest number of chunks. The default is `CHUNK_LIMITS.maxChunks`. */
-  readonly maxChunks?: number | undefined;
-}
-/**
- * This function estimates the number of tokens of a part of a text. A word is one token for each
- * six letters. A Chinese, Japanese or Thai character is one token. Each other character that is
- * not a space is one token. The estimate is a little high for English text, so a chunk fits.
- */
-declare const estimateTokens: (text: string, start?: number, end?: number) => number;
-/**
- * This function cuts a text into chunks for an embedder. A chunk is a window of whole sentences.
- * It ends at the end of a paragraph when the paragraph fills half of the window or more. A
- * sentence that is longer than the window is cut at its spaces. Two chunks next to each other share
- * a few sentences (`overlapTokens`), so a passage at a cut is in one chunk. Each chunk keeps its
- * position in the text, so a hit points at a passage.
- */
-declare const chunkText: (text: string, options: ChunkOptions) => TextChunk[];
-/**
- * This function gives the number of tokens of a chunk for an embedder. It is the input limit with a
- * margin for the estimate, and `CHUNK_LIMITS.maxTokens` at most.
- */
-declare const chunkTokensOf: (embedder: Pick<Embedder, "maxInputTokens">) => number;
-/** This function gives a copy of a vector with a length of 1. A vector of zeros stays zeros. A value that is not finite throws a `RangeError`. */
-declare const unitVector: (vector: ArrayLike<number>) => Float32Array;
-/** These are the defaults of a hybrid search. */
-declare const HYBRID_DEFAULTS: {
-  /** The full-text search gives this number of candidates. Each candidate gets vectors when it has none. */
-  readonly candidates: number;
-  /** The vector search gives this number of documents: the documents with the closest chunks. */
-  readonly neighbors: number;
-  /** This value is the constant k of reciprocal rank fusion. A larger k gives less weight to the top of each list. */
-  readonly fusionK: number;
-};
-/** This is one item of a fused list. */
-interface FusedRank {
-  readonly id: string;
-  /** This value is the sum of 1 / (k + rank) over the lists that hold the item. */
-  readonly score: number;
-  /** These values are the rank of the item in each list, from 1, or null when a list does not hold it. */
-  readonly ranks: ReadonlyArray<number | null>;
-}
-/**
- * This function fuses ranked lists with reciprocal rank fusion. Each item gets the sum of
- * 1 / (k + rank) over the lists that hold it, with the rank from 1. The best item is first. A tie goes to
- * the item with the best rank in a list, then to the earlier list, then to the smaller id. An item
- * that is two times in one list counts at its first place.
- */
-declare const fuseRanks: (lists: ReadonlyArray<readonly string[]>, k?: number) => FusedRank[];
-//#endregion
-//#region src/extract.d.ts
-/**
- * The field check takes the values that a caller proposes for the fields of a stored page. It finds
- * each value in the stored text or in the JSON-LD. A model or a person gives the values. The check
- * does not make values: it only checks and locates the values that it gets.
- *
- * Each field gets one status:
- *
- * - `verified`: the stored text holds the exact value. The result gives the span (or the JSON-LD
- *   path) and the capture.
- * - `normalized`: the stored text holds the value under a stated normalization, for example a
- *   different number format. The result names the normalization and gives the span of the stored text.
- * - `ambiguous`: the value matches, but the stored text does not give one meaning. The result gives
- *   each span, and it does not choose one.
- * - `not-found`: the stored text does not hold the value.
- * - `held-back`: the page asked not to be AI input. The result tells nothing about the value.
- * - `invalid`: the value is not a value of its kind, so the check did not search for it.
- *
- * The time of the check is linear in the length of the text, for a fixed set of fields.
- */
-/** The kinds of field that the check reads. `json-ld:<path>` reads a value at a path of the JSON-LD, for example `json-ld:Product.offers.price`. */
-type FieldKind = "text" | "number" | "money" | "date" | "url" | "email" | "phone" | `json-ld:${string}`;
-/** These kinds read the text of the page. The kind `json-ld:<path>` reads the JSON-LD. */
-declare const FIELD_KINDS: readonly string[];
-/** This is the value that a caller proposes for one field. */
-interface FieldClaim {
-  /** This value is the proposal. The check removes the spaces at its start and at its end. */
-  readonly value: string | number;
-  /** This value is the kind of the field. The default is `number` for a number and `text` for a text. */
-  readonly kind?: FieldKind | undefined;
-}
-/** The status of one field. The module comment of `extract.ts` and the README tell what each status means. */
-type FieldStatus = "verified" | "normalized" | "ambiguous" | "not-found" | "held-back" | "invalid";
-/** These positions locate a value in the text of a version. The end is not part of the value. */
-interface FieldSpan {
-  readonly start: number;
-  readonly end: number;
-  /** This value is `full_text` when the value is only in the whole text of a page that the reader cut. */
-  readonly in: "text" | "full_text";
-}
-/** This is one place that an ambiguous field matches. */
-interface FieldCandidate {
-  readonly span: FieldSpan | null;
-  /** This path locates a value of the JSON-LD (`jsonld[0].offers[1].price`) or a stored link (`links[3]`). */
-  readonly path: string | null;
-  /** This value is the exact stored text, or the stored value of the JSON-LD. */
-  readonly raw: string | number | boolean;
-  readonly excerpt: string;
-  /** These are the meanings that the stored text can have, for example two dates for `03/04/2026`. */
-  readonly readings: readonly string[];
-}
-/** This is the check of one field, before the runtime adds the capture. */
-interface FieldCheck {
-  readonly status: FieldStatus;
-  /** This value is the kind that the check used. */
-  readonly kind: string;
-  readonly span: FieldSpan | null;
-  readonly path: string | null;
-  /** This value is the exact stored text of the match, or the stored value of the JSON-LD. */
-  readonly raw: string | number | boolean | null;
-  /** This text is the stored text around the match. */
-  readonly excerpt: string | null;
-  /** This value names the normalization of a `normalized` match, for example `thousands-separator`. Two or more names are joined with `+`. */
-  readonly normalization: string | null;
-  /** This value is the meaning of the stored text that matched, for example `1234.5` or `2026-03-04`. */
-  readonly reading: string | null;
-  /** This value is the number of places that match. */
-  readonly count: number;
-  /** These are the places of an ambiguous field, one for each meaning, 5 or fewer. */
-  readonly candidates: readonly FieldCandidate[];
-  readonly note: string | null;
-}
-/** This is the result of one field: the check, and the capture of the text that the check read. */
-interface FieldResult extends FieldCheck {
-  readonly capture: string | null;
-  /** This time (ISO 8601) is the fetch time of that capture. */
-  readonly fetched_at: string | null;
-}
-/** These are the counts of the statuses of the fields. */
-interface ExtractSummary {
-  readonly fields: number;
-  readonly verified: number;
-  readonly normalized: number;
-  readonly ambiguous: number;
-  readonly not_found: number;
-  readonly held_back: number;
-  readonly invalid: number;
-}
-/** This is the text of a document that `checkFields` reads, with its metadata. */
-interface ExtractSource {
-  readonly text: string;
-  /** This text is the whole text of a page that the reader cut. The check reads it when the value is not in `text`. */
-  readonly full_text?: string | undefined;
-  /** This URL is the address of the page. The check resolves a relative URL against it. */
-  readonly url?: string | null | undefined;
-  /** The check reads `jsonld`, `jsonld_dropped`, `links` and `truncated` of this metadata. */
-  readonly meta?: Readonly<Record<string, unknown>> | undefined;
-}
-/** This is the result of `checkFields`. */
-interface FieldsChecked {
-  readonly fields: Readonly<Record<string, FieldCheck>>;
-  readonly summary: ExtractSummary;
-  /** This value counts the units of work of the check. It grows in proportion to the length of the text. */
-  readonly steps: number;
-}
-/** These options control a check of fields against a stored document. */
-interface ExtractOptions {
-  /** This id checks the version of one capture, not the current version. */
-  readonly capture?: string | undefined;
-  /**
-   * This time checks the version that was current at that time: an ISO 8601 text or a `Date`. A date
-   * with no time is the start of that day in UTC. Give `capture` or `at`, not the two.
-   */
-  readonly at?: string | Date | undefined;
-  /** This value is true when a person gave this URL directly. It is the only way to check a held-back document. The crawler logs it. */
-  readonly userProvided?: boolean | undefined;
-}
-/** This is the result of a check of fields against a stored document. */
-interface ExtractResult {
-  readonly doc: string;
-  readonly url: string | null;
-  readonly captures: readonly string[];
-  /** This id is the capture of the text that the check read. A held-back document gives null. */
-  readonly capture: string | null;
-  /** This hash is the blob of that capture. */
-  readonly blob: string | null;
-  /** This time (ISO 8601) is the fetch time of that capture. */
-  readonly fetched_at: string | null;
-  readonly held_back: boolean;
-  /** These are the results of the fields, with the names that the caller gave. */
-  readonly fields: Readonly<Record<string, FieldResult>>;
-  readonly summary: ExtractSummary;
-  readonly note: string | null;
-  /** This time (ISO 8601) is the `at` of the check. The result has it only when the check gave `at`. */
-  readonly at?: string | undefined;
-  /** This value is true when `at` is before the first capture of the document. Then no field is found. */
-  readonly before_first?: boolean | undefined;
-  /**
-   * This value tells that the checked text is an archived copy, with the archive and the date of the
-   * copy. The note says so too. A citation of a value names the archive and that date.
-   */
-  readonly archived?: Archived | undefined;
-}
-/**
- * This function checks the proposed values of fields against the text and the metadata of a
- * document. It sends no request and writes nothing. It does not make values. A field that is not
- * valid throws a `RangeError`. The work grows in proportion to the length of the text.
- */
-declare const checkFields: (source: ExtractSource, fields: Readonly<Record<string, FieldClaim>>) => FieldsChecked;
-/**
- * This function checks the proposed values of fields against a stored document. It does not make
- * values. It applies the rules of `verify`:
- *
- * - With no option, it checks the current version. `capture` checks the version of one capture,
- *   and `at` checks the version that was current at a time.
- * - A held-back document gives `held-back` for each field, with no position and no capture, unless
- *   `userProvided`. The crawler then logs an override.
- * - A field that is not valid throws a `RangeError`, and an unknown document throws an error.
- */
-declare const extractDocument: (runtime: CrawlRuntime, id: string, fields: Readonly<Record<string, FieldClaim>>, options?: ExtractOptions) => Promise<ExtractResult>;
-//#endregion
-//#region src/frontier.d.ts
-/**
- * The frontier is a small crawl of a site. It starts at the seeds. It follows the links of each page
- * that it fetched, to a maximum of pages, a maximum depth and a maximum time.
- *
- * Each page is a fetch of the crawler, so each polite rule applies. These rules are robots.txt, one
- * request at a time for each host, the pace, the stops, the egress policy and the capture log. The
- * frontier does not start a second request to a host with a request in progress. The crawler
- * limits the hosts in progress (`maxActive`).
- *
- * The state of the frontier is in memory, for one call. A later version can make it resumable.
- */
-/** These are the default limits of `crawl`, and the limits of the memory of the frontier. */
-declare const CRAWL_LIMITS: {
-  /** The most pages that one crawl gives. */
-  readonly maxPages: 100;
-  /** The most links from a seed to a page. A seed has the depth 0. */
-  readonly maxDepth: 3;
-  /** The longest time of one crawl: 10 minutes. */
-  readonly budgetMs: 600_000;
-  /** The most hosts that get a request of the crawl at one time. */
-  readonly hostsAtOnce: 16;
-  /** The most URLs that wait in the frontier. */
-  readonly maxQueue: 10_000;
-  /** The most URLs that the frontier remembers, so it fetches none of them two times. */
-  readonly maxSeen: 100_000;
-};
-/**
- * A pattern for the URLs of a crawl. A text that starts with `/` matches the path and the query.
- * A different text matches the whole URL. In a text, `*` matches any characters and a `$` at the
- * end matches the end, as in robots.txt. A regular expression tests the whole URL.
- */
-type UrlPattern = string | RegExp;
-/** These options control a crawl. Each limit has a default in `CRAWL_LIMITS`. */
-interface CrawlOptions extends ScopeOptions {
-  /** This value is the most pages that the crawl gives. The default is 100. */
-  readonly maxPages?: number | undefined;
-  /** This value is the most links from a seed to a page. The default is 3, and 0 fetches only the seeds. */
-  readonly maxDepth?: number | undefined;
-  /** This value is the longest time of the crawl in milliseconds. No request starts after it. The default is 10 minutes. */
-  readonly budgetMs?: number | undefined;
-  /** When this list is not empty, the crawl follows only a URL that matches a pattern of it. */
-  readonly include?: readonly UrlPattern[] | undefined;
-  /** The crawl follows no URL that matches a pattern of this list. */
-  readonly exclude?: readonly UrlPattern[] | undefined;
-  /** With the value true, the crawl adds the pages of the sitemaps of each site of the seeds (`discover`). */
-  readonly useSitemaps?: boolean | undefined;
-  /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */
-  readonly render?: "auto" | "never" | "always" | undefined;
-  /** A stored copy younger than this age answers, with no request. The default is `24h`. */
-  readonly maxAge?: Duration | undefined;
-  /** These agents get their robots.txt rules applied to each fetch. */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** When this signal aborts, the crawl stops the fetches in progress and ends. */
-  readonly signal?: AbortSignal | undefined;
-  /** This label goes into the capture records of the crawl. */
-  readonly scope?: string | undefined;
-  /** These labels go into the capture records of the crawl. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/** These options control `crawlSite`: the options of `crawl`, and a step before the first fetch. */
-interface CrawlRunOptions extends CrawlOptions {
-  /** This function operates before the first fetch. `openCrawler` waits there for the open of the store. */
-  readonly ready?: (() => Promise<void>) | undefined;
-}
-/** A page of a crawl: the result of its fetch, and the place of the page in the crawl. */
-interface CrawlPage extends PageResult {
-  /** This value is the number of links from a seed to the page. */
-  readonly depth: number;
-  /** This value tells if the page is a seed, a page of a sitemap, or the target of a link. */
-  readonly source: "seed" | "sitemap" | "link";
-  /** This URL is the page whose link the crawl followed, or null. */
-  readonly referrer: string | null;
-}
-/** The reason that a crawl stopped. */
-type CrawlStop = "done" | "max-pages" | "budget" | "cancelled" | "closed" | "error";
-/** The summary of a crawl. */
-interface CrawlSummary {
-  /** This value is the number of pages that the crawl gave. */
-  readonly pages: number;
-  /** This value counts the pages with a document: `ok` or `not-modified`, from the store too. */
-  readonly fetched: number;
-  /** This value counts the pages that a stored copy answered, with no request. */
-  readonly from_cache: number;
-  /**
-   * This value counts the refusals. These are the links that robots.txt does not permit, and the
-   * pages that the crawler declined (`skipped`). The pages that the site refused (`blocked`,
-   * `challenge`, `payment-required`) are refusals too.
-   */
-  readonly refused: number;
-  /** This value counts the pages with a different outcome, for example `error` or `not-found`. */
-  readonly failed: number;
-  /** This value counts the links that the crawl did not follow. `skips` gives the reasons. */
-  readonly skipped: number;
-  readonly skips: {
-    /** The links to a host that is not in scope. */
-    readonly scope: number;
-    /** The links that `include` or `exclude` left out. */
-    readonly pattern: number;
-    /** The links deeper than `maxDepth`. */
-    readonly depth: number;
-    /** The pages that asked not to be AI input: the crawl does not follow their links. */
-    readonly held_back: number;
-    /** The links over the limits of the memory of the frontier. */
-    readonly limit: number;
-    /** The URLs that a fetched page had as its final or its canonical URL. */
-    readonly duplicate: number;
-  };
-  /** This value counts the URLs that waited in the frontier at the stop. */
-  readonly queued: number;
-  /** This value counts the URLs that the sitemaps gave, with `useSitemaps`. */
-  readonly sitemap_urls: number;
-  readonly stopped: CrawlStop;
-  /** This text tells why the crawl stopped. */
-  readonly reason: string;
-  /** This value is the time of the crawl in milliseconds, on the clock of the crawler. */
-  readonly elapsed_ms: number;
-}
-/**
- * A crawl in progress. A `for await` loop gets each page when its fetch ends. `summary` resolves
- * when the loop ends: at the last page, at a stop or at a `break`. The crawl starts at the first
- * step of the loop, and each crawl operates one time only.
- */
-interface CrawlRun extends AsyncIterable<CrawlPage> {
-  readonly summary: Promise<CrawlSummary>;
-}
-/**
- * This function crawls a site from its seeds. `PoliteCrawler.crawl` uses it. It gives each page
- * when its fetch ends, and a summary at the end. The seeds come first, then the pages of the
- * sitemaps (the newest first), then the links, the lower depth first.
- */
-declare const crawlSite: (runtime: CrawlRuntime, seeds: string | readonly string[], options?: CrawlRunOptions) => CrawlRun;
-//#endregion
-//#region src/vectors.d.ts
-/**
- * The vector store keeps the vectors of hybrid search in `vectors.sqlite`, beside the index. It is
- * a projection, as the index is. Delete the file, and the next hybrid search or `embed` makes the
- * vectors again from the stored texts.
- *
- * - A row is one chunk of one text for one embedder. Its key is the SHA-256 of the text, the
- *   number of the chunk and the id of the embedder. Thus a reindex keeps each vector, and the same
- *   text gets its vectors one time for each model, also when two documents hold it.
- * - A text is complete when the table `texts` has it, with the plan of its chunks. A text with a
- *   different plan gets new chunks.
- * - The table `docs` points each document at the text that has its vectors. A search uses only
- *   the texts that a document points at, and the caller checks each document against the index.
- * - A search compares the vector of the question with each stored vector (a brute-force cosine).
- *   The store keeps the vectors of each embedder in memory after the first search. It reads them
- *   again when a different process writes the file.
- *
- * Each connection waits for the lock of a different process (`busy_timeout`), and each write is
- * one transaction (`BEGIN IMMEDIATE`). The file has a schema version (`VECTORS_SCHEMA`). A file of
- * a different version opens empty.
- */
-/** This value is the version of the schema of the vector file. */
-declare const VECTORS_SCHEMA = 1;
-/** This is one chunk of a text with its vector, as the store keeps it. */
-interface StoredChunk {
-  readonly chunk: number;
-  /** These values are the positions of the chunk in the text, in UTF-16 units. */
-  readonly start: number;
-  readonly end: number;
-  /** This vector has a length of 1, or it is zeros. */
-  readonly vector: Float32Array;
-}
-/** This is the best chunk of one text for a question. */
-interface VectorHit {
-  /** This value is the SHA-256 of the text, in hex. */
-  readonly sha: string;
-  readonly chunk: number;
-  readonly start: number;
-  readonly end: number;
-  /** This value is the cosine of the chunk and the question, from -1 to 1. */
-  readonly similarity: number;
-  /** These documents point at the text. The caller checks each one against the index. */
-  readonly docs: readonly string[];
-}
-/** These are the counts of the store, for one embedder or for each embedder. */
-interface VectorCounts {
-  readonly texts: number;
-  readonly chunks: number;
-  readonly docs: number;
-  /** This value is the size of the vectors in bytes, with no overhead of SQLite. */
-  readonly bytes: number;
-}
-/** These options open a vector store. */
-interface VectorStoreOptions {
-  /** This value is the time in milliseconds that a write waits for the lock of a different process. The default is 10 seconds. */
-  readonly busyTimeoutMs?: number | undefined;
-}
-/** These are the parts of one text in a write of the store. */
-interface VectorWrite {
-  readonly embedder: string;
-  readonly dimensions: number;
-  /** This value is the SHA-256 of the text, in hex. */
-  readonly sha: string;
-  /** This text names the rule of the chunks. A text with a different plan gets new chunks. */
-  readonly plan: string;
-  /** This value is the length of the text. */
-  readonly chars: number;
-  readonly chunks: ReadonlyArray<Omit<StoredChunk, "chunk">>;
-  /** These documents hold the text as their current text. The store points each one at it. */
-  readonly docs: readonly string[];
-  /** This time (ISO 8601) is the time of the write. */
-  readonly at: string;
-}
-declare class VectorStore {
-  readonly path: string;
-  private readonly db;
-  private readonly statements;
-  private readonly matrices;
-  private closed;
-  constructor(path: string, options?: VectorStoreOptions);
-  private stmt;
-  private write;
-  private live;
-  /** This method gives the plan of the chunks of a text when the store has each of its vectors, or undefined. */
-  plan(embedder: string, sha: string): string | undefined;
-  /** This method gives the text that a document points at, or undefined. */
-  pointed(embedder: string, doc: string): string | undefined;
-  /** This method gives the dimensions that the store keeps for an embedder, or undefined. */
-  dimensions(embedder: string): number | undefined;
-  /**
-   * This method writes the chunks of one text and points its documents at it, in one transaction.
-   * A vector with a length that is different from the dimensions of the embedder throws an error.
-   */
-  put(input: VectorWrite): void;
-  private pointRow;
-  /** This method points a document at a text that the store has. */
-  point(embedder: string, doc: string, sha: string, at: string): void;
-  /** This method removes the pointer of a document, for example when the document is held back. Its text keeps its vectors. */
-  forget(embedder: string, doc: string): void;
-  /** This method gives the chunks of a text, in their sequence. */
-  chunks(embedder: string, sha: string): StoredChunk[];
-  private pointMatrix;
-  private append;
-  /** This method gives the vectors of an embedder in memory. It reads them again when a different process wrote the file. */
-  private matrix;
-  private load;
-  /** This method gives the best chunk of one text for a question, or undefined when the store has no vector of the text. */
-  best(embedder: string, sha: string, query: Float32Array): VectorHit | undefined;
-  private hitOf;
-  /**
-   * This method compares a question with each stored vector of an embedder (a brute-force cosine).
-   * It gives the best chunk of each text that a document points at, the best first. With `within`,
-   * it compares only the texts of those documents.
-   */
-  search(embedder: string, query: Float32Array, options?: {
-    readonly within?: Iterable<string> | undefined;
-  }): VectorHit[];
-  /** This method gives the counts of the store, for one embedder or for each embedder. */
-  counts(embedder?: string): VectorCounts;
-  /** This method closes the file. A second close does nothing. */
-  close(): void;
-}
-//#endregion
-//#region src/hybrid.d.ts
-/**
- * Hybrid search ranks the stored documents two times and fuses the two lists:
- *
- * 1. The full-text index gives the BM25 candidates (`candidates`, 50 by default). Each candidate
- *    whose text has no vectors gets them from the embedder first.
- * 2. The vector store gives the documents with the closest chunks (`neighbors`, 50 by default),
- *    with a brute-force cosine over the vectors of the embedder.
- * 3. Reciprocal rank fusion (`fusionK`, 60 by default) makes one list of the two.
- *
- * The snippet and the `passage` of each hit come from its best chunk. A held-back document gets
- * no vectors, and no result gives it, exactly as for `find`. The text of a held-back version does not
- * goes to the embedder.
- */
-/** These options change a hybrid search. */
-interface HybridOptions {
-  /** This value is the number of BM25 candidates. The default is `HYBRID_DEFAULTS.candidates`. */
-  readonly candidates?: number | undefined;
-  /** This value is the number of documents from the vector search. The default is `HYBRID_DEFAULTS.neighbors`. */
-  readonly neighbors?: number | undefined;
-  /** This value is the constant k of reciprocal rank fusion. The default is `HYBRID_DEFAULTS.fusionK`. */
-  readonly fusionK?: number | undefined;
-  /** When this signal aborts, the search stops the embedder and rejects. */
-  readonly signal?: AbortSignal | undefined;
-}
-/** These options control a hybrid search of the store. */
-interface HybridFindOptions extends HybridOptions {
-  /** This value is the maximum number of results. The default is 10, and the maximum is 1000. */
-  readonly limit?: number | undefined;
-  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
-  readonly kind?: string | null | undefined;
-}
-/** This is one document of a ranking of a set of documents (`rankWithin`). */
-interface Ranked {
-  readonly doc: string;
-  /** This value is the score. A larger score is a better match. In the mode `hybrid`, it is the fused score. */
-  readonly score: number;
-  /** This value is the best chunk of the document, in the mode `hybrid`. */
-  readonly passage?: Passage | undefined;
-}
-/** These options control `embed`, which gives vectors to the stored documents that have none. */
-interface EmbedOptions {
-  /** This value is the longest time of the work, for example `"10m"` (the default). No embedding starts after it. */
-  readonly budget?: Duration | undefined;
-  /** This value is the largest number of texts that get new vectors. The default has no limit. */
-  readonly maxDocs?: number | undefined;
-  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
-  readonly kind?: string | null | undefined;
-  /** When this signal aborts, the work stops. The result then has `stopped: "signal"`. */
-  readonly signal?: AbortSignal | undefined;
-}
-/** The reason of the end of `embed`. */
-type EmbedStop = "done" | "budget" | "max-docs" | "signal";
-/** The counts of `embed`. */
-interface EmbedResult {
-  /** This value is the id of the embedder. */
-  readonly embedder: string;
-  /** This value is the number of documents that the work examined. */
-  readonly docs: number;
-  /** This value is the number of texts that got new vectors. */
-  readonly embedded: number;
-  /** This value is the number of documents whose text had vectors already. */
-  readonly reused: number;
-  /** This value is the number of documents that the work left out, because they are held back. */
-  readonly held_back: number;
-  /** This value is the number of new vectors: one for each chunk. */
-  readonly chunks: number;
-  readonly stopped: EmbedStop;
-  /** This value is the time of the work in milliseconds. */
-  readonly ms: number;
-}
-/** Hybrid search over a runtime and a vector store. `openCrawler` makes one when it has an embedder. */
-interface HybridSearch {
-  readonly embedder: Embedder;
-  /** This method searches the stored documents in the mode `hybrid`. It sends no request to a site. */
-  find(q: string, options?: HybridFindOptions): Promise<Found[]>;
-  /** This method ranks a set of documents in the mode `hybrid`, the best first. */
-  rankWithin(q: string, docs: readonly string[], options?: HybridOptions): Promise<Ranked[]>;
-  /** This method gives vectors to the stored documents that have none, within a budget. */
-  embed(options?: EmbedOptions): Promise<EmbedResult>;
-}
-/** These options make a hybrid search. */
-interface HybridSearchOptions {
-  readonly embedder: Embedder;
-  /** This store keeps the vectors. The caller opens it and closes it. */
-  readonly store: VectorStore;
-}
-/** This function makes a hybrid search over a runtime. The vectors go into the store, with the id of the embedder. */
-declare const createHybridSearch: (runtime: CrawlRuntime, options: HybridSearchOptions) => HybridSearch;
-//#endregion
-//#region src/recipe.d.ts
-/** The capture that an extractor reads: the final URL of the response and its content type. */
-interface RecipeCapture {
-  readonly url: string;
-  readonly contentType: string | null;
-}
-/**
- * This function reads one captured body into the documents that it holds. It is pure: the same
- * body and capture always give the same documents. It sends no request. It can answer later.
- */
-type RecipeExtract = (body: Uint8Array, capture: RecipeCapture) => ReadonlyArray<ExtractedDoc> | Promise<ReadonlyArray<ExtractedDoc>>;
-/** The request of a recipe for one input: the URL, and the parts that change with the input. */
-interface RecipeRequest {
-  readonly url: string;
-  /** This value replaces the `accept` of the recipe for this request. */
-  readonly accept?: string | undefined;
-  /** These headers go with the request. The crawler does not write them into the capture. */
-  readonly headers?: Readonly<Record<string, string>> | undefined;
-  /** These query parameters hold secrets. The capture shows `REDACTED` in their place. */
-  readonly secretParams?: readonly string[] | undefined;
-}
-/** These values declare a recipe. Only `name` and `extract` are necessary. */
-interface RecipeDefinition<Input = string> {
-  /**
-   * This name goes into each capture record of the recipe, so it is part of each capture id.
-   * `reindex` finds the extractor by it. Do not change it after the first capture.
-   */
-  readonly name: string;
-  /** This text tells what the recipe reads. */
-  readonly description?: string | undefined;
-  /**
-   * The value `page` (the default) is a web page: robots.txt applies. The value `api` is a
-   * documented API with terms of its own: robots.txt does not apply, and the pace still does.
-   */
-  readonly mode?: "api" | "page" | undefined;
-  /** This value is the `Accept` header. The defaults are HTML for a page and JSON for an API. */
-  readonly accept?: string | undefined;
-  /** This value is the documented pace of the source: the shortest time between two of its requests. */
-  readonly minInterval?: Duration | undefined;
-  /** A stored copy younger than this age answers with no request. The default is `24h`. */
-  readonly maxAge?: Duration | undefined;
-  /** This value is the most bytes of a body that the crawler keeps. */
-  readonly maxBytes?: number | undefined;
-  /** This function gives the request for an input. With no function, the input is the URL. */
-  readonly fetch?: ((input: Input) => string | RecipeRequest) | undefined;
-  /** This function reads a captured body into documents. It is pure, and it sends no request. */
-  readonly extract: RecipeExtract;
-}
-/** These options build the request of a recipe for one input (`Recipe.request`). */
-interface RecipeRequestOptions {
-  /** This value replaces the `maxAge` of the recipe. The value `0` asks the source again. */
-  readonly maxAge?: Duration | undefined;
-  /** These agents get their robots.txt rules applied too, for a recipe of the mode `page`. */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** This name connects the captures to the job that asked for them. */
-  readonly job?: string | undefined;
-  /** When this signal aborts, the crawler stops the fetch. */
-  readonly signal?: AbortSignal | undefined;
-  /** This label goes into each capture record of the request. */
-  readonly scope?: string | undefined;
-  /** These labels go into each capture record of the request. They are not part of the capture id. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-  /** No request starts at or after this time (milliseconds since the epoch, on the clock of the crawler). */
-  readonly deadline?: number | undefined;
-}
-/** A recipe that `defineRecipe` made. Give it to `runRecipe`, to `registerRecipe` or to `openCrawler`. */
-interface Recipe<Input = string> {
-  readonly name: string;
-  readonly description: string | null;
-  readonly mode: "api" | "page";
-  readonly accept: string;
-  /** This extractor reads the captures of the recipe. It checks the documents of the definition. */
-  readonly extractor: Extractor;
-  /** This method gives the request of the crawler for one input. A value that is not a URL throws a `TypeError`. */
-  request(input: Input, options?: RecipeRequestOptions): FetchRequest;
-}
-/**
- * This function reads a body as JSON, in the encoding that the response declares. A body that is
- * not JSON (a page about maintenance, or a login page) throws an error that names the URL and the
- * content type. In an extractor, the runtime then reports that error.
- */
-declare const readJson: (body: Uint8Array, capture: RecipeCapture) => unknown;
-/**
- * These recipe names belong to the crawler: the page reader, the archived fallback, robots.txt,
- * and the sitemaps and feeds of `discover`. A recipe cannot use them.
- */
-declare const RESERVED_RECIPES: readonly string[];
-/** These are the limits of a recipe. */
-declare const RECIPE_LIMITS: {
-  /** The longest recipe name. */
-  readonly nameChars: number;
-  /** The longest document id. */
-  readonly docIdChars: number;
-};
-/**
- * This function declares a recipe. It checks the definition at once: a name that is not valid, a
- * reserved name or a duration that it cannot read throws an error. The extractor of the recipe
- * checks each document of `extract`. A document that is not valid makes the extractor throw, and
- * the runtime reports it as an error of the extractor.
- */
-declare const defineRecipe: <Input = string>(definition: RecipeDefinition<Input>) => Recipe<Input>;
-/**
- * This function gives the extractor of a recipe to the extractors of a runtime. A second
- * registration of the same recipe does nothing. A different extractor with the same name throws
- * an error, because `reindex` must read each capture with one extractor.
- */
-declare const registerRecipe: <Input>(extractors: Extractors, recipe: Recipe<Input>) => void;
-/** These options control one run of a recipe. */
-interface RecipeRunOptions extends Omit<RecipeRequestOptions, "deadline"> {
-  /** No request starts after this time from the start of the run, for example `"30s"`. */
-  readonly budget?: Duration | undefined;
-  /** This value is true when a person gave this input directly. It is the only way to see a held-back document. The crawler logs it. */
-  readonly userProvided?: boolean | undefined;
-  /**
-   * The value `archive` gives a copy from a public web archive when the site refuses the page or is
-   * down. Only a recipe of the mode `page` takes it. The page reader reads the copy.
-   */
-  readonly fallback?: "archive" | "none" | undefined;
-  /** The value true also gives an archived copy of a page that the current robots.txt disallows. */
-  readonly archiveDespiteRobots?: boolean | undefined;
-}
-/** One document of a run of a recipe: the version of the capture of the run. */
-interface RecipeDoc {
-  readonly doc: string;
-  readonly kind: string;
-  /** This value is the title. A held-back document shows no title. */
-  readonly title: string;
-  readonly url: string | null;
-  /** This value is the metadata. A held-back document shows only the structural fields. */
-  readonly meta: Readonly<Record<string, unknown>>;
-  /** This value is the length of the text. `read` gives the text. */
-  readonly chars: number;
-  /** This id is the capture of this version. A citation names it. */
-  readonly capture: string;
-  readonly fetched_at: string;
-  readonly held_back: boolean;
-  /** This value tells that the document is an archived copy, with the archive and the date of the copy. */
-  readonly archived?: Archived | undefined;
-}
-/** The result of `runRecipe`. */
-interface RecipeResult {
-  readonly recipe: string;
-  /** This URL is the URL of the capture. Its secrets show as `REDACTED`. */
-  readonly url: string;
-  readonly outcome: string;
-  readonly status: number;
-  /** These documents came from the capture of the run, in the order of the extractor. */
-  readonly docs: readonly RecipeDoc[];
-  /** This id is the capture of the documents. */
-  readonly capture: string | null;
-  readonly fetched_at: string | null;
-  /** This value is true when the stored copy answered, and the crawler sent no request. */
-  readonly from_cache: boolean;
-  /** This value is true when a document is held back. */
-  readonly held_back: boolean;
-  /** This text is the error of the extractor, when it threw for the capture. */
-  readonly error: string | null;
-  /** This value tells that the documents are an archived copy, not a fresh fetch. */
-  readonly archived?: Archived | undefined;
-  readonly note: string | null;
-}
-/** A recipe operates on a runtime, or on a crawler that has one (`crawler.runtime`). */
-type RecipeTarget = CrawlRuntime | {
-  readonly runtime: CrawlRuntime;
-};
-/**
- * This function starts a recipe for one input. It fetches the request through the crawler, and it
- * reads the capture with the extractor of the recipe. It gives the documents of that capture.
- * These are its rules:
- *
- * - The runtime registers the recipe the first time that it gets it. Give the recipe to
- *   `openCrawler` too, so that `reindex` reads its captures before the first fetch.
- * - A stored copy younger than `maxAge` answers with no request. When the extractor cannot read a
- *   stored copy, the crawler asks the source one more time.
- * - An extractor that throws gives no document, and `error` tells why. The fetch still succeeds.
- * - A document that asked not to be AI input shows no title and only its structural metadata,
- *   unless `userProvided`. The crawler logs each `userProvided` read of a held-back document.
- * - With `fallback: "archive"`, a page that the site refuses gets an archived copy, with
- *   `archived`. The page reader reads the copy. The extractor of the recipe reads only live
- *   captures. Thus its documents do not mix a live answer and an old copy of a third party.
- */
-declare const runRecipe: <Input>(target: RecipeTarget, recipe: Recipe<Input>, input: Input, options?: RecipeRunOptions) => Promise<RecipeResult>;
-//#endregion
-//#region src/watch.d.ts
-/**
- * A watch list keeps the pages that the crawler checks again on a schedule. `watch.run` fetches the
- * pages that are due through the normal fetch, so each polite rule applies. It gives an event for
- * each page: new, changed, unchanged, gone, refused or error.
- *
- * These are the rules of a watch:
- *
- * - A watch compares the title, the text and the JSON-LD of the page with the last confirmed version.
- * - With `confirm: 2` (the default), a change is suspected when a check first sees it. The next
- *   check confirms it when it sees the same new version. A check that sees the earlier version
- *   again clears the suspicion, because one read can be a glitch. With `confirm: 1`, each check confirms.
- * - `every` has a floor of 1 hour by default. A check that a stop of the host refused is due again
- *   at the end of the stop, or later.
- * - The list is in `watch.sqlite` in the store, so it stays after a restart. A run claims each entry
- *   with a lease in that file, so two processes do not check one entry two times.
- *
- * A scheduler of the operator starts `watch.run`, or the daemon (`startDaemon`) starts it on a schedule.
- */
-/** This value is the version of the schema of `watch.sqlite`. */
-declare const WATCH_SCHEMA = 1;
-/** These are the defaults and the limits of the watch list. */
-declare const WATCH_LIMITS: {
-  /** The shortest `every` of a watch by default: 1 hour, so a watch cannot hammer a site. */
-  readonly floorMs: 3_600_000;
-  /** The default `every`: 24 hours. */
-  readonly everyMs: 86_400_000;
-  /** The default time of one run: 10 minutes. No request starts after it. */
-  readonly budgetMs: 600_000;
-  /** The default number of entries that one run checks. */
-  readonly maxPages: 100;
-  /** The most hosts that get a check of one run at one time. */
-  readonly hostsAtOnce: 8;
-  /** A run holds the lease of an entry until the end of its budget and this time more: 2 minutes. */
-  readonly leaseMs: 120_000;
-};
-/** The kind of a check: the page is new, changed, unchanged or gone, or the check got no version. */
-type WatchKind = "new" | "changed" | "unchanged" | "gone" | "refused" | "error";
-/** These options add a page to the watch list, or change its watch. */
-interface WatchOptions {
-  /** This value is the time between two checks. The default is `24h`, and the floor is `1h` (`watchFloor`). */
-  readonly every?: Duration | undefined;
-  /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */
-  readonly render?: "auto" | "never" | "always" | undefined;
-  /** These labels go into the capture records of each check. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-  /** This label goes into the capture records of each check. */
-  readonly scope?: string | undefined;
-  /** These agents get their robots.txt rules applied to each check. */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** This value is the number of checks that must see a change before it is confirmed. The default is 2. */
-  readonly confirm?: 1 | 2 | undefined;
-}
-/** A change that one check saw, and that the next check can confirm. */
-interface WatchPending {
-  /** This value is `changed`, `gone` or `new` (a page that came back). */
-  readonly kind: "changed" | "gone" | "new";
-  /** This id is the capture of the suspected version, or null for a page that is gone. */
-  readonly capture: string | null;
-  /** This time (ISO 8601) is the time of the check that first saw the change. */
-  readonly since: string;
-}
-/** This is one entry of the watch list. */
-interface WatchEntry {
-  /** This URL is the page that the watch fetches. */
-  readonly url: string;
-  /** This value is the time between two checks in milliseconds. */
-  readonly every_ms: number;
-  readonly render: "auto" | "never" | "always";
-  readonly confirm: 1 | 2;
-  readonly scope: string | null;
-  readonly labels: Readonly<Record<string, string>> | null;
-  readonly on_behalf_of: readonly string[];
-  /** This time (ISO 8601) is the time of the first add. */
-  readonly added_at: string;
-  /** This time (ISO 8601) is the time when the entry is due. */
-  readonly next_at: string;
-  /** This time (ISO 8601) is the time of the last check, or null before the first check. */
-  readonly checked_at: string | null;
-  readonly last_kind: WatchKind | null;
-  readonly last_note: string | null;
-  /** This value counts the checks. */
-  readonly checks: number;
-  /** This value counts the confirmed changes. */
-  readonly changes: number;
-  /** This id is the document of the page. */
-  readonly doc: string | null;
-  /** This id is the capture of the last confirmed version, or null. */
-  readonly capture: string | null;
-  /** This time (ISO 8601) is the fetch time of the last confirmed version, or null. */
-  readonly fetched_at: string | null;
-  /** This value is true when the last confirmed state is that the page is gone. */
-  readonly gone: boolean;
-  /** This value is the change that the last check suspected, or null. */
-  readonly pending: WatchPending | null;
-  /** This time (ISO 8601) is the end of the lease of a run that checks the entry, or null. */
-  readonly leased_until: string | null;
-}
-/** These options control a run of the watch list. */
-interface WatchRunOptions {
-  /** This value is the longest time of the run in milliseconds. No request starts after it. The default is 10 minutes. */
-  readonly budgetMs?: number | undefined;
-  /** This value is the most entries that the run checks. The default is 100. */
-  readonly maxPages?: number | undefined;
-  /** When this signal aborts, the run stops the checks in progress and ends. */
-  readonly signal?: AbortSignal | undefined;
-  /**
-   * When this signal aborts, the run starts no new check. The checks in progress end and write
-   * their state, and the run stops with `cancelled`. The daemon uses it for a clean stop.
-   */
-  readonly drain?: AbortSignal | undefined;
-}
-/** These options control `runWatch`: the options of a run, and the parts of `openCrawler`. */
-interface WatchRunnerOptions extends WatchRunOptions {
-  /** These agents get their robots.txt rules applied to each check, before the agents of the entry. */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** This function operates before the first check. `openCrawler` waits there for the open of the store. */
-  readonly ready?: (() => Promise<void>) | undefined;
-}
-/** This is the event of one check of a run. */
-interface WatchEvent {
-  /** This URL is the page of the watch. */
-  readonly url: string;
-  readonly doc: string | null;
-  readonly kind: WatchKind;
-  /** This value is true when the check saw a change for the first time, and the next check must confirm it. */
-  readonly suspected: boolean;
-  /** This value is true when the check confirmed a change. Its version is then the last confirmed version. */
-  readonly confirmed: boolean;
-  /** This id is the capture that the check got, or null. */
-  readonly capture: string | null;
-  /** This time (ISO 8601) is the fetch time of that capture, or null. */
-  readonly fetched_at: string | null;
-  /** This id is the capture of the last confirmed version before the check, or null. */
-  readonly previous: string | null;
-  /** This time (ISO 8601) is the fetch time of that version, or null. */
-  readonly previous_fetched_at: string | null;
-  readonly outcome: string;
-  readonly status: number;
-  /** These are the counts of the diff from the last confirmed version, for a change. A held-back page gives null. */
-  readonly diff: DiffSummary | null;
-  /** This value is true when the page asked not to be AI input. */
-  readonly held_back: boolean;
-  /** This time (ISO 8601) is the time when the entry is due again. */
-  readonly next_at: string;
-  readonly note: string | null;
-}
-/** The reason that a run stopped. */
-type WatchStop = "done" | "max-pages" | "budget" | "cancelled" | "closed" | "error";
-/** The summary of a run. */
-interface WatchSummary {
-  /** This value counts the checks that gave an event. */
-  readonly checked: number;
-  readonly new: number;
-  readonly changed: number;
-  readonly unchanged: number;
-  readonly gone: number;
-  readonly refused: number;
-  readonly error: number;
-  /** This value counts the changes that wait for the next check. */
-  readonly suspected: number;
-  /** This value counts the changes that the run confirmed. */
-  readonly confirmed: number;
-  /** This value counts the entries that are due at the end of the run. */
-  readonly due: number;
-  readonly stopped: WatchStop;
-  /** This text tells why the run stopped. */
-  readonly reason: string;
-  /** This value is the time of the run in milliseconds, on the clock of the crawler. */
-  readonly elapsed_ms: number;
-}
-/** A run in progress. A `for await` loop gets each event. `summary` resolves when the loop ends. */
-interface WatchRun extends AsyncIterable<WatchEvent> {
-  readonly summary: Promise<WatchSummary>;
-}
-/** This is the watch list of `openCrawler`. Each method gives a promise, except `run`. */
-interface WatchList {
-  /** This method adds a page, or changes the options of its watch. A new entry is due at once. */
-  add(url: string, options?: WatchOptions): Promise<WatchEntry>;
-  /** This method removes a page from the list. It gives false when the list does not have the page. */
-  remove(url: string): Promise<boolean>;
-  /** This method gives the entries, the first due first. */
-  list(): Promise<WatchEntry[]>;
-  /** This method checks the entries that are due, to a budget. A `for await` loop gets each event. */
-  run(options?: WatchRunOptions): WatchRun;
-}
-/** These options open a watch store. */
-interface WatchStoreOptions {
-  /** This value is the time in milliseconds that a step waits for the lock of a different process. The default is 10 seconds. */
-  readonly busyTimeoutMs?: number | undefined;
-  /** This value is the shortest `every` in milliseconds. The default is 1 hour. */
-  readonly floorMs?: number | undefined;
-}
-interface Row {
-  url: string;
-  target: string;
-  host: string;
-  every_ms: number;
-  render: string;
-  confirm: number;
-  scope: string | null;
-  labels: string | null;
-  on_behalf_of: string | null;
-  added_at: number;
-  next_at: number;
-  lease_until: number;
-  lease_owner: string | null;
-  checks: number;
-  changes: number;
-  last_at: number | null;
-  last_kind: string | null;
-  last_note: string | null;
-  doc: string | null;
-  base_state: string | null;
-  base_sha: string | null;
-  base_capture: string | null;
-  base_fetched_at: string | null;
-  pend_state: string | null;
-  pend_sha: string | null;
-  pend_capture: string | null;
-  pend_fetched_at: string | null;
-  pend_since: number | null;
-}
-/** The state of a page after a check: what a later check compares. */
-interface State {
-  readonly state: "present" | "gone";
-  readonly sha: string | null;
-  readonly capture: string | null;
-  readonly fetched_at: string | null;
-}
-/** This is the change of an entry after a check. */
-interface Update {
-  readonly next_at: number;
-  readonly last_at: number;
-  readonly last_kind: WatchKind;
-  readonly last_note: string | null;
-  readonly changed: boolean;
-  readonly doc: string | null;
-  /** The new confirmed state, or undefined when it stays. */
-  readonly base?: State | undefined;
-  /** The new suspected state, null to clear it, or undefined when it stays. */
-  readonly pending?: (State & {
-    readonly since: number;
-  }) | null | undefined;
-}
-/**
- * This store keeps the watch list in a SQLite file (`watch.sqlite`). Two or more processes on one
- * store share it. Each step is one transaction (`BEGIN IMMEDIATE`), and it waits for the lock of a
- * different process (`busy_timeout`).
- *
- * The list is the decision of the operator, not a projection of the capture log. Thus it has its
- * own file: a deletion of `hosts.sqlite` or of the index does not lose it.
- */
-declare class WatchStore {
-  readonly path: string;
-  private readonly db;
-  private readonly statements;
-  private closed;
-  /** This value is the shortest `every` in milliseconds. */
-  readonly floorMs: number;
-  constructor(path: string, options?: WatchStoreOptions);
-  private stmt;
-  /** This method does one step in one transaction. */
-  private step;
-  private row;
-  /**
-   * This method adds a page, or changes the options of its watch and keeps its state. A new entry
-   * is due at once. A changed entry is due one `every` after its last check. A bad option throws
-   * a `RangeError`.
-   */
-  add(url: string, options: WatchOptions, now: number): WatchEntry;
-  /** This method removes a page. It gives false when the list does not have the page. */
-  remove(url: string): boolean;
-  /** This method gives the entry of a page, or undefined. */
-  get(url: string, now: number): WatchEntry | undefined;
-  /** This method gives the entries, the first due first. */
-  list(now: number): WatchEntry[];
-  /** This method counts the entries that are due at a time. */
-  due(now: number): number;
-  /**
-   * This method claims the first due entry with no lease, on a host that is not in `busy`. The
-   * lease of `owner` holds until `until`. Thus two processes do not get one entry at one time.
-   */
-  claim(now: number, owner: string, until: number, busy: readonly string[]): Row | undefined;
-  /** This method ends the lease of `owner` and changes nothing else, so the entry stays due. A closed store does nothing. */
-  release(key: string, owner: string): void;
-  /**
-   * This method writes the result of a check and ends the lease. It gives false when `owner` does
-   * not hold the lease, or when the store is closed.
-   */
-  settle(key: string, owner: string, update: Update): boolean;
-  /** This method closes the file. A second close does nothing. */
-  close(): void;
-}
-/**
- * This function checks the entries of a watch store that are due. `PoliteCrawler.watch.run` uses
- * it. Each check is a fetch of the page with `fetchPage`, with no stored copy (`maxAgeMs: 0`), so
- * each polite rule applies. A check that the budget stops before its request stays due. The store
- * can be a function that opens it: the run uses it before the first check.
- */
-declare const runWatch: (runtime: CrawlRuntime, source: WatchStore | (() => Promise<WatchStore>), options?: WatchRunnerOptions) => WatchRun;
-//#endregion
-//#region src/open.d.ts
-/**
- * This function gives the default `User-Agent` header. The header names the crawler, its version
- * and the page about it, so a site owner can find out what made each request.
- */
-declare const defaultUserAgent: (contact?: string) => string;
-/**
- * This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,
- * section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in
- * the `User-Agent` only.
- */
-declare const isMailbox: (contact: string) => boolean;
-/**
- * These options make a runtime. A store needs a log, a blob store and an index. Give `dir`, and
- * the runtime makes the three parts in it, or give the parts. A part that you give replaces the
- * part in `dir`.
- */
-interface RuntimeOptions {
-  /** This folder keeps the capture log (`captures/`), the bodies (`blobs/`), the index (`index.sqlite`) and the state of each host (`hosts.sqlite`). */
-  readonly dir?: string | undefined;
-  /** This log replaces `<dir>/captures`. */
-  readonly log?: CaptureLog | undefined;
-  /** This blob store replaces `<dir>/blobs`. */
-  readonly blobs?: BlobStore | undefined;
-  /** This index replaces `<dir>/index.sqlite`. A runtime takes any `ReadingIndex`. */
-  readonly index?: ReadingIndex | undefined;
-  /**
-   * This store keeps the pace and the stops of each host. The default with `dir` is a SQLite file
-   * (`hosts.sqlite`), so the processes on one store share the pace. With no `dir`, it is in memory.
-   */
-  readonly hostState?: HostStateStore | undefined;
-  /** This token is the name that robots.txt rules match. The default is `PoliteCrawler`. */
-  readonly token?: string | undefined;
-  /** This text is the full `User-Agent` header. The default names the crawler and its page. */
-  readonly userAgent?: string | undefined;
-  /**
-   * This address goes into the `User-Agent` header. A mailbox (`ops@example.com`) also goes into the
-   * `From` header, and a URL does not. The crawler sends no contact without it.
-   */
-  readonly contact?: string | undefined;
-  /**
-   * This browser renders the pages that need JavaScript. The value `false` turns rendering off. The
-   * default checks one time that `playwright-core` and its browser are installed. With no browser,
-   * each page that needs JavaScript gets a note at once, with no wait.
-   */
-  readonly renderer?: Renderer | false | undefined;
-  /** This value is the number of sites that the crawler fetches from at one time. The default is 16. */
-  readonly maxActive?: number | undefined;
-  /** This value is the number of pages that the browser renders at one time. The default is 2: each render holds a browser context. */
-  readonly maxRenders?: number | undefined;
-  /** This `undici` dispatcher sends the requests. Tests give a mock agent, and a proxy is possible. */
-  readonly dispatcher?: Dispatcher | undefined;
-  /** This policy controls the addresses that the crawler fetches. By default, it fetches only public addresses. */
-  readonly egress?: EgressOptions | undefined;
-  /** These values change the limits of the politeness policy, for each host or for some hosts. */
-  readonly politeness?: PolicyOptions | undefined;
-  /** This clock gives the time to the crawler and to the index. Tests give a clock that they control. */
-  readonly clock?: Clock | undefined;
-  /** This function gives the random numbers of the jitter. The default is `Math.random`. */
-  readonly random?: (() => number) | undefined;
-  /** This label goes into each capture record. */
-  readonly scope?: string | undefined;
-  /** With the value false, the crawler does not also obey the group of `PoliteCrawler` when its `User-Agent` links the bot page (`CrawlerOptions.obeyDefaultToken`). */
-  readonly obeyDefaultToken?: boolean | undefined;
-  /** This function ranks the scopes, the most private highest. A shared fetch of a robots.txt records the scope with the highest rank (`CrawlerOptions.scopeRank`). */
-  readonly scopeRank?: ((scope: string) => number) | undefined;
-  /** This function gets an event for each request, response, wait, stop and skip. */
-  readonly onEvent?: ((event: CrawlEvent) => void) | undefined;
-  /** These extractors read the captures of the recipes of an application. The recipes `page` and `archive-copy` are built in. */
-  readonly extractors?: Readonly<Record<string, Extractor>> | undefined;
-  /**
-   * These recipes of `defineRecipe` get their extractors at open. Thus `reindex` reads their
-   * captures before the first fetch. A recipe with the name of an entry of `extractors` throws an error.
-   */
-  readonly recipes?: ReadonlyArray<Recipe<never>> | undefined;
-  /**
-   * This signer signs each request with Web Bot Auth (`CrawlerOptions.signer`). Give the options of
-   * `createWebBotAuthSigner`, or a `RequestSigner`. Without it, the crawler signs nothing.
-   */
-  readonly signer?: RequestSigner | WebBotAuthOptions | undefined;
-}
-/** A runtime that a store made. It owns the parts that it made, and it closes them. */
-interface StoreRuntime extends CrawlRuntime {
-  readonly hostState: HostStateStore;
-  /** This method closes the crawler, the browser, the log, the index and the host state store that the runtime made. A second close does nothing. */
-  close(): Promise<void>;
-}
-/**
- * This function makes a runtime. It is the one place that puts the log, the blob store, the index,
- * the host state, the crawler and the extractors together. The function `openCrawler` uses it.
- * An application that needs the parts uses it too. The runtime and the crawler share one clock.
- */
-declare const createRuntime: (options: RuntimeOptions) => StoreRuntime;
-/** These options open a crawler on a folder. Only `dir` is necessary. */
-interface OpenOptions extends Omit<RuntimeOptions, "dir" | "log" | "blobs" | "index"> {
-  /** This folder keeps the capture log, the response bodies, the search index and the state of each host. */
-  readonly dir: string;
-  /** These agents get their robots.txt rules applied to each fetch, for example `Claude-User`. */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** This value is the shortest `every` of a watch. The default is `1h`, so a watch cannot hammer a site. */
-  readonly watchFloor?: Duration | undefined;
-  /**
-   * This embedder makes the vectors of hybrid search (`rank: "hybrid"`): `transformersEmbedder`,
-   * `ollamaEmbedder` or an embedder of your own. The vectors go into `vectors.sqlite` in the store.
-   * Without it, `rank: "hybrid"` throws an error. `close` also closes the embedder.
-   */
-  readonly embedder?: Embedder | undefined;
-}
-/** These options control one fetch. */
-interface FetchOptions {
-  /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */
-  readonly render?: "auto" | "never" | "always" | undefined;
-  /**
-   * A stored copy younger than this age answers, and the crawler sends no request. The default is
-   * `24h`. The value `auto` uses the freshness that the server gave. The value `0s` asks again.
-   */
-  readonly maxAge?: Duration | undefined;
-  /** These agents get their robots.txt rules applied to this fetch, in addition to the defaults. */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** When this signal aborts, the crawler stops the fetch and the render. */
-  readonly signal?: AbortSignal | undefined;
-  /** This label goes into the capture records of this fetch. */
-  readonly scope?: string | undefined;
-  /** These labels go into the capture records of this fetch. They are not part of the capture ids. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-  /**
-   * The value `archive` gives a copy from a public web archive (the Wayback Machine, then Common
-   * Crawl) when the site refuses the page or is down. The result then has `archived`. The default is `none`.
-   */
-  readonly fallback?: "archive" | "none" | undefined;
-  /**
-   * The value true also gives an archived copy of a page that the current robots.txt of its site
-   * disallows for this agent. The default is false: only a stop, an error or a dead host gets a copy.
-   */
-  readonly archiveDespiteRobots?: boolean | undefined;
-}
-/** These options control a search. */
-interface FindOptions {
-  /** This value is the maximum number of results. The default is 10. */
-  readonly limit?: number | undefined;
-  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
-  readonly kind?: string | null | undefined;
-  /**
-   * This value chooses the ranking. The default `exact` ranks each match with BM25. `fast` leaves
-   * the common words out of the second part of the search, so a long question is faster (`RankMode`).
-   */
-  readonly rank?: RankMode | undefined;
-}
-/**
- * These options control a search of `openCrawler`: the options of `find`, and `rank: "hybrid"`
- * with the options of a hybrid search. The mode `hybrid` needs `OpenOptions.embedder`.
- */
-interface SearchOptions extends Omit<FindOptions, "rank">, HybridFindOptions {
-  /** This value chooses the ranking. `hybrid` fuses BM25 with the ranking of the vectors (`SearchRank`). */
-  readonly rank?: SearchRank | undefined;
-}
-/** These options control `rankWithin` of a crawler. */
-interface RankOptions extends HybridOptions {
-  /** This value chooses the ranking. The default is `exact`. */
-  readonly rank?: SearchRank | undefined;
-}
-/** A polite crawler fetches each source one time, keeps it, and answers the next questions from the disk. */
-interface PoliteCrawler {
-  /** The runtime gives the parts to an application that needs more control. */
-  readonly runtime: StoreRuntime;
-  /** This method fetches a page, reads it and adds it to the index. */
-  fetch(url: string, options?: FetchOptions): Promise<PageResult>;
-  /**
-   * This method searches the stored documents. It sends no request. A held-back document is not
-   * in the results. The syntax of the query is the syntax of `parseQuery`.
-   */
-  find(q: string, options?: FindOptions): Promise<Found[]>;
-  /** This method reads a range of characters of one stored document, the current version or the version of one capture. */
-  read(doc: string, options?: ReadOptions): Promise<ReadResult>;
-  /**
-   * This method finds a quote in a stored document. It ignores case, spaces, typographic
-   * punctuation and Unicode forms. It tells which capture holds the quote.
-   */
-  verify(doc: string, quote: string, options?: VerifyOptions): Promise<VerifyResult>;
-  /** This method gives the versions of a stored document, the newest first. A held-back version shows no title. */
-  versions(doc: string, options?: {
-    readonly limit?: number;
-  }): Promise<DocVersion[]>;
-  /**
-   * This method builds the index again from the capture log and the stored bodies, with no request.
-   * When the crawler rebuilt a stale index at open, the first reindex gives the counts of that
-   * rebuild, unless a fetch came before it.
-   */
-  reindex(): Promise<ReindexResult>;
-  /** This method closes the browser, the log files and the index. A second close does nothing. */
-  close(): Promise<void>;
-  /**
-   * This optional method finds the pages of a site from its sitemaps and its feeds. Each file is a
-   * polite fetch with its capture. It keeps the URLs on the site that robots.txt permits, to its
-   * limits. `openCrawler` gives it. Over a runtime of your own, use `discoverSite`.
-   */
-  discover?(site: string, options?: DiscoverOptions): Promise<DiscoverResult>;
-  /**
-   * This optional method crawls a site from its seeds. It follows the links of each page to a
-   * maximum of pages, a maximum depth and a maximum time. A `for await` loop gets each page.
-   * `summary` tells why the crawl stopped. `openCrawler` gives it. Over a runtime of your own, use
-   * `crawlSite`.
-   */
-  crawl?(seeds: string | readonly string[], options?: CrawlOptions): CrawlRun;
-  /**
-   * This optional method compares two versions of a stored document: the lines and the words of
-   * the text, and the metadata at its paths. It sends no request. `openCrawler` gives it.
-   */
-  diff?(doc: string, options?: DiffOptions): Promise<DiffResult>;
-  /** This optional list keeps the pages that the crawler checks again on a schedule. `openCrawler` gives it. */
-  readonly watch?: WatchList;
-  /**
-   * This optional method checks the proposed values of fields against a stored document, and it
-   * locates each value. It does not make values. It sends no request. `openCrawler` gives it.
-   */
-  extract?(doc: string, fields: Readonly<Record<string, FieldClaim>>, options?: ExtractOptions): Promise<ExtractResult>;
-  /** This optional search ranks with the vectors of an embedder. `openCrawler` gives it when it has an embedder. */
-  readonly hybrid?: HybridSearch | undefined;
-  /** This optional method gives vectors to the stored documents that have none, within a budget. */
-  embed?(options?: EmbedOptions): Promise<EmbedResult>;
-  /** This optional method ranks a set of stored documents against a query, the best first. */
-  rankWithin?(q: string, docs: readonly string[], options?: RankOptions): Promise<Ranked[]>;
-}
-/** This crawler is the result of `openCrawler` in version 0.1.2: a `PoliteCrawler` with `discover`, `crawl`, `diff`, `watch` and `extract`. */
-interface OpenedCrawler extends PoliteCrawler {
-  /** This method finds the pages of a site from its sitemaps and its feeds. */
-  discover(site: string, options?: DiscoverOptions): Promise<DiscoverResult>;
-  /** This method crawls a site from its seeds, within its limits. A `for await` loop gets each page. */
-  crawl(seeds: string | readonly string[], options?: CrawlOptions): CrawlRun;
-  /** This method compares two versions of a stored document. It sends no request. */
-  diff(doc: string, options?: DiffOptions): Promise<DiffResult>;
-  /** This list keeps the pages that the crawler checks again on a schedule, in `watch.sqlite`. */
-  readonly watch: WatchList;
-  /**
-   * This method checks the proposed values of fields against a stored document. Each field gets
-   * `verified`, `normalized`, `ambiguous`, `not-found`, `held-back` or `invalid`.
-   */
-  extract(doc: string, fields: Readonly<Record<string, FieldClaim>>, options?: ExtractOptions): Promise<ExtractResult>;
-}
-/**
- * This crawler is the result of `openCrawler`: an `OpenedCrawler` with hybrid search. Its `find`
- * takes `rank: "hybrid"`, and it has `hybrid`, `embed` and `rankWithin`. `OpenedCrawler` does not
- * change, so an object of your own that implements the `OpenedCrawler` of 0.1.2 stays correct.
- */
-interface HybridCrawler extends OpenedCrawler {
-  /** This method searches the stored documents. It also takes `rank: "hybrid"`, when the crawler has an embedder. */
-  find(q: string, options?: SearchOptions): Promise<Found[]>;
-  /** This search ranks with the vectors of the embedder, or it is undefined when the crawler has no embedder. */
-  readonly hybrid: HybridSearch | undefined;
-  /** This method gives vectors to the stored documents that have none, within a budget. It rejects when the crawler has no embedder. */
-  embed(options?: EmbedOptions): Promise<EmbedResult>;
-  /**
-   * This method ranks a set of stored documents against a query, the best first. The modes
-   * `exact` and `fast` are `DocIndex.rankWithin`. The mode `hybrid` needs an embedder.
-   */
-  rankWithin(q: string, docs: readonly string[], options?: RankOptions): Promise<Ranked[]>;
-}
-/**
- * This function opens a crawler on a folder. The folder keeps the capture log (`captures/`), the
- * response bodies (`blobs/`), the search index (`index.sqlite`) and the state of each host
- * (`hosts.sqlite`). The index is a projection: `reindex` makes it again from the log and the bodies.
- * At open, `catchUpIndex` rebuilds an index of an earlier version, and gives an index that is
- * behind the log the captures that it missed.
- *
- * Each method gives a promise, and none throws before its promise. After `close`, each method
- * rejects.
- */
-declare const openCrawler: (options: OpenOptions) => HybridCrawler;
-//#endregion
-//#region src/daemon.d.ts
-/** These are the defaults of the daemon. */
-declare const DAEMON_DEFAULTS: {
-  /** The longest wait between two ticks: 1 minute. A tick comes sooner when an entry is due sooner. */
-  readonly tickMs: 60_000;
-  /** The most random time that the daemon adds to each wait: 10 seconds. */
-  readonly jitterMs: 10_000;
-  /** The budget of the checks of one tick: 10 minutes. No request starts after it. */
-  readonly budgetMs: 600_000;
-  /** The most checks of one tick. */
-  readonly maxPages: 100;
-  /** At a stop, the checks in progress get this time to end: 10 seconds. Then they abort. */
-  readonly graceMs: 10_000;
-  /** The shortest wait between two ticks: 1 second. */
-  readonly minGapMs: 1_000;
-  /** The shortest tick: 1 second. */
-  readonly minTickMs: 1_000;
-  /** The status lists this number of the next due entries. */
-  readonly dueShown: 10;
-  /** An on-change command that runs longer than this time stops: 1 minute. */
-  readonly hookTimeoutMs: 60_000;
-};
-/** This value is the version of the format of the state file. */
-declare const DAEMON_STATE_SCHEMA = 1;
-/** This is a watch entry that is due soon, as the status shows it. */
-interface DaemonDue {
-  readonly url: string;
-  /** This time (ISO 8601) is the time when the entry is due. */
-  readonly next_at: string;
-  readonly every_ms: number;
-  /** This value is the kind of the last check, or null before the first check. */
-  readonly last_kind: string | null;
-}
-/** This is one tick of the schedule: one run of the watch list. */
-interface DaemonTick {
-  /** This time (ISO 8601) is the start of the run. */
-  readonly started_at: string;
-  /** This time (ISO 8601) is the end of the run. */
-  readonly ended_at: string;
-  /** This value is the summary of the run, or null when the run failed. */
-  readonly summary: WatchSummary | null;
-  /** This text tells why the run failed, or null. */
-  readonly error: string | null;
-  /** This time (ISO 8601) is the time of the next tick, or null when the schedule stops. */
-  readonly next_at: string | null;
-  /** These entries are the next due entries after the run, the first due first. */
-  readonly due: readonly DaemonDue[];
-}
-/** These options control the schedule of the watch list. */
-interface ScheduleOptions {
-  /** This value is the longest wait between two ticks. The default is `1m`, and the floor is `1s`. */
-  readonly tick?: Duration | undefined;
-  /** This value is the most random time that the schedule adds to each wait. The default is `10s`. */
-  readonly jitter?: Duration | undefined;
-  /** This value is the budget of the checks of one tick. The default is `10m`. */
-  readonly budget?: Duration | undefined;
-  /** This value is the most checks of one tick. The default is 100. */
-  readonly maxPages?: number | undefined;
-  /** This clock gives the time and the waits. Give the clock of the crawler. The default is the real clock. */
-  readonly clock?: Clock | undefined;
-  /** This function gives the random numbers of the jitter. The default is `Math.random`. */
-  readonly random?: (() => number) | undefined;
-  /** This function gets each event of each check. */
-  readonly onEvent?: ((event: WatchEvent) => void) | undefined;
-  /** This function gets each tick when its run ends. */
-  readonly onTick?: ((tick: DaemonTick) => void) | undefined;
-  /** This function gets the error of a run that failed. The schedule goes on at the next tick. */
-  readonly onError?: ((error: Error) => void) | undefined;
-}
-/** This is the result of a schedule: the counts so far, and the last tick. */
-interface ScheduleResult {
-  readonly ticks: number;
-  /** This value counts the checks that gave an event. */
-  readonly checked: number;
-  /** This value counts the confirmed changes. */
-  readonly changes: number;
-  readonly last: DaemonTick | null;
-}
-/** A schedule that runs. `stop` ends it, and `done` resolves when it ends. */
-interface WatchSchedule {
-  /** This promise resolves when the schedule ends. It does not reject. */
-  readonly done: Promise<ScheduleResult>;
-  /**
-   * This method stops the schedule. The run in progress starts no new check. The checks in
-   * progress get `grace` (10 seconds by default) to end and write their state. Then they abort,
-   * and the run releases their leases. A second call with a shorter grace shortens it.
-   */
-  stop(grace?: Duration): Promise<ScheduleResult>;
-  /** This method gives the counts so far. */
-  result(): ScheduleResult;
-}
-/**
- * This function checks a watch list on a schedule. The first tick starts at once. Each tick is one
- * `watch.run` with the budget and `maxPages` of a tick. Thus each polite rule of the run applies:
- * one request at a time to each host, the stops of a host, `Retry-After` and robots.txt.
- *
- * After a run, the schedule waits for the next tick (`nextTickAt`). An error of a run goes to
- * `onError`, and the schedule goes on.
- */
-declare const scheduleWatches: (watch: WatchList, options?: ScheduleOptions) => WatchSchedule;
-/** This is a task of the MCP server, as the status shows it. */
-interface DaemonTask {
-  readonly id: string;
-  /** This value is the name of the tool, for example `crawl_site`. */
-  readonly tool: string;
-  /** This value is `working`, `input_required`, `completed`, `failed` or `cancelled`. */
-  readonly status: string;
-  /** This text tells the progress, for example `3 of 10 done`, or null. */
-  readonly message: string | null;
-  readonly created_at: string;
-  readonly updated_at: string;
-}
-/** This is the status of a daemon. The state file (`<store>/daemon/state.json`) holds the same JSON. */
-interface DaemonStatus {
-  /** This value is the version of the format (`DAEMON_STATE_SCHEMA`). */
-  readonly schema: number;
-  /** This value is the version of polite-crawler. */
-  readonly version: string;
-  /** This value is the id of the process of the daemon. */
-  readonly pid: number;
-  readonly store: string;
-  /** This value is false after the daemon stopped. */
-  readonly running: boolean;
-  readonly started_at: string;
-  readonly updated_at: string;
-  readonly stopped_at: string | null;
-  readonly tick_ms: number;
-  readonly jitter_ms: number;
-  readonly budget_ms: number;
-  readonly max_pages: number;
-  /** This value counts the ticks. */
-  readonly ticks: number;
-  /** This value counts the checks. */
-  readonly checked: number;
-  /** This value counts the confirmed changes. */
-  readonly changes: number;
-  readonly last_tick: DaemonTick | null;
-  /** This time (ISO 8601) is the time of the next tick, or null. */
-  readonly next_tick_at: string | null;
-  /** These entries are the next due entries at the last tick, the first due first. */
-  readonly next_due: readonly DaemonDue[];
-  /** These tasks of the MCP server did not end. */
-  readonly tasks: readonly DaemonTask[];
-  /** This value is the MCP endpoint and the file of its token, or null with no MCP server. */
-  readonly mcp: {
-    readonly url: string;
-    readonly token_file: string | null;
-  } | null;
-  /** This value is the last error: of a tick, of the event log, of an on-change hook or of the MCP server. */
-  readonly last_error: {
-    readonly at: string;
-    readonly message: string;
-  } | null;
-}
-/** This is a status that `readDaemonState` read from a state file. */
-interface DaemonState extends DaemonStatus {
-  /** This value is true when the daemon runs: the state says so, and its process is alive. */
-  readonly alive: boolean;
-}
-/** This function gives the path of the state file of a store. */
-declare const daemonStateFile: (dir: string) => string;
-/**
- * This function reads the state file of a daemon. It gives undefined when the file is not there,
- * or when it is not a state file. `alive` tells if the daemon runs.
- */
-declare const readDaemonState: (dir: string, file?: string) => DaemonState | undefined;
-/** These options start an on-change command. */
-interface ChangeCommandOptions {
-  /** A command that runs longer than this time stops. The default is `1m`. */
-  readonly timeout?: Duration | undefined;
-  /** This folder is the working folder of the command. The default is the working folder of the daemon. */
-  readonly cwd?: string | undefined;
-}
-/**
- * This function gives an `onChange` hook that starts a command of the operator for each change.
- * The command operates in the shell of the system, as the user of the daemon. It gets the event as
- * one JSON line on its standard input, not in its command line. Thus no text of a page goes into
- * the command line. The output of the command goes nowhere. A command that exits with a code that
- * is not 0, or that runs too long, rejects with the end of its standard error.
- *
- * The command is the decision of the operator. Do not take it from a page or from an assistant.
- */
-declare const changeCommand: (command: string, options?: ChangeCommandOptions) => ((event: WatchEvent) => Promise<void>);
-/** These options serve the MCP tools over HTTP. */
-interface DaemonMcpOptions {
-  /** This value is the port. The default is 9307. The value 0 takes a free port. */
-  readonly port?: number | undefined;
-  /** This value is the host to bind. The default is `127.0.0.1`. A host that is not loopback needs `allowRemote`. */
-  readonly host?: string | undefined;
-  /** The value true permits a host that is not loopback. Read the warning in the README first. */
-  readonly allowRemote?: boolean | undefined;
-  /** This value is the bearer token. The default makes a new token at each start, in `<store>/daemon/mcp-token`. */
-  readonly token?: string | undefined;
-  /** This file holds the bearer token of the operator. The daemon reads it and writes nothing. */
-  readonly tokenFile?: string | undefined;
-  /** These tokens get their robots.txt rules applied to each fetch of a tool. The default is `["Claude-User"]`. */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** The value true lets `user_provided` release held-back text when the client cannot ask the user. */
-  readonly allowUserProvided?: boolean | undefined;
-  /** This text tells why the server renders no page. */
-  readonly renderOff?: string | undefined;
-}
-/** These functions connect an MCP host to the daemon. */
-interface DaemonMcpHooks {
-  /** This function gives the status of the daemon, for the status route. */
-  readonly status: () => DaemonStatus;
-  /** This function operates when a task starts, changes or ends. */
-  readonly onTasks: () => void;
-  /** This function writes a message of the server. */
-  readonly log: (line: string) => void;
-}
-/** This is the MCP server of the daemon, as the MCP host gives it. */
-interface DaemonMcpService {
-  /** This URL is the MCP endpoint. */
-  readonly url: string;
-  /** This method gives the tasks that did not end. */
-  tasks(): readonly DaemonTask[];
-  /** This method stops the server and cancels its tasks. */
-  close(): Promise<void>;
-}
-/**
- * This is the seam between the daemon and its MCP server. The default host (`mcpHttpHost` of
- * `polite-crawler/mcp`) serves the tools over Streamable HTTP. A different MCP host can replace it.
- */
-interface DaemonMcpHost {
-  start(crawler: OpenedCrawler, options: DaemonMcpOptions, hooks: DaemonMcpHooks): Promise<DaemonMcpService>;
-}
-/** These options start a daemon on a store. Only `dir` is necessary. */
-interface DaemonOptions extends OpenOptions {
-  /** This value is the longest wait between two ticks. The default is `1m`, and the floor is `1s`. */
-  readonly tick?: Duration | undefined;
-  /** This value is the most random time that the daemon adds to each wait. The default is `10s`. */
-  readonly jitter?: Duration | undefined;
-  /** This value is the budget of the checks of one tick. The default is `10m`. */
-  readonly budget?: Duration | undefined;
-  /** This value is the most checks of one tick. The default is 100. */
-  readonly maxPages?: number | undefined;
-  /** At a stop, the checks in progress get this time to end. The default is `10s`. */
-  readonly grace?: Duration | undefined;
-  /**
-   * The daemon writes each change event as one JSON line here: a file, a function, or the standard
-   * output (the default). The value false writes no line. A change is an event with `confirmed: true`.
-   */
-  readonly events?: string | ((line: string) => void) | false | undefined;
-  /** This function gets each change event. The daemon starts it in order, one at a time, and logs its error. */
-  readonly onChange?: ((event: WatchEvent) => void | Promise<void>) | undefined;
-  /** These options serve the MCP tools over HTTP. Without them, the daemon serves no MCP. */
-  readonly mcp?: DaemonMcpOptions | undefined;
-  /** This host serves the MCP tools. The default serves them over Streamable HTTP. */
-  readonly mcpHost?: DaemonMcpHost | undefined;
-  /** The value false installs no handler of `SIGINT` and `SIGTERM`. The default is true. */
-  readonly signals?: boolean | undefined;
-  /** This file keeps the state of the daemon. The default is `<store>/daemon/state.json`. */
-  readonly stateFile?: string | undefined;
-  /** This function writes the messages of the daemon. The default writes them to the standard error. */
-  readonly log?: ((line: string) => void) | undefined;
-}
-/** This is a daemon that runs. */
-interface Daemon {
-  /** This crawler is the crawler of the daemon. The daemon closes it when it stops. */
-  readonly crawler: OpenedCrawler;
-  /** This file keeps the state of the daemon. */
-  readonly stateFile: string;
-  /** This value is the MCP server, or undefined. */
-  readonly mcp: DaemonMcpService | undefined;
-  /** This method gives the status at this time. */
-  status(): DaemonStatus;
-  /**
-   * This method stops the daemon. The checks in progress get the grace time to end, then they
-   * abort and the run releases their leases. The MCP server stops and cancels its tasks. A second
-   * call with `0` aborts the checks at once.
-   */
-  stop(grace?: Duration): Promise<DaemonStatus>;
-  /** This promise resolves with the last status when the daemon stopped. */
-  readonly done: Promise<DaemonStatus>;
-}
-/**
- * This function starts a daemon on a store. It opens the crawler, serves MCP when `mcp` is given,
- * and checks the watch list on a schedule (`scheduleWatches`). It writes each change event as one
- * JSON line, and it gives each change to `onChange`. The state file tells `daemon status` what runs.
- *
- * Only one daemon operates on a store: a second start rejects. At `SIGINT` or `SIGTERM`, the daemon
- * stops cleanly (`stop`). A second signal aborts the checks at once.
- */
-declare const startDaemon: (options: DaemonOptions) => Promise<Daemon>;
-//#endregion
-export { RECIPE_LIMITS as $, FetchOutcome as $n, ReadOptions as $r, checkFields as $t, RuntimeOptions as A, PlaywrightRenderer as Ai, DiffHunk as An, Found as Ar, VectorWrite as At, WatchEvent as B, VersionChoice as Bn, Archived as Br, ExtractOptions as Bt, FetchOptions as C, ScanOptions as Ci, SitemapEntry as Cn, DocDigest as Cr, createHybridSearch as Ct, OpenedCrawler as D, Duration as Di, feedDate as Dn, ExtractedDoc as Dr, VectorHit as Dt, OpenOptions as E, makeRecord as Ei, SitemapRead as En, DocVersion as Er, VectorCounts as Et, isMailbox as F, Renderer as Fi, DiffResult as Fn, ReadingIndex as Fr, CrawlRunOptions as Ft, WatchRun as G, CacheEntry as Gn, CatchUpResult as Gr, FieldCandidate as Gt, WatchList as H, diffDocument as Hn, archiveExtractor as Hr, ExtractSource as Ht, openCrawler as I, playwrightRenderer as Ii, DiffSide as In, SearchRank as Ir, CrawlStop as It, WatchStop as J, CrawlEvent as Jn, Extractors as Jr, FieldKind as Jt, WatchRunOptions as K, CacheIndex as Kn, CrawlRuntime as Kr, FieldCheck as Kt, WATCH_LIMITS as L, BlobStore as Li, DiffSummary as Ln, Variant as Lr, CrawlSummary as Lt, StoreRuntime as M, RenderOptions as Mi, DiffLimits as Mn, NO_EMBEDDER as Mr, CrawlOptions as Mt, createRuntime as N, RenderRefused as Ni, DiffLine as Nn, Passage as Nr, CrawlPage as Nt, PoliteCrawler as O, durationMs as Oi, readSitemap as On, FAST_RANK_SHARE as Or, VectorStore as Ot, defaultUserAgent as P, Rendered as Pi, DiffOptions as Pn, RankMode as Pr, CrawlRun as Pt, runWatch as Q, DEFAULT_SECRET_PARAMS as Qn, PageResult as Qr, FieldsChecked as Qt, WATCH_SCHEMA as R, FileBlobStore as Ri, MetaChange as Rn, VersionAt as Rr, UrlPattern as Rt, startDaemon as S, DamagedLine as Si, SITEMAP_LIMITS as Sn, Doc as Sr, Ranked as St, HybridCrawler as T, RecordInput as Ti, SitemapLimits as Tn, DocInput as Tr, VECTORS_SCHEMA as Tt, WatchOptions as U, diffTexts as Un, archivedDocId as Ur, ExtractSummary as Ut, WatchKind as V, WordPart as Vn, WebArchive as Vr, ExtractResult as Vt, WatchPending as W, CAPTURE_SCHEMA as Wn, CapturePayload as Wr, FIELD_KINDS as Wt, WatchStoreOptions as X, CrawlerOptions as Xn, Indexed as Xr, FieldSpan as Xt, WatchStore as Y, Crawler as Yn, HELD_BACK as Yr, FieldResult as Yt, WatchSummary as Z, DEADLINE_NOTE as Zn, PageOptions as Zr, FieldStatus as Zt, WatchSchedule as _, verifyQuote as _i, DiscoverSource as _n, verifySignature as _r, EmbedStop as _t, DaemonDue as a, fetchAndIndex as ai, FusedRank as an, KEY_DIRECTORY_PATH as ar, RecipeExtract as at, readDaemonState as b, CaptureLog as bi, discoverSite as bn, egressLookup as br, HybridSearch as bt, DaemonMcpOptions as c, heldBackMeta as ci, chunkText as cn, KeyInput as cr, RecipeResult as ct, DaemonState as d, pageDocId as di, fuseRanks as dn, SignatureCheck as dr, defineRecipe as dt, ReadResult as ei, extractDocument as en, FetchRequest as er, RESERVED_RECIPES as et, DaemonStatus as f, pageExtractor as fi, unitVector as fn, SignedRequest as fr, readJson as ft, ScheduleResult as g, renderAndIndex as gi, DiscoverRunOptions as gn, keyDirectory as gr, EmbedResult as gt, ScheduleOptions as h, reindex as hi, DiscoverResult as hn, createWebBotAuthSigner as hr, EmbedOptions as ht, Daemon as i, catchUpIndex as ii, Embedder as in, DirectoryKey as ir, RecipeDoc as it, SearchOptions as j, RefusedReason as ji, DiffInput as jn, Latest as jr, CRAWL_LIMITS as jt, RankOptions as k, NO_PLAYWRIGHT as ki, DIFF_LIMITS as kn, FindIndexOptions as kr, VectorStoreOptions as kt, DaemonMcpService as l, isHeldBack as li, chunkTokensOf as ln, RequestSigner as lr, RecipeRunOptions as lt, DaemonTick as m, recordOverride as mi, DiscoverOptions as mn, WebBotAuthSigner as mr, runRecipe as mt, DAEMON_DEFAULTS as n, VerifyOptions as ni, ChunkOptions as nn, RenderRequest as nr, RecipeCapture as nt, DaemonMcpHooks as o, fetchPage as oi, HYBRID_DEFAULTS as on, KEY_DIRECTORY_TYPE as or, RecipeRequest as ot, DaemonTask as p, readDocument as pi, DISCOVER_LIMITS as pn, WebBotAuthOptions as pr, registerRecipe as pt, WatchRunnerOptions as q, CaptureWriter as qn, Extractor as qr, FieldClaim as qt, DAEMON_STATE_SCHEMA as r, VerifyResult as ri, EmbedCallOptions as rn, renderedKey as rr, RecipeDefinition as rt, DaemonMcpHost as s, findDocuments as si, TextChunk as sn, KeyDirectory as sr, RecipeRequestOptions as st, ChangeCommandOptions as t, ReindexResult as ti, CHUNK_LIMITS as tn, Fetched as tr, Recipe as tt, DaemonOptions as u, isoTime as ui, estimateTokens as un, SignatureAgentType as ur, RecipeTarget as ut, changeCommand as v, versionAtTime as vi, DiscoverStop as vn, EgressOptions as vr, HybridFindOptions as vt, FindOptions as w, CaptureRecord as wi, SitemapKind as wn, DocIndex as wr, StoredChunk as wt, scheduleWatches as x, CaptureReader as xi, Changefreq as xn, DigestOptions as xr, HybridSearchOptions as xt, daemonStateFile as y, CaptureFilter as yi, DiscoveredUrl as yn, EgressRefused as yr, HybridOptions as yt, WatchEntry as z, TextDiff as zn, VersionText as zr, crawlSite as zt };
-//# sourceMappingURL=daemon-DP4EqjFL.d.mts.map