polite-crawler 0.1.1 → 0.1.3

package/build/open-eHcriTds.d.mts

@@ -1,1499 +0,0 @@
-import { n as Outcome } from "./classify-CrfyXcwo.mjs";
-import { r as Signals, t as AiInput } from "./signals-BHpy2xb2.mjs";
-import { f as PolicyOptions, o as HostStateStore, t as Clock } from "./politeness-yI3xBEx9.mjs";
-import { a as Robots } from "./robots-R-qaEBOb.mjs";
-import { Dispatcher } from "undici";
-import { LookupAddress, LookupAllOptions } from "node:dns";
-import { LookupFunction } from "node:net";
-//#region src/blobs.d.ts
-/**
- * A blob store keeps the bytes of each response, and the hash of the bytes is the key. The crawler
- * needs only these two operations, so an application can give its own store.
- */
-interface BlobStore {
-  put(bytes: Uint8Array): Promise<{
-    readonly hash: string;
-  }>;
-  get(hash: string): Promise<Uint8Array | undefined>;
-}
-/** These options change a file blob store. */
-interface FileBlobStoreOptions {
-  /** When this value is true (the default), a blob is on the disk (`fsync`) before `put` finishes. */
-  readonly sync?: boolean | undefined;
-}
-/**
- * This store keeps each blob in a file: `<dir>/ab/cd/abcd…`, where the name is the SHA-256 hash
- * of the bytes. These are its rules:
- *
- * - Equal bytes are written one time only. A file of the wrong size (a write that a crash cut) is
- *   written again.
- * - A write goes to a temporary file with a random name, goes to the disk, and then gets its name.
- *   A crash thus cannot leave half a blob under its name.
- * - Two writers can put the same bytes at the same time. When the rename of one fails because the
- *   other was first, the put succeeds if the stored file has the same hash.
- * - The method `get` examines the hash of the bytes. A damaged file gets a new name
- *   (`.damaged`), and `get` gives no bytes, so the crawler fetches the body again.
- */
-declare class FileBlobStore implements BlobStore {
-  readonly dir: string;
-  private readonly writing;
-  private readonly sync;
-  constructor(dir: string, options?: FileBlobStoreOptions);
-  put(bytes: Uint8Array): Promise<{
-    readonly hash: string;
-    readonly existed: boolean;
-  }>;
-  /** This method tells if a file holds exactly the bytes of a hash. */
-  private static holds;
-  private write;
-  get(hash: string): Promise<Uint8Array | undefined>;
-}
-//#endregion
-//#region src/render.d.ts
-/**
- * The renderer opens the headless browser only for a page that shows almost nothing without
- * JavaScript. The settings:
- *
- * - The default `chromium-headless-shell` of Playwright, the lighter build.
- * - One browser for many pages, and a new context for each page. Thus no cookie or storage goes
- *   from one site to the next. A context has its own HTTP cache, so routing each request costs
- *   no cache across sites.
- * - Each request of the page goes through a route, the first navigation too. The route refuses
- *   images, fonts and media by their resource type. It asks `allowRequest` about each other
- *   request, and it sends `headers` with it.
- * - Playwright routes only the first request of a redirect chain. Thus with `allowRequest`, the
- *   route sends each request itself, follows each redirect itself and asks about each step. The
- *   browser then gets each body whole, not as a stream.
- * - No download, no popup, no service worker and no dialog.
- * - A wait until the visible text stops growing, with a limit. Playwright discourages
- *   `networkidle`. The text includes open shadow roots and the frames of the same origin.
- * - One deadline for the whole render: the navigation, the wait and the reading of the page.
- * - After a crash or a disconnection, the next page launches a new browser.
- * - The browser closes after two idle minutes. An idle Chromium holds hundreds of megabytes, so
- *   it stays open only while pages need it. The open browser keeps the process alive, so a
- *   script uses `close()` when it is done.
- */
-interface Rendered {
-  readonly html: string;
-  readonly finalUrl: string;
-  readonly status: number | null;
-  readonly headers: Readonly<Record<string, string>>;
-}
-/** These are the options of one render. */
-interface RenderOptions {
-  readonly userAgent: string;
-  /** This value is the deadline of the whole render, in milliseconds. */
-  readonly timeoutMs: number;
-  /** These headers go with each request of the page, for example the `From` header. */
-  readonly headers?: Readonly<Record<string, string>> | undefined;
-  /**
-   * This function decides each request that the page makes, the first navigation included. When
-   * it gives false, the browser does not send the request. The crawler uses it to apply the
-   * egress policy and robots.txt to the requests of the page.
-   */
-  readonly allowRequest?: ((url: string) => Promise<boolean> | boolean) | undefined;
-  /** This value is the maximum number of redirects of one request of the page. The default is 5, as for a plain fetch. */
-  readonly maxRedirects?: number | undefined;
-  /** When this signal aborts, the renderer stops the page: it closes the context, and the render rejects. */
-  readonly signal?: AbortSignal | undefined;
-}
-interface Renderer {
-  render(url: string, options: RenderOptions): Promise<Rendered>;
-  close(): Promise<void>;
-}
-/** The reason of a `RenderRefused`. */
-type RefusedReason = "refused" | "download" | "scheme" | "browser";
-/**
- * This error tells that the renderer did not render the page, and that the page or the site is
- * not at fault. The reason is one of these:
- *
- * - `refused`: `allowRequest` refused the first navigation, a step of its redirects, or a later
- *   navigation of the page that left it on an error page. `url` names the refused URL.
- * - `download`: the URL gives a file, not a page.
- * - `scheme`: the URL is not an http or https URL.
- * - `browser`: the renderer has no browser (`playwright-core` or its browser is not installed). The
- *   crawler then renders no more pages, and gives each one the note at once.
- *
- * A crawler does not count this error as a failure of the host.
- */
-declare class RenderRefused extends Error {
-  readonly reason: RefusedReason;
-  /** This URL is the request that `allowRequest` or the limit of redirects refused, when the reason is `refused`. */
-  readonly url: string | undefined;
-  constructor(reason: RefusedReason, message: string, url?: string);
-}
-/** The text of the error when the optional package `playwright-core` is not installed. */
-declare const NO_PLAYWRIGHT = "rendering needs the package playwright-core and a browser: npm install playwright-core, then npx playwright-core install chromium-headless-shell. For the MCP server with npx, start it with npx -y -p polite-crawler -p playwright-core polite-crawler mcp";
-interface PlaywrightRenderer extends Renderer {
-  /** This function tells if a browser is open. */
-  readonly open: () => boolean;
-}
-declare const playwrightRenderer: (options?: {
-  readonly idleMs?: number;
-  /**
-   * This function starts the browser: it gives a `Browser` of `playwright-core`. The default is
-   * the headless shell of Playwright. The type names no type of `playwright-core`, so a program
-   * with no `playwright-core` can check its types.
-   */
-  readonly launch?: (() => Promise<object>) | undefined;
-}) => PlaywrightRenderer;
-//#endregion
-//#region src/docs.d.ts
-/** This is one document, as an extractor gives it. */
-interface ExtractedDoc {
-  readonly doc: string;
-  readonly kind: string;
-  readonly title: string;
-  /** This text is the readable text. For a paper, it is the title and the abstract. */
-  readonly text: string;
-  readonly url: string | null;
-  readonly meta: Readonly<Record<string, unknown>>;
-  /** This rule chooses between the texts of two recipes. The default is `richest`: the longer text. */
-  readonly merge?: "latest" | "richest" | undefined;
-  /** When the extractor cut the text, this text is the whole text. The function `verify` looks for a quote here too. */
-  readonly full_text?: string | undefined;
-}
-/** This is one version of a document: the document as one capture gave it. */
-interface DocInput extends ExtractedDoc {
-  /** These ids are the capture records that this version came from. The first id names the version. */
-  readonly captures: readonly string[];
-  readonly ai_input?: AiInput | undefined;
-  /** This id is the capture of the version. The default is the first id of `captures`. */
-  readonly capture?: string | undefined;
-  /** This time (ISO 8601) is the time of the fetch. The default is the time of the upsert. */
-  readonly fetched_at?: string | undefined;
-  readonly blob?: string | null | undefined;
-  /** This value is the recipe that read the capture. A version with no recipe has the recipe `""`. */
-  readonly recipe?: string | undefined;
-  /** This value is true when the headless browser made the capture. */
-  readonly rendered?: boolean | undefined;
-  /** This label comes from the capture. The index writes it as it is and does not read it. */
-  readonly scope?: string | undefined;
-  /** These labels come from the capture. The index writes them as they are and does not read them. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/** This is the current version of a document. */
-interface Doc {
-  readonly doc: string;
-  readonly kind: string;
-  readonly title: string;
-  readonly text: string;
-  readonly url: string | null;
-  readonly meta: Readonly<Record<string, unknown>>;
-  /** These ids are the captures behind the current text, the newest first, `MAX_CAPTURES` or fewer. */
-  readonly captures: readonly string[];
-  /** This value is the number of versions of the document. */
-  readonly capture_count: number;
-  /** This id is the capture of the current text. */
-  readonly capture: string;
-  readonly blob: string | null;
-  /** This time (ISO 8601) is the fetch time of the current text. */
-  readonly fetched_at: string;
-  readonly recipe: string;
-  readonly rendered: boolean;
-  readonly ai_input: AiInput;
-  /** This value is true when the extractor cut the text, and the index keeps the whole text for `verify`. */
-  readonly cut: boolean;
-  readonly scope?: string | undefined;
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-  readonly updated_at: string;
-}
-/** This is one version of a document, with no text. */
-interface DocVersion {
-  readonly doc: string;
-  readonly capture: string;
-  readonly recipe: string;
-  readonly rendered: boolean;
-  readonly fetched_at: string;
-  readonly blob: string | null;
-  readonly kind: string;
-  readonly title: string;
-  readonly url: string | null;
-  readonly ai_input: AiInput;
-  /** This value is the length of the text of the version. */
-  readonly chars: number;
-  /** This value is true when the index keeps a whole text that is longer than the text. */
-  readonly cut: boolean;
-}
-/** This is the text of one version, with its metadata. */
-interface VersionText extends DocVersion {
-  readonly text: string;
-  /** This text is the whole text when the extractor cut `text`. */
-  readonly full_text: string | undefined;
-  readonly meta: Readonly<Record<string, unknown>>;
-}
-/**
- * This value chooses the ranking of a search:
- *
- * - `exact` (the default): BM25 ranks each document that matches.
- * - `fast`: the form of the query that needs one part leaves out the common parts. A common part
- *   is in more than `FAST_RANK_SHARE` of the documents. The rarest part stays when each part is common.
- *
- * The documents that hold each part come first in the two modes, with the same rank. In the mode
- * `fast`, the rest of the results leaves out a document that holds only common parts. The common
- * parts also do not add to the score of the rest. Thus `fast` is faster for a long question, and
- * its results can be different. The README gives the measured difference.
- */
-type RankMode = "exact" | "fast";
-/** In the mode `fast`, a part is common when more than this share of the documents hold it. */
-declare const FAST_RANK_SHARE = .2;
-/** These are the options of a search of the index. */
-interface FindIndexOptions {
-  /** This value is the maximum number of results. The index gives 1000 or fewer. */
-  readonly limit: number;
-  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
-  readonly kind?: string | null | undefined;
-  /** This value chooses the ranking. The default is `exact`. */
-  readonly rank?: RankMode | undefined;
-}
-interface Found {
-  readonly doc: string;
-  readonly kind: string;
-  readonly title: string;
-  readonly url: string | null;
-  readonly snippet: string;
-  readonly score: number;
-  readonly ai_input: AiInput;
-  /** This time (ISO 8601) is the fetch time of the current text. */
-  readonly fetched_at: string;
-  /** This id is the capture of the current text. */
-  readonly capture: string;
-}
-/** The variant of a cache entry: the plain response, or the page that the browser rendered. */
-type Variant = "plain" | "rendered";
-interface Latest {
-  readonly url: string;
-  readonly capture: string;
-  readonly blob: string | null;
-  readonly etag: string | null;
-  readonly last_modified: string | null;
-  readonly content_type: string | null;
-  readonly fetched_at: string;
-  readonly status: number;
-  /** This value is what the page said about AI input at the time of the fetch. */
-  readonly ai_input: AiInput;
-  /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */
-  readonly location?: string | null | undefined;
-  /** This value is true when the stored body is only the first part of the response. */
-  readonly truncated?: boolean | undefined;
-  /** This time (ISO 8601) is the end of the freshness that the server gave. */
-  readonly fresh_until?: string | null | undefined;
-  /** This text holds the request headers that the response varies on (`Vary`), with their values. */
-  readonly vary_key?: string | null | undefined;
-  /** The default is `plain`. A key that ends in `#rendered` (the form of version 0.1) is `rendered`. */
-  readonly variant?: Variant | undefined;
-}
-/** These are the counts of the index. */
-interface Counts {
-  readonly docs: number;
-  readonly urls: number;
-}
-/** A rebuild of the index writes a new generation of the tables. The old generation answers until the commit. */
-interface IndexBuild {
-  /** This method adds a version to the new generation. The build writes the versions in batches. */
-  upsert(input: DocInput, now: string): void;
-  /** This method sets a cache entry in the new generation, when it is newer than the stored entry. */
-  setLatest(entry: Latest): void;
-  /** This method reads a cache entry of the new generation. */
-  latest(url: string, variant?: Variant): Latest | undefined;
-  /** This method writes the waiting versions in one transaction. */
-  flush(): void;
-  /**
-   * This method copies the versions of each recipe that is not in `recipes` from the current
-   * generation. A runtime with no extractor for a recipe thus keeps the documents of that recipe.
-   * It gives the number of documents that it copied.
-   */
-  carry(recipes: ReadonlySet<string>): number;
-  /**
-   * This method makes the new generation current. It first copies the rows that a different writer
-   * added to the current generation since `since`. Then it removes the old generation.
-   */
-  commit(since: string): Counts;
-  /** This method removes the new generation. The current generation stays. */
-  abort(): void;
-}
-/**
- * A reading index keeps the documents, their versions, the search and the cache entries. `DocIndex`
- * is the SQLite index. A runtime takes any object with these methods, for example a search index
- * of a different kind that wraps a `DocIndex`.
- */
-interface ReadingIndex {
-  /** This value is true when the index opened empty and a rebuild did not fill it yet. */
-  readonly stale: boolean;
-  get(doc: string): Doc | undefined;
-  upsert(input: DocInput, now: string): Doc;
-  versions(doc: string, options?: {
-    readonly limit?: number;
-  }): DocVersion[];
-  versionText(doc: string, capture?: string): VersionText | undefined;
-  earlierTexts(doc: string, options?: {
-    readonly limit?: number;
-  }): VersionText[];
-  docsOfCapture(capture: string): Doc[];
-  /** This method tells if a capture gave a version of a document. */
-  hasVersion(capture: string): boolean;
-  find(q: string, options: FindIndexOptions): Found[];
-  rankWithin(q: string, docs: readonly string[], options?: {
-    readonly rank?: RankMode | undefined;
-  }): Array<{
-    readonly doc: string;
-    readonly score: number;
-  }>;
-  latest(url: string, variant?: Variant): Latest | undefined;
-  setLatest(entry: Latest): void;
-  /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
-  newestFetch(): string | undefined;
-  counts(): Counts;
-  beginRebuild(options?: {
-    readonly batch?: number;
-  }): IndexBuild;
-}
-/** These are the options of a reading index. */
-interface DocIndexOptions {
-  /** This value is the time in milliseconds that a write waits for the lock of a different process. The default is 10 seconds. */
-  readonly busyTimeoutMs?: number | undefined;
-}
-declare class DocIndex implements ReadingIndex {
-  readonly path: string;
-  private readonly db;
-  private readonly statements;
-  private depth;
-  private closed;
-  private empty;
-  /**
-   * This value is true when the file had a different schema, or no tables, so the index opened
-   * empty. A store with captures then needs `reindex`. The commit of a rebuild makes it false.
-   */
-  get stale(): boolean;
-  constructor(path: string, options?: DocIndexOptions);
-  /** This method makes the tables of the schema. It gives true when the index opened empty. */
-  private prepareSchema;
-  private setMeta;
-  private getMeta;
-  private stmt;
-  /**
-   * This method does a write in one transaction. A transaction in progress takes the write in,
-   * also a transaction that the caller opened on the connection.
-   */
-  private write;
-  private generation;
-  /**
-   * This method does a step on the tables of the current generation. A rebuild of a different
-   * process can remove those tables in the meantime. Then the method reads the generation again
-   * and does the step once more.
-   */
-  private live;
-  /** This method removes the tables of a generation. */
-  private dropGeneration;
-  /** This method removes the generations of builds that a stopped process left. */
-  private dropAbandoned;
-  private putText;
-  private textOf;
-  private putVersion;
-  private newest;
-  /** This method chooses the current version of a document from its versions, and writes it with its search entry. */
-  private recompute;
-  private docOf;
-  private toDoc;
-  private versionTextOf;
-  get(doc: string): Doc | undefined;
-  /**
-   * This method adds one version of a document, and it gives the current document. The rules of
-   * the class choose the current version. An upsert of a known capture replaces that version.
-   */
-  upsert(input: DocInput, now: string): Doc;
-  /** This method gives the versions of a document, the newest first. */
-  versions(doc: string, options?: {
-    readonly limit?: number;
-  }): DocVersion[];
-  /**
-   * This method gives the text of one version of a document. With no capture, it gives the current
-   * version. It also gives the text of a held-back version: the caller applies `isHeldBack` to the
-   * `ai_input` of the version.
-   */
-  versionText(doc: string, capture?: string): VersionText | undefined;
-  /**
-   * This method gives the texts of the earlier versions of a document, the newest first. Each
-   * text comes one time, with the newest version that has it. It skips the current text. It also
-   * gives held-back texts: the caller applies `isHeldBack` to the `ai_input` of each version.
-   */
-  earlierTexts(doc: string, options?: {
-    readonly limit?: number;
-  }): VersionText[];
-  /** This method tells if a capture gave a version of a document. */
-  hasVersion(capture: string): boolean;
-  /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
-  newestFetch(): string | undefined;
-  /** This method gives the current documents that a capture gave. */
-  docsOfCapture(capture: string): Doc[];
-  /**
-   * This method searches the documents with BM25, the best first. The syntax of the query is the
-   * syntax of `parseQuery`. It first asks for each part. When that gives too few documents, it
-   * adds the documents that match one part. A document that asked not to be AI input is not in
-   * the results, so a search cannot show its text. The option `rank` chooses the ranking (`RankMode`).
-   */
-  find(q: string, options: FindIndexOptions): Found[];
-  /**
-   * This method scores a set of documents against a query, with the best first. It leaves out a
-   * document that matches no term, and a document that asked not to be AI input. It ranks the
-   * results of a source again, because the order of the source is only as good as its search engine.
-   * In the mode `fast`, it also leaves out a document that holds only common parts (`RankMode`).
-   */
-  rankWithin(q: string, docs: readonly string[], options?: {
-    readonly rank?: RankMode | undefined;
-  }): Array<{
-    readonly doc: string;
-    readonly score: number;
-  }>;
-  /**
-   * This method gives the form of a query that needs one part, without the common parts (the mode
-   * `fast` of `RankMode`). It counts the documents that hold each part. When each part is common,
-   * it keeps the rarest part.
-   */
-  private rareParts;
-  /** This method gives the cache entry of a URL. The key is canonical, so two spellings of one URL give one entry. */
-  latest(url: string, variant?: Variant): Latest | undefined;
-  private putLatest;
-  /** This method keeps the last answer for a URL. An entry that is older than the stored entry changes nothing. */
-  setLatest(entry: Latest): void;
-  counts(): Counts;
-  /** This method empties the current generation. A rebuild uses `beginRebuild`, which keeps the index until its commit. */
-  reset(): void;
-  /**
-   * This method starts a rebuild into a new generation of the tables. The current generation
-   * answers each read and takes each write until the commit. A failed rebuild thus leaves the
-   * index as it was.
-   */
-  beginRebuild(options?: {
-    readonly batch?: number;
-  }): IndexBuild;
-  /** This method closes the file. A second close does nothing. */
-  close(): void;
-}
-//#endregion
-//#region src/egress.d.ts
-/**
- * The egress policy keeps the crawler on the public internet. A page or an agent can ask for an
- * internal address. Examples are a loopback service, a router on the LAN and the metadata
- * service of a cloud host. The crawler refuses these addresses by default.
- *
- * - The crawler fetches only `http:` and `https:` URLs.
- * - The check uses the address after DNS. The `lookup` hook of the connection checks the address
- *   that the socket then uses, so a DNS answer that changes cannot pass the check.
- * - The crawler examines a literal IP host directly, in each spelling that the URL parser reads.
- * - The crawler examines each redirect hop again.
- *
- * A refusal is a skipped fetch with a clear note. It does not throw an error.
- */
-/** These options change the egress policy. With no options, the crawler fetches only public addresses. */
-interface EgressOptions {
-  /** The value true lets the crawler fetch loopback, private and link-local addresses. The default is false. */
-  readonly allowPrivate?: boolean | undefined;
-  /**
-   * These hosts are permitted although their address is not public. An entry is a host name, a
-   * name pattern (`*.corp.example`), an IP address or a CIDR range (`10.0.0.0/8`).
-   */
-  readonly allowHosts?: readonly string[] | undefined;
-}
-/** The connection hook gives this error when the policy refuses each address of a host. */
-declare class EgressRefused extends Error {
-  readonly address: string;
-  readonly kind: string;
-  readonly code: string;
-  constructor(address: string, kind: string);
-}
-/** This is the form of `dns.lookup` that the egress hook uses. Tests give a resolver of their own. */
-type Resolve = (host: string, options: LookupAllOptions, callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void) => void;
-/**
- * This function gives a `lookup` hook for the connection options of an undici `Agent`. The hook
- * resolves the name, removes each address that the policy refuses, and gives the rest to the
- * socket. When no address stays, the connection fails with `EgressRefused`.
- */
-declare const egressLookup: (policy: EgressOptions, resolve?: Resolve) => LookupFunction;
-//#endregion
-//#region src/record.d.ts
-/**
- * A capture record is one line of the capture log. Each fetch decision makes one record, and a
- * fetch that the crawler declines makes one too.
- *
- * The record id comes from a hash of its content, so the same decision always gets the same id.
- * A citation names this id.
- *
- * The id has 48 bits: the source and the first 12 hexadecimal digits of the hash. Approximately
- * 1 million records give a chance of a collision of 0.2 percent. The package keeps this length on purpose,
- * because the ids of the earlier capture logs and their citations must stay the same. A later
- * version can make the ids of new records longer behind a record version. It must still read a
- * 12-digit id.
- */
-interface CaptureRecord {
-  readonly id: string;
-  /** The source of the record. The crawler writes `fetch`, and an application can add its own sources. */
-  readonly source: string;
-  readonly kind: string;
-  readonly external_id?: string | undefined;
-  readonly occurred_at?: string | undefined;
-  readonly received_at: string;
-  readonly payload: unknown;
-  readonly content_hash: string;
-  /** This label comes from the application. The crawler writes it as it is and does not read it. */
-  readonly scope?: string | undefined;
-  /**
-   * These labels come from the application, for example the lineage of a request. The crawler writes
-   * them as they are and does not read them. Like `scope`, they are not part of the hash.
-   */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/** These are the fields of a new record. The function `makeRecord` calculates the id and the hash. */
-interface RecordInput {
-  readonly source: string;
-  readonly kind: string;
-  readonly payload: unknown;
-  readonly received_at: string;
-  readonly external_id?: string | undefined;
-  readonly occurred_at?: string | undefined;
-  readonly scope?: string | undefined;
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/**
- * This function makes a capture record. The hash covers the source, the kind, the external id
- * and the payload. The id is the source and the first 12 digits of the hash. The times, `scope`
- * and `labels` are not part of the hash.
- */
-declare const makeRecord: (input: RecordInput) => CaptureRecord;
-//#endregion
-//#region src/crawler.d.ts
-/**
- * The crawler is a polite fetcher. It sends one request at a time to each host, and it obeys
- * robots.txt for web pages. Its `User-Agent` header tells what it is. It asks again with the
- * validators of the stored copy, and it stops when a site refuses. Each decision goes into the
- * capture log, and a fetch that the crawler declines goes there too.
- *
- * There are two kinds of request. A documented public API (`api`) has its own terms and rate
- * limits. A web page (`page`) obeys robots.txt. The settings come from measurements:
- *
- * - The `request` function of undici on one `Agent`.
- * - A keep-alive time longer than the delay for each host. With the default of 4 seconds, each
- *   polite request opens a new connection.
- * - A backoff with jitter in the queue, not the retry handler of undici. That handler limits
- *   Retry-After to 30 seconds and adds no jitter.
- * - A maximum size for each body, after decompression. The crawler keeps the first part of a
- *   longer body and marks it as truncated.
- *
- * A fetch and a render go through one admission. It examines the URL, the egress policy, the
- * product tokens, robots.txt, the cache and the stops of the host. Only the transport is
- * different: an HTTP request or the headless browser.
- *
- * The egress policy (`EgressOptions`) keeps the crawler on public addresses by default. The
- * politeness policy (`PolitenessPolicy`) holds each limit, and a host can have its own values.
- * The fetch and the render do not throw an error: each problem is an outcome with a note.
- */
-/**
- * The outcome of a fetch or a render. A `skipped` request did not go out. A render of a URL that
- * gives a file and not a page is `not-html`.
- */
-type FetchOutcome = Outcome | "skipped" | "not-html";
-interface FetchRequest {
-  readonly url: string;
-  /** The value `api` is a documented API with its own terms. The value `page` is a web page, and robots.txt applies. */
-  readonly mode: "api" | "page";
-  /** This recipe reads the capture again. The function `reindex` uses it to replay the capture. */
-  readonly recipe: string;
-  readonly accept?: string | undefined;
-  /**
-   * These are more request headers. The crawler does not write them into the capture. They
-   * cannot replace `User-Agent`, `From`, `Accept-Encoding` or the validators. A redirect to a
-   * different origin removes `Authorization`, `Cookie` and `Proxy-Authorization`.
-   */
-  readonly headers?: Readonly<Record<string, string>> | undefined;
-  /**
-   * These query parameters contain secrets. Each capture shows `REDACTED` in their place. The
-   * names in `DEFAULT_SECRET_PARAMS` get the same treatment with no list.
-   */
-  readonly secretParams?: readonly string[] | undefined;
-  /** This value is the documented pace of the source: the shortest time between two of its requests. */
-  readonly minIntervalMs?: number | undefined;
-  /**
-   * A stored capture younger than this age answers, and the crawler sends no request. The value
-   * `auto` uses the freshness that the server gave (RFC 9111): `Cache-Control`, `Expires`, or 10
-   * percent of the age of `Last-Modified`.
-   */
-  readonly maxAgeMs?: number | "auto" | undefined;
-  readonly maxBytes?: number | undefined;
-  /**
-   * These are the product tokens of an agent that the fetch acts for, for example `Claude-User`.
-   * The robots.txt rules for these tokens apply too.
-   */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** This name connects the captures to the job that asked for them. */
-  readonly job?: string | undefined;
-  /** When this signal aborts, the crawler stops the fetch. The result is then `skipped` or `error`, with a note. */
-  readonly signal?: AbortSignal | undefined;
-  /** This label goes into each capture record of the request. It replaces the `scope` of the crawler. */
-  readonly scope?: string | undefined;
-  /** These labels go into each capture record of the request, for example its lineage. They are not part of the capture id. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-interface Fetched {
-  /** This URL is the URL of the capture. Its secrets show as `REDACTED`. */
-  readonly url: string;
-  readonly status: number;
-  readonly outcome: FetchOutcome;
-  /** This value is true when a stored capture answered, with no request. */
-  readonly fromCache: boolean;
-  /** This id is the capture record behind the body. A citation names it. */
-  readonly capture: string | undefined;
-  readonly blob: string | undefined;
-  readonly body: Uint8Array | undefined;
-  readonly contentType: string | undefined;
-  readonly signals: Signals;
-  readonly note: string | undefined;
-  /** This value is true when the body was longer than the maximum, so the crawler kept only its first part. */
-  readonly truncated?: boolean | undefined;
-  /** This time (ISO 8601) is the time of the fetch of the body. For a stored copy, it is the time of that copy. */
-  readonly fetchedAt?: string | undefined;
-  /** This label is the scope of the capture. */
-  readonly scope?: string | undefined;
-  /** These labels are the labels of the capture. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/** The crawler writes each record to a capture writer. The class `CaptureLog` is one. */
-interface CaptureWriter {
-  append(record: CaptureRecord): Promise<void>;
-  close?(): Promise<void>;
-}
-/** A cache entry is the last answer for a URL: its capture, its body and its validators. */
-interface CacheEntry {
-  /** This text is the cache key: the canonical URL, with its secrets as `REDACTED`. */
-  readonly url: string;
-  readonly capture: string;
-  readonly blob: string | null;
-  readonly etag: string | null;
-  readonly last_modified: string | null;
-  readonly content_type: string | null;
-  readonly fetched_at: string;
-  readonly status: number;
-  /** This value is what the page said about AI input at the time of the fetch. */
-  readonly ai_input: AiInput;
-  /** This URL is the target of a stored redirect. Its secrets show as `REDACTED`. */
-  readonly location?: string | null | undefined;
-  /** This value is true when the stored body is only the first part of the response. */
-  readonly truncated?: boolean | undefined;
-  /** This time (ISO 8601) is the end of the freshness that the server gave. */
-  readonly fresh_until?: string | null | undefined;
-  /** This text holds the request headers that the response varies on (`Vary`), with their values. */
-  readonly vary_key?: string | null | undefined;
-  /** This value tells a plain response from a page that the browser rendered. The default is `plain`. */
-  readonly variant?: Variant | undefined;
-}
-/**
- * The cache index holds the last answer for each URL. The crawler reads it before a request, and
- * it writes it after an answer. The class `DocIndex` is one. An index that keeps only the
- * necessary fields of `CacheEntry` works, and the crawler keeps the other fields in memory.
- */
-interface CacheIndex {
-  latest(url: string, variant?: Variant): CacheEntry | undefined;
-  setLatest(entry: CacheEntry): void;
-}
-/** An event tells a listener what the crawler does. The listener cannot change it. */
-type CrawlEvent = {
-  readonly type: "request";
-  readonly url: string;
-  readonly host: string;
-  readonly transport: "http" | "browser";
-} | {
-  readonly type: "response";
-  readonly url: string;
-  readonly host: string;
-  readonly status: number;
-  readonly outcome: FetchOutcome;
-  readonly elapsedMs: number;
-} | {
-  readonly type: "wait";
-  readonly host: string;
-  readonly ms: number;
-  readonly reason: string;
-} | {
-  readonly type: "stop";
-  readonly host: string;
-  readonly until: number;
-  readonly reason: string;
-} | {
-  readonly type: "skip";
-  readonly url: string;
-  readonly note: string;
-};
-interface CrawlerOptions {
-  /** This product token is the name that robots.txt rules match. It contains only letters, `_` and `-` (RFC 9309). */
-  readonly token: string;
-  /** This text is the full `User-Agent` header. */
-  readonly userAgent: string;
-  /** This contact goes into the `From` header. Without it, the crawler sends no `From` header. */
-  readonly from?: string | undefined;
-  readonly log: CaptureWriter;
-  readonly blobs: BlobStore;
-  readonly index: CacheIndex;
-  readonly clock?: Clock | undefined;
-  readonly random?: (() => number) | undefined;
-  /** This value is the number of hosts that get requests at one time. The default is 16. */
-  readonly maxActive?: number | undefined;
-  /**
-   * This value tells if the crawler also obeys the robots.txt group of `PoliteCrawler`. That applies
-   * when the `User-Agent` links the bot page of polite-crawler (`BOT_PAGE`) and the token is different.
-   * Site owners read that page, and they write their rules for `PoliteCrawler`. The most restrictive
-   * of the groups applies, as for `onBehalfOf`. The default is true. Set false for a bot page of your own.
-   */
-  readonly obeyDefaultToken?: boolean | undefined;
-  /**
-   * This value is the number of pages that the browser renders at one time. The default is 2. Each
-   * render is a browser context of its own, which holds much memory. A render waits for a slot before
-   * the lease of its host, so the wait does not count toward the time of the host.
-   */
-  readonly maxRenders?: number | undefined;
-  /**
-   * This undici dispatcher sends the requests. With a dispatcher of your own, the crawler checks
-   * only literal addresses. Put `egressLookup` in its connection options to check the address after DNS.
-   */
-  readonly dispatcher?: Dispatcher | undefined;
-  /** This label goes into each capture record as it is. */
-  readonly scope?: string | undefined;
-  /**
-   * This function ranks the scopes of the requests, the most private scope highest. One fetch of a
-   * robots.txt can serve requests of different scopes. Its record then takes the scope with the
-   * highest rank, and the labels of each request. With no function, the first request gives the scope.
-   */
-  readonly scopeRank?: ((scope: string) => number) | undefined;
-  /** This browser renders the pages that need JavaScript. Without it, the crawler renders no page. */
-  readonly renderer?: Renderer | undefined;
-  /** This policy controls the addresses that the crawler fetches. By default, the crawler fetches only public addresses. */
-  readonly egress?: EgressOptions | undefined;
-  /** These values change the limits of the politeness policy, for each host or for some hosts. */
-  readonly politeness?: PolicyOptions | undefined;
-  /** This store keeps the pace, the counts and the stops of each host. The default keeps them in memory. */
-  readonly hostState?: HostStateStore | undefined;
-  /** This function gets an event for each request, response, wait, stop and skip. An error in it does not stop the crawler. */
-  readonly onEvent?: ((event: CrawlEvent) => void) | undefined;
-}
-interface RenderRequest {
-  readonly url: string;
-  readonly recipe: string;
-  /** A stored render younger than this age answers. The value `auto` uses the freshness that the server gave. */
-  readonly maxAgeMs?: number | "auto" | undefined;
-  readonly onBehalfOf?: readonly string[] | undefined;
-  readonly job?: string | undefined;
-  /** When this signal aborts, the crawler stops the render, and the browser stops the page. */
-  readonly signal?: AbortSignal | undefined;
-  /** This label goes into the capture record of the render. It replaces the `scope` of the crawler. */
-  readonly scope?: string | undefined;
-  /** These labels go into the capture record of the render. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/**
- * This function gives the key of the cache entry of a rendered page in the form of version 0.1.
- * The index keeps the variant in a column of its own, and it reads this form too.
- */
-declare const renderedKey: (url: string) => string;
-/** This value is the version of the payload of the records that the crawler writes. A record with no `schema` is version 1. */
-declare const CAPTURE_SCHEMA = 2;
-/**
- * These query parameters hold secrets in many APIs. The crawler writes `REDACTED` in place of
- * their values in each capture, in addition to the names of `secretParams`.
- */
-declare const DEFAULT_SECRET_PARAMS: readonly string[];
-declare class Crawler {
-  private readonly options;
-  private readonly clock;
-  private readonly random;
-  private readonly scheduler;
-  private readonly dispatcher;
-  private readonly ownsDispatcher;
-  private readonly egress;
-  private readonly robots;
-  /** The fields of a cache entry that the index does not keep, and the raw target of a redirect. */
-  private readonly meta;
-  private readonly inFlight;
-  private readonly running;
-  private readonly closing;
-  private closed;
-  private closingDone;
-  /** This note tells why the renderer has no browser. After the first page that finds it, the crawler renders nothing. */
-  private renderOff;
-  /** The renders that run at one time (`maxRenders`). */
-  private readonly renders;
-  /** These tokens apply because the `User-Agent` links the bot page (`obeyDefaultToken`). */
-  private readonly pageTokens;
-  constructor(options: CrawlerOptions);
-  private iso;
-  private policyOf;
-  private emit;
-  private track;
-  /** This method gives the validator memo of a host. The memo is a hint, so an error of its store gives no memo. */
-  private memoOf;
-  private stopHost;
-  private record;
-  private skip;
-  /** This method reads the cache entry of a URL, with the fields that the crawler keeps in memory. */
-  private lookup;
-  private store;
-  /** This method gives the target of a stored redirect, when the crawler can follow it. */
-  private followable;
-  /** This method answers from the cache, when the stored entry is young or fresh enough. */
-  private fromCache;
-  private requestHeaders;
-  /**
-   * This method records the result of a request for the host, and it gives the rest of the host.
-   * The rest comes from `restAfter`. A 429 or 503 waits, an error counts toward a stop, and a
-   * refusal stops the host for a time.
-   */
-  private settle;
-  /**
-   * This method stops the host of a URL as a challenge does: for the `challengeStopMs` of the policy.
-   * The runtime uses it when a page that answered 200 is a bot check (`meta.blocked`).
-   */
-  reportBlocked(url: string, what: string): void;
-  /**
-   * This method applies the rules that a fetch and a render share. The sequence is the URL, the
-   * options, the egress policy, the product tokens and the rules for an agent. Then come the
-   * cache, the stops of the host and robots.txt. It gives the answer when a rule decides, or the
-   * facts that the transport needs.
-   */
-  private admit;
-  /** This method fetches a URL. It follows the redirects up to the maximum, and each step is a separate decision. */
-  fetch(request_: FetchRequest): Promise<Fetched>;
-  private follow;
-  /** This method joins a request to an equal request in progress, so one URL gets one request at a time. */
-  private hop;
-  private knownRobots;
-  /**
-   * This method gives the robots.txt of the origin of a URL. A good copy stays for the TTL of the
-   * policy. An unreachable verdict stays for some minutes, with a backoff and the `Retry-After`
-   * of the server. During that time the last good copy applies, when there is one (RFC 9309 §2.4).
-   */
-  private robotsFor;
-  /**
-   * This method gives the robots.txt rules that apply to a URL, as the crawler applies them. They
-   * come from its memory, or from a fetch of `/robots.txt` with its redirects. A robots.txt that
-   * the crawler cannot read gives rules that permit nothing, and `unreachable` gives the reason.
-   */
-  readRobots(url: string): Promise<{
-    readonly robots: Robots;
-    readonly capture: string | undefined;
-    readonly stale?: string | undefined;
-  }>;
-  /** This method reads the stored copy of a robots.txt, when it is younger than the limit of the policy. */
-  private storedRobots;
-  /** This method adds the scope and the labels of one request to the requests that wait for a robots.txt. The most private scope wins. */
-  private join;
-  private fetchRobots;
-  private fetchOnce;
-  /**
-   * This method gives the function that decides each request of a rendered page. It refuses an
-   * address that the egress policy refuses, and a URL of the page's host that robots.txt
-   * disallows. For a different host, it uses a robots.txt that the crawler holds, when there is one.
-   */
-  private allowRequestFor;
-  /**
-   * This method tells why the browser did not go on to a URL that the page moved to: the egress
-   * policy or robots.txt. For a different site, it uses a robots.txt that the crawler holds. It
-   * gives undefined for the page itself, or for a URL that it cannot read.
-   */
-  private movedRefusal;
-  /**
-   * This method renders a page in the headless browser and captures what the page shows. The rules
-   * of a fetch apply:
-   *
-   * - the egress policy and robots.txt, for the page, its final URL and each request of the page
-   * - one request at a time for each host
-   * - the backoff after a 429 or a 503
-   * - a capture for each decision
-   *
-   * A render costs the site more than a fetch, so the host rests longer after one. A render that
-   * the browser refuses (`RenderRefused`) does not count against the host.
-   */
-  render(request_: RenderRequest): Promise<Fetched>;
-  private renderOnce;
-  /**
-   * This method removes the stops and the failure counts of one host, or of each host. It also
-   * forgets the robots.txt verdicts that came from an unreachable robots.txt. The pace of each host
-   * stays. A process that runs for a long time uses it, for example after the network comes back.
-   */
-  resetHosts(host?: string): void;
-  /**
-   * This method closes the crawler. Work in the queue ends as `skipped` with the note
-   * `crawler closed`, and the crawler writes no capture after the close. Then it closes the
-   * browser, the log and its own dispatcher. A second close waits for the first.
-   */
-  close(): Promise<void>;
-}
-//#endregion
-//#region src/log.d.ts
-/** A scan filter selects the records of one source, one kind, or the records after one time. */
-interface CaptureFilter {
-  readonly source?: string | undefined;
-  readonly kind?: string | undefined;
-  /** This value is an ISO 8601 time. The scan gives the records received at this time or later. */
-  readonly since?: string | undefined;
-}
-/** A damaged line is a line of the log that is not a record: a torn append, or bytes that a crash left. */
-interface DamagedLine {
-  /** This value is the path of the file. */
-  readonly path: string;
-  /** This value is the number of the line in the file, from 1. */
-  readonly line: number;
-}
-/** These options change a scan. */
-interface ScanOptions {
-  /** The scan skips a damaged line, and it gives the line to this function. */
-  readonly onDamaged?: ((damaged: DamagedLine) => void) | undefined;
-}
-/** The read side of the capture log is what `reindex` replays. */
-interface CaptureReader {
-  scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;
-}
-/** These options change a capture log. */
-interface CaptureLogOptions {
-  /**
-   * When this value is true (the default), the log writes each batch of appends to the disk
-   * (`fdatasync`) before the appends finish. Thus a power cut does not lose a record that the
-   * crawler reported.
-   */
-  readonly sync?: boolean | undefined;
-}
-/**
- * The capture log keeps each fetch decision as one line of JSON in
- * `<dir>/<source>/<yyyy>/<mm>.ndjson`. It is the record of each access that the crawler makes.
- *
- * These are the rules of the log:
- *
- * - The log appends a batch of records with one write, and it writes the batch to the disk before
- *   the appends finish.
- * - An append always starts on a new line. After a crash leaves a partial line, the next append
- *   starts after a line break. The partial line stays as a damaged line, and the record is good.
- * - A scan reads each file as a stream. It skips a damaged line, counts it, and continues.
- * - The log does not remove or change a line.
- *
- * Two processes can append to one log, because each batch is one write to a file in append mode.
- */
-declare class CaptureLog implements CaptureReader {
-  readonly dir: string;
-  private readonly handles;
-  private readonly pending;
-  private draining;
-  private readonly sync;
-  constructor(dir: string, options?: CaptureLogOptions);
-  private pathOf;
-  private handleFor;
-  /** This method tells if the last byte of a file is not a line break: a partial line that a crash left. */
-  private static needsBreak;
-  append(record: CaptureRecord): Promise<void>;
-  /** This method starts the writer when it is not in progress. A record that comes as the writer stops starts it again. */
-  private kick;
-  /** This method writes the waiting records in batches: one write and one sync for each file. */
-  private drain;
-  /** This method waits until each append in progress is on the disk. */
-  flush(): Promise<void>;
-  /** This method gives the record files of one source, in the sequence of time. It skips the months before `since`. */
-  private filesOf;
-  /** This method reads the records of one source in the sequence of the files. */
-  private readSource;
-  /**
-   * This method reads the records that agree with the filter. It first waits for the appends that
-   * are in progress. It reads each file as a stream, so its memory does not grow with the log.
-   *
-   * The records of one source come in the sequence of the files. The records of two or more sources
-   * merge in the sequence of `received_at`.
-   */
-  scan(filter?: CaptureFilter, options?: ScanOptions): AsyncIterable<CaptureRecord>;
-  /** This method writes the appends in progress, then closes the files. A later append opens them again. */
-  close(): Promise<void>;
-}
-//#endregion
-//#region src/runtime.d.ts
-/**
- * An extractor changes one captured body into the documents that it holds. It has no side effects,
- * so `reindex` can replay each capture through it and build the index again with no request. It can
- * answer later, because the crawler reads a PDF asynchronously.
- *
- * An extractor that throws does not stop a fetch, a render or a rebuild. The runtime records the
- * error and gives no document for that capture.
- */
-type Extractor = (body: Uint8Array, capture: {
-  readonly url: string;
-  readonly contentType: string | null;
-}) => ReadonlyArray<ExtractedDoc> | Promise<ReadonlyArray<ExtractedDoc>>;
-/** This function gives the document id of a web page. Each page has one id. */
-declare const pageDocId: (url: string) => string;
-/**
- * This extractor is the built-in recipe for a web page, plain or rendered. It keeps the visible
- * text, the JSON-LD (whole objects, up to a limit) and the links. It decodes the body in the
- * encoding that the page declares, with `decodeBody`. It does not throw: a page that it cannot read
- * gives a document that names the error. When the reader cut the text, the document keeps the whole
- * text in `full_text`, so `verify` finds a quote past the cut.
- */
-declare const pageExtractor: Extractor;
-declare class Extractors {
-  private readonly byRecipe;
-  register(recipe: string, extractor: Extractor): void;
-  get(recipe: string): Extractor | undefined;
-  /** This method gives the names of the recipes that have an extractor. */
-  recipes(): string[];
-}
-interface CrawlRuntime {
-  readonly crawler: Crawler;
-  /** This index keeps the documents and the cache entries. `DocIndex` is the SQLite index. */
-  readonly index: ReadingIndex;
-  readonly blobs: BlobStore;
-  /** This reader is the read side of the capture log. The function `reindex` replays it. */
-  readonly captures: CaptureReader;
-  /** This log is the write side of the capture log. */
-  readonly log: CaptureLog;
-  readonly extractors: Extractors;
-  readonly now: () => string;
-}
-/** This is the payload of a capture record, as the crawler writes it. */
-interface CapturePayload {
-  readonly url: string;
-  readonly recipe: string;
-  readonly mode: "api" | "page";
-  readonly status?: number;
-  readonly outcome?: string;
-  readonly blob?: string | null;
-  readonly content_type?: string | null;
-  readonly etag?: string | null;
-  readonly last_modified?: string | null;
-  readonly fetched_at: string;
-  readonly signals?: {
-    readonly ai_input?: AiInput;
-  };
-  /** This value is true when the headless browser made the capture, not a plain request. */
-  readonly rendered?: boolean;
-  /** This value is true when the body is only the first part of the response. */
-  readonly truncated?: boolean;
-  /** This value is true when the server cut a compressed body. The capture is not a copy to keep. */
-  readonly cut?: boolean;
-  /** This value is true when an API answered with a web page that the request did not accept. The capture is not a copy to keep. */
-  readonly unasked?: boolean;
-  /** This time (ISO 8601) is the end of the freshness that the server gave. Captures before version 0.2 do not have it. */
-  readonly fresh_until?: string | null;
-  /** This text holds the request headers that the response varies on. Captures before version 0.2 do not have it. */
-  readonly vary_key?: string | null;
-  /** This URL is the target of a redirect, with its secrets as `REDACTED`. */
-  readonly location?: string | null;
-  readonly note?: string;
-}
-/** The note of a document that is held back. */
-declare const HELD_BACK = "This page asked not to be used as AI input, so its text is held back. Open the URL to read it, or read again with userProvided when a person gave this URL directly (the crawler logs that).";
-/**
- * This function tells if a document is held back: the page asked not to be AI input, and no person
- * gave its URL directly. The only way to read a held-back document is `userProvided`.
- */
-declare const isHeldBack: (doc: {
-  readonly ai_input: AiInput;
-}, userProvided?: boolean) => boolean;
-/** This function gives the metadata that a held-back document shows: only the structural fields, which hold no text of the page. */
-declare const heldBackMeta: (meta: Readonly<Record<string, unknown>>) => Record<string, unknown>;
-/** The result of `fetchAndIndex` and `renderAndIndex`. */
-interface Indexed {
-  readonly fetched: Fetched;
-  readonly docs: readonly Doc[];
-  /** This text tells why the capture gave no document: the error of the extractor, or a bot check. */
-  readonly error?: string | undefined;
-  /** This value names the bot check or the firewall that sent its page in place of the content. */
-  readonly blocked?: string | undefined;
-}
-/**
- * This function fetches one URL and adds its documents to the index. Each recipe uses it. A fetch
- * from the cache gives the documents that the index has for that capture. Thus a repeated
- * question sends no request and reads nothing again. An extractor that throws gives no document and an
- * `error`, and the fetch still succeeds.
- */
-declare const fetchAndIndex: (runtime: CrawlRuntime, request: FetchRequest) => Promise<Indexed>;
-/** This function renders a page in the headless browser and adds what the page shows to the index. */
-declare const renderAndIndex: (runtime: CrawlRuntime, request: RenderRequest) => Promise<Indexed>;
-interface PageResult {
-  readonly url: string;
-  readonly doc: string | null;
-  readonly outcome: string;
-  readonly status: number;
-  /** This value is the title of the page. A held-back page shows no title. */
-  readonly title: string | null;
-  readonly chars: number;
-  readonly rendered: boolean;
-  /** These are the schema.org types that the JSON-LD of the page declares. */
-  readonly structured: readonly string[];
-  readonly links: number;
-  /** These ids are the captures behind the current text, the newest first, 10 or fewer. */
-  readonly captures: readonly string[];
-  /** This id is the capture of the text that the document shows. A citation names it. */
-  readonly capture: string | null;
-  /** This value is the number of captures of the document. */
-  readonly capture_count: number;
-  /** This time (ISO 8601) is the fetch time of the text that the document shows. */
-  readonly fetched_at: string | null;
-  /** This value is true when the stored copy answered, and the crawler sent no request. */
-  readonly from_cache: boolean;
-  /** This value is true when the page asked not to be AI input. Its title and text are then held back. */
-  readonly held_back: boolean;
-  readonly note: string | null;
-}
-/** These options control one page. */
-interface PageOptions {
-  readonly render: "auto" | "never" | "always";
-  readonly maxAgeMs?: number | "auto" | undefined;
-  readonly onBehalfOf?: readonly string[] | undefined;
-  readonly job?: string | undefined;
-  /** When this signal aborts, the crawler stops the fetch and the render. */
-  readonly signal?: AbortSignal | undefined;
-  readonly scope?: string | undefined;
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/**
- * This function reads one web page. It first sends a plain request. It opens the browser only when
- * the page shows almost nothing without JavaScript, or when the caller asks for it. It renders
- * only an HTML page: not a PDF, a different file or a bot check. A page that sends the reader on
- * with a `<meta http-equiv="refresh">` gets one more plain fetch, of its target.
- */
-declare const fetchPage: (runtime: CrawlRuntime, url: string, options: PageOptions) => Promise<PageResult>;
-/** This result is a part of one stored document. */
-interface ReadResult {
-  readonly doc: string;
-  readonly kind: string;
-  /** This value is the title of the document. A held-back document shows no title. */
-  readonly title: string;
-  readonly url: string | null;
-  /** This value is the metadata of the document. A held-back document shows only the structural fields. */
-  readonly meta: Readonly<Record<string, unknown>>;
-  readonly from: number;
-  readonly to: number;
-  readonly total: number;
-  readonly text: string;
-  readonly captures: readonly string[];
-  /** This id is the capture of the text that the result reads. */
-  readonly capture: string;
-  /** This time (ISO 8601) is the fetch time of the text that the result reads. */
-  readonly fetched_at: string;
-  readonly ai_input: AiInput;
-  /** This value is true when the page asked not to be AI input. The text is then empty. */
-  readonly held_back: boolean;
-  readonly note: string | null;
-}
-/** This result tells if a quote is in a stored document, and where. */
-interface VerifyResult {
-  readonly doc: string;
-  /** This value is true when the quote is in the text. A held-back document gives false. */
-  readonly found: boolean;
-  /** These values are the positions of the quote in the text of `capture`. */
-  readonly start: number | null;
-  readonly end: number | null;
-  readonly excerpt: string | null;
-  readonly url: string | null;
-  readonly captures: readonly string[];
-  /** This id is the capture whose text holds the quote, or the capture that the result examined. */
-  readonly capture: string | null;
-  /** This hash is the blob of that capture. */
-  readonly blob: string | null;
-  /** This time (ISO 8601) is the fetch time of that capture. */
-  readonly fetched_at: string | null;
-  /** This value is true when only an earlier version of the document holds the quote. */
-  readonly stale: boolean;
-  /** This value is true when the quote is in the whole text of a page that the reader cut. The positions are then in that whole text. */
-  readonly in_full_text: boolean;
-  /** This value is false when the examined text is not complete (a PDF that the reader stopped). A quote that is not found can then be in the file. */
-  readonly complete: boolean;
-  /** This value is the number of places where the quote is. */
-  readonly count: number;
-  readonly held_back: boolean;
-  readonly note: string | null;
-}
-/** These options control a read. */
-interface ReadOptions {
-  /** This value is the first character to read. The default is 0. */
-  readonly from?: number | undefined;
-  /** This value is the number of characters to read. The default is 4000. */
-  readonly length?: number | undefined;
-  /** This value is true when a person gave this URL directly. It is the only way to read a held-back document. The crawler logs it. */
-  readonly userProvided?: boolean | undefined;
-  /** This id reads the version of one capture, not the current version. */
-  readonly capture?: string | undefined;
-}
-/** These options control a verification. */
-interface VerifyOptions {
-  /** This id examines the version of one capture only. A citation of that capture thus stays true after a new fetch. */
-  readonly capture?: string | undefined;
-  /** This value is true when a person gave this URL directly. It is the only way to verify a quote in a held-back document. */
-  readonly userProvided?: boolean | undefined;
-}
-/**
- * This function writes an `override` record to the capture log. A person gave the URL of a
- * held-back document directly, and the hold stops for one action (`read` or `verify`). The record
- * takes the scope and the labels of `marks`, or else those of the document, as its captures do.
- */
-declare const recordOverride: (runtime: CrawlRuntime, doc: {
-  readonly doc: string;
-  readonly url: string | null;
-  readonly scope?: string | undefined;
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}, action: string, marks?: {
-  readonly scope?: string | undefined;
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}) => Promise<void>;
-/**
- * This function reads a range of characters of one stored document. It applies the rule for a
- * held-back document: no title, no text, and only the structural metadata, unless `userProvided`.
- * The positions do not cut a character in two.
- */
-declare const readDocument: (runtime: CrawlRuntime, id: string, options?: ReadOptions) => Promise<ReadResult>;
-/**
- * This function finds a quote in a stored document, with `locate`. It tells which capture holds the
- * quote, with its blob, its fetch time and the positions:
- *
- * - With `capture`, it examines the version of that capture only.
- * - With no capture, it examines the current text first. When the quote is not there, it examines
- *   the earlier texts, the newest first. A match there has `stale: true`.
- * - When the reader cut a text, it also examines the whole text.
- *
- * A held-back document gives `found: false` and no positions, so a quote cannot test its text,
- * unless `userProvided`. The same rule applies to each version. A version that asked not to be AI
- * input stays held back after a later capture drops the signal. The crawler logs an override when
- * `userProvided` opens a held-back version.
- */
-declare const verifyQuote: (runtime: CrawlRuntime, id: string, quote: string, options?: VerifyOptions) => Promise<VerifyResult>;
-/**
- * This function searches the stored documents. The syntax of the query is the syntax of
- * `parseQuery`. A held-back document is not in the results. The option `rank` chooses the
- * ranking (`RankMode`), and the default is `exact`.
- */
-declare const findDocuments: (runtime: CrawlRuntime, q: string, options?: {
-  readonly limit?: number | undefined;
-  readonly kind?: string | null | undefined;
-  readonly rank?: RankMode | undefined;
-}) => Promise<Found[]>;
-/** The counts of a rebuild. */
-interface ReindexResult {
-  /** This value is the number of captures that the rebuild read again. */
-  readonly captures: number;
-  readonly docs: number;
-  readonly urls: number;
-  /** This value is the number of captures whose extractor threw. */
-  readonly failed: number;
-  /** This value is the number of captures whose blob is missing or damaged. */
-  readonly missing: number;
-  /** This value is the number of damaged lines of the log that the rebuild skipped. */
-  readonly damaged: number;
-  /** This value is the number of documents of recipes with no extractor, kept from the old index. */
-  readonly carried: number;
-  /** These recipes have captures in the log and no extractor in this runtime. */
-  readonly unknown_recipes: readonly string[];
-  /** These are the first errors of the extractors. */
-  readonly errors: ReadonlyArray<{
-    readonly capture: string;
-    readonly error: string;
-  }>;
-}
-/**
- * This function builds the reading index again from the capture log and the blobs. The index is a
- * projection, and this function is the proof that its deletion loses nothing. These are its rules:
- *
- * - It builds a new generation of the tables and commits it in batches. The old index answers
- *   until the end, and a failed rebuild leaves it as it was.
- * - It reads the log as a stream, so its memory does not grow with the log.
- * - It counts and skips a damaged line, a missing blob and an extractor that throws.
- * - The documents of a recipe that has no extractor in this runtime stay, from the old index.
- * - A capture that a different writer adds during the rebuild goes into the new index.
- */
-declare const reindex: (runtime: CrawlRuntime, options?: {
-  readonly batch?: number | undefined;
-}) => Promise<ReindexResult>;
-/** The result of `catchUpIndex`. */
-interface CatchUpResult extends ReindexResult {
-  /**
-   * The value `rebuild` tells that the index was stale, and the function built it again from the
-   * whole log. The value `tail` tells that it read only the captures that the index missed.
-   */
-  readonly mode: "rebuild" | "tail";
-}
-/**
- * This function brings the reading index up to the capture log, with no request. `openCrawler`
- * uses it at open. An application that uses `createRuntime` uses it the same way:
- *
- * - A stale index (a new schema, or a deleted file) is built again with `reindex`.
- * - An index that is behind the log gets the captures that it missed. A process can stop between
- *   the log and the index, and SQLite can lose its last transactions after a power cut. The
- *   function reads the log from a short time before the newest cache entry.
- *
- * It gives the counts, or undefined when the index missed nothing.
- */
-declare const catchUpIndex: (runtime: CrawlRuntime) => Promise<CatchUpResult | undefined>;
-//#endregion
-//#region src/open.d.ts
-/**
- * This function gives the default `User-Agent` header. The header names the crawler, its version
- * and the page about it, so a site owner can find out what made each request.
- */
-declare const defaultUserAgent: (contact?: string) => string;
-/**
- * This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,
- * section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in
- * the `User-Agent` only.
- */
-declare const isMailbox: (contact: string) => boolean;
-/** A duration is a count of milliseconds, or a text, for example `"30m"`, `"24h"` or `"7d"`. */
-type Duration = number | string;
-/**
- * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,
- * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`
- * is zero. The function throws an error for a text that it cannot read.
- */
-declare const durationMs: (duration: Duration) => number;
-/**
- * These options make a runtime. A store needs a log, a blob store and an index. Give `dir`, and
- * the runtime makes the three parts in it, or give the parts. A part that you give replaces the
- * part in `dir`.
- */
-interface RuntimeOptions {
-  /** This folder keeps the capture log (`captures/`), the bodies (`blobs/`), the index (`index.sqlite`) and the state of each host (`hosts.sqlite`). */
-  readonly dir?: string | undefined;
-  /** This log replaces `<dir>/captures`. */
-  readonly log?: CaptureLog | undefined;
-  /** This blob store replaces `<dir>/blobs`. */
-  readonly blobs?: BlobStore | undefined;
-  /** This index replaces `<dir>/index.sqlite`. A runtime takes any `ReadingIndex`. */
-  readonly index?: ReadingIndex | undefined;
-  /**
-   * This store keeps the pace and the stops of each host. The default with `dir` is a SQLite file
-   * (`hosts.sqlite`), so the processes on one store share the pace. With no `dir`, it is in memory.
-   */
-  readonly hostState?: HostStateStore | undefined;
-  /** This token is the name that robots.txt rules match. The default is `PoliteCrawler`. */
-  readonly token?: string | undefined;
-  /** This text is the full `User-Agent` header. The default names the crawler and its page. */
-  readonly userAgent?: string | undefined;
-  /**
-   * This address goes into the `User-Agent` header. A mailbox (`ops@example.com`) also goes into the
-   * `From` header, and a URL does not. The crawler sends no contact without it.
-   */
-  readonly contact?: string | undefined;
-  /**
-   * This browser renders the pages that need JavaScript. The value `false` turns rendering off. The
-   * default checks one time that `playwright-core` and its browser are installed. With no browser,
-   * each page that needs JavaScript gets a note at once, with no wait.
-   */
-  readonly renderer?: Renderer | false | undefined;
-  /** This value is the number of sites that the crawler fetches from at one time. The default is 16. */
-  readonly maxActive?: number | undefined;
-  /** This value is the number of pages that the browser renders at one time. The default is 2: each render holds a browser context. */
-  readonly maxRenders?: number | undefined;
-  /** This `undici` dispatcher sends the requests. Tests give a mock agent, and a proxy is possible. */
-  readonly dispatcher?: Dispatcher | undefined;
-  /** This policy controls the addresses that the crawler fetches. By default, it fetches only public addresses. */
-  readonly egress?: EgressOptions | undefined;
-  /** These values change the limits of the politeness policy, for each host or for some hosts. */
-  readonly politeness?: PolicyOptions | undefined;
-  /** This clock gives the time to the crawler and to the index. Tests give a clock that they control. */
-  readonly clock?: Clock | undefined;
-  /** This function gives the random numbers of the jitter. The default is `Math.random`. */
-  readonly random?: (() => number) | undefined;
-  /** This label goes into each capture record. */
-  readonly scope?: string | undefined;
-  /** With the value false, the crawler does not also obey the group of `PoliteCrawler` when its `User-Agent` links the bot page (`CrawlerOptions.obeyDefaultToken`). */
-  readonly obeyDefaultToken?: boolean | undefined;
-  /** This function ranks the scopes, the most private highest. A shared fetch of a robots.txt records the scope with the highest rank (`CrawlerOptions.scopeRank`). */
-  readonly scopeRank?: ((scope: string) => number) | undefined;
-  /** This function gets an event for each request, response, wait, stop and skip. */
-  readonly onEvent?: ((event: CrawlEvent) => void) | undefined;
-  /** These extractors read the captures of the recipes of an application. The recipe `page` is built in. */
-  readonly extractors?: Readonly<Record<string, Extractor>> | undefined;
-}
-/** A runtime that a store made. It owns the parts that it made, and it closes them. */
-interface StoreRuntime extends CrawlRuntime {
-  readonly hostState: HostStateStore;
-  /** This method closes the crawler, the browser, the log, the index and the host state store that the runtime made. A second close does nothing. */
-  close(): Promise<void>;
-}
-/**
- * This function makes a runtime. It is the one place that puts the log, the blob store, the index,
- * the host state, the crawler and the extractors together. The function `openCrawler` uses it.
- * An application that needs the parts uses it too. The runtime and the crawler share one clock.
- */
-declare const createRuntime: (options: RuntimeOptions) => StoreRuntime;
-/** These options open a crawler on a folder. Only `dir` is necessary. */
-interface OpenOptions extends Omit<RuntimeOptions, "dir" | "log" | "blobs" | "index"> {
-  /** This folder keeps the capture log, the response bodies, the search index and the state of each host. */
-  readonly dir: string;
-  /** These agents get their robots.txt rules applied to each fetch, for example `Claude-User`. */
-  readonly onBehalfOf?: readonly string[] | undefined;
-}
-/** These options control one fetch. */
-interface FetchOptions {
-  /** The value `auto` renders a page only when it shows almost nothing without JavaScript. */
-  readonly render?: "auto" | "never" | "always" | undefined;
-  /**
-   * A stored copy younger than this age answers, and the crawler sends no request. The default is
-   * `24h`. The value `auto` uses the freshness that the server gave. The value `0s` asks again.
-   */
-  readonly maxAge?: Duration | undefined;
-  /** These agents get their robots.txt rules applied to this fetch, in addition to the defaults. */
-  readonly onBehalfOf?: readonly string[] | undefined;
-  /** When this signal aborts, the crawler stops the fetch and the render. */
-  readonly signal?: AbortSignal | undefined;
-  /** This label goes into the capture records of this fetch. */
-  readonly scope?: string | undefined;
-  /** These labels go into the capture records of this fetch. They are not part of the capture ids. */
-  readonly labels?: Readonly<Record<string, string>> | undefined;
-}
-/** These options control a search. */
-interface FindOptions {
-  /** This value is the maximum number of results. The default is 10. */
-  readonly limit?: number | undefined;
-  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
-  readonly kind?: string | null | undefined;
-  /**
-   * This value chooses the ranking. The default `exact` ranks each match with BM25. `fast` leaves
-   * the common words out of the second part of the search, so a long question is faster (`RankMode`).
-   */
-  readonly rank?: RankMode | undefined;
-}
-/** A polite crawler fetches each source one time, keeps it, and answers the next questions from the disk. */
-interface PoliteCrawler {
-  /** The runtime gives the parts to an application that needs more control. */
-  readonly runtime: StoreRuntime;
-  /** This method fetches a page, reads it and adds it to the index. */
-  fetch(url: string, options?: FetchOptions): Promise<PageResult>;
-  /**
-   * This method searches the stored documents. It sends no request. A held-back document is not
-   * in the results. The syntax of the query is the syntax of `parseQuery`.
-   */
-  find(q: string, options?: FindOptions): Promise<Found[]>;
-  /** This method reads a range of characters of one stored document, the current version or the version of one capture. */
-  read(doc: string, options?: ReadOptions): Promise<ReadResult>;
-  /**
-   * This method finds a quote in a stored document. It ignores case, spaces, typographic
-   * punctuation and Unicode forms. It tells which capture holds the quote.
-   */
-  verify(doc: string, quote: string, options?: VerifyOptions): Promise<VerifyResult>;
-  /** This method gives the versions of a stored document, the newest first. A held-back version shows no title. */
-  versions(doc: string, options?: {
-    readonly limit?: number;
-  }): Promise<DocVersion[]>;
-  /**
-   * This method builds the index again from the capture log and the stored bodies, with no request.
-   * When the crawler rebuilt a stale index at open, the first reindex gives the counts of that
-   * rebuild, unless a fetch came before it.
-   */
-  reindex(): Promise<ReindexResult>;
-  /** This method closes the browser, the log files and the index. A second close does nothing. */
-  close(): Promise<void>;
-}
-/**
- * This function opens a crawler on a folder. The folder keeps the capture log (`captures/`), the
- * response bodies (`blobs/`), the search index (`index.sqlite`) and the state of each host
- * (`hosts.sqlite`). The index is a projection: `reindex` makes it again from the log and the bodies.
- * At open, `catchUpIndex` rebuilds an index of an earlier version, and gives an index that is
- * behind the log the captures that it missed.
- *
- * Each method gives a promise, and none throws before its promise. After `close`, each method
- * rejects.
- */
-declare const openCrawler: (options: OpenOptions) => PoliteCrawler;
-//#endregion
-export { FetchOutcome as $, findDocuments as A, playwrightRenderer as At, CaptureFilter as B, ReadResult as C, NO_PLAYWRIGHT as Ct, catchUpIndex as D, RenderRefused as Dt, VerifyResult as E, RenderOptions as Et, readDocument as F, CAPTURE_SCHEMA as G, CaptureReader as H, recordOverride as I, CaptureWriter as J, CacheEntry as K, reindex as L, isHeldBack as M, FileBlobStore as Mt, pageDocId as N, fetchAndIndex as O, Rendered as Ot, pageExtractor as P, DEFAULT_SECRET_PARAMS as Q, renderAndIndex as R, ReadOptions as S, VersionText as St, VerifyOptions as T, RefusedReason as Tt, DamagedLine as U, CaptureLog as V, ScanOptions as W, Crawler as X, CrawlEvent as Y, CrawlerOptions as Z, Extractors as _, Found as _t, PoliteCrawler as a, RecordInput as at, PageOptions as b, ReadingIndex as bt, createRuntime as c, EgressRefused as ct, isMailbox as d, DocIndex as dt, FetchRequest as et, openCrawler as f, DocInput as ft, Extractor as g, FindIndexOptions as gt, CrawlRuntime as h, FAST_RANK_SHARE as ht, OpenOptions as i, CaptureRecord as it, heldBackMeta as j, BlobStore as jt, fetchPage as k, Renderer as kt, defaultUserAgent as l, egressLookup as lt, CatchUpResult as m, ExtractedDoc as mt, FetchOptions as n, RenderRequest as nt, RuntimeOptions as o, makeRecord as ot, CapturePayload as p, DocVersion as pt, CacheIndex as q, FindOptions as r, renderedKey as rt, StoreRuntime as s, EgressOptions as st, Duration as t, Fetched as tt, durationMs as u, Doc as ut, HELD_BACK as v, Latest as vt, ReindexResult as w, PlaywrightRenderer as wt, PageResult as x, Variant as xt, Indexed as y, RankMode as yt, verifyQuote as z };
-//# sourceMappingURL=open-eHcriTds.d.mts.map