polite-crawler 0.1.2 → 0.1.3

package/build/{open-Ca1lcWR3.d.mts → daemon-DP4EqjFL.d.mts}

@@ -146,6 +146,16 @@ declare const playwrightRenderer: (options?: {
   readonly launch?: (() => Promise<object>) | undefined;
 }) => PlaywrightRenderer;
 //#endregion
+//#region src/duration.d.ts
+/** A duration is a count of milliseconds, or a text, for example `"30m"`, `"24h"` or `"7d"`. */
+type Duration = number | string;
+/**
+ * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,
+ * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`
+ * is zero. The function throws an error for a text that it cannot read.
+ */
+declare const durationMs: (duration: Duration) => number;
+//#endregion
 //#region src/record.d.ts
 /**
  * A capture record is one line of the capture log. Each fetch decision makes one record, and a
@@ -806,6 +816,14 @@ interface VersionAt {
 type RankMode = "exact" | "fast";
 /** In the mode `fast`, a part is common when more than this share of the documents hold it. */
 declare const FAST_RANK_SHARE = .2;
+/**
+ * This value chooses the ranking of a search of the crawler: a `RankMode` of the index, or
+ * `hybrid`. The mode `hybrid` fuses the BM25 ranking with the ranking of the vectors of an
+ * embedder (`OpenOptions.embedder`). Without an embedder, it throws an error.
+ */
+type SearchRank = RankMode | "hybrid";
+/** The text of the error of hybrid search with no embedder. */
+declare const NO_EMBEDDER = "hybrid search needs an embedder: open the crawler with the option embedder (transformersEmbedder or ollamaEmbedder), or give --embedder transformers or ollama to the command line";
 /** These are the options of a search of the index. */
 interface FindIndexOptions {
   /** This value is the maximum number of results. The index gives 1000 or fewer. */
@@ -815,6 +833,14 @@ interface FindIndexOptions {
   /** This value chooses the ranking. The default is `exact`. */
   readonly rank?: RankMode | undefined;
 }
+/** This is the passage of a hit of a hybrid search: the chunk of the text that is closest to the question. */
+interface Passage {
+  /** These values are the positions of the chunk in the text of the capture of the hit, in UTF-16 units. */
+  readonly start: number;
+  readonly end: number;
+  /** This value is the cosine of the vectors of the chunk and the question, from -1 to 1. */
+  readonly similarity: number;
+}
 interface Found {
   readonly doc: string;
   readonly kind: string;
@@ -829,6 +855,25 @@ interface Found {
   readonly capture: string;
   /** This value tells that the document is an archived copy (the kind `archived`), with the archive and the date of the copy. */
   readonly archived?: Archived | undefined;
+  /** This value is the passage that a hybrid search found. A search of the other modes does not give it. */
+  readonly passage?: Passage | undefined;
+}
+/** This is the current text of a document in short: its hash, with no text. */
+interface DocDigest {
+  readonly doc: string;
+  readonly kind: string;
+  readonly ai_input: AiInput;
+  /** This value is the SHA-256 of the current text of the document, in hex. */
+  readonly text_sha: string;
+}
+/** These options choose the documents of `ReadingIndex.digests`. */
+interface DigestOptions {
+  /** These documents, in place of a list of each document. A document that is not in the index gives nothing. */
+  readonly docs?: readonly string[] | undefined;
+  /** In a list of each document, the documents with an id after this id, in the sequence of the ids. */
+  readonly after?: string | undefined;
+  /** In a list of each document, this number of documents at most. The default is 500. */
+  readonly limit?: number | undefined;
 }
 /** The variant of a cache entry: the plain response, or the page that the browser rendered. */
 type Variant = "plain" | "rendered";
@@ -909,6 +954,12 @@ interface ReadingIndex {
   docsOfCapture(capture: string): Doc[];
   /** This method tells if a capture gave a version of a document. */
   hasVersion(capture: string): boolean;
+  /**
+   * This optional method gives the hash of the current text of documents, with no text. Hybrid
+   * search uses it to find the documents whose text has no vectors. An index without it gives the
+   * same facts from `get`, more slowly.
+   */
+  digests?(options: DigestOptions): DocDigest[];
   find(q: string, options: FindIndexOptions): Found[];
   rankWithin(q: string, docs: readonly string[], options?: {
     readonly rank?: RankMode | undefined;
@@ -1013,6 +1064,11 @@ declare class DocIndex implements ReadingIndex {
   }): VersionText[];
   /** This method tells if a capture gave a version of a document. */
   hasVersion(capture: string): boolean;
+  /**
+   * This method gives the hash of the current text of documents, with no text. It gives the
+   * documents of `docs`, or each document after `after` in the sequence of the ids, to `limit`.
+   */
+  digests(options?: DigestOptions): DocDigest[];
   /** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
   newestFetch(): string | undefined;
   /** This method gives the current documents that a capture gave. */
@@ -1844,16 +1900,6 @@ interface DiffResult {
  */
 declare const diffDocument: (runtime: CrawlRuntime, id: string, options?: DiffOptions) => Promise<DiffResult>;
 //#endregion
-//#region src/duration.d.ts
-/** A duration is a count of milliseconds, or a text, for example `"30m"`, `"24h"` or `"7d"`. */
-type Duration = number | string;
-/**
- * This function changes a duration into milliseconds. The units are `ms`, `s`, `m` (minutes), `h`,
- * `d` and `w`, in any case. The unit `M` is an error, because a month is not a unit. A bare `0`
- * is zero. The function throws an error for a text that it cannot read.
- */
-declare const durationMs: (duration: Duration) => number;
-//#endregion
 //#region src/sitemaps.d.ts
 /**
  * The sitemap reader reads the files that list the pages of a site. These are an XML sitemap, a
@@ -2059,6 +2105,122 @@ interface DiscoverResult {
  */
 declare const discoverSite: (runtime: CrawlRuntime, site: string, options?: DiscoverRunOptions) => Promise<DiscoverResult>;
 //#endregion
+//#region src/embed.d.ts
+/**
+ * The parts of hybrid search that need no store: the embedder interface, the chunks of a text, and
+ * the fusion of ranked lists.
+ *
+ * An embedder changes texts into vectors. Two texts with a similar meaning get vectors that point
+ * in a similar direction. Hybrid search ranks the stored documents two times: one time with the
+ * full-text index (BM25), one time by the cosine of the vectors. Reciprocal rank fusion (RRF) then
+ * makes one list of the two.
+ *
+ * `Embedder` is the extension point. `transformersEmbedder` and `ollamaEmbedder` are two adapters,
+ * and an application can give an embedder of its own, as it can give a renderer of its own.
+ */
+/** These options go to each call of `Embedder.embed`. */
+interface EmbedCallOptions {
+  /** When this signal aborts, the embedder stops and rejects with the reason of the signal. */
+  readonly signal?: AbortSignal | undefined;
+  /**
+   * This value tells what the texts are: `document` (parts of stored texts, the default) or `query`
+   * (the words of a search). Some models want a different prefix for each.
+   */
+  readonly purpose?: "document" | "query" | undefined;
+}
+/**
+ * An embedder changes texts into vectors. The vector store keeps each vector with the `id` of its
+ * embedder, so the vectors of two models do not mix.
+ */
+interface Embedder {
+  /** This text names the model and its version. Change it when the vectors of the model change. */
+  readonly id: string;
+  /** This value is the length of each vector. The value 0 tells that the embedder learns it from its first answer. */
+  readonly dimensions: number;
+  /** This value is the largest input of the model, in tokens. The chunks of a text fit in it. The default is 256. */
+  readonly maxInputTokens?: number | undefined;
+  /** This method gives one vector for each text, in the same sequence. */
+  embed(texts: readonly string[], options: EmbedCallOptions): Promise<Float32Array[]>;
+  /** This optional method releases the model. A later call of `embed` loads it again. */
+  close?(): Promise<void>;
+}
+/** These are the limits of the chunks of a text. */
+declare const CHUNK_LIMITS: {
+  /** A chunk has this number of tokens at most, also for a model with a larger input. */
+  readonly maxTokens: number;
+  /** A chunk has this number of tokens when the embedder does not give `maxInputTokens`. */
+  readonly defaultTokens: number;
+  /** A chunk fills this share of its limit, because the count of the tokens is an estimate. */
+  readonly fill: number;
+  /** Two chunks next to each other share this share of the tokens of a chunk. */
+  readonly overlap: number;
+  /** A text gives this number of chunks at most. The rest of a longer text gets no vector. */
+  readonly maxChunks: number;
+};
+/** This is one chunk of a text: a window of paragraphs, with its position in the text. */
+interface TextChunk {
+  /** This value is the position of the chunk in the list, from 0. */
+  readonly index: number;
+  /** These values are the positions of the chunk in the text, in UTF-16 units. */
+  readonly start: number;
+  readonly end: number;
+  readonly text: string;
+}
+/** These options control the chunks of a text. */
+interface ChunkOptions {
+  /** This value is the largest number of tokens in a chunk. */
+  readonly maxTokens: number;
+  /** This value is the number of tokens that two chunks next to each other can share. The default is 10 percent of `maxTokens`. */
+  readonly overlapTokens?: number | undefined;
+  /** This value is the largest number of chunks. The default is `CHUNK_LIMITS.maxChunks`. */
+  readonly maxChunks?: number | undefined;
+}
+/**
+ * This function estimates the number of tokens of a part of a text. A word is one token for each
+ * six letters. A Chinese, Japanese or Thai character is one token. Each other character that is
+ * not a space is one token. The estimate is a little high for English text, so a chunk fits.
+ */
+declare const estimateTokens: (text: string, start?: number, end?: number) => number;
+/**
+ * This function cuts a text into chunks for an embedder. A chunk is a window of whole sentences.
+ * It ends at the end of a paragraph when the paragraph fills half of the window or more. A
+ * sentence that is longer than the window is cut at its spaces. Two chunks next to each other share
+ * a few sentences (`overlapTokens`), so a passage at a cut is in one chunk. Each chunk keeps its
+ * position in the text, so a hit points at a passage.
+ */
+declare const chunkText: (text: string, options: ChunkOptions) => TextChunk[];
+/**
+ * This function gives the number of tokens of a chunk for an embedder. It is the input limit with a
+ * margin for the estimate, and `CHUNK_LIMITS.maxTokens` at most.
+ */
+declare const chunkTokensOf: (embedder: Pick<Embedder, "maxInputTokens">) => number;
+/** This function gives a copy of a vector with a length of 1. A vector of zeros stays zeros. A value that is not finite throws a `RangeError`. */
+declare const unitVector: (vector: ArrayLike<number>) => Float32Array;
+/** These are the defaults of a hybrid search. */
+declare const HYBRID_DEFAULTS: {
+  /** The full-text search gives this number of candidates. Each candidate gets vectors when it has none. */
+  readonly candidates: number;
+  /** The vector search gives this number of documents: the documents with the closest chunks. */
+  readonly neighbors: number;
+  /** This value is the constant k of reciprocal rank fusion. A larger k gives less weight to the top of each list. */
+  readonly fusionK: number;
+};
+/** This is one item of a fused list. */
+interface FusedRank {
+  readonly id: string;
+  /** This value is the sum of 1 / (k + rank) over the lists that hold the item. */
+  readonly score: number;
+  /** These values are the rank of the item in each list, from 1, or null when a list does not hold it. */
+  readonly ranks: ReadonlyArray<number | null>;
+}
+/**
+ * This function fuses ranked lists with reciprocal rank fusion. Each item gets the sum of
+ * 1 / (k + rank) over the lists that hold it, with the rank from 1. The best item is first. A tie goes to
+ * the item with the best rank in a list, then to the earlier list, then to the smaller id. An item
+ * that is two times in one list counts at its first place.
+ */
+declare const fuseRanks: (lists: ReadonlyArray<readonly string[]>, k?: number) => FusedRank[];
+//#endregion
 //#region src/extract.d.ts
 /**
  * The field check takes the values that a caller proposes for the fields of a stored page. It finds
@@ -2353,6 +2515,402 @@ interface CrawlRun extends AsyncIterable<CrawlPage> {
  */
 declare const crawlSite: (runtime: CrawlRuntime, seeds: string | readonly string[], options?: CrawlRunOptions) => CrawlRun;
 //#endregion
+//#region src/vectors.d.ts
+/**
+ * The vector store keeps the vectors of hybrid search in `vectors.sqlite`, beside the index. It is
+ * a projection, as the index is. Delete the file, and the next hybrid search or `embed` makes the
+ * vectors again from the stored texts.
+ *
+ * - A row is one chunk of one text for one embedder. Its key is the SHA-256 of the text, the
+ *   number of the chunk and the id of the embedder. Thus a reindex keeps each vector, and the same
+ *   text gets its vectors one time for each model, also when two documents hold it.
+ * - A text is complete when the table `texts` has it, with the plan of its chunks. A text with a
+ *   different plan gets new chunks.
+ * - The table `docs` points each document at the text that has its vectors. A search uses only
+ *   the texts that a document points at, and the caller checks each document against the index.
+ * - A search compares the vector of the question with each stored vector (a brute-force cosine).
+ *   The store keeps the vectors of each embedder in memory after the first search. It reads them
+ *   again when a different process writes the file.
+ *
+ * Each connection waits for the lock of a different process (`busy_timeout`), and each write is
+ * one transaction (`BEGIN IMMEDIATE`). The file has a schema version (`VECTORS_SCHEMA`). A file of
+ * a different version opens empty.
+ */
+/** This value is the version of the schema of the vector file. */
+declare const VECTORS_SCHEMA = 1;
+/** This is one chunk of a text with its vector, as the store keeps it. */
+interface StoredChunk {
+  readonly chunk: number;
+  /** These values are the positions of the chunk in the text, in UTF-16 units. */
+  readonly start: number;
+  readonly end: number;
+  /** This vector has a length of 1, or it is zeros. */
+  readonly vector: Float32Array;
+}
+/** This is the best chunk of one text for a question. */
+interface VectorHit {
+  /** This value is the SHA-256 of the text, in hex. */
+  readonly sha: string;
+  readonly chunk: number;
+  readonly start: number;
+  readonly end: number;
+  /** This value is the cosine of the chunk and the question, from -1 to 1. */
+  readonly similarity: number;
+  /** These documents point at the text. The caller checks each one against the index. */
+  readonly docs: readonly string[];
+}
+/** These are the counts of the store, for one embedder or for each embedder. */
+interface VectorCounts {
+  readonly texts: number;
+  readonly chunks: number;
+  readonly docs: number;
+  /** This value is the size of the vectors in bytes, with no overhead of SQLite. */
+  readonly bytes: number;
+}
+/** These options open a vector store. */
+interface VectorStoreOptions {
+  /** This value is the time in milliseconds that a write waits for the lock of a different process. The default is 10 seconds. */
+  readonly busyTimeoutMs?: number | undefined;
+}
+/** These are the parts of one text in a write of the store. */
+interface VectorWrite {
+  readonly embedder: string;
+  readonly dimensions: number;
+  /** This value is the SHA-256 of the text, in hex. */
+  readonly sha: string;
+  /** This text names the rule of the chunks. A text with a different plan gets new chunks. */
+  readonly plan: string;
+  /** This value is the length of the text. */
+  readonly chars: number;
+  readonly chunks: ReadonlyArray<Omit<StoredChunk, "chunk">>;
+  /** These documents hold the text as their current text. The store points each one at it. */
+  readonly docs: readonly string[];
+  /** This time (ISO 8601) is the time of the write. */
+  readonly at: string;
+}
+declare class VectorStore {
+  readonly path: string;
+  private readonly db;
+  private readonly statements;
+  private readonly matrices;
+  private closed;
+  constructor(path: string, options?: VectorStoreOptions);
+  private stmt;
+  private write;
+  private live;
+  /** This method gives the plan of the chunks of a text when the store has each of its vectors, or undefined. */
+  plan(embedder: string, sha: string): string | undefined;
+  /** This method gives the text that a document points at, or undefined. */
+  pointed(embedder: string, doc: string): string | undefined;
+  /** This method gives the dimensions that the store keeps for an embedder, or undefined. */
+  dimensions(embedder: string): number | undefined;
+  /**
+   * This method writes the chunks of one text and points its documents at it, in one transaction.
+   * A vector with a length that is different from the dimensions of the embedder throws an error.
+   */
+  put(input: VectorWrite): void;
+  private pointRow;
+  /** This method points a document at a text that the store has. */
+  point(embedder: string, doc: string, sha: string, at: string): void;
+  /** This method removes the pointer of a document, for example when the document is held back. Its text keeps its vectors. */
+  forget(embedder: string, doc: string): void;
+  /** This method gives the chunks of a text, in their sequence. */
+  chunks(embedder: string, sha: string): StoredChunk[];
+  private pointMatrix;
+  private append;
+  /** This method gives the vectors of an embedder in memory. It reads them again when a different process wrote the file. */
+  private matrix;
+  private load;
+  /** This method gives the best chunk of one text for a question, or undefined when the store has no vector of the text. */
+  best(embedder: string, sha: string, query: Float32Array): VectorHit | undefined;
+  private hitOf;
+  /**
+   * This method compares a question with each stored vector of an embedder (a brute-force cosine).
+   * It gives the best chunk of each text that a document points at, the best first. With `within`,
+   * it compares only the texts of those documents.
+   */
+  search(embedder: string, query: Float32Array, options?: {
+    readonly within?: Iterable<string> | undefined;
+  }): VectorHit[];
+  /** This method gives the counts of the store, for one embedder or for each embedder. */
+  counts(embedder?: string): VectorCounts;
+  /** This method closes the file. A second close does nothing. */
+  close(): void;
+}
+//#endregion
+//#region src/hybrid.d.ts
+/**
+ * Hybrid search ranks the stored documents two times and fuses the two lists:
+ *
+ * 1. The full-text index gives the BM25 candidates (`candidates`, 50 by default). Each candidate
+ *    whose text has no vectors gets them from the embedder first.
+ * 2. The vector store gives the documents with the closest chunks (`neighbors`, 50 by default),
+ *    with a brute-force cosine over the vectors of the embedder.
+ * 3. Reciprocal rank fusion (`fusionK`, 60 by default) makes one list of the two.
+ *
+ * The snippet and the `passage` of each hit come from its best chunk. A held-back document gets
+ * no vectors, and no result gives it, exactly as for `find`. The text of a held-back version does not
+ * goes to the embedder.
+ */
+/** These options change a hybrid search. */
+interface HybridOptions {
+  /** This value is the number of BM25 candidates. The default is `HYBRID_DEFAULTS.candidates`. */
+  readonly candidates?: number | undefined;
+  /** This value is the number of documents from the vector search. The default is `HYBRID_DEFAULTS.neighbors`. */
+  readonly neighbors?: number | undefined;
+  /** This value is the constant k of reciprocal rank fusion. The default is `HYBRID_DEFAULTS.fusionK`. */
+  readonly fusionK?: number | undefined;
+  /** When this signal aborts, the search stops the embedder and rejects. */
+  readonly signal?: AbortSignal | undefined;
+}
+/** These options control a hybrid search of the store. */
+interface HybridFindOptions extends HybridOptions {
+  /** This value is the maximum number of results. The default is 10, and the maximum is 1000. */
+  readonly limit?: number | undefined;
+  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
+  readonly kind?: string | null | undefined;
+}
+/** This is one document of a ranking of a set of documents (`rankWithin`). */
+interface Ranked {
+  readonly doc: string;
+  /** This value is the score. A larger score is a better match. In the mode `hybrid`, it is the fused score. */
+  readonly score: number;
+  /** This value is the best chunk of the document, in the mode `hybrid`. */
+  readonly passage?: Passage | undefined;
+}
+/** These options control `embed`, which gives vectors to the stored documents that have none. */
+interface EmbedOptions {
+  /** This value is the longest time of the work, for example `"10m"` (the default). No embedding starts after it. */
+  readonly budget?: Duration | undefined;
+  /** This value is the largest number of texts that get new vectors. The default has no limit. */
+  readonly maxDocs?: number | undefined;
+  /** This value keeps only the documents of one kind, for example `page` or `pdf`. */
+  readonly kind?: string | null | undefined;
+  /** When this signal aborts, the work stops. The result then has `stopped: "signal"`. */
+  readonly signal?: AbortSignal | undefined;
+}
+/** The reason of the end of `embed`. */
+type EmbedStop = "done" | "budget" | "max-docs" | "signal";
+/** The counts of `embed`. */
+interface EmbedResult {
+  /** This value is the id of the embedder. */
+  readonly embedder: string;
+  /** This value is the number of documents that the work examined. */
+  readonly docs: number;
+  /** This value is the number of texts that got new vectors. */
+  readonly embedded: number;
+  /** This value is the number of documents whose text had vectors already. */
+  readonly reused: number;
+  /** This value is the number of documents that the work left out, because they are held back. */
+  readonly held_back: number;
+  /** This value is the number of new vectors: one for each chunk. */
+  readonly chunks: number;
+  readonly stopped: EmbedStop;
+  /** This value is the time of the work in milliseconds. */
+  readonly ms: number;
+}
+/** Hybrid search over a runtime and a vector store. `openCrawler` makes one when it has an embedder. */
+interface HybridSearch {
+  readonly embedder: Embedder;
+  /** This method searches the stored documents in the mode `hybrid`. It sends no request to a site. */
+  find(q: string, options?: HybridFindOptions): Promise<Found[]>;
+  /** This method ranks a set of documents in the mode `hybrid`, the best first. */
+  rankWithin(q: string, docs: readonly string[], options?: HybridOptions): Promise<Ranked[]>;
+  /** This method gives vectors to the stored documents that have none, within a budget. */
+  embed(options?: EmbedOptions): Promise<EmbedResult>;
+}
+/** These options make a hybrid search. */
+interface HybridSearchOptions {
+  readonly embedder: Embedder;
+  /** This store keeps the vectors. The caller opens it and closes it. */
+  readonly store: VectorStore;
+}
+/** This function makes a hybrid search over a runtime. The vectors go into the store, with the id of the embedder. */
+declare const createHybridSearch: (runtime: CrawlRuntime, options: HybridSearchOptions) => HybridSearch;
+//#endregion
+//#region src/recipe.d.ts
+/** The capture that an extractor reads: the final URL of the response and its content type. */
+interface RecipeCapture {
+  readonly url: string;
+  readonly contentType: string | null;
+}
+/**
+ * This function reads one captured body into the documents that it holds. It is pure: the same
+ * body and capture always give the same documents. It sends no request. It can answer later.
+ */
+type RecipeExtract = (body: Uint8Array, capture: RecipeCapture) => ReadonlyArray<ExtractedDoc> | Promise<ReadonlyArray<ExtractedDoc>>;
+/** The request of a recipe for one input: the URL, and the parts that change with the input. */
+interface RecipeRequest {
+  readonly url: string;
+  /** This value replaces the `accept` of the recipe for this request. */
+  readonly accept?: string | undefined;
+  /** These headers go with the request. The crawler does not write them into the capture. */
+  readonly headers?: Readonly<Record<string, string>> | undefined;
+  /** These query parameters hold secrets. The capture shows `REDACTED` in their place. */
+  readonly secretParams?: readonly string[] | undefined;
+}
+/** These values declare a recipe. Only `name` and `extract` are necessary. */
+interface RecipeDefinition<Input = string> {
+  /**
+   * This name goes into each capture record of the recipe, so it is part of each capture id.
+   * `reindex` finds the extractor by it. Do not change it after the first capture.
+   */
+  readonly name: string;
+  /** This text tells what the recipe reads. */
+  readonly description?: string | undefined;
+  /**
+   * The value `page` (the default) is a web page: robots.txt applies. The value `api` is a
+   * documented API with terms of its own: robots.txt does not apply, and the pace still does.
+   */
+  readonly mode?: "api" | "page" | undefined;
+  /** This value is the `Accept` header. The defaults are HTML for a page and JSON for an API. */
+  readonly accept?: string | undefined;
+  /** This value is the documented pace of the source: the shortest time between two of its requests. */
+  readonly minInterval?: Duration | undefined;
+  /** A stored copy younger than this age answers with no request. The default is `24h`. */
+  readonly maxAge?: Duration | undefined;
+  /** This value is the most bytes of a body that the crawler keeps. */
+  readonly maxBytes?: number | undefined;
+  /** This function gives the request for an input. With no function, the input is the URL. */
+  readonly fetch?: ((input: Input) => string | RecipeRequest) | undefined;
+  /** This function reads a captured body into documents. It is pure, and it sends no request. */
+  readonly extract: RecipeExtract;
+}
+/** These options build the request of a recipe for one input (`Recipe.request`). */
+interface RecipeRequestOptions {
+  /** This value replaces the `maxAge` of the recipe. The value `0` asks the source again. */
+  readonly maxAge?: Duration | undefined;
+  /** These agents get their robots.txt rules applied too, for a recipe of the mode `page`. */
+  readonly onBehalfOf?: readonly string[] | undefined;
+  /** This name connects the captures to the job that asked for them. */
+  readonly job?: string | undefined;
+  /** When this signal aborts, the crawler stops the fetch. */
+  readonly signal?: AbortSignal | undefined;
+  /** This label goes into each capture record of the request. */
+  readonly scope?: string | undefined;
+  /** These labels go into each capture record of the request. They are not part of the capture id. */
+  readonly labels?: Readonly<Record<string, string>> | undefined;
+  /** No request starts at or after this time (milliseconds since the epoch, on the clock of the crawler). */
+  readonly deadline?: number | undefined;
+}
+/** A recipe that `defineRecipe` made. Give it to `runRecipe`, to `registerRecipe` or to `openCrawler`. */
+interface Recipe<Input = string> {
+  readonly name: string;
+  readonly description: string | null;
+  readonly mode: "api" | "page";
+  readonly accept: string;
+  /** This extractor reads the captures of the recipe. It checks the documents of the definition. */
+  readonly extractor: Extractor;
+  /** This method gives the request of the crawler for one input. A value that is not a URL throws a `TypeError`. */
+  request(input: Input, options?: RecipeRequestOptions): FetchRequest;
+}
+/**
+ * This function reads a body as JSON, in the encoding that the response declares. A body that is
+ * not JSON (a page about maintenance, or a login page) throws an error that names the URL and the
+ * content type. In an extractor, the runtime then reports that error.
+ */
+declare const readJson: (body: Uint8Array, capture: RecipeCapture) => unknown;
+/**
+ * These recipe names belong to the crawler: the page reader, the archived fallback, robots.txt,
+ * and the sitemaps and feeds of `discover`. A recipe cannot use them.
+ */
+declare const RESERVED_RECIPES: readonly string[];
+/** These are the limits of a recipe. */
+declare const RECIPE_LIMITS: {
+  /** The longest recipe name. */
+  readonly nameChars: number;
+  /** The longest document id. */
+  readonly docIdChars: number;
+};
+/**
+ * This function declares a recipe. It checks the definition at once: a name that is not valid, a
+ * reserved name or a duration that it cannot read throws an error. The extractor of the recipe
+ * checks each document of `extract`. A document that is not valid makes the extractor throw, and
+ * the runtime reports it as an error of the extractor.
+ */
+declare const defineRecipe: <Input = string>(definition: RecipeDefinition<Input>) => Recipe<Input>;
+/**
+ * This function gives the extractor of a recipe to the extractors of a runtime. A second
+ * registration of the same recipe does nothing. A different extractor with the same name throws
+ * an error, because `reindex` must read each capture with one extractor.
+ */
+declare const registerRecipe: <Input>(extractors: Extractors, recipe: Recipe<Input>) => void;
+/** These options control one run of a recipe. */
+interface RecipeRunOptions extends Omit<RecipeRequestOptions, "deadline"> {
+  /** No request starts after this time from the start of the run, for example `"30s"`. */
+  readonly budget?: Duration | undefined;
+  /** This value is true when a person gave this input directly. It is the only way to see a held-back document. The crawler logs it. */
+  readonly userProvided?: boolean | undefined;
+  /**
+   * The value `archive` gives a copy from a public web archive when the site refuses the page or is
+   * down. Only a recipe of the mode `page` takes it. The page reader reads the copy.
+   */
+  readonly fallback?: "archive" | "none" | undefined;
+  /** The value true also gives an archived copy of a page that the current robots.txt disallows. */
+  readonly archiveDespiteRobots?: boolean | undefined;
+}
+/** One document of a run of a recipe: the version of the capture of the run. */
+interface RecipeDoc {
+  readonly doc: string;
+  readonly kind: string;
+  /** This value is the title. A held-back document shows no title. */
+  readonly title: string;
+  readonly url: string | null;
+  /** This value is the metadata. A held-back document shows only the structural fields. */
+  readonly meta: Readonly<Record<string, unknown>>;
+  /** This value is the length of the text. `read` gives the text. */
+  readonly chars: number;
+  /** This id is the capture of this version. A citation names it. */
+  readonly capture: string;
+  readonly fetched_at: string;
+  readonly held_back: boolean;
+  /** This value tells that the document is an archived copy, with the archive and the date of the copy. */
+  readonly archived?: Archived | undefined;
+}
+/** The result of `runRecipe`. */
+interface RecipeResult {
+  readonly recipe: string;
+  /** This URL is the URL of the capture. Its secrets show as `REDACTED`. */
+  readonly url: string;
+  readonly outcome: string;
+  readonly status: number;
+  /** These documents came from the capture of the run, in the order of the extractor. */
+  readonly docs: readonly RecipeDoc[];
+  /** This id is the capture of the documents. */
+  readonly capture: string | null;
+  readonly fetched_at: string | null;
+  /** This value is true when the stored copy answered, and the crawler sent no request. */
+  readonly from_cache: boolean;
+  /** This value is true when a document is held back. */
+  readonly held_back: boolean;
+  /** This text is the error of the extractor, when it threw for the capture. */
+  readonly error: string | null;
+  /** This value tells that the documents are an archived copy, not a fresh fetch. */
+  readonly archived?: Archived | undefined;
+  readonly note: string | null;
+}
+/** A recipe operates on a runtime, or on a crawler that has one (`crawler.runtime`). */
+type RecipeTarget = CrawlRuntime | {
+  readonly runtime: CrawlRuntime;
+};
+/**
+ * This function starts a recipe for one input. It fetches the request through the crawler, and it
+ * reads the capture with the extractor of the recipe. It gives the documents of that capture.
+ * These are its rules:
+ *
+ * - The runtime registers the recipe the first time that it gets it. Give the recipe to
+ *   `openCrawler` too, so that `reindex` reads its captures before the first fetch.
+ * - A stored copy younger than `maxAge` answers with no request. When the extractor cannot read a
+ *   stored copy, the crawler asks the source one more time.
+ * - An extractor that throws gives no document, and `error` tells why. The fetch still succeeds.
+ * - A document that asked not to be AI input shows no title and only its structural metadata,
+ *   unless `userProvided`. The crawler logs each `userProvided` read of a held-back document.
+ * - With `fallback: "archive"`, a page that the site refuses gets an archived copy, with
+ *   `archived`. The page reader reads the copy. The extractor of the recipe reads only live
+ *   captures. Thus its documents do not mix a live answer and an old copy of a third party.
+ */
+declare const runRecipe: <Input>(target: RecipeTarget, recipe: Recipe<Input>, input: Input, options?: RecipeRunOptions) => Promise<RecipeResult>;
+//#endregion
 //#region src/watch.d.ts
 /**
  * A watch list keeps the pages that the crawler checks again on a schedule. `watch.run` fetches the
@@ -2370,7 +2928,7 @@ declare const crawlSite: (runtime: CrawlRuntime, seeds: string | readonly string
  * - The list is in `watch.sqlite` in the store, so it stays after a restart. A run claims each entry
  *   with a lease in that file, so two processes do not check one entry two times.
  *
- * The package has no daemon. A scheduler of the operator starts `watch.run`.
+ * A scheduler of the operator starts `watch.run`, or the daemon (`startDaemon`) starts it on a schedule.
  */
 /** This value is the version of the schema of `watch.sqlite`. */
 declare const WATCH_SCHEMA = 1;
@@ -2459,6 +3017,11 @@ interface WatchRunOptions {
   readonly maxPages?: number | undefined;
   /** When this signal aborts, the run stops the checks in progress and ends. */
   readonly signal?: AbortSignal | undefined;
+  /**
+   * When this signal aborts, the run starts no new check. The checks in progress end and write
+   * their state, and the run stops with `cancelled`. The daemon uses it for a clean stop.
+   */
+  readonly drain?: AbortSignal | undefined;
 }
 /** These options control `runWatch`: the options of a run, and the parts of `openCrawler`. */
 interface WatchRunnerOptions extends WatchRunOptions {
@@ -2720,6 +3283,11 @@ interface RuntimeOptions {
   readonly onEvent?: ((event: CrawlEvent) => void) | undefined;
   /** These extractors read the captures of the recipes of an application. The recipes `page` and `archive-copy` are built in. */
   readonly extractors?: Readonly<Record<string, Extractor>> | undefined;
+  /**
+   * These recipes of `defineRecipe` get their extractors at open. Thus `reindex` reads their
+   * captures before the first fetch. A recipe with the name of an entry of `extractors` throws an error.
+   */
+  readonly recipes?: ReadonlyArray<Recipe<never>> | undefined;
   /**
    * This signer signs each request with Web Bot Auth (`CrawlerOptions.signer`). Give the options of
    * `createWebBotAuthSigner`, or a `RequestSigner`. Without it, the crawler signs nothing.
@@ -2746,6 +3314,12 @@ interface OpenOptions extends Omit<RuntimeOptions, "dir" | "log" | "blobs" | "in
   readonly onBehalfOf?: readonly string[] | undefined;
   /** This value is the shortest `every` of a watch. The default is `1h`, so a watch cannot hammer a site. */
   readonly watchFloor?: Duration | undefined;
+  /**
+   * This embedder makes the vectors of hybrid search (`rank: "hybrid"`): `transformersEmbedder`,
+   * `ollamaEmbedder` or an embedder of your own. The vectors go into `vectors.sqlite` in the store.
+   * Without it, `rank: "hybrid"` throws an error. `close` also closes the embedder.
+   */
+  readonly embedder?: Embedder | undefined;
 }
 /** These options control one fetch. */
 interface FetchOptions {
@@ -2787,6 +3361,19 @@ interface FindOptions {
    */
   readonly rank?: RankMode | undefined;
 }
+/**
+ * These options control a search of `openCrawler`: the options of `find`, and `rank: "hybrid"`
+ * with the options of a hybrid search. The mode `hybrid` needs `OpenOptions.embedder`.
+ */
+interface SearchOptions extends Omit<FindOptions, "rank">, HybridFindOptions {
+  /** This value chooses the ranking. `hybrid` fuses BM25 with the ranking of the vectors (`SearchRank`). */
+  readonly rank?: SearchRank | undefined;
+}
+/** These options control `rankWithin` of a crawler. */
+interface RankOptions extends HybridOptions {
+  /** This value chooses the ranking. The default is `exact`. */
+  readonly rank?: SearchRank | undefined;
+}
 /** A polite crawler fetches each source one time, keeps it, and answers the next questions from the disk. */
 interface PoliteCrawler {
   /** The runtime gives the parts to an application that needs more control. */
@@ -2842,8 +3429,14 @@ interface PoliteCrawler {
    * locates each value. It does not make values. It sends no request. `openCrawler` gives it.
    */
   extract?(doc: string, fields: Readonly<Record<string, FieldClaim>>, options?: ExtractOptions): Promise<ExtractResult>;
-}
-/** This crawler is the result of `openCrawler`: a `PoliteCrawler` with `discover`, `crawl`, `diff`, `watch` and `extract`. */
+  /** This optional search ranks with the vectors of an embedder. `openCrawler` gives it when it has an embedder. */
+  readonly hybrid?: HybridSearch | undefined;
+  /** This optional method gives vectors to the stored documents that have none, within a budget. */
+  embed?(options?: EmbedOptions): Promise<EmbedResult>;
+  /** This optional method ranks a set of stored documents against a query, the best first. */
+  rankWithin?(q: string, docs: readonly string[], options?: RankOptions): Promise<Ranked[]>;
+}
+/** This crawler is the result of `openCrawler` in version 0.1.2: a `PoliteCrawler` with `discover`, `crawl`, `diff`, `watch` and `extract`. */
 interface OpenedCrawler extends PoliteCrawler {
   /** This method finds the pages of a site from its sitemaps and its feeds. */
   discover(site: string, options?: DiscoverOptions): Promise<DiscoverResult>;
@@ -2859,6 +3452,24 @@ interface OpenedCrawler extends PoliteCrawler {
    */
   extract(doc: string, fields: Readonly<Record<string, FieldClaim>>, options?: ExtractOptions): Promise<ExtractResult>;
 }
+/**
+ * This crawler is the result of `openCrawler`: an `OpenedCrawler` with hybrid search. Its `find`
+ * takes `rank: "hybrid"`, and it has `hybrid`, `embed` and `rankWithin`. `OpenedCrawler` does not
+ * change, so an object of your own that implements the `OpenedCrawler` of 0.1.2 stays correct.
+ */
+interface HybridCrawler extends OpenedCrawler {
+  /** This method searches the stored documents. It also takes `rank: "hybrid"`, when the crawler has an embedder. */
+  find(q: string, options?: SearchOptions): Promise<Found[]>;
+  /** This search ranks with the vectors of the embedder, or it is undefined when the crawler has no embedder. */
+  readonly hybrid: HybridSearch | undefined;
+  /** This method gives vectors to the stored documents that have none, within a budget. It rejects when the crawler has no embedder. */
+  embed(options?: EmbedOptions): Promise<EmbedResult>;
+  /**
+   * This method ranks a set of stored documents against a query, the best first. The modes
+   * `exact` and `fast` are `DocIndex.rankWithin`. The mode `hybrid` needs an embedder.
+   */
+  rankWithin(q: string, docs: readonly string[], options?: RankOptions): Promise<Ranked[]>;
+}
 /**
  * This function opens a crawler on a folder. The folder keeps the capture log (`captures/`), the
  * response bodies (`blobs/`), the search index (`index.sqlite`) and the state of each host
@@ -2869,7 +3480,293 @@ interface OpenedCrawler extends PoliteCrawler {
  * Each method gives a promise, and none throws before its promise. After `close`, each method
  * rejects.
  */
-declare const openCrawler: (options: OpenOptions) => OpenedCrawler;
+declare const openCrawler: (options: OpenOptions) => HybridCrawler;
+//#endregion
+//#region src/daemon.d.ts
+/** These are the defaults of the daemon. */
+declare const DAEMON_DEFAULTS: {
+  /** The longest wait between two ticks: 1 minute. A tick comes sooner when an entry is due sooner. */
+  readonly tickMs: 60_000;
+  /** The most random time that the daemon adds to each wait: 10 seconds. */
+  readonly jitterMs: 10_000;
+  /** The budget of the checks of one tick: 10 minutes. No request starts after it. */
+  readonly budgetMs: 600_000;
+  /** The most checks of one tick. */
+  readonly maxPages: 100;
+  /** At a stop, the checks in progress get this time to end: 10 seconds. Then they abort. */
+  readonly graceMs: 10_000;
+  /** The shortest wait between two ticks: 1 second. */
+  readonly minGapMs: 1_000;
+  /** The shortest tick: 1 second. */
+  readonly minTickMs: 1_000;
+  /** The status lists this number of the next due entries. */
+  readonly dueShown: 10;
+  /** An on-change command that runs longer than this time stops: 1 minute. */
+  readonly hookTimeoutMs: 60_000;
+};
+/** This value is the version of the format of the state file. */
+declare const DAEMON_STATE_SCHEMA = 1;
+/** This is a watch entry that is due soon, as the status shows it. */
+interface DaemonDue {
+  readonly url: string;
+  /** This time (ISO 8601) is the time when the entry is due. */
+  readonly next_at: string;
+  readonly every_ms: number;
+  /** This value is the kind of the last check, or null before the first check. */
+  readonly last_kind: string | null;
+}
+/** This is one tick of the schedule: one run of the watch list. */
+interface DaemonTick {
+  /** This time (ISO 8601) is the start of the run. */
+  readonly started_at: string;
+  /** This time (ISO 8601) is the end of the run. */
+  readonly ended_at: string;
+  /** This value is the summary of the run, or null when the run failed. */
+  readonly summary: WatchSummary | null;
+  /** This text tells why the run failed, or null. */
+  readonly error: string | null;
+  /** This time (ISO 8601) is the time of the next tick, or null when the schedule stops. */
+  readonly next_at: string | null;
+  /** These entries are the next due entries after the run, the first due first. */
+  readonly due: readonly DaemonDue[];
+}
+/** These options control the schedule of the watch list. */
+interface ScheduleOptions {
+  /** This value is the longest wait between two ticks. The default is `1m`, and the floor is `1s`. */
+  readonly tick?: Duration | undefined;
+  /** This value is the most random time that the schedule adds to each wait. The default is `10s`. */
+  readonly jitter?: Duration | undefined;
+  /** This value is the budget of the checks of one tick. The default is `10m`. */
+  readonly budget?: Duration | undefined;
+  /** This value is the most checks of one tick. The default is 100. */
+  readonly maxPages?: number | undefined;
+  /** This clock gives the time and the waits. Give the clock of the crawler. The default is the real clock. */
+  readonly clock?: Clock | undefined;
+  /** This function gives the random numbers of the jitter. The default is `Math.random`. */
+  readonly random?: (() => number) | undefined;
+  /** This function gets each event of each check. */
+  readonly onEvent?: ((event: WatchEvent) => void) | undefined;
+  /** This function gets each tick when its run ends. */
+  readonly onTick?: ((tick: DaemonTick) => void) | undefined;
+  /** This function gets the error of a run that failed. The schedule goes on at the next tick. */
+  readonly onError?: ((error: Error) => void) | undefined;
+}
+/** This is the result of a schedule: the counts so far, and the last tick. */
+interface ScheduleResult {
+  readonly ticks: number;
+  /** This value counts the checks that gave an event. */
+  readonly checked: number;
+  /** This value counts the confirmed changes. */
+  readonly changes: number;
+  readonly last: DaemonTick | null;
+}
+/** A schedule that runs. `stop` ends it, and `done` resolves when it ends. */
+interface WatchSchedule {
+  /** This promise resolves when the schedule ends. It does not reject. */
+  readonly done: Promise<ScheduleResult>;
+  /**
+   * This method stops the schedule. The run in progress starts no new check. The checks in
+   * progress get `grace` (10 seconds by default) to end and write their state. Then they abort,
+   * and the run releases their leases. A second call with a shorter grace shortens it.
+   */
+  stop(grace?: Duration): Promise<ScheduleResult>;
+  /** This method gives the counts so far. */
+  result(): ScheduleResult;
+}
+/**
+ * This function checks a watch list on a schedule. The first tick starts at once. Each tick is one
+ * `watch.run` with the budget and `maxPages` of a tick. Thus each polite rule of the run applies:
+ * one request at a time to each host, the stops of a host, `Retry-After` and robots.txt.
+ *
+ * After a run, the schedule waits for the next tick (`nextTickAt`). An error of a run goes to
+ * `onError`, and the schedule goes on.
+ */
+declare const scheduleWatches: (watch: WatchList, options?: ScheduleOptions) => WatchSchedule;
+/** This is a task of the MCP server, as the status shows it. */
+interface DaemonTask {
+  readonly id: string;
+  /** This value is the name of the tool, for example `crawl_site`. */
+  readonly tool: string;
+  /** This value is `working`, `input_required`, `completed`, `failed` or `cancelled`. */
+  readonly status: string;
+  /** This text tells the progress, for example `3 of 10 done`, or null. */
+  readonly message: string | null;
+  readonly created_at: string;
+  readonly updated_at: string;
+}
+/** This is the status of a daemon. The state file (`<store>/daemon/state.json`) holds the same JSON. */
+interface DaemonStatus {
+  /** This value is the version of the format (`DAEMON_STATE_SCHEMA`). */
+  readonly schema: number;
+  /** This value is the version of polite-crawler. */
+  readonly version: string;
+  /** This value is the id of the process of the daemon. */
+  readonly pid: number;
+  readonly store: string;
+  /** This value is false after the daemon stopped. */
+  readonly running: boolean;
+  readonly started_at: string;
+  readonly updated_at: string;
+  readonly stopped_at: string | null;
+  readonly tick_ms: number;
+  readonly jitter_ms: number;
+  readonly budget_ms: number;
+  readonly max_pages: number;
+  /** This value counts the ticks. */
+  readonly ticks: number;
+  /** This value counts the checks. */
+  readonly checked: number;
+  /** This value counts the confirmed changes. */
+  readonly changes: number;
+  readonly last_tick: DaemonTick | null;
+  /** This time (ISO 8601) is the time of the next tick, or null. */
+  readonly next_tick_at: string | null;
+  /** These entries are the next due entries at the last tick, the first due first. */
+  readonly next_due: readonly DaemonDue[];
+  /** These tasks of the MCP server did not end. */
+  readonly tasks: readonly DaemonTask[];
+  /** This value is the MCP endpoint and the file of its token, or null with no MCP server. */
+  readonly mcp: {
+    readonly url: string;
+    readonly token_file: string | null;
+  } | null;
+  /** This value is the last error: of a tick, of the event log, of an on-change hook or of the MCP server. */
+  readonly last_error: {
+    readonly at: string;
+    readonly message: string;
+  } | null;
+}
+/** This is a status that `readDaemonState` read from a state file. */
+interface DaemonState extends DaemonStatus {
+  /** This value is true when the daemon runs: the state says so, and its process is alive. */
+  readonly alive: boolean;
+}
+/** This function gives the path of the state file of a store. */
+declare const daemonStateFile: (dir: string) => string;
+/**
+ * This function reads the state file of a daemon. It gives undefined when the file is not there,
+ * or when it is not a state file. `alive` tells if the daemon runs.
+ */
+declare const readDaemonState: (dir: string, file?: string) => DaemonState | undefined;
+/** These options start an on-change command. */
+interface ChangeCommandOptions {
+  /** A command that runs longer than this time stops. The default is `1m`. */
+  readonly timeout?: Duration | undefined;
+  /** This folder is the working folder of the command. The default is the working folder of the daemon. */
+  readonly cwd?: string | undefined;
+}
+/**
+ * This function gives an `onChange` hook that starts a command of the operator for each change.
+ * The command operates in the shell of the system, as the user of the daemon. It gets the event as
+ * one JSON line on its standard input, not in its command line. Thus no text of a page goes into
+ * the command line. The output of the command goes nowhere. A command that exits with a code that
+ * is not 0, or that runs too long, rejects with the end of its standard error.
+ *
+ * The command is the decision of the operator. Do not take it from a page or from an assistant.
+ */
+declare const changeCommand: (command: string, options?: ChangeCommandOptions) => ((event: WatchEvent) => Promise<void>);
+/** These options serve the MCP tools over HTTP. */
+interface DaemonMcpOptions {
+  /** This value is the port. The default is 9307. The value 0 takes a free port. */
+  readonly port?: number | undefined;
+  /** This value is the host to bind. The default is `127.0.0.1`. A host that is not loopback needs `allowRemote`. */
+  readonly host?: string | undefined;
+  /** The value true permits a host that is not loopback. Read the warning in the README first. */
+  readonly allowRemote?: boolean | undefined;
+  /** This value is the bearer token. The default makes a new token at each start, in `<store>/daemon/mcp-token`. */
+  readonly token?: string | undefined;
+  /** This file holds the bearer token of the operator. The daemon reads it and writes nothing. */
+  readonly tokenFile?: string | undefined;
+  /** These tokens get their robots.txt rules applied to each fetch of a tool. The default is `["Claude-User"]`. */
+  readonly onBehalfOf?: readonly string[] | undefined;
+  /** The value true lets `user_provided` release held-back text when the client cannot ask the user. */
+  readonly allowUserProvided?: boolean | undefined;
+  /** This text tells why the server renders no page. */
+  readonly renderOff?: string | undefined;
+}
+/** These functions connect an MCP host to the daemon. */
+interface DaemonMcpHooks {
+  /** This function gives the status of the daemon, for the status route. */
+  readonly status: () => DaemonStatus;
+  /** This function operates when a task starts, changes or ends. */
+  readonly onTasks: () => void;
+  /** This function writes a message of the server. */
+  readonly log: (line: string) => void;
+}
+/** This is the MCP server of the daemon, as the MCP host gives it. */
+interface DaemonMcpService {
+  /** This URL is the MCP endpoint. */
+  readonly url: string;
+  /** This method gives the tasks that did not end. */
+  tasks(): readonly DaemonTask[];
+  /** This method stops the server and cancels its tasks. */
+  close(): Promise<void>;
+}
+/**
+ * This is the seam between the daemon and its MCP server. The default host (`mcpHttpHost` of
+ * `polite-crawler/mcp`) serves the tools over Streamable HTTP. A different MCP host can replace it.
+ */
+interface DaemonMcpHost {
+  start(crawler: OpenedCrawler, options: DaemonMcpOptions, hooks: DaemonMcpHooks): Promise<DaemonMcpService>;
+}
+/** These options start a daemon on a store. Only `dir` is necessary. */
+interface DaemonOptions extends OpenOptions {
+  /** This value is the longest wait between two ticks. The default is `1m`, and the floor is `1s`. */
+  readonly tick?: Duration | undefined;
+  /** This value is the most random time that the daemon adds to each wait. The default is `10s`. */
+  readonly jitter?: Duration | undefined;
+  /** This value is the budget of the checks of one tick. The default is `10m`. */
+  readonly budget?: Duration | undefined;
+  /** This value is the most checks of one tick. The default is 100. */
+  readonly maxPages?: number | undefined;
+  /** At a stop, the checks in progress get this time to end. The default is `10s`. */
+  readonly grace?: Duration | undefined;
+  /**
+   * The daemon writes each change event as one JSON line here: a file, a function, or the standard
+   * output (the default). The value false writes no line. A change is an event with `confirmed: true`.
+   */
+  readonly events?: string | ((line: string) => void) | false | undefined;
+  /** This function gets each change event. The daemon starts it in order, one at a time, and logs its error. */
+  readonly onChange?: ((event: WatchEvent) => void | Promise<void>) | undefined;
+  /** These options serve the MCP tools over HTTP. Without them, the daemon serves no MCP. */
+  readonly mcp?: DaemonMcpOptions | undefined;
+  /** This host serves the MCP tools. The default serves them over Streamable HTTP. */
+  readonly mcpHost?: DaemonMcpHost | undefined;
+  /** The value false installs no handler of `SIGINT` and `SIGTERM`. The default is true. */
+  readonly signals?: boolean | undefined;
+  /** This file keeps the state of the daemon. The default is `<store>/daemon/state.json`. */
+  readonly stateFile?: string | undefined;
+  /** This function writes the messages of the daemon. The default writes them to the standard error. */
+  readonly log?: ((line: string) => void) | undefined;
+}
+/** This is a daemon that runs. */
+interface Daemon {
+  /** This crawler is the crawler of the daemon. The daemon closes it when it stops. */
+  readonly crawler: OpenedCrawler;
+  /** This file keeps the state of the daemon. */
+  readonly stateFile: string;
+  /** This value is the MCP server, or undefined. */
+  readonly mcp: DaemonMcpService | undefined;
+  /** This method gives the status at this time. */
+  status(): DaemonStatus;
+  /**
+   * This method stops the daemon. The checks in progress get the grace time to end, then they
+   * abort and the run releases their leases. The MCP server stops and cancels its tasks. A second
+   * call with `0` aborts the checks at once.
+   */
+  stop(grace?: Duration): Promise<DaemonStatus>;
+  /** This promise resolves with the last status when the daemon stopped. */
+  readonly done: Promise<DaemonStatus>;
+}
+/**
+ * This function starts a daemon on a store. It opens the crawler, serves MCP when `mcp` is given,
+ * and checks the watch list on a schedule (`scheduleWatches`). It writes each change event as one
+ * JSON line, and it gives each change to `onChange`. The state file tells `daemon status` what runs.
+ *
+ * Only one daemon operates on a store: a second start rejects. At `SIGINT` or `SIGTERM`, the daemon
+ * stops cleanly (`stop`). A second signal aborts the checks at once.
+ */
+declare const startDaemon: (options: DaemonOptions) => Promise<Daemon>;
 //#endregion
-export { DiscoverOptions as $, verifyQuote as $n, SignedRequest as $t, CrawlPage as A, Extractors as An, diffTexts as At, ExtractSummary as B, catchUpIndex as Bn, FetchOutcome as Bt, WatchStop as C, WebArchive as Cn, DiffSide as Ct, runWatch as D, CatchUpResult as Dn, VersionChoice as Dt, WatchSummary as E, CapturePayload as En, TextDiff as Et, UrlPattern as F, ReadOptions as Fn, CrawlEvent as Ft, FieldKind as G, isHeldBack as Gn, DirectoryKey as Gt, FieldCandidate as H, fetchPage as Hn, Fetched as Ht, crawlSite as I, ReadResult as In, Crawler as It, FieldStatus as J, pageExtractor as Jn, KeyDirectory as Jt, FieldResult as K, isoTime as Kn, KEY_DIRECTORY_PATH as Kt, ExtractOptions as L, ReindexResult as Ln, CrawlerOptions as Lt, CrawlRunOptions as M, Indexed as Mn, CacheEntry as Mt, CrawlStop as N, PageOptions as Nn, CacheIndex as Nt, CRAWL_LIMITS as O, CrawlRuntime as On, WordPart as Ot, CrawlSummary as P, PageResult as Pn, CaptureWriter as Pt, DISCOVER_LIMITS as Q, renderAndIndex as Qn, SignatureCheck as Qt, ExtractResult as R, VerifyOptions as Rn, DEADLINE_NOTE as Rt, WatchRunnerOptions as S, Archived as Sn, DiffResult as St, WatchStoreOptions as T, archivedDocId as Tn, MetaChange as Tt, FieldCheck as U, findDocuments as Un, RenderRequest as Ut, FIELD_KINDS as V, fetchAndIndex as Vn, FetchRequest as Vt, FieldClaim as W, heldBackMeta as Wn, renderedKey as Wt, checkFields as X, recordOverride as Xn, RequestSigner as Xt, FieldsChecked as Y, readDocument as Yn, KeyInput as Yt, extractDocument as Z, reindex as Zn, SignatureAgentType as Zt, WatchList as _, RankMode as _n, BlobStore as _r, DiffHunk as _t, PoliteCrawler as a, EgressOptions as an, ScanOptions as ar, discoverSite as at, WatchRun as b, VersionAt as bn, DiffLine as bt, createRuntime as c, Doc as cn, makeRecord as cr, SitemapEntry as ct, openCrawler as d, DocVersion as dn, RefusedReason as dr, SitemapRead as dt, WebBotAuthOptions as en, versionAtTime as er, DiscoverResult as et, WATCH_LIMITS as f, ExtractedDoc as fn, RenderOptions as fr, feedDate as ft, WatchKind as g, Latest as gn, playwrightRenderer as gr, DIFF_LIMITS as gt, WatchEvent as h, Found as hn, Renderer as hr, durationMs as ht, OpenedCrawler as i, verifySignature as in, DamagedLine as ir, DiscoveredUrl as it, CrawlRun as j, HELD_BACK as jn, CAPTURE_SCHEMA as jt, CrawlOptions as k, Extractor as kn, diffDocument as kt, defaultUserAgent as l, DocIndex as ln, NO_PLAYWRIGHT as lr, SitemapKind as lt, WatchEntry as m, FindIndexOptions as mn, Rendered as mr, Duration as mt, FindOptions as n, createWebBotAuthSigner as nn, CaptureLog as nr, DiscoverSource as nt, RuntimeOptions as o, EgressRefused as on, CaptureRecord as or, Changefreq as ot, WATCH_SCHEMA as p, FAST_RANK_SHARE as pn, RenderRefused as pr, readSitemap as pt, FieldSpan as q, pageDocId as qn, KEY_DIRECTORY_TYPE as qt, OpenOptions as r, keyDirectory as rn, CaptureReader as rr, DiscoverStop as rt, StoreRuntime as s, egressLookup as sn, RecordInput as sr, SITEMAP_LIMITS as st, FetchOptions as t, WebBotAuthSigner as tn, CaptureFilter as tr, DiscoverRunOptions as tt, isMailbox as u, DocInput as un, PlaywrightRenderer as ur, SitemapLimits as ut, WatchOptions as v, ReadingIndex as vn, FileBlobStore as vr, DiffInput as vt, WatchStore as w, archiveExtractor as wn, DiffSummary as wt, WatchRunOptions as x, VersionText as xn, DiffOptions as xt, WatchPending as y, Variant as yn, DiffLimits as yt, ExtractSource as z, VerifyResult as zn, DEFAULT_SECRET_PARAMS as zt };
-//# sourceMappingURL=open-Ca1lcWR3.d.mts.map
+export { RECIPE_LIMITS as $, FetchOutcome as $n, ReadOptions as $r, checkFields as $t, RuntimeOptions as A, PlaywrightRenderer as Ai, DiffHunk as An, Found as Ar, VectorWrite as At, WatchEvent as B, VersionChoice as Bn, Archived as Br, ExtractOptions as Bt, FetchOptions as C, ScanOptions as Ci, SitemapEntry as Cn, DocDigest as Cr, createHybridSearch as Ct, OpenedCrawler as D, Duration as Di, feedDate as Dn, ExtractedDoc as Dr, VectorHit as Dt, OpenOptions as E, makeRecord as Ei, SitemapRead as En, DocVersion as Er, VectorCounts as Et, isMailbox as F, Renderer as Fi, DiffResult as Fn, ReadingIndex as Fr, CrawlRunOptions as Ft, WatchRun as G, CacheEntry as Gn, CatchUpResult as Gr, FieldCandidate as Gt, WatchList as H, diffDocument as Hn, archiveExtractor as Hr, ExtractSource as Ht, openCrawler as I, playwrightRenderer as Ii, DiffSide as In, SearchRank as Ir, CrawlStop as It, WatchStop as J, CrawlEvent as Jn, Extractors as Jr, FieldKind as Jt, WatchRunOptions as K, CacheIndex as Kn, CrawlRuntime as Kr, FieldCheck as Kt, WATCH_LIMITS as L, BlobStore as Li, DiffSummary as Ln, Variant as Lr, CrawlSummary as Lt, StoreRuntime as M, RenderOptions as Mi, DiffLimits as Mn, NO_EMBEDDER as Mr, CrawlOptions as Mt, createRuntime as N, RenderRefused as Ni, DiffLine as Nn, Passage as Nr, CrawlPage as Nt, PoliteCrawler as O, durationMs as Oi, readSitemap as On, FAST_RANK_SHARE as Or, VectorStore as Ot, defaultUserAgent as P, Rendered as Pi, DiffOptions as Pn, RankMode as Pr, CrawlRun as Pt, runWatch as Q, DEFAULT_SECRET_PARAMS as Qn, PageResult as Qr, FieldsChecked as Qt, WATCH_SCHEMA as R, FileBlobStore as Ri, MetaChange as Rn, VersionAt as Rr, UrlPattern as Rt, startDaemon as S, DamagedLine as Si, SITEMAP_LIMITS as Sn, Doc as Sr, Ranked as St, HybridCrawler as T, RecordInput as Ti, SitemapLimits as Tn, DocInput as Tr, VECTORS_SCHEMA as Tt, WatchOptions as U, diffTexts as Un, archivedDocId as Ur, ExtractSummary as Ut, WatchKind as V, WordPart as Vn, WebArchive as Vr, ExtractResult as Vt, WatchPending as W, CAPTURE_SCHEMA as Wn, CapturePayload as Wr, FIELD_KINDS as Wt, WatchStoreOptions as X, CrawlerOptions as Xn, Indexed as Xr, FieldSpan as Xt, WatchStore as Y, Crawler as Yn, HELD_BACK as Yr, FieldResult as Yt, WatchSummary as Z, DEADLINE_NOTE as Zn, PageOptions as Zr, FieldStatus as Zt, WatchSchedule as _, verifyQuote as _i, DiscoverSource as _n, verifySignature as _r, EmbedStop as _t, DaemonDue as a, fetchAndIndex as ai, FusedRank as an, KEY_DIRECTORY_PATH as ar, RecipeExtract as at, readDaemonState as b, CaptureLog as bi, discoverSite as bn, egressLookup as br, HybridSearch as bt, DaemonMcpOptions as c, heldBackMeta as ci, chunkText as cn, KeyInput as cr, RecipeResult as ct, DaemonState as d, pageDocId as di, fuseRanks as dn, SignatureCheck as dr, defineRecipe as dt, ReadResult as ei, extractDocument as en, FetchRequest as er, RESERVED_RECIPES as et, DaemonStatus as f, pageExtractor as fi, unitVector as fn, SignedRequest as fr, readJson as ft, ScheduleResult as g, renderAndIndex as gi, DiscoverRunOptions as gn, keyDirectory as gr, EmbedResult as gt, ScheduleOptions as h, reindex as hi, DiscoverResult as hn, createWebBotAuthSigner as hr, EmbedOptions as ht, Daemon as i, catchUpIndex as ii, Embedder as in, DirectoryKey as ir, RecipeDoc as it, SearchOptions as j, RefusedReason as ji, DiffInput as jn, Latest as jr, CRAWL_LIMITS as jt, RankOptions as k, NO_PLAYWRIGHT as ki, DIFF_LIMITS as kn, FindIndexOptions as kr, VectorStoreOptions as kt, DaemonMcpService as l, isHeldBack as li, chunkTokensOf as ln, RequestSigner as lr, RecipeRunOptions as lt, DaemonTick as m, recordOverride as mi, DiscoverOptions as mn, WebBotAuthSigner as mr, runRecipe as mt, DAEMON_DEFAULTS as n, VerifyOptions as ni, ChunkOptions as nn, RenderRequest as nr, RecipeCapture as nt, DaemonMcpHooks as o, fetchPage as oi, HYBRID_DEFAULTS as on, KEY_DIRECTORY_TYPE as or, RecipeRequest as ot, DaemonTask as p, readDocument as pi, DISCOVER_LIMITS as pn, WebBotAuthOptions as pr, registerRecipe as pt, WatchRunnerOptions as q, CaptureWriter as qn, Extractor as qr, FieldClaim as qt, DAEMON_STATE_SCHEMA as r, VerifyResult as ri, EmbedCallOptions as rn, renderedKey as rr, RecipeDefinition as rt, DaemonMcpHost as s, findDocuments as si, TextChunk as sn, KeyDirectory as sr, RecipeRequestOptions as st, ChangeCommandOptions as t, ReindexResult as ti, CHUNK_LIMITS as tn, Fetched as tr, Recipe as tt, DaemonOptions as u, isoTime as ui, estimateTokens as un, SignatureAgentType as ur, RecipeTarget as ut, changeCommand as v, versionAtTime as vi, DiscoverStop as vn, EgressOptions as vr, HybridFindOptions as vt, FindOptions as w, CaptureRecord as wi, SitemapKind as wn, DocIndex as wr, StoredChunk as wt, scheduleWatches as x, CaptureReader as xi, Changefreq as xn, DigestOptions as xr, HybridSearchOptions as xt, daemonStateFile as y, CaptureFilter as yi, DiscoveredUrl as yn, EgressRefused as yr, HybridOptions as yt, WatchEntry as z, TextDiff as zn, VersionText as zr, crawlSite as zt };
+//# sourceMappingURL=daemon-DP4EqjFL.d.mts.map