polite-crawler 0.1.2 → 0.1.3

package/build/{open-DnrWgkC0.mjs → open-C4fhXJgD.mjs}

@@ -2,14 +2,14 @@ import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
 import { n as FileBlobStore, t as CaptureLog } from "./log-Dx-wXjJl.mjs";
 import { r as playwrightRenderer } from "./render-C1CUraWF.mjs";
 import { r as checkedRenderer } from "./browser-CKRNbIDT.mjs";
-import { A as Crawler, D as parseQuery, F as sha256, I as stableStringify, O as snippetOf, a as fetchPage, c as isHeldBack, f as readDocument, g as verifyQuote, i as fetchAndIndex, j as DEADLINE_NOTE, m as reindex, o as findDocuments, r as catchUpIndex, t as Extractors, w as fold } from "./runtime-k8mo1_7U.mjs";
+import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-ntbY-6wh.mjs";
+import { A as snippetOf, C as fallbackReason, E as fold, L as sha256$1, M as Crawler, N as DEADLINE_NOTE, R as stableStringify, S as archivedOf, a as fetchPage, c as isHeldBack, f as readDocument, g as verifyQuote, i as fetchAndIndex, k as parseQuery, m as reindex, n as HELD_BACK, o as findDocuments, p as recordOverride, r as catchUpIndex, s as heldBackMeta, t as Extractors, y as archivedCopy } from "./runtime-JToRO63o.mjs";
 import { MemoryHostStateStore, nextValidatorMemo, realClock } from "./politeness.mjs";
 import { matchesPattern } from "./robots.mjs";
-import { i as VERSION, n as BOT_PAGE, r as DEFAULT_TOKEN, t as durationMs } from "./duration-iIuMMofi.mjs";
 import { n as canonicalUrl, r as scopeTest, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
 import { o as decodeBody, t as extractPage } from "./html-DkQUAE_d.mjs";
-import { n as diffDocument, r as diffTexts } from "./diff-Dk2eSvi-.mjs";
-import { r as extractDocument } from "./extract-DcwNrTK7.mjs";
+import { n as diffDocument, r as diffTexts } from "./diff-NQcQqKe9.mjs";
+import { r as extractDocument } from "./extract-ByFfc8Zk.mjs";
 import { createHash, randomUUID } from "node:crypto";
 import { dirname, join } from "node:path";
 import { existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
@@ -19,10 +19,13 @@ import { DatabaseSync } from "node:sqlite";
 const MAX_INDEX_CHARS = 2e5;
 /** In the mode `fast`, a part is common when more than this share of the documents hold it. */
 const FAST_RANK_SHARE = .2;
+/** The text of the error of hybrid search with no embedder. */
+const NO_EMBEDDER = "hybrid search needs an embedder: open the crawler with the option embedder (transformersEmbedder or ollamaEmbedder), or give --embedder transformers or ollama to the command line";
 /** This function gives the rank mode of an option. A value that is not a rank mode throws a `RangeError`. */
 const rankModeOf = (value) => {
 	if (value === void 0 || value === "exact") return "exact";
 	if (value === "fast") return "fast";
+	if (value === "hybrid") throw new RangeError("the index ranks with \"exact\" and \"fast\" only: rank \"hybrid\" needs the vectors of an embedder, so use find or rankWithin of a crawler with the option embedder");
 	throw new RangeError(`rank must be "exact" or "fast", not ${JSON.stringify(value) ?? "undefined"}`);
 };
 const namesOf = (generation) => ({
@@ -539,6 +542,20 @@ var DocIndex = class {
 	hasVersion(capture) {
 		return this.live((n) => this.stmt(`SELECT 1 AS found FROM ${n.versions} WHERE capture = ? LIMIT 1`).get(capture) !== void 0);
 	}
+	/**
+	* This method gives the hash of the current text of documents, with no text. It gives the
+	* documents of `docs`, or each document after `after` in the sequence of the ids, to `limit`.
+	*/
+	digests(options = {}) {
+		return this.live((n) => {
+			if (options.docs !== void 0) {
+				if (options.docs.length === 0) return [];
+				return this.stmt(`SELECT doc, kind, ai_input, text_sha FROM ${n.docs} WHERE doc IN (SELECT value FROM json_each(?))`).all(JSON.stringify(options.docs));
+			}
+			const limit = Math.max(1, Math.floor(options.limit ?? 500));
+			return this.stmt(`SELECT doc, kind, ai_input, text_sha FROM ${n.docs} WHERE doc > ? ORDER BY doc LIMIT ?`).all(options.after ?? "", limit);
+		});
+	}
 	/** This method gives the newest fetch time of the cache entries, or undefined for an empty index. */
 	newestFetch() {
 		return this.live((n) => this.stmt(`SELECT max(fetched_at) AS newest FROM ${n.latest}`).get()?.newest ?? void 0);
@@ -830,7 +847,7 @@ const XML_ENTITIES = {
 };
 /** A tag keeps this number of attributes at most. The reader reads the others and keeps nothing of them. */
 const MAX_ATTRIBUTES = 64;
-const NAME = /^[A-Za-z_][\w.:-]*$/;
+const NAME$1 = /^[A-Za-z_][\w.:-]*$/;
 const W3C_DATE = /^\d{4}(?:-\d{2}(?:-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?)?)?$/;
 const MAIL_DATE = /^(?:[A-Za-z]{3},?\s+)?\d{1,2}\s+[A-Za-z]{3,9}\s+\d{2,4}\s+\d{1,2}:\d{2}(?::\d{2})?(?:\s*(?:[A-Za-z]{1,5}|[+-]\d{4}))?$/;
 const short = (name) => name === void 0 ? "nothing" : name.length > 40 ? `${name.slice(0, 40)}...` : name;
@@ -903,7 +920,7 @@ const tagAt = (text, from) => {
 		at += 1;
 	}
 	const name = text.slice(from, Math.min(at, from + 256));
-	if (at - from > 256 || !NAME.test(name)) return void 0;
+	if (at - from > 256 || !NAME$1.test(name)) return void 0;
 	const attributes = [];
 	for (;;) {
 		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
@@ -928,7 +945,7 @@ const tagAt = (text, from) => {
 			at += 1;
 		}
 		const key = text.slice(start, Math.min(at, start + 256));
-		if (at - start > 256 || !NAME.test(key)) return void 0;
+		if (at - start > 256 || !NAME$1.test(key)) return void 0;
 		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
 		if (text.charCodeAt(at) !== 61) return void 0;
 		at += 1;
@@ -1322,7 +1339,7 @@ const DISCOVER_LIMITS = {
 	/** The most feeds that discovery reads from the links of the page. */
 	maxLinkedFeeds: 5
 };
-const PAGE_ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
+const PAGE_ACCEPT$1 = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
 const SITEMAP_ACCEPT = "application/xml,text/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5";
 const FEED_ACCEPT = "application/rss+xml,application/atom+xml,application/xml;q=0.9,text/xml;q=0.8,*/*;q=0.5";
 const FEED_KINDS = /* @__PURE__ */ new Set([
@@ -1517,7 +1534,7 @@ const discoverSite = async (runtime, site, options = {}) => {
 			url: start.href,
 			mode: "page",
 			recipe: "page",
-			accept: PAGE_ACCEPT,
+			accept: PAGE_ACCEPT$1,
 			maxAgeMs,
 			onBehalfOf: options.onBehalfOf,
 			signal,
@@ -2332,6 +2349,1326 @@ var SqliteHostStateStore = class {
 	}
 };
 //#endregion
+//#region src/embed.ts
+/** These are the limits of the chunks of a text. */
+const CHUNK_LIMITS = {
+	maxTokens: 512,
+	defaultTokens: 256,
+	fill: .9,
+	overlap: .1,
+	maxChunks: 64
+};
+/** These scripts write no space between words, so the estimate counts each character as a token. */
+const SEGMENTED = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;
+const WORD = /[\p{L}\p{N}\p{M}]/u;
+const SPACE = /\s/u;
+/** A word of this number of letters or fewer is one token in the estimate. A longer word is more tokens. */
+const LETTERS_PER_TOKEN = 6;
+const isAsciiWord = (code) => code >= 48 && code <= 57 || code >= 65 && code <= 90 || code >= 97 && code <= 122;
+const isAsciiSpace = (code) => code === 32 || code >= 9 && code <= 13;
+const kindOf = (point) => {
+	if (point < 128) return isAsciiWord(point) ? "word" : isAsciiSpace(point) ? "space" : "other";
+	const char = String.fromCodePoint(point);
+	if (SEGMENTED.test(char)) return "segmented";
+	if (WORD.test(char)) return "word";
+	return SPACE.test(char) ? "space" : "other";
+};
+/**
+* This function estimates the number of tokens of a part of a text. A word is one token for each
+* six letters. A Chinese, Japanese or Thai character is one token. Each other character that is
+* not a space is one token. The estimate is a little high for English text, so a chunk fits.
+*/
+const estimateTokens = (text, start = 0, end = text.length) => {
+	let tokens = 0;
+	let run = 0;
+	for (let at = start; at < end;) {
+		const point = text.codePointAt(at);
+		at += point > 65535 ? 2 : 1;
+		const kind = kindOf(point);
+		if (kind === "word") {
+			run += 1;
+			continue;
+		}
+		if (run > 0) tokens += Math.ceil(run / LETTERS_PER_TOKEN);
+		run = 0;
+		if (kind !== "space") tokens += 1;
+	}
+	if (run > 0) tokens += Math.ceil(run / LETTERS_PER_TOKEN);
+	return tokens;
+};
+/** These characters end a sentence when a space follows them. */
+const STOP = /[.!?…]+["'”’)\]]*(?=\s|$)|[。！？]+/gu;
+/** This function gives the ranges of the sentences of a part of a text, with no space at the ends. */
+const sentencesOf = (text, start, end) => {
+	const part = text.slice(start, end);
+	const out = [];
+	let from = 0;
+	for (const match of part.matchAll(STOP)) {
+		const to = match.index + match[0].length;
+		if (to > from) out.push([from, to]);
+		from = to;
+	}
+	if (from < part.length) out.push([from, part.length]);
+	return out.map(([a, b]) => {
+		let first = a;
+		let last = b;
+		while (first < last && SPACE.test(part[first])) first += 1;
+		while (last > first && SPACE.test(part[last - 1])) last -= 1;
+		return [start + first, start + last];
+	}).filter(([a, b]) => b > a);
+};
+/** This function cuts a part of a text that is longer than the limit. It cuts at the spaces, and in a word that is too long. */
+const cutLong = (text, start, end, size) => {
+	const out = [];
+	let from = start;
+	let tokens = 0;
+	let run = 0;
+	let lastSpace = -1;
+	let at = start;
+	const emit = (to) => {
+		let first = from;
+		while (first < to && SPACE.test(text[first])) first += 1;
+		let last = to;
+		while (last > first && SPACE.test(text[last - 1])) last -= 1;
+		if (last > first) out.push({
+			start: first,
+			end: last,
+			tokens: estimateTokens(text, first, last),
+			paragraph: false
+		});
+		from = to;
+		tokens = 0;
+		run = 0;
+		lastSpace = -1;
+	};
+	while (at < end) {
+		const point = text.codePointAt(at);
+		const width = point > 65535 ? 2 : 1;
+		const kind = kindOf(point);
+		let cost = 0;
+		if (kind === "word") {
+			run += 1;
+			if (run % LETTERS_PER_TOKEN === 1) cost = 1;
+		} else {
+			run = 0;
+			if (kind !== "space") cost = 1;
+		}
+		if (tokens + cost > size && at > from) {
+			if (lastSpace > from) {
+				const to = lastSpace;
+				emit(to);
+				at = to;
+				continue;
+			}
+			emit(at);
+		}
+		if (kind === "space") lastSpace = at;
+		tokens += cost;
+		at += width;
+	}
+	emit(end);
+	return out;
+};
+/** This function gives the pieces of a text: its sentences, in paragraphs, and the parts of a sentence that is longer than the limit. */
+const piecesOf = (text, size) => {
+	const pieces = [];
+	for (const line of text.matchAll(/[^\r\n]+/g)) {
+		let first = true;
+		for (const [start, end] of sentencesOf(text, line.index, line.index + line[0].length)) {
+			const tokens = estimateTokens(text, start, end);
+			const parts = tokens <= size ? [{
+				start,
+				end,
+				tokens,
+				paragraph: false
+			}] : cutLong(text, start, end, size);
+			for (const part of parts) {
+				pieces.push({
+					...part,
+					paragraph: first
+				});
+				first = false;
+			}
+		}
+	}
+	return pieces;
+};
+const positive = (name, value) => {
+	if (!Number.isFinite(value) || value < 1) throw new RangeError(`${name} must be a number of 1 or more, not ${String(value)}`);
+	return Math.floor(value);
+};
+/**
+* This function cuts a text into chunks for an embedder. A chunk is a window of whole sentences.
+* It ends at the end of a paragraph when the paragraph fills half of the window or more. A
+* sentence that is longer than the window is cut at its spaces. Two chunks next to each other share
+* a few sentences (`overlapTokens`), so a passage at a cut is in one chunk. Each chunk keeps its
+* position in the text, so a hit points at a passage.
+*/
+const chunkText = (text, options) => {
+	const size = positive("maxTokens", options.maxTokens);
+	const overlap = Math.min(Math.floor(size / 2), Math.max(0, Math.floor(options.overlapTokens ?? size * CHUNK_LIMITS.overlap)));
+	const maxChunks = positive("maxChunks", options.maxChunks ?? CHUNK_LIMITS.maxChunks);
+	const pieces = piecesOf(text, size);
+	const chunks = [];
+	let first = 0;
+	let fresh = 0;
+	while (first < pieces.length && chunks.length < maxChunks) {
+		let last = first;
+		let tokens = pieces[first].tokens;
+		while (last + 1 < pieces.length && tokens + pieces[last + 1].tokens <= size) {
+			last += 1;
+			tokens += pieces[last].tokens;
+		}
+		if (last + 1 < pieces.length && !pieces[last + 1].paragraph) {
+			let filled = 0;
+			let cut = -1;
+			for (let at = first; at < last; at += 1) {
+				filled += pieces[at].tokens;
+				if (at >= fresh && pieces[at + 1].paragraph && filled * 2 >= size) cut = at;
+			}
+			if (cut !== -1) last = cut;
+		}
+		fresh = last + 1;
+		const start = pieces[first].start;
+		const end = pieces[last].end;
+		chunks.push({
+			index: chunks.length,
+			start,
+			end,
+			text: text.slice(start, end)
+		});
+		if (last + 1 >= pieces.length) break;
+		let next = last + 1;
+		let shared = 0;
+		while (next - 1 > first && shared + pieces[next - 1].tokens <= overlap) {
+			next -= 1;
+			shared += pieces[next].tokens;
+		}
+		if (shared + pieces[last + 1].tokens > size) next = last + 1;
+		first = next;
+	}
+	return chunks;
+};
+/**
+* This function gives the number of tokens of a chunk for an embedder. It is the input limit with a
+* margin for the estimate, and `CHUNK_LIMITS.maxTokens` at most.
+*/
+const chunkTokensOf = (embedder) => {
+	const limit = embedder.maxInputTokens;
+	const tokens = limit === void 0 || !Number.isFinite(limit) || limit < 1 ? CHUNK_LIMITS.defaultTokens : Math.min(limit, CHUNK_LIMITS.maxTokens);
+	return Math.max(8, Math.floor(tokens * CHUNK_LIMITS.fill));
+};
+/** This function gives a copy of a vector with a length of 1. A vector of zeros stays zeros. A value that is not finite throws a `RangeError`. */
+const unitVector = (vector) => {
+	const out = Float32Array.from(vector);
+	let sum = 0;
+	for (const value of out) {
+		if (!Number.isFinite(value)) throw new RangeError("the embedder gave a value that is not finite");
+		sum += value * value;
+	}
+	if (sum === 0) return out;
+	const scale = 1 / Math.sqrt(sum);
+	for (let at = 0; at < out.length; at += 1) out[at] = out[at] * scale;
+	return out;
+};
+/** These are the defaults of a hybrid search. */
+const HYBRID_DEFAULTS = {
+	candidates: 50,
+	neighbors: 50,
+	fusionK: 60
+};
+/**
+* This function fuses ranked lists with reciprocal rank fusion. Each item gets the sum of
+* 1 / (k + rank) over the lists that hold it, with the rank from 1. The best item is first. A tie goes to
+* the item with the best rank in a list, then to the earlier list, then to the smaller id. An item
+* that is two times in one list counts at its first place.
+*/
+const fuseRanks = (lists, k = HYBRID_DEFAULTS.fusionK) => {
+	if (!Number.isFinite(k) || k < 0) throw new RangeError(`the fusion constant k must be a number of 0 or more, not ${String(k)}`);
+	const ranks = /* @__PURE__ */ new Map();
+	lists.forEach((list, which) => {
+		list.forEach((id, at) => {
+			let entry = ranks.get(id);
+			if (entry === void 0) {
+				entry = lists.map(() => null);
+				ranks.set(id, entry);
+			}
+			entry[which] ??= at + 1;
+		});
+	});
+	const keyed = [...ranks].map(([id, entry]) => {
+		let score = 0;
+		let best = Number.POSITIVE_INFINITY;
+		for (const rank of entry) {
+			if (rank === null) continue;
+			score += 1 / (k + rank);
+			best = Math.min(best, rank);
+		}
+		return {
+			id,
+			score,
+			ranks: entry,
+			best,
+			first: entry.findIndex((rank) => rank !== null)
+		};
+	});
+	keyed.sort((a, b) => b.score - a.score || a.best - b.best || a.first - b.first || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+	return keyed.map(({ id, score, ranks: entry }) => ({
+		id,
+		score,
+		ranks: entry
+	}));
+};
+//#endregion
+//#region src/hybrid.ts
+/** The embedder gets this number of texts in one call at most. */
+const BATCH = 32;
+/** The search keeps the vectors of this number of questions. */
+const QUERY_CACHE = 32;
+/** The search checks this number of texts of the vector search against the index at one time. */
+const PAGE = 64;
+const sha256 = (text) => createHash("sha256").update(text, "utf8").digest("hex");
+const count = (name, value, fallback, max) => {
+	if (value === void 0) return fallback;
+	if (!Number.isFinite(value) || value < 1) throw new RangeError(`${name} must be a number of 1 or more, not ${String(value)}`);
+	return Math.min(max, Math.floor(value));
+};
+/** This function checks the constant k of reciprocal rank fusion. */
+const fusionKOf = (value) => {
+	if (value === void 0) return HYBRID_DEFAULTS.fusionK;
+	if (!Number.isFinite(value) || value < 0) throw new RangeError(`fusionK must be a number of 0 or more, not ${String(value)}`);
+	return value;
+};
+/** This function gives a promise with its resolve and its reject. A rejection with no listener is not an error of the process. */
+const deferred = () => {
+	let resolve = () => void 0;
+	let reject = () => void 0;
+	const promise = new Promise((yes, no) => {
+		resolve = yes;
+		reject = no;
+	});
+	promise.catch(() => void 0);
+	return {
+		promise,
+		resolve,
+		reject
+	};
+};
+const round = (value, digits) => {
+	const scale = 10 ** digits;
+	return Math.round(value * scale) / scale;
+};
+/** This function makes a hybrid search over a runtime. The vectors go into the store, with the id of the embedder. */
+const createHybridSearch = (runtime, options) => {
+	const { embedder, store } = options;
+	const tokens = chunkTokensOf(embedder);
+	const plan = `chunks-1/${String(tokens)}/${String(Math.floor(tokens * CHUNK_LIMITS.overlap))}/${String(CHUNK_LIMITS.maxChunks)}`;
+	const working = /* @__PURE__ */ new Map();
+	const queries = /* @__PURE__ */ new Map();
+	let dimensions = embedder.dimensions > 0 ? embedder.dimensions : store.dimensions(embedder.id);
+	/** This function gives the heads of documents, from `digests` when the index has it. */
+	const headsOf = (docs) => {
+		const heads = /* @__PURE__ */ new Map();
+		if (docs.length === 0) return heads;
+		const index = runtime.index;
+		if (index.digests !== void 0) {
+			for (const head of index.digests({ docs })) heads.set(head.doc, head);
+			return heads;
+		}
+		for (const doc of docs) {
+			const found = index.get(doc);
+			if (found !== void 0) heads.set(doc, headOfDoc(found));
+		}
+		return heads;
+	};
+	const headOfDoc = (doc) => ({
+		doc: doc.doc,
+		kind: doc.kind,
+		ai_input: doc.ai_input,
+		text_sha: sha256(doc.text)
+	});
+	/** This function asks the embedder for the vectors of texts, in batches, and checks them. */
+	const vectorsOf = async (texts, purpose, signal) => {
+		const out = [];
+		for (let at = 0; at < texts.length; at += BATCH) {
+			signal?.throwIfAborted();
+			const part = texts.slice(at, at + BATCH);
+			const given = await embedder.embed(part, {
+				signal,
+				purpose
+			});
+			if (!Array.isArray(given) || given.length !== part.length) throw new Error(`the embedder ${embedder.id} gave ${String(Array.isArray(given) ? given.length : 0)} vectors for ${String(part.length)} texts`);
+			for (const vector of given) {
+				const unit = unitVector(vector);
+				dimensions ??= unit.length;
+				if (unit.length !== dimensions || unit.length === 0) throw new Error(`the embedder ${embedder.id} gave a vector of ${String(unit.length)} dimensions, not ${String(dimensions)}`);
+				out.push(unit);
+			}
+		}
+		return out;
+	};
+	const queryVector = async (q, signal) => {
+		const text = q.replace(/["*]/g, " ").replace(/\s+/g, " ").trim();
+		const known = queries.get(text);
+		if (known !== void 0) {
+			queries.delete(text);
+			queries.set(text, known);
+			return known;
+		}
+		const [vector] = await vectorsOf([text], "query", signal);
+		queries.set(text, vector);
+		if (queries.size > QUERY_CACHE) queries.delete(queries.keys().next().value);
+		return vector;
+	};
+	/**
+	* This function gives vectors to the texts of documents that have none, and points each document
+	* at its current text. It reads each text from the index, and it does not send the text
+	* of a held-back document. Two calls for one text share the work. It gives the number of new vectors.
+	*/
+	const ensure = async (docs, signal) => {
+		const tally = {
+			embedded: 0,
+			reused: 0,
+			held: 0,
+			chunks: 0
+		};
+		const now = runtime.now();
+		const heads = headsOf(docs);
+		const waiting = /* @__PURE__ */ new Map();
+		const shared = [];
+		for (const id of docs) {
+			const head = heads.get(id);
+			if (head === void 0) continue;
+			if (isHeldBack(head)) {
+				tally.held += 1;
+				if (store.pointed(embedder.id, id) !== void 0) store.forget(embedder.id, id);
+				continue;
+			}
+			if (store.plan(embedder.id, head.text_sha) === plan) {
+				tally.reused += 1;
+				if (store.pointed(embedder.id, id) !== head.text_sha) store.point(embedder.id, id, head.text_sha, now);
+				continue;
+			}
+			const doc = runtime.index.get(id);
+			if (doc === void 0 || isHeldBack(doc)) continue;
+			const sha = sha256(doc.text);
+			const entry = waiting.get(sha);
+			if (entry !== void 0) {
+				entry.docs.push(id);
+				continue;
+			}
+			const busy = working.get(sha);
+			if (busy !== void 0) {
+				shared.push(busy.then((made) => {
+					store.point(embedder.id, id, sha, now);
+					return made;
+				}));
+				continue;
+			}
+			waiting.set(sha, {
+				text: doc.text,
+				docs: [id]
+			});
+		}
+		const jobs = [...waiting].map(([sha, entry]) => {
+			const job = {
+				sha,
+				entry,
+				chunks: chunkText(entry.text, { maxTokens: tokens }),
+				...deferred()
+			};
+			working.set(sha, job.promise);
+			return job;
+		});
+		try {
+			for (let at = 0; at < jobs.length;) {
+				const batch = [];
+				let size = 0;
+				while (at < jobs.length && (batch.length === 0 || size + jobs[at].chunks.length <= BATCH)) {
+					batch.push(jobs[at]);
+					size += jobs[at].chunks.length;
+					at += 1;
+				}
+				const vectors = await vectorsOf(batch.flatMap((job) => job.chunks.map((chunk) => chunk.text)), "document", signal);
+				let next = 0;
+				for (const job of batch) {
+					const chunks = job.chunks.map((chunk) => ({
+						start: chunk.start,
+						end: chunk.end,
+						vector: vectors[next++]
+					}));
+					store.put({
+						embedder: embedder.id,
+						dimensions: dimensions ?? chunks[0]?.vector.length ?? 0,
+						sha: job.sha,
+						plan,
+						chars: job.entry.text.length,
+						chunks,
+						docs: job.entry.docs,
+						at: now
+					});
+					tally.embedded += 1;
+					tally.chunks += chunks.length;
+					working.delete(job.sha);
+					job.resolve(chunks.length);
+				}
+			}
+		} catch (error) {
+			for (const job of jobs) {
+				if (working.get(job.sha) === job.promise) working.delete(job.sha);
+				job.reject(error);
+			}
+			throw error;
+		}
+		await Promise.all(shared);
+		return tally;
+	};
+	/**
+	* This function gives the documents of the vector search, the best first, to `limit`. It checks
+	* each document against the index. Its current text must be the text of the hit, it must not be
+	* held back, and it must have the kind. With `within`, it ranks only those documents.
+	*/
+	const neighborsOf = (query, limit, kind, within) => {
+		const hits = store.search(embedder.id, query, within === void 0 ? {} : { within });
+		const out = [];
+		for (let at = 0; at < hits.length && out.length < limit; at += PAGE) {
+			const page = hits.slice(at, at + PAGE);
+			const docs = [...new Set(page.flatMap((hit) => within === void 0 ? hit.docs : hit.docs.filter((doc) => within.has(doc))))];
+			const heads = headsOf(docs);
+			for (const hit of page) for (const doc of hit.docs) {
+				if (out.length >= limit) break;
+				if (within !== void 0 && !within.has(doc)) continue;
+				const head = heads.get(doc);
+				if (head === void 0 || head.text_sha !== hit.sha || isHeldBack(head)) continue;
+				if (kind !== void 0 && kind !== null && head.kind !== kind) continue;
+				out.push({
+					doc,
+					hit
+				});
+			}
+		}
+		return out;
+	};
+	const passageOf = (hit) => ({
+		start: hit.start,
+		end: hit.end,
+		similarity: round(hit.similarity, 4)
+	});
+	return {
+		embedder,
+		async find(q, findOptions = {}) {
+			const query = parseQuery(q);
+			if (query === void 0) return [];
+			const limit = Math.max(1, Math.min(1e3, Math.floor(findOptions.limit ?? 10) || 10));
+			const candidates = count("candidates", findOptions.candidates, HYBRID_DEFAULTS.candidates, 1e3);
+			const neighbors = count("neighbors", findOptions.neighbors, HYBRID_DEFAULTS.neighbors, 1e3);
+			const k = fusionKOf(findOptions.fusionK);
+			const signal = findOptions.signal;
+			const vector = await queryVector(q, signal);
+			const kind = findOptions.kind ?? null;
+			const lexical = runtime.index.find(q, {
+				limit: candidates,
+				kind,
+				rank: "exact"
+			});
+			await ensure(lexical.map((hit) => hit.doc), signal);
+			const near = neighborsOf(vector, neighbors, kind, void 0);
+			const fused = fuseRanks([lexical.map((hit) => hit.doc), near.map((one) => one.doc)], k).slice(0, limit);
+			const byDoc = new Map(lexical.map((hit) => [hit.doc, hit]));
+			const nearBy = new Map(near.map((one) => [one.doc, one.hit]));
+			const results = [];
+			for (const one of fused) {
+				const doc = runtime.index.get(one.id);
+				if (doc === void 0 || isHeldBack(doc)) continue;
+				const best = nearBy.get(one.id) ?? store.best(embedder.id, sha256(doc.text), vector);
+				const lexicalHit = byDoc.get(one.id);
+				const archived = doc.kind === "archived" ? archivedOf(doc.meta) : void 0;
+				results.push({
+					doc: doc.doc,
+					kind: doc.kind,
+					title: doc.title,
+					url: doc.url,
+					snippet: best === void 0 ? lexicalHit?.snippet ?? snippetOf(doc.text, query.terms) : snippetOf(doc.text.slice(best.start, best.end), query.terms),
+					score: round(one.score, 6),
+					ai_input: doc.ai_input,
+					fetched_at: doc.fetched_at,
+					capture: doc.capture,
+					...archived === void 0 ? {} : { archived },
+					...best === void 0 ? {} : { passage: passageOf(best) }
+				});
+			}
+			return results;
+		},
+		async rankWithin(q, docs, rankOptions = {}) {
+			if (parseQuery(q) === void 0 || docs.length === 0) return [];
+			const candidates = count("candidates", rankOptions.candidates, HYBRID_DEFAULTS.candidates, 1e3);
+			const neighbors = count("neighbors", rankOptions.neighbors, HYBRID_DEFAULTS.neighbors, 1e5);
+			const k = fusionKOf(rankOptions.fusionK);
+			const signal = rankOptions.signal;
+			const vector = await queryVector(q, signal);
+			const lexical = runtime.index.rankWithin(q, docs, { rank: "exact" });
+			const first = [.../* @__PURE__ */ new Set([...lexical.map((hit) => hit.doc), ...docs])].slice(0, candidates);
+			await ensure(first, signal);
+			const within = new Set(docs);
+			const near = neighborsOf(vector, neighbors, void 0, within);
+			const nearBy = new Map(near.map((one) => [one.doc, one.hit]));
+			const heads = headsOf(lexical.map((hit) => hit.doc).filter((doc) => !nearBy.has(doc)));
+			return fuseRanks([lexical.map((hit) => hit.doc), near.map((one) => one.doc)], k).map((one) => {
+				const sha = heads.get(one.id)?.text_sha;
+				const best = nearBy.get(one.id) ?? (sha === void 0 ? void 0 : store.best(embedder.id, sha, vector));
+				return {
+					doc: one.id,
+					score: round(one.score, 6),
+					...best === void 0 ? {} : { passage: passageOf(best) }
+				};
+			});
+		},
+		async embed(embedOptions = {}) {
+			const started = Date.now();
+			const budgetMs = durationMs(embedOptions.budget ?? "10m");
+			const maxDocs = embedOptions.maxDocs === void 0 ? Number.POSITIVE_INFINITY : count("maxDocs", embedOptions.maxDocs, 1, Number.MAX_SAFE_INTEGER);
+			const kind = embedOptions.kind ?? null;
+			const timer = new AbortController();
+			const clock = setTimeout(() => timer.abort(/* @__PURE__ */ new Error("the budget of embed is over")), Math.min(budgetMs, 2147483647));
+			clock.unref();
+			const signal = embedOptions.signal === void 0 ? timer.signal : AbortSignal.any([embedOptions.signal, timer.signal]);
+			const tally = {
+				docs: 0,
+				embedded: 0,
+				reused: 0,
+				held_back: 0,
+				chunks: 0
+			};
+			let stopped = "done";
+			const stopOf = () => embedOptions.signal?.aborted === true ? "signal" : timer.signal.aborted || Date.now() - started >= budgetMs ? "budget" : tally.embedded >= maxDocs ? "max-docs" : void 0;
+			try {
+				for await (const page of pagesOf(runtime)) {
+					const docs = page.filter((head) => kind === null || head.kind === kind);
+					for (let at = 0; at < docs.length;) {
+						const stop = stopOf();
+						if (stop !== void 0) {
+							stopped = stop;
+							break;
+						}
+						const room = Math.max(1, Math.min(BATCH, maxDocs - tally.embedded));
+						const group = docs.slice(at, at + room).map((head) => head.doc);
+						at += group.length;
+						try {
+							const done = await ensure(group, signal);
+							tally.docs += group.length;
+							tally.embedded += done.embedded;
+							tally.reused += done.reused;
+							tally.held_back += done.held;
+							tally.chunks += done.chunks;
+						} catch (error) {
+							const stop = stopOf();
+							if (stop === void 0) throw error;
+							stopped = stop;
+							break;
+						}
+					}
+					if (stopped !== "done") break;
+				}
+			} finally {
+				clearTimeout(clock);
+			}
+			return {
+				embedder: embedder.id,
+				...tally,
+				stopped,
+				ms: Date.now() - started
+			};
+		}
+	};
+};
+/**
+* This function gives the heads of each stored document, in pages. It uses `digests` when the
+* index has it. Else it reads the capture log and asks the index for the documents of each capture.
+*/
+async function* pagesOf(runtime) {
+	const index = runtime.index;
+	if (index.digests !== void 0) {
+		let after = "";
+		for (;;) {
+			const page = index.digests({
+				after,
+				limit: 500
+			});
+			if (page.length === 0) return;
+			yield page;
+			after = page.at(-1).doc;
+		}
+	}
+	const seen = /* @__PURE__ */ new Set();
+	let page = [];
+	for await (const envelope of runtime.captures.scan({ source: "fetch" })) for (const doc of index.docsOfCapture(envelope.id)) {
+		if (seen.has(doc.doc)) continue;
+		seen.add(doc.doc);
+		page.push({
+			doc: doc.doc,
+			kind: doc.kind,
+			ai_input: doc.ai_input,
+			text_sha: sha256(doc.text)
+		});
+		if (page.length >= 500) {
+			yield page;
+			page = [];
+		}
+	}
+	if (page.length > 0) yield page;
+}
+//#endregion
+//#region src/recipe.ts
+/**
+* This function reads a body as JSON, in the encoding that the response declares. A body that is
+* not JSON (a page about maintenance, or a login page) throws an error that names the URL and the
+* content type. In an extractor, the runtime then reports that error.
+*/
+const readJson = (body, capture) => {
+	const text = decodeBody(body, capture.contentType);
+	try {
+		return JSON.parse(text);
+	} catch {
+		throw new Error(`${capture.url} answered ${capture.contentType ?? "a body of no stated type"}, not JSON`);
+	}
+};
+/**
+* These recipe names belong to the crawler: the page reader, the archived fallback, robots.txt,
+* and the sitemaps and feeds of `discover`. A recipe cannot use them.
+*/
+const RESERVED_RECIPES = [
+	"page",
+	"archive-copy",
+	"archive-index",
+	"robots.txt",
+	"sitemap",
+	"feed"
+];
+/** These are the limits of a recipe. */
+const RECIPE_LIMITS = {
+	nameChars: 64,
+	docIdChars: 2048
+};
+const NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
+const PAGE_ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
+const API_ACCEPT = "application/json";
+const AI_INPUTS = [
+	"yes",
+	"no",
+	"unset"
+];
+/** This function tells what is wrong with one document of an extractor, or gives undefined. */
+const faultOf = (value, seen) => {
+	if (value === null || typeof value !== "object" || Array.isArray(value)) return "a document must be an object";
+	const doc = value;
+	const id = doc["doc"];
+	if (typeof id !== "string" || id.length === 0 || id.length > RECIPE_LIMITS.docIdChars) return `doc must be a text of 1 to ${String(RECIPE_LIMITS.docIdChars)} characters`;
+	if (id.startsWith("archive:")) return `the id ${id} starts with archive:, which is for the archived copies`;
+	if (seen.has(id)) return `two documents have the id ${id}`;
+	seen.add(id);
+	const kind = doc["kind"];
+	if (typeof kind !== "string" || kind.length === 0) return "kind must be a text that is not empty";
+	if (kind === "archived") return "the kind archived is for the archived copies";
+	if (typeof doc["title"] !== "string") return "title must be a text";
+	if (typeof doc["text"] !== "string") return "text must be a text";
+	if (doc["url"] !== null && typeof doc["url"] !== "string") return "url must be a text or null";
+	const meta = doc["meta"];
+	if (meta === null || typeof meta !== "object" || Array.isArray(meta)) return "meta must be an object";
+	try {
+		JSON.stringify(meta);
+	} catch {
+		return "meta must be JSON: no cycle and no BigInt";
+	}
+	const merge = doc["merge"];
+	if (merge !== void 0 && merge !== "latest" && merge !== "richest") return "merge must be \"latest\" or \"richest\"";
+	if (doc["full_text"] !== void 0 && typeof doc["full_text"] !== "string") return "full_text must be a text";
+	const aiInput = doc["ai_input"];
+	if (aiInput !== void 0 && !AI_INPUTS.includes(aiInput)) return "ai_input must be \"yes\", \"no\" or \"unset\"";
+};
+/** This function gives the documents of an extractor, or throws an error that names the first fault. */
+const checkedDocs = (name, docs) => {
+	if (!Array.isArray(docs)) throw new TypeError(`the recipe ${name} gave ${typeof docs}, not an array of documents`);
+	const seen = /* @__PURE__ */ new Set();
+	for (const [index, doc] of docs.entries()) {
+		const fault = faultOf(doc, seen);
+		if (fault !== void 0) throw new TypeError(`the recipe ${name} gave a document that is not valid (${String(index)}): ${fault}`);
+	}
+	return docs;
+};
+/**
+* This function declares a recipe. It checks the definition at once: a name that is not valid, a
+* reserved name or a duration that it cannot read throws an error. The extractor of the recipe
+* checks each document of `extract`. A document that is not valid makes the extractor throw, and
+* the runtime reports it as an error of the extractor.
+*/
+const defineRecipe = (definition) => {
+	const name = definition.name;
+	if (typeof name !== "string" || name.length > RECIPE_LIMITS.nameChars || !NAME.test(name)) throw new RangeError(`a recipe name is 1 to ${String(RECIPE_LIMITS.nameChars)} letters, digits and . _ : -, and it starts with a letter or a digit, not ${JSON.stringify(name) ?? "undefined"}`);
+	if (RESERVED_RECIPES.includes(name)) throw new RangeError(`the recipe name ${name} belongs to the crawler: give the recipe a name of its own`);
+	if (typeof definition.extract !== "function") throw new TypeError(`the recipe ${name} needs extract, a function that reads a body into documents`);
+	const mode = definition.mode ?? "page";
+	if (mode !== "page" && mode !== "api") throw new RangeError(`the mode of the recipe ${name} must be "page" or "api", not ${String(mode)}`);
+	const accept = definition.accept ?? (mode === "api" ? API_ACCEPT : PAGE_ACCEPT);
+	const minIntervalMs = definition.minInterval === void 0 ? void 0 : durationMs(definition.minInterval);
+	const maxAgeOf = (value) => value === "auto" ? "auto" : durationMs(value);
+	const maxAge = maxAgeOf(definition.maxAge ?? "24h");
+	const maxBytes = definition.maxBytes;
+	if (maxBytes !== void 0 && (!Number.isSafeInteger(maxBytes) || maxBytes < 1)) throw new RangeError(`maxBytes of the recipe ${name} must be a whole number of 1 or more`);
+	const extract = definition.extract;
+	const extractor = async (body, capture) => checkedDocs(name, await extract(body, {
+		url: capture.url,
+		contentType: capture.contentType
+	}));
+	const fetch = definition.fetch;
+	return {
+		name,
+		description: definition.description ?? null,
+		mode,
+		accept,
+		extractor,
+		request(input, options = {}) {
+			const asked = fetch === void 0 ? input : fetch(input);
+			const target = typeof asked === "string" ? { url: asked } : asked;
+			const url = target?.url;
+			let parsed;
+			try {
+				parsed = typeof url === "string" ? new URL(url) : void 0;
+			} catch {
+				parsed = void 0;
+			}
+			if (parsed === void 0 || parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new TypeError(`the recipe ${name} needs an http or https URL for its input, not ${JSON.stringify(url) ?? String(url)}`);
+			return {
+				url: parsed.href,
+				mode,
+				recipe: name,
+				accept: target?.accept ?? accept,
+				headers: target?.headers,
+				secretParams: target?.secretParams,
+				minIntervalMs,
+				maxAgeMs: options.maxAge === void 0 ? maxAge : maxAgeOf(options.maxAge),
+				maxBytes,
+				onBehalfOf: options.onBehalfOf,
+				job: options.job,
+				signal: options.signal,
+				scope: options.scope,
+				labels: options.labels,
+				deadline: options.deadline
+			};
+		}
+	};
+};
+/**
+* This function gives the extractor of a recipe to the extractors of a runtime. A second
+* registration of the same recipe does nothing. A different extractor with the same name throws
+* an error, because `reindex` must read each capture with one extractor.
+*/
+const registerRecipe = (extractors, recipe) => {
+	const known = extractors.get(recipe.name);
+	if (known === recipe.extractor) return;
+	if (known !== void 0) throw new Error(`the recipe ${recipe.name} has a different extractor in this runtime: give each recipe a name of its own`);
+	extractors.register(recipe.name, recipe.extractor);
+};
+/** This function gives the runtime of a target. */
+const runtimeOf = (target) => "crawler" in target ? target : target.runtime;
+/** This function gives the deadline of a budget, on the clock of the runtime. */
+const deadlineOf = (runtime, budget) => budget === void 0 ? void 0 : Date.parse(runtime.now()) + durationMs(budget);
+/** This function gives one document of a run, with the rule for a held-back document. */
+const recipeDoc = async (runtime, doc, capture, userProvided, action) => {
+	const current = runtime.index.get(doc.doc) ?? doc;
+	const version = (capture === void 0 ? void 0 : runtime.index.versionText(doc.doc, capture)) ?? current;
+	const aiInput = current.ai_input === "no" || version.ai_input === "no" ? "no" : version.ai_input;
+	const held = isHeldBack({ ai_input: aiInput }, userProvided);
+	if (aiInput === "no" && userProvided) await recordOverride(runtime, current, action);
+	const archived = archivedOf(version.meta);
+	return {
+		doc: doc.doc,
+		kind: version.kind,
+		title: held ? "" : version.title,
+		url: version.url,
+		meta: held ? heldBackMeta(version.meta) : version.meta,
+		chars: version.text.length,
+		capture: version.capture,
+		fetched_at: version.fetched_at,
+		held_back: held,
+		...archived === void 0 ? {} : { archived }
+	};
+};
+const noteOf = (parts) => {
+	const kept = parts.filter((part) => typeof part === "string" && part.length > 0);
+	return kept.length > 0 ? kept.join("; ") : null;
+};
+/**
+* This function starts a recipe for one input. It fetches the request through the crawler, and it
+* reads the capture with the extractor of the recipe. It gives the documents of that capture.
+* These are its rules:
+*
+* - The runtime registers the recipe the first time that it gets it. Give the recipe to
+*   `openCrawler` too, so that `reindex` reads its captures before the first fetch.
+* - A stored copy younger than `maxAge` answers with no request. When the extractor cannot read a
+*   stored copy, the crawler asks the source one more time.
+* - An extractor that throws gives no document, and `error` tells why. The fetch still succeeds.
+* - A document that asked not to be AI input shows no title and only its structural metadata,
+*   unless `userProvided`. The crawler logs each `userProvided` read of a held-back document.
+* - With `fallback: "archive"`, a page that the site refuses gets an archived copy, with
+*   `archived`. The page reader reads the copy. The extractor of the recipe reads only live
+*   captures. Thus its documents do not mix a live answer and an old copy of a third party.
+*/
+const runRecipe = async (target, recipe, input, options = {}) => {
+	const runtime = runtimeOf(target);
+	if (options.fallback === "archive" && recipe.mode !== "page") throw new RangeError(`the recipe ${recipe.name} reads an API: an archived copy is for a page, so give fallback only to a recipe of the mode page`);
+	registerRecipe(runtime.extractors, recipe);
+	const deadline = deadlineOf(runtime, options.budget);
+	const request = recipe.request(input, {
+		...options,
+		deadline
+	});
+	let indexed = await fetchAndIndex(runtime, request);
+	if (indexed.error !== void 0 && indexed.blocked === void 0 && indexed.fetched.fromCache) indexed = await fetchAndIndex(runtime, {
+		...request,
+		maxAgeMs: 0
+	});
+	const userProvided = options.userProvided ?? false;
+	const action = options.job ?? `recipe ${recipe.name}`;
+	const fetched = indexed.fetched;
+	const notes = [fetched.note, indexed.error];
+	if (indexed.docs.length === 0 && options.fallback === "archive") {
+		const reason = fallbackReason(indexed);
+		if (reason !== void 0) {
+			const page = {
+				render: "never",
+				maxAgeMs: request.maxAgeMs,
+				onBehalfOf: request.onBehalfOf,
+				job: request.job,
+				signal: request.signal,
+				scope: request.scope,
+				labels: request.labels,
+				deadline,
+				fallback: "archive",
+				archiveDespiteRobots: options.archiveDespiteRobots
+			};
+			const answer = await archivedCopy(runtime, request.url, reason, page);
+			if (answer.doc !== void 0 && answer.archived !== void 0) {
+				const doc = await recipeDoc(runtime, answer.doc, answer.doc.capture, userProvided, action);
+				return {
+					recipe: recipe.name,
+					url: fetched.url,
+					outcome: fetched.outcome,
+					status: fetched.status,
+					docs: [doc],
+					capture: doc.capture,
+					fetched_at: doc.fetched_at,
+					from_cache: answer.fromCache,
+					held_back: doc.held_back,
+					error: indexed.error ?? null,
+					archived: answer.archived,
+					note: noteOf([
+						...notes,
+						...answer.notes,
+						"the page reader read the archived copy, not the extractor of the recipe",
+						doc.held_back ? HELD_BACK : void 0
+					])
+				};
+			}
+			notes.push(...answer.notes);
+		}
+	}
+	const capture = fetched.capture;
+	const docs = [];
+	for (const doc of indexed.docs) docs.push(await recipeDoc(runtime, doc, capture, userProvided, action));
+	const held = docs.some((doc) => doc.held_back);
+	return {
+		recipe: recipe.name,
+		url: fetched.url,
+		outcome: fetched.outcome,
+		status: fetched.status,
+		docs,
+		capture: docs[0]?.capture ?? capture ?? null,
+		fetched_at: docs[0]?.fetched_at ?? fetched.fetchedAt ?? null,
+		from_cache: fetched.fromCache,
+		held_back: held,
+		error: indexed.error ?? null,
+		note: noteOf([...notes, held ? HELD_BACK : void 0])
+	};
+};
+//#endregion
+//#region src/vectors.ts
+/**
+* The vector store keeps the vectors of hybrid search in `vectors.sqlite`, beside the index. It is
+* a projection, as the index is. Delete the file, and the next hybrid search or `embed` makes the
+* vectors again from the stored texts.
+*
+* - A row is one chunk of one text for one embedder. Its key is the SHA-256 of the text, the
+*   number of the chunk and the id of the embedder. Thus a reindex keeps each vector, and the same
+*   text gets its vectors one time for each model, also when two documents hold it.
+* - A text is complete when the table `texts` has it, with the plan of its chunks. A text with a
+*   different plan gets new chunks.
+* - The table `docs` points each document at the text that has its vectors. A search uses only
+*   the texts that a document points at, and the caller checks each document against the index.
+* - A search compares the vector of the question with each stored vector (a brute-force cosine).
+*   The store keeps the vectors of each embedder in memory after the first search. It reads them
+*   again when a different process writes the file.
+*
+* Each connection waits for the lock of a different process (`busy_timeout`), and each write is
+* one transaction (`BEGIN IMMEDIATE`). The file has a schema version (`VECTORS_SCHEMA`). A file of
+* a different version opens empty.
+*/
+/** This value is the version of the schema of the vector file. */
+const VECTORS_SCHEMA = 1;
+const DDL = `
+CREATE TABLE IF NOT EXISTS vectors_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+CREATE TABLE IF NOT EXISTS embedders (
+  id TEXT PRIMARY KEY,
+  dimensions INTEGER NOT NULL,
+  first_at TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS texts (
+  sha TEXT NOT NULL,
+  embedder TEXT NOT NULL,
+  plan TEXT NOT NULL,
+  chunks INTEGER NOT NULL,
+  chars INTEGER NOT NULL,
+  embedded_at TEXT NOT NULL,
+  PRIMARY KEY (sha, embedder)
+);
+CREATE TABLE IF NOT EXISTS chunks (
+  sha TEXT NOT NULL,
+  chunk INTEGER NOT NULL,
+  embedder TEXT NOT NULL,
+  char_start INTEGER NOT NULL,
+  char_end INTEGER NOT NULL,
+  vector BLOB NOT NULL,
+  PRIMARY KEY (sha, chunk, embedder)
+);
+CREATE INDEX IF NOT EXISTS chunks_embedder ON chunks (embedder, sha, chunk);
+CREATE TABLE IF NOT EXISTS docs (
+  doc TEXT NOT NULL,
+  embedder TEXT NOT NULL,
+  sha TEXT NOT NULL,
+  pointed_at TEXT NOT NULL,
+  PRIMARY KEY (doc, embedder)
+);
+CREATE INDEX IF NOT EXISTS docs_sha ON docs (embedder, sha);
+`;
+/** This function gives the bytes of a vector for a BLOB. The bytes are in the order of the platform (little-endian on each common platform). */
+const bytesOf = (vector) => new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength);
+/** This function gives the vector of the bytes of a BLOB. It copies the bytes, so an odd offset of the BLOB is not a problem. */
+const vectorOf = (bytes) => {
+	const out = new Float32Array(Math.floor(bytes.byteLength / 4));
+	new Uint8Array(out.buffer).set(bytes.subarray(0, out.length * 4));
+	return out;
+};
+const dot = (data, row, query, dims) => {
+	let sum = 0;
+	const base = row * dims;
+	for (let at = 0; at < dims; at += 1) sum += data[base + at] * query[at];
+	return sum;
+};
+var VectorStore = class {
+	path;
+	db;
+	statements = /* @__PURE__ */ new Map();
+	matrices = /* @__PURE__ */ new Map();
+	closed = false;
+	constructor(path, options = {}) {
+		this.path = path;
+		if (path !== ":memory:") mkdirSync(dirname(path), {
+			recursive: true,
+			mode: 448
+		});
+		this.db = new DatabaseSync(path);
+		const wait = Math.max(0, Math.floor(options.busyTimeoutMs ?? 1e4));
+		this.db.exec(`PRAGMA busy_timeout = ${String(wait)};`);
+		if (path !== ":memory:") this.db.exec("PRAGMA journal_mode = WAL;");
+		this.db.exec("PRAGMA synchronous = NORMAL;");
+		this.write(() => {
+			if (this.db.prepare("PRAGMA user_version").get().user_version !== 1) {
+				const tables = this.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all();
+				for (const { name } of tables) this.db.exec(`DROP TABLE IF EXISTS "${name}"`);
+			}
+			this.db.exec(DDL);
+			this.db.exec(`PRAGMA user_version = ${String(1)}`);
+		});
+	}
+	stmt(sql) {
+		let statement = this.statements.get(sql);
+		if (statement === void 0) {
+			statement = this.db.prepare(sql);
+			this.statements.set(sql, statement);
+		}
+		return statement;
+	}
+	write(work) {
+		if (this.closed) throw new Error("the vector store is closed");
+		this.db.exec("BEGIN IMMEDIATE");
+		try {
+			const result = work();
+			this.db.exec("COMMIT");
+			return result;
+		} catch (error) {
+			if (this.db.isTransaction) this.db.exec("ROLLBACK");
+			throw error;
+		}
+	}
+	live() {
+		if (this.closed) throw new Error("the vector store is closed");
+	}
+	/** This method gives the plan of the chunks of a text when the store has each of its vectors, or undefined. */
+	plan(embedder, sha) {
+		this.live();
+		return this.stmt("SELECT plan FROM texts WHERE sha = ? AND embedder = ?").get(sha, embedder)?.plan;
+	}
+	/** This method gives the text that a document points at, or undefined. */
+	pointed(embedder, doc) {
+		this.live();
+		return this.stmt("SELECT sha FROM docs WHERE doc = ? AND embedder = ?").get(doc, embedder)?.sha;
+	}
+	/** This method gives the dimensions that the store keeps for an embedder, or undefined. */
+	dimensions(embedder) {
+		this.live();
+		return this.stmt("SELECT dimensions FROM embedders WHERE id = ?").get(embedder)?.dimensions;
+	}
+	/**
+	* This method writes the chunks of one text and points its documents at it, in one transaction.
+	* A vector with a length that is different from the dimensions of the embedder throws an error.
+	*/
+	put(input) {
+		const dims = input.dimensions;
+		for (const chunk of input.chunks) if (chunk.vector.length !== dims) throw new Error(`the embedder ${input.embedder} gave a vector of ${String(chunk.vector.length)} dimensions, not ${String(dims)}`);
+		let replaced = false;
+		this.write(() => {
+			const known = this.stmt("SELECT dimensions FROM embedders WHERE id = ?").get(input.embedder);
+			if (known === void 0 && input.chunks.length > 0) this.stmt("INSERT INTO embedders (id, dimensions, first_at) VALUES (?, ?, ?)").run(input.embedder, dims, input.at);
+			else if (known !== void 0 && input.chunks.length > 0 && known.dimensions !== dims) throw new Error(`vectors.sqlite holds vectors of ${String(known.dimensions)} dimensions for the embedder ${input.embedder}, and it gave ${String(dims)}: give the embedder a new id, or delete vectors.sqlite`);
+			const before = this.stmt("SELECT plan FROM texts WHERE sha = ? AND embedder = ?").get(input.sha, input.embedder);
+			replaced = before !== void 0 && before.plan !== input.plan;
+			this.stmt("DELETE FROM chunks WHERE sha = ? AND embedder = ?").run(input.sha, input.embedder);
+			const insert = this.stmt("INSERT INTO chunks (sha, chunk, embedder, char_start, char_end, vector) VALUES (?, ?, ?, ?, ?, ?)");
+			input.chunks.forEach((chunk, at) => {
+				insert.run(input.sha, at, input.embedder, chunk.start, chunk.end, bytesOf(chunk.vector));
+			});
+			this.stmt(`INSERT INTO texts (sha, embedder, plan, chunks, chars, embedded_at) VALUES (?, ?, ?, ?, ?, ?)
+         ON CONFLICT(sha, embedder) DO UPDATE SET plan = excluded.plan, chunks = excluded.chunks,
+           chars = excluded.chars, embedded_at = excluded.embedded_at`).run(input.sha, input.embedder, input.plan, input.chunks.length, input.chars, input.at);
+			for (const doc of input.docs) this.pointRow(input.embedder, doc, input.sha, input.at);
+		});
+		const matrix = this.matrices.get(input.embedder);
+		if (matrix === void 0) return;
+		if (replaced || matrix.texts.has(input.sha)) {
+			this.matrices.delete(input.embedder);
+			return;
+		}
+		this.append(matrix, input.sha, input.chunks.map((chunk, at) => ({
+			...chunk,
+			chunk: at
+		})));
+		for (const doc of input.docs) this.pointMatrix(matrix, doc, input.sha);
+	}
+	pointRow(embedder, doc, sha, at) {
+		this.stmt(`INSERT INTO docs (doc, embedder, sha, pointed_at) VALUES (?, ?, ?, ?)
+       ON CONFLICT(doc, embedder) DO UPDATE SET sha = excluded.sha, pointed_at = excluded.pointed_at`).run(doc, embedder, sha, at);
+	}
+	/** This method points a document at a text that the store has. */
+	point(embedder, doc, sha, at) {
+		this.write(() => this.pointRow(embedder, doc, sha, at));
+		const matrix = this.matrices.get(embedder);
+		if (matrix === void 0) return;
+		if (!matrix.texts.has(sha)) this.append(matrix, sha, this.chunks(embedder, sha));
+		this.pointMatrix(matrix, doc, sha);
+	}
+	/** This method removes the pointer of a document, for example when the document is held back. Its text keeps its vectors. */
+	forget(embedder, doc) {
+		this.write(() => this.stmt("DELETE FROM docs WHERE doc = ? AND embedder = ?").run(doc, embedder));
+		const matrix = this.matrices.get(embedder);
+		if (matrix === void 0) return;
+		const sha = matrix.shaOf.get(doc);
+		if (sha !== void 0) matrix.docsOf.get(sha)?.delete(doc);
+		matrix.shaOf.delete(doc);
+	}
+	/** This method gives the chunks of a text, in their sequence. */
+	chunks(embedder, sha) {
+		this.live();
+		return this.stmt("SELECT chunk, char_start, char_end, vector FROM chunks WHERE sha = ? AND embedder = ? ORDER BY chunk").all(sha, embedder).map((row) => ({
+			chunk: row.chunk,
+			start: row.char_start,
+			end: row.char_end,
+			vector: vectorOf(row.vector)
+		}));
+	}
+	pointMatrix(matrix, doc, sha) {
+		const before = matrix.shaOf.get(doc);
+		if (before !== void 0 && before !== sha) matrix.docsOf.get(before)?.delete(doc);
+		matrix.shaOf.set(doc, sha);
+		let docs = matrix.docsOf.get(sha);
+		if (docs === void 0) {
+			docs = /* @__PURE__ */ new Set();
+			matrix.docsOf.set(sha, docs);
+		}
+		docs.add(doc);
+	}
+	append(matrix, sha, chunks) {
+		const dims = matrix.dimensions;
+		const need = matrix.rows + chunks.length;
+		if (need > matrix.chunk.length) {
+			const size = Math.max(need, matrix.chunk.length * 2, 64);
+			const data = new Float32Array(size * dims);
+			data.set(matrix.data.subarray(0, matrix.rows * dims));
+			const grow = (old) => {
+				const next = new Int32Array(size);
+				next.set(old.subarray(0, matrix.rows));
+				return next;
+			};
+			matrix.data = data;
+			matrix.chunk = grow(matrix.chunk);
+			matrix.start = grow(matrix.start);
+			matrix.end = grow(matrix.end);
+		}
+		const from = matrix.rows;
+		for (const chunk of chunks) {
+			const row = matrix.rows;
+			if (chunk.vector.length === dims) matrix.data.set(chunk.vector, row * dims);
+			matrix.chunk[row] = chunk.chunk;
+			matrix.start[row] = chunk.start;
+			matrix.end[row] = chunk.end;
+			matrix.rows += 1;
+		}
+		matrix.texts.set(sha, {
+			from,
+			to: matrix.rows
+		});
+	}
+	/** This method gives the vectors of an embedder in memory. It reads them again when a different process wrote the file. */
+	matrix(embedder) {
+		this.live();
+		const version = this.db.prepare("PRAGMA data_version").get().data_version;
+		const cached = this.matrices.get(embedder);
+		if (cached !== void 0 && cached.version === version) return cached;
+		const dims = this.dimensions(embedder);
+		if (dims === void 0) return void 0;
+		this.db.exec("BEGIN");
+		try {
+			const matrix = this.load(embedder, dims, version);
+			this.matrices.set(embedder, matrix);
+			return matrix;
+		} finally {
+			if (this.db.isTransaction) this.db.exec("COMMIT");
+		}
+	}
+	load(embedder, dims, version) {
+		const count = this.db.prepare("SELECT count(*) AS n FROM chunks WHERE embedder = ? AND sha IN (SELECT sha FROM docs WHERE embedder = ?)").get(embedder, embedder).n;
+		const matrix = {
+			dimensions: dims,
+			rows: 0,
+			data: new Float32Array(Math.max(1, count) * dims),
+			chunk: new Int32Array(Math.max(1, count)),
+			start: new Int32Array(Math.max(1, count)),
+			end: new Int32Array(Math.max(1, count)),
+			texts: /* @__PURE__ */ new Map(),
+			docsOf: /* @__PURE__ */ new Map(),
+			shaOf: /* @__PURE__ */ new Map(),
+			version
+		};
+		const rows = this.db.prepare(`SELECT sha, chunk, char_start, char_end, vector FROM chunks
+          WHERE embedder = ? AND sha IN (SELECT sha FROM docs WHERE embedder = ?)
+          ORDER BY sha, chunk`).iterate(embedder, embedder);
+		let sha;
+		let from = 0;
+		for (const row of rows) {
+			if (row.sha !== sha) {
+				if (sha !== void 0) matrix.texts.set(sha, {
+					from,
+					to: matrix.rows
+				});
+				sha = row.sha;
+				from = matrix.rows;
+			}
+			if (matrix.rows >= matrix.chunk.length) break;
+			const at = matrix.rows;
+			if (row.vector.byteLength === dims * 4) new Uint8Array(matrix.data.buffer, at * dims * 4, dims * 4).set(row.vector);
+			matrix.chunk[at] = row.chunk;
+			matrix.start[at] = row.char_start;
+			matrix.end[at] = row.char_end;
+			matrix.rows += 1;
+		}
+		if (sha !== void 0) matrix.texts.set(sha, {
+			from,
+			to: matrix.rows
+		});
+		for (const row of this.db.prepare("SELECT doc, sha FROM docs WHERE embedder = ?").all(embedder)) this.pointMatrix(matrix, row.doc, row.sha);
+		return matrix;
+	}
+	/** This method gives the best chunk of one text for a question, or undefined when the store has no vector of the text. */
+	best(embedder, sha, query) {
+		const matrix = this.matrix(embedder);
+		const range = matrix?.texts.get(sha);
+		if (matrix === void 0 || range === void 0 || range.to === range.from) return void 0;
+		return this.hitOf(matrix, sha, range, query);
+	}
+	hitOf(matrix, sha, range, query) {
+		let bestRow = range.from;
+		let bestScore = Number.NEGATIVE_INFINITY;
+		for (let row = range.from; row < range.to; row += 1) {
+			const score = dot(matrix.data, row, query, matrix.dimensions);
+			if (score > bestScore) {
+				bestScore = score;
+				bestRow = row;
+			}
+		}
+		return {
+			sha,
+			chunk: matrix.chunk[bestRow],
+			start: matrix.start[bestRow],
+			end: matrix.end[bestRow],
+			similarity: bestScore,
+			docs: [...matrix.docsOf.get(sha) ?? []]
+		};
+	}
+	/**
+	* This method compares a question with each stored vector of an embedder (a brute-force cosine).
+	* It gives the best chunk of each text that a document points at, the best first. With `within`,
+	* it compares only the texts of those documents.
+	*/
+	search(embedder, query, options = {}) {
+		const matrix = this.matrix(embedder);
+		if (matrix === void 0) return [];
+		if (query.length !== matrix.dimensions) throw new Error(`the question has a vector of ${String(query.length)} dimensions, and the store holds ${String(matrix.dimensions)} for the embedder ${embedder}`);
+		const shas = /* @__PURE__ */ new Set();
+		if (options.within === void 0) {
+			for (const [sha, docs] of matrix.docsOf) if (docs.size > 0) shas.add(sha);
+		} else for (const doc of options.within) {
+			const sha = matrix.shaOf.get(doc);
+			if (sha !== void 0) shas.add(sha);
+		}
+		const hits = [];
+		for (const sha of shas) {
+			const range = matrix.texts.get(sha);
+			if (range === void 0 || range.to === range.from) continue;
+			hits.push(this.hitOf(matrix, sha, range, query));
+		}
+		return hits.sort((a, b) => b.similarity - a.similarity || (a.sha < b.sha ? -1 : 1));
+	}
+	/** This method gives the counts of the store, for one embedder or for each embedder. */
+	counts(embedder) {
+		this.live();
+		const where = embedder === void 0 ? "" : " WHERE embedder = ?";
+		const params = embedder === void 0 ? [] : [embedder];
+		const one = (sql) => this.stmt(sql).get(...params).n ?? 0;
+		return {
+			texts: one(`SELECT count(*) AS n FROM texts${where}`),
+			chunks: one(`SELECT count(*) AS n FROM chunks${where}`),
+			docs: one(`SELECT count(*) AS n FROM docs${where}`),
+			bytes: one(`SELECT sum(length(vector)) AS n FROM chunks${where}`)
+		};
+	}
+	/** This method closes the file. A second close does nothing. */
+	close() {
+		if (this.closed) return;
+		this.closed = true;
+		this.statements.clear();
+		this.matrices.clear();
+		this.db.close();
+	}
+};
+//#endregion
 //#region src/watch.ts
 /**
 * A watch list keeps the pages that the crawler checks again on a schedule. `watch.run` fetches the
@@ -2349,7 +3686,7 @@ var SqliteHostStateStore = class {
 * - The list is in `watch.sqlite` in the store, so it stays after a restart. A run claims each entry
 *   with a lease in that file, so two processes do not check one entry two times.
 *
-* The package has no daemon. A scheduler of the operator starts `watch.run`.
+* A scheduler of the operator starts `watch.run`, or the daemon (`startDaemon`) starts it on a schedule.
 */
 /** This value is the version of the schema of `watch.sqlite`. */
 const WATCH_SCHEMA = 1;
@@ -2619,7 +3956,7 @@ const REFUSALS = /* @__PURE__ */ new Set([
 	"payment-required"
 ]);
 /** This function gives the fingerprint of a version: a hash of its title, its text and its JSON-LD. */
-const watchFingerprint = (version) => sha256(stableStringify([
+const watchFingerprint = (version) => sha256$1(stableStringify([
 	version.title,
 	version.text,
 	version.meta["jsonld"] ?? null
@@ -2836,6 +4173,10 @@ const runWatch = (runtime, source, options = {}) => {
 			const deadline = startedAt + budgetMs;
 			for (;;) {
 				if (stopping === void 0 && abort.signal.aborted) stopping = "cancelled";
+				if (stopping === void 0 && options.drain?.aborted === true) {
+					stopping = "cancelled";
+					reason = "the run stopped for a shutdown: the checks in progress ended first";
+				}
 				if (stopping === void 0 && now() >= deadline) stopping = "budget";
 				while (stopping === void 0 && inflight.size < WATCH_LIMITS.hostsAtOnce && started < maxPages) {
 					const claimed = store.claim(now(), owner, Math.max(now(), deadline) + WATCH_LIMITS.leaseMs, [...inflight.keys()]);
@@ -2926,7 +4267,8 @@ const STORE_PARTS = [
 	"blobs",
 	"index.sqlite",
 	"hosts.sqlite",
-	"watch.sqlite"
+	"watch.sqlite",
+	"vectors.sqlite"
 ];
 /**
 * This function makes the folder of a store for the owner only, where the system has modes. It
@@ -2960,6 +4302,9 @@ const createRuntime = (options) => {
 	if (dir === void 0 && (options.log === void 0 || options.blobs === void 0 || options.index === void 0)) throw new Error("a runtime needs dir, or a log, a blob store and an index");
 	const contact = options.contact;
 	if (contact !== void 0 && !/^[\t -~]*$/.test(contact)) throw new Error(`contact "${contact}" has a character that an HTTP header cannot hold: use printable ASCII`);
+	const extractors = new Extractors();
+	for (const [recipe, extractor] of Object.entries(options.extractors ?? {})) extractors.register(recipe, extractor);
+	for (const recipe of options.recipes ?? []) registerRecipe(extractors, recipe);
 	if (dir !== void 0) guardStore(dir);
 	const owned = [];
 	const log = options.log ?? new CaptureLog(join(dir, "captures"));
@@ -3003,8 +4348,6 @@ const createRuntime = (options) => {
 		onEvent: options.onEvent,
 		signer: options.signer
 	});
-	const extractors = new Extractors();
-	for (const [recipe, extractor] of Object.entries(options.extractors ?? {})) extractors.register(recipe, extractor);
 	let closing;
 	return {
 		crawler,
@@ -3057,6 +4400,26 @@ const openCrawler = (options) => {
 		return watches;
 	};
 	const clockNow = () => Date.parse(runtime.now());
+	const embedder = options.embedder;
+	let vectors;
+	let search;
+	const hybridSearch = async () => {
+		await live();
+		if (embedder === void 0) throw new Error(NO_EMBEDDER);
+		vectors ??= new VectorStore(join(options.dir, "vectors.sqlite"));
+		search ??= createHybridSearch(runtime, {
+			embedder,
+			store: vectors
+		});
+		return search;
+	};
+	const hybrid = embedder === void 0 ? void 0 : {
+		embedder,
+		find: async (q, findOptions) => (await hybridSearch()).find(q, findOptions),
+		rankWithin: async (q, docs, rankOptions) => (await hybridSearch()).rankWithin(q, docs, rankOptions),
+		embed: async (embedOptions) => (await hybridSearch()).embed(embedOptions)
+	};
+	let closing;
 	return {
 		runtime,
 		async fetch(url, fetchOptions = {}) {
@@ -3076,7 +4439,22 @@ const openCrawler = (options) => {
 		},
 		async find(q, findOptions = {}) {
 			await live();
-			return findDocuments(runtime, q, findOptions);
+			const { rank, ...rest } = findOptions;
+			if (rank === "hybrid") return (await hybridSearch()).find(q, rest);
+			return findDocuments(runtime, q, {
+				...rest,
+				...rank === void 0 ? {} : { rank }
+			});
+		},
+		hybrid,
+		async embed(embedOptions = {}) {
+			return (await hybridSearch()).embed(embedOptions);
+		},
+		async rankWithin(q, docs, rankOptions = {}) {
+			await live();
+			const { rank, ...rest } = rankOptions;
+			if (rank === "hybrid") return (await hybridSearch()).rankWithin(q, docs, rest);
+			return runtime.index.rankWithin(q, docs, rank === void 0 ? {} : { rank });
 		},
 		async read(id, readOptions = {}) {
 			await live();
@@ -3147,15 +4525,20 @@ const openCrawler = (options) => {
 				});
 			}
 		},
-		async close() {
+		close() {
 			closed = true;
-			await ready;
-			await runtime.close();
-			watches?.close();
+			closing ??= (async () => {
+				await ready;
+				await runtime.close();
+				watches?.close();
+				vectors?.close();
+				await embedder?.close?.();
+			})();
+			return closing;
 		}
 	};
 };
 //#endregion
-export { readSitemap as _, open_exports as a, WatchStore as c, CRAWL_LIMITS as d, crawlSite as f, feedDate as g, SITEMAP_LIMITS as h, openCrawler as i, runWatch as l, discoverSite as m, defaultUserAgent as n, WATCH_LIMITS as o, DISCOVER_LIMITS as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, SqliteHostStateStore as u, DocIndex as v, FAST_RANK_SHARE as y };
+export { crawlSite as A, chunkText as C, unitVector as D, fuseRanks as E, readSitemap as F, DocIndex as I, FAST_RANK_SHARE as L, discoverSite as M, SITEMAP_LIMITS as N, SqliteHostStateStore as O, feedDate as P, NO_EMBEDDER as R, HYBRID_DEFAULTS as S, estimateTokens as T, registerRecipe as _, open_exports as a, createHybridSearch as b, WatchStore as c, VectorStore as d, RECIPE_LIMITS as f, readJson as g, defineRecipe as h, openCrawler as i, DISCOVER_LIMITS as j, CRAWL_LIMITS as k, runWatch as l, deadlineOf as m, defaultUserAgent as n, WATCH_LIMITS as o, RESERVED_RECIPES as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, VECTORS_SCHEMA as u, runRecipe as v, chunkTokensOf as w, CHUNK_LIMITS as x, runtimeOf as y };
 
-//# sourceMappingURL=open-DnrWgkC0.mjs.map
+//# sourceMappingURL=open-C4fhXJgD.mjs.map