polite-crawler 0.1.3 → 0.2.0

package/build/{open-C4fhXJgD.mjs → open-CEo22Hpu.mjs}

@@ -1,15 +1,16 @@
-import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
-import { n as FileBlobStore, t as CaptureLog } from "./log-Dx-wXjJl.mjs";
-import { r as playwrightRenderer } from "./render-C1CUraWF.mjs";
-import { r as checkedRenderer } from "./browser-CKRNbIDT.mjs";
-import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-ntbY-6wh.mjs";
-import { A as snippetOf, C as fallbackReason, E as fold, L as sha256$1, M as Crawler, N as DEADLINE_NOTE, R as stableStringify, S as archivedOf, a as fetchPage, c as isHeldBack, f as readDocument, g as verifyQuote, i as fetchAndIndex, k as parseQuery, m as reindex, n as HELD_BACK, o as findDocuments, p as recordOverride, r as catchUpIndex, s as heldBackMeta, t as Extractors, y as archivedCopy } from "./runtime-JToRO63o.mjs";
+import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
+import { n as FileBlobStore, t as CaptureLog } from "./log-CmBWcyQm.mjs";
+import { r as playwrightRenderer } from "./render-CKE2LILH.mjs";
+import { r as checkedRenderer } from "./browser-56dBBLec.mjs";
+import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-Djva7JkN.mjs";
+import { A as DEADLINE_NOTE, C as fold, D as snippetOf, E as parseQuery, N as sha256$1, P as stableStringify, a as fetchPage, c as isHeldBack, f as reindex, i as fetchAndIndex, k as Crawler, o as findDocuments, p as verifyQuote, r as catchUpIndex, t as Extractors, u as readDocument, y as archivedOf } from "./runtime-C_2BQibH.mjs";
 import { MemoryHostStateStore, nextValidatorMemo, realClock } from "./politeness.mjs";
 import { matchesPattern } from "./robots.mjs";
 import { n as canonicalUrl, r as scopeTest, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
 import { o as decodeBody, t as extractPage } from "./html-DkQUAE_d.mjs";
-import { n as diffDocument, r as diffTexts } from "./diff-NQcQqKe9.mjs";
-import { r as extractDocument } from "./extract-ByFfc8Zk.mjs";
+import { n as diffDocument, r as diffTexts } from "./diff-CpKN0Jdc.mjs";
+import { r as extractDocument } from "./extract-DqeqiNHb.mjs";
+import { o as registerRecipe } from "./recipe-B2SkepDZ.mjs";
 import { createHash, randomUUID } from "node:crypto";
 import { dirname, join } from "node:path";
 import { existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
@@ -17,7 +18,11 @@ import { createGunzip } from "node:zlib";
 import { DatabaseSync } from "node:sqlite";
 /** The search index reads this many characters of each text, at most. */
 const MAX_INDEX_CHARS = 2e5;
-/** In the mode `fast`, a part is common when more than this share of the documents hold it. */
+/**
+* In the mode `fast`, a part is common when more than this share of the documents hold it.
+*
+* @beta
+*/
 const FAST_RANK_SHARE = .2;
 /** The text of the error of hybrid search with no embedder. */
 const NO_EMBEDDER = "hybrid search needs an embedder: open the crawler with the option embedder (transformersEmbedder or ollamaEmbedder), or give --embedder transformers or ollama to the command line";
@@ -163,6 +168,7 @@ const toVersion = (row) => ({
 	cut: row.full_sha !== null
 });
 const messageOf = (error) => error instanceof Error ? error.message : String(error);
+/** @beta */
 var DocIndex = class {
 	path;
 	db;
@@ -813,7 +819,11 @@ var DocIndex = class {
 };
 //#endregion
 //#region src/sitemaps.ts
-/** These are the default limits of the reader, and the maximum length of one value. */
+/**
+* These are the default limits of the reader, and the maximum length of one value.
+*
+* @beta
+*/
 const SITEMAP_LIMITS = {
 	maxEntries: 5e4,
 	maxDepth: 32,
@@ -847,7 +857,7 @@ const XML_ENTITIES = {
 };
 /** A tag keeps this number of attributes at most. The reader reads the others and keeps nothing of them. */
 const MAX_ATTRIBUTES = 64;
-const NAME$1 = /^[A-Za-z_][\w.:-]*$/;
+const NAME = /^[A-Za-z_][\w.:-]*$/;
 const W3C_DATE = /^\d{4}(?:-\d{2}(?:-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?)?)?$/;
 const MAIL_DATE = /^(?:[A-Za-z]{3},?\s+)?\d{1,2}\s+[A-Za-z]{3,9}\s+\d{2,4}\s+\d{1,2}:\d{2}(?::\d{2})?(?:\s*(?:[A-Za-z]{1,5}|[+-]\d{4}))?$/;
 const short = (name) => name === void 0 ? "nothing" : name.length > 40 ? `${name.slice(0, 40)}...` : name;
@@ -855,6 +865,8 @@ const short = (name) => name === void 0 ? "nothing" : name.length > 40 ? `${name
 * This function changes a date of a sitemap or a feed into ISO 8601. It reads the W3C form of the
 * sitemap protocol and Atom, and the mail form of RSS. A time with no zone is UTC. It gives null
 * for a text that it cannot read.
+*
+* @beta
 */
 const feedDate = (value) => {
 	const text = value.trim();
@@ -920,7 +932,7 @@ const tagAt = (text, from) => {
 		at += 1;
 	}
 	const name = text.slice(from, Math.min(at, from + 256));
-	if (at - from > 256 || !NAME$1.test(name)) return void 0;
+	if (at - from > 256 || !NAME.test(name)) return void 0;
 	const attributes = [];
 	for (;;) {
 		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
@@ -945,7 +957,7 @@ const tagAt = (text, from) => {
 			at += 1;
 		}
 		const key = text.slice(start, Math.min(at, start + 256));
-		if (at - start > 256 || !NAME$1.test(key)) return void 0;
+		if (at - start > 256 || !NAME.test(key)) return void 0;
 		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
 		if (text.charCodeAt(at) !== 61) return void 0;
 		at += 1;
@@ -1304,6 +1316,8 @@ const scanSitemap = (text, base, limits = {}, meter = { steps: 0 }) => {
 * This function reads a sitemap, a sitemap index, an RSS or Atom feed, or a text sitemap. The
 * root element of the file tells its kind. The entries of a feed can have URLs relative to `base`
 * (the URL of the file). The entries of a sitemap must have absolute URLs.
+*
+* @beta
 */
 const readSitemap = (text, base, limits) => scanSitemap(text, base, limits);
 //#endregion
@@ -1322,7 +1336,11 @@ const readSitemap = (text, base, limits) => scanSitemap(text, base, limits);
 * Discovery keeps a URL only when it is in scope (the same site by default) and robots.txt permits
 * it. It stops at a maximum of files, a maximum of URLs and a maximum time.
 */
-/** These are the default limits of `discover`. */
+/**
+* These are the default limits of `discover`.
+*
+* @beta
+*/
 const DISCOVER_LIMITS = {
 	/** The most sitemap and feed files that one discovery reads. */
 	maxFiles: 20,
@@ -1339,7 +1357,7 @@ const DISCOVER_LIMITS = {
 	/** The most feeds that discovery reads from the links of the page. */
 	maxLinkedFeeds: 5
 };
-const PAGE_ACCEPT$1 = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
+const PAGE_ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
 const SITEMAP_ACCEPT = "application/xml,text/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5";
 const FEED_ACCEPT = "application/rss+xml,application/atom+xml,application/xml;q=0.9,text/xml;q=0.8,*/*;q=0.5";
 const FEED_KINDS = /* @__PURE__ */ new Set([
@@ -1412,6 +1430,8 @@ const pastDeadline = (note) => note?.startsWith(DEADLINE_NOTE) === true;
 * This function finds the pages of a site from its sitemaps and its feeds. `PoliteCrawler.discover`
 * uses it. Each file is a fetch of the crawler with the recipe `sitemap` or `feed`, so the
 * capture log records it, and robots.txt applies to it.
+*
+* @beta
 */
 const discoverSite = async (runtime, site, options = {}) => {
 	const start = parsedUrl(site);
@@ -1534,7 +1554,7 @@ const discoverSite = async (runtime, site, options = {}) => {
 			url: start.href,
 			mode: "page",
 			recipe: "page",
-			accept: PAGE_ACCEPT$1,
+			accept: PAGE_ACCEPT,
 			maxAgeMs,
 			onBehalfOf: options.onBehalfOf,
 			signal,
@@ -1721,7 +1741,11 @@ const discoverSite = async (runtime, site, options = {}) => {
 *
 * The state of the frontier is in memory, for one call. A later version can make it resumable.
 */
-/** These are the default limits of `crawl`, and the limits of the memory of the frontier. */
+/**
+* These are the default limits of `crawl`, and the limits of the memory of the frontier.
+*
+* @beta
+*/
 const CRAWL_LIMITS = {
 	/** The most pages that one crawl gives. */
 	maxPages: 100,
@@ -1819,6 +1843,8 @@ const REFUSALS$1 = /* @__PURE__ */ new Set([
 * This function crawls a site from its seeds. `PoliteCrawler.crawl` uses it. It gives each page
 * when its fetch ends, and a summary at the end. The seeds come first, then the pages of the
 * sitemaps (the newest first), then the links, the lower depth first.
+*
+* @beta
 */
 const crawlSite = (runtime, seeds, options = {}) => {
 	let resolveSummary = () => void 0;
@@ -2169,6 +2195,8 @@ const memoOf = (row) => ({
 *
 * The state is not a projection of the capture log. A deletion of the file loses only the pace,
 * the counts, the stops and the memos, and the crawler starts them again.
+*
+* @beta
 */
 var SqliteHostStateStore = class {
 	path;
@@ -2350,7 +2378,11 @@ var SqliteHostStateStore = class {
 };
 //#endregion
 //#region src/embed.ts
-/** These are the limits of the chunks of a text. */
+/**
+* These are the limits of the chunks of a text.
+*
+* @beta
+*/
 const CHUNK_LIMITS = {
 	maxTokens: 512,
 	defaultTokens: 256,
@@ -2377,6 +2409,8 @@ const kindOf = (point) => {
 * This function estimates the number of tokens of a part of a text. A word is one token for each
 * six letters. A Chinese, Japanese or Thai character is one token. Each other character that is
 * not a space is one token. The estimate is a little high for English text, so a chunk fits.
+*
+* @beta
 */
 const estimateTokens = (text, start = 0, end = text.length) => {
 	let tokens = 0;
@@ -2503,6 +2537,8 @@ const positive = (name, value) => {
 * sentence that is longer than the window is cut at its spaces. Two chunks next to each other share
 * a few sentences (`overlapTokens`), so a passage at a cut is in one chunk. Each chunk keeps its
 * position in the text, so a hit points at a passage.
+*
+* @beta
 */
 const chunkText = (text, options) => {
 	const size = positive("maxTokens", options.maxTokens);
@@ -2552,13 +2588,19 @@ const chunkText = (text, options) => {
 /**
 * This function gives the number of tokens of a chunk for an embedder. It is the input limit with a
 * margin for the estimate, and `CHUNK_LIMITS.maxTokens` at most.
+*
+* @beta
 */
 const chunkTokensOf = (embedder) => {
 	const limit = embedder.maxInputTokens;
 	const tokens = limit === void 0 || !Number.isFinite(limit) || limit < 1 ? CHUNK_LIMITS.defaultTokens : Math.min(limit, CHUNK_LIMITS.maxTokens);
 	return Math.max(8, Math.floor(tokens * CHUNK_LIMITS.fill));
 };
-/** This function gives a copy of a vector with a length of 1. A vector of zeros stays zeros. A value that is not finite throws a `RangeError`. */
+/**
+* This function gives a copy of a vector with a length of 1. A vector of zeros stays zeros. A value that is not finite throws a `RangeError`.
+*
+* @beta
+*/
 const unitVector = (vector) => {
 	const out = Float32Array.from(vector);
 	let sum = 0;
@@ -2582,6 +2624,8 @@ const HYBRID_DEFAULTS = {
 * 1 / (k + rank) over the lists that hold it, with the rank from 1. The best item is first. A tie goes to
 * the item with the best rank in a list, then to the earlier list, then to the smaller id. An item
 * that is two times in one list counts at its first place.
+*
+* @beta
 */
 const fuseRanks = (lists, k = HYBRID_DEFAULTS.fusionK) => {
 	if (!Number.isFinite(k) || k < 0) throw new RangeError(`the fusion constant k must be a number of 0 or more, not ${String(k)}`);
@@ -2658,7 +2702,11 @@ const round = (value, digits) => {
 	const scale = 10 ** digits;
 	return Math.round(value * scale) / scale;
 };
-/** This function makes a hybrid search over a runtime. The vectors go into the store, with the id of the embedder. */
+/**
+* This function makes a hybrid search over a runtime. The vectors go into the store, with the id of the embedder.
+*
+* @beta
+*/
 const createHybridSearch = (runtime, options) => {
 	const { embedder, store } = options;
 	const tokens = chunkTokensOf(embedder);
@@ -3018,279 +3066,6 @@ async function* pagesOf(runtime) {
 	if (page.length > 0) yield page;
 }
 //#endregion
-//#region src/recipe.ts
-/**
-* This function reads a body as JSON, in the encoding that the response declares. A body that is
-* not JSON (a page about maintenance, or a login page) throws an error that names the URL and the
-* content type. In an extractor, the runtime then reports that error.
-*/
-const readJson = (body, capture) => {
-	const text = decodeBody(body, capture.contentType);
-	try {
-		return JSON.parse(text);
-	} catch {
-		throw new Error(`${capture.url} answered ${capture.contentType ?? "a body of no stated type"}, not JSON`);
-	}
-};
-/**
-* These recipe names belong to the crawler: the page reader, the archived fallback, robots.txt,
-* and the sitemaps and feeds of `discover`. A recipe cannot use them.
-*/
-const RESERVED_RECIPES = [
-	"page",
-	"archive-copy",
-	"archive-index",
-	"robots.txt",
-	"sitemap",
-	"feed"
-];
-/** These are the limits of a recipe. */
-const RECIPE_LIMITS = {
-	nameChars: 64,
-	docIdChars: 2048
-};
-const NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
-const PAGE_ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
-const API_ACCEPT = "application/json";
-const AI_INPUTS = [
-	"yes",
-	"no",
-	"unset"
-];
-/** This function tells what is wrong with one document of an extractor, or gives undefined. */
-const faultOf = (value, seen) => {
-	if (value === null || typeof value !== "object" || Array.isArray(value)) return "a document must be an object";
-	const doc = value;
-	const id = doc["doc"];
-	if (typeof id !== "string" || id.length === 0 || id.length > RECIPE_LIMITS.docIdChars) return `doc must be a text of 1 to ${String(RECIPE_LIMITS.docIdChars)} characters`;
-	if (id.startsWith("archive:")) return `the id ${id} starts with archive:, which is for the archived copies`;
-	if (seen.has(id)) return `two documents have the id ${id}`;
-	seen.add(id);
-	const kind = doc["kind"];
-	if (typeof kind !== "string" || kind.length === 0) return "kind must be a text that is not empty";
-	if (kind === "archived") return "the kind archived is for the archived copies";
-	if (typeof doc["title"] !== "string") return "title must be a text";
-	if (typeof doc["text"] !== "string") return "text must be a text";
-	if (doc["url"] !== null && typeof doc["url"] !== "string") return "url must be a text or null";
-	const meta = doc["meta"];
-	if (meta === null || typeof meta !== "object" || Array.isArray(meta)) return "meta must be an object";
-	try {
-		JSON.stringify(meta);
-	} catch {
-		return "meta must be JSON: no cycle and no BigInt";
-	}
-	const merge = doc["merge"];
-	if (merge !== void 0 && merge !== "latest" && merge !== "richest") return "merge must be \"latest\" or \"richest\"";
-	if (doc["full_text"] !== void 0 && typeof doc["full_text"] !== "string") return "full_text must be a text";
-	const aiInput = doc["ai_input"];
-	if (aiInput !== void 0 && !AI_INPUTS.includes(aiInput)) return "ai_input must be \"yes\", \"no\" or \"unset\"";
-};
-/** This function gives the documents of an extractor, or throws an error that names the first fault. */
-const checkedDocs = (name, docs) => {
-	if (!Array.isArray(docs)) throw new TypeError(`the recipe ${name} gave ${typeof docs}, not an array of documents`);
-	const seen = /* @__PURE__ */ new Set();
-	for (const [index, doc] of docs.entries()) {
-		const fault = faultOf(doc, seen);
-		if (fault !== void 0) throw new TypeError(`the recipe ${name} gave a document that is not valid (${String(index)}): ${fault}`);
-	}
-	return docs;
-};
-/**
-* This function declares a recipe. It checks the definition at once: a name that is not valid, a
-* reserved name or a duration that it cannot read throws an error. The extractor of the recipe
-* checks each document of `extract`. A document that is not valid makes the extractor throw, and
-* the runtime reports it as an error of the extractor.
-*/
-const defineRecipe = (definition) => {
-	const name = definition.name;
-	if (typeof name !== "string" || name.length > RECIPE_LIMITS.nameChars || !NAME.test(name)) throw new RangeError(`a recipe name is 1 to ${String(RECIPE_LIMITS.nameChars)} letters, digits and . _ : -, and it starts with a letter or a digit, not ${JSON.stringify(name) ?? "undefined"}`);
-	if (RESERVED_RECIPES.includes(name)) throw new RangeError(`the recipe name ${name} belongs to the crawler: give the recipe a name of its own`);
-	if (typeof definition.extract !== "function") throw new TypeError(`the recipe ${name} needs extract, a function that reads a body into documents`);
-	const mode = definition.mode ?? "page";
-	if (mode !== "page" && mode !== "api") throw new RangeError(`the mode of the recipe ${name} must be "page" or "api", not ${String(mode)}`);
-	const accept = definition.accept ?? (mode === "api" ? API_ACCEPT : PAGE_ACCEPT);
-	const minIntervalMs = definition.minInterval === void 0 ? void 0 : durationMs(definition.minInterval);
-	const maxAgeOf = (value) => value === "auto" ? "auto" : durationMs(value);
-	const maxAge = maxAgeOf(definition.maxAge ?? "24h");
-	const maxBytes = definition.maxBytes;
-	if (maxBytes !== void 0 && (!Number.isSafeInteger(maxBytes) || maxBytes < 1)) throw new RangeError(`maxBytes of the recipe ${name} must be a whole number of 1 or more`);
-	const extract = definition.extract;
-	const extractor = async (body, capture) => checkedDocs(name, await extract(body, {
-		url: capture.url,
-		contentType: capture.contentType
-	}));
-	const fetch = definition.fetch;
-	return {
-		name,
-		description: definition.description ?? null,
-		mode,
-		accept,
-		extractor,
-		request(input, options = {}) {
-			const asked = fetch === void 0 ? input : fetch(input);
-			const target = typeof asked === "string" ? { url: asked } : asked;
-			const url = target?.url;
-			let parsed;
-			try {
-				parsed = typeof url === "string" ? new URL(url) : void 0;
-			} catch {
-				parsed = void 0;
-			}
-			if (parsed === void 0 || parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new TypeError(`the recipe ${name} needs an http or https URL for its input, not ${JSON.stringify(url) ?? String(url)}`);
-			return {
-				url: parsed.href,
-				mode,
-				recipe: name,
-				accept: target?.accept ?? accept,
-				headers: target?.headers,
-				secretParams: target?.secretParams,
-				minIntervalMs,
-				maxAgeMs: options.maxAge === void 0 ? maxAge : maxAgeOf(options.maxAge),
-				maxBytes,
-				onBehalfOf: options.onBehalfOf,
-				job: options.job,
-				signal: options.signal,
-				scope: options.scope,
-				labels: options.labels,
-				deadline: options.deadline
-			};
-		}
-	};
-};
-/**
-* This function gives the extractor of a recipe to the extractors of a runtime. A second
-* registration of the same recipe does nothing. A different extractor with the same name throws
-* an error, because `reindex` must read each capture with one extractor.
-*/
-const registerRecipe = (extractors, recipe) => {
-	const known = extractors.get(recipe.name);
-	if (known === recipe.extractor) return;
-	if (known !== void 0) throw new Error(`the recipe ${recipe.name} has a different extractor in this runtime: give each recipe a name of its own`);
-	extractors.register(recipe.name, recipe.extractor);
-};
-/** This function gives the runtime of a target. */
-const runtimeOf = (target) => "crawler" in target ? target : target.runtime;
-/** This function gives the deadline of a budget, on the clock of the runtime. */
-const deadlineOf = (runtime, budget) => budget === void 0 ? void 0 : Date.parse(runtime.now()) + durationMs(budget);
-/** This function gives one document of a run, with the rule for a held-back document. */
-const recipeDoc = async (runtime, doc, capture, userProvided, action) => {
-	const current = runtime.index.get(doc.doc) ?? doc;
-	const version = (capture === void 0 ? void 0 : runtime.index.versionText(doc.doc, capture)) ?? current;
-	const aiInput = current.ai_input === "no" || version.ai_input === "no" ? "no" : version.ai_input;
-	const held = isHeldBack({ ai_input: aiInput }, userProvided);
-	if (aiInput === "no" && userProvided) await recordOverride(runtime, current, action);
-	const archived = archivedOf(version.meta);
-	return {
-		doc: doc.doc,
-		kind: version.kind,
-		title: held ? "" : version.title,
-		url: version.url,
-		meta: held ? heldBackMeta(version.meta) : version.meta,
-		chars: version.text.length,
-		capture: version.capture,
-		fetched_at: version.fetched_at,
-		held_back: held,
-		...archived === void 0 ? {} : { archived }
-	};
-};
-const noteOf = (parts) => {
-	const kept = parts.filter((part) => typeof part === "string" && part.length > 0);
-	return kept.length > 0 ? kept.join("; ") : null;
-};
-/**
-* This function starts a recipe for one input. It fetches the request through the crawler, and it
-* reads the capture with the extractor of the recipe. It gives the documents of that capture.
-* These are its rules:
-*
-* - The runtime registers the recipe the first time that it gets it. Give the recipe to
-*   `openCrawler` too, so that `reindex` reads its captures before the first fetch.
-* - A stored copy younger than `maxAge` answers with no request. When the extractor cannot read a
-*   stored copy, the crawler asks the source one more time.
-* - An extractor that throws gives no document, and `error` tells why. The fetch still succeeds.
-* - A document that asked not to be AI input shows no title and only its structural metadata,
-*   unless `userProvided`. The crawler logs each `userProvided` read of a held-back document.
-* - With `fallback: "archive"`, a page that the site refuses gets an archived copy, with
-*   `archived`. The page reader reads the copy. The extractor of the recipe reads only live
-*   captures. Thus its documents do not mix a live answer and an old copy of a third party.
-*/
-const runRecipe = async (target, recipe, input, options = {}) => {
-	const runtime = runtimeOf(target);
-	if (options.fallback === "archive" && recipe.mode !== "page") throw new RangeError(`the recipe ${recipe.name} reads an API: an archived copy is for a page, so give fallback only to a recipe of the mode page`);
-	registerRecipe(runtime.extractors, recipe);
-	const deadline = deadlineOf(runtime, options.budget);
-	const request = recipe.request(input, {
-		...options,
-		deadline
-	});
-	let indexed = await fetchAndIndex(runtime, request);
-	if (indexed.error !== void 0 && indexed.blocked === void 0 && indexed.fetched.fromCache) indexed = await fetchAndIndex(runtime, {
-		...request,
-		maxAgeMs: 0
-	});
-	const userProvided = options.userProvided ?? false;
-	const action = options.job ?? `recipe ${recipe.name}`;
-	const fetched = indexed.fetched;
-	const notes = [fetched.note, indexed.error];
-	if (indexed.docs.length === 0 && options.fallback === "archive") {
-		const reason = fallbackReason(indexed);
-		if (reason !== void 0) {
-			const page = {
-				render: "never",
-				maxAgeMs: request.maxAgeMs,
-				onBehalfOf: request.onBehalfOf,
-				job: request.job,
-				signal: request.signal,
-				scope: request.scope,
-				labels: request.labels,
-				deadline,
-				fallback: "archive",
-				archiveDespiteRobots: options.archiveDespiteRobots
-			};
-			const answer = await archivedCopy(runtime, request.url, reason, page);
-			if (answer.doc !== void 0 && answer.archived !== void 0) {
-				const doc = await recipeDoc(runtime, answer.doc, answer.doc.capture, userProvided, action);
-				return {
-					recipe: recipe.name,
-					url: fetched.url,
-					outcome: fetched.outcome,
-					status: fetched.status,
-					docs: [doc],
-					capture: doc.capture,
-					fetched_at: doc.fetched_at,
-					from_cache: answer.fromCache,
-					held_back: doc.held_back,
-					error: indexed.error ?? null,
-					archived: answer.archived,
-					note: noteOf([
-						...notes,
-						...answer.notes,
-						"the page reader read the archived copy, not the extractor of the recipe",
-						doc.held_back ? HELD_BACK : void 0
-					])
-				};
-			}
-			notes.push(...answer.notes);
-		}
-	}
-	const capture = fetched.capture;
-	const docs = [];
-	for (const doc of indexed.docs) docs.push(await recipeDoc(runtime, doc, capture, userProvided, action));
-	const held = docs.some((doc) => doc.held_back);
-	return {
-		recipe: recipe.name,
-		url: fetched.url,
-		outcome: fetched.outcome,
-		status: fetched.status,
-		docs,
-		capture: docs[0]?.capture ?? capture ?? null,
-		fetched_at: docs[0]?.fetched_at ?? fetched.fetchedAt ?? null,
-		from_cache: fetched.fromCache,
-		held_back: held,
-		error: indexed.error ?? null,
-		note: noteOf([...notes, held ? HELD_BACK : void 0])
-	};
-};
-//#endregion
 //#region src/vectors.ts
 /**
 * The vector store keeps the vectors of hybrid search in `vectors.sqlite`, beside the index. It is
@@ -3312,7 +3087,11 @@ const runRecipe = async (target, recipe, input, options = {}) => {
 * one transaction (`BEGIN IMMEDIATE`). The file has a schema version (`VECTORS_SCHEMA`). A file of
 * a different version opens empty.
 */
-/** This value is the version of the schema of the vector file. */
+/**
+* This value is the version of the schema of the vector file.
+*
+* @beta
+*/
 const VECTORS_SCHEMA = 1;
 const DDL = `
 CREATE TABLE IF NOT EXISTS vectors_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
@@ -3363,6 +3142,7 @@ const dot = (data, row, query, dims) => {
 	for (let at = 0; at < dims; at += 1) sum += data[base + at] * query[at];
 	return sum;
 };
+/** @beta */
 var VectorStore = class {
 	path;
 	db;
@@ -3768,6 +3548,8 @@ const targetOf = (url) => {
 *
 * The list is the decision of the operator, not a projection of the capture log. Thus it has its
 * own file: a deletion of `hosts.sqlite` or of the index does not lose it.
+*
+* @beta
 */
 var WatchStore = class {
 	path;
@@ -3967,6 +3749,8 @@ const sameState = (a, b) => a !== void 0 && a.state === b.state && (a.state ===
 * it. Each check is a fetch of the page with `fetchPage`, with no stored copy (`maxAgeMs: 0`), so
 * each polite rule applies. A check that the budget stops before its request stays due. The store
 * can be a function that opens it: the run uses it before the first check.
+*
+* @beta
 */
 const runWatch = (runtime, source, options = {}) => {
 	let resolveSummary = () => void 0;
@@ -4259,6 +4043,8 @@ const MAILBOX = /^(?:[^\s<>@:/]+@[^\s<>@/]+|[^<>]*<[^\s<>@:/]+@[^\s<>@/]+>)$/;
 * This function tells if a contact is a mailbox. The `From` header holds only a mailbox (RFC 9110,
 * section 10.1.2). Thus the crawler sends a mailbox in `From` and in the `User-Agent`, and a URL in
 * the `User-Agent` only.
+*
+* @beta
 */
 const isMailbox = (contact) => MAILBOX.test(contact.trim());
 /** The parts of a store that can be a symbolic link to a different place only by mistake or by attack. */
@@ -4296,6 +4082,8 @@ const guardStore = (dir) => {
 * This function makes a runtime. It is the one place that puts the log, the blob store, the index,
 * the host state, the crawler and the extractors together. The function `openCrawler` uses it.
 * An application that needs the parts uses it too. The runtime and the crawler share one clock.
+*
+* @beta
 */
 const createRuntime = (options) => {
 	const dir = options.dir;
@@ -4539,6 +4327,6 @@ const openCrawler = (options) => {
 	};
 };
 //#endregion
-export { crawlSite as A, chunkText as C, unitVector as D, fuseRanks as E, readSitemap as F, DocIndex as I, FAST_RANK_SHARE as L, discoverSite as M, SITEMAP_LIMITS as N, SqliteHostStateStore as O, feedDate as P, NO_EMBEDDER as R, HYBRID_DEFAULTS as S, estimateTokens as T, registerRecipe as _, open_exports as a, createHybridSearch as b, WatchStore as c, VectorStore as d, RECIPE_LIMITS as f, readJson as g, defineRecipe as h, openCrawler as i, DISCOVER_LIMITS as j, CRAWL_LIMITS as k, runWatch as l, deadlineOf as m, defaultUserAgent as n, WATCH_LIMITS as o, RESERVED_RECIPES as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, VECTORS_SCHEMA as u, runRecipe as v, chunkTokensOf as w, CHUNK_LIMITS as x, runtimeOf as y };
+export { NO_EMBEDDER as A, DISCOVER_LIMITS as C, readSitemap as D, feedDate as E, DocIndex as O, crawlSite as S, SITEMAP_LIMITS as T, estimateTokens as _, open_exports as a, SqliteHostStateStore as b, WatchStore as c, VectorStore as d, createHybridSearch as f, chunkTokensOf as g, chunkText as h, openCrawler as i, FAST_RANK_SHARE as k, runWatch as l, HYBRID_DEFAULTS as m, defaultUserAgent as n, WATCH_LIMITS as o, CHUNK_LIMITS as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, VECTORS_SCHEMA as u, fuseRanks as v, discoverSite as w, CRAWL_LIMITS as x, unitVector as y };
 
-//# sourceMappingURL=open-C4fhXJgD.mjs.map
+//# sourceMappingURL=open-CEo22Hpu.mjs.map