polite-crawler 0.1.3 → 0.1.4

package/build/mcp.d.mts

@@ -1,4 +1,4 @@
-import { E as OpenOptions, O as PoliteCrawler, s as DaemonMcpHost } from "./daemon-DP4EqjFL.mjs";
+import { E as OpenOptions, O as PoliteCrawler, s as DaemonMcpHost } from "./daemon-wwIWsUmU.mjs";
 import { McpServer, McpServer as McpServer$1 } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { Request, RequestId, Result, Task } from "@modelcontextprotocol/sdk/types.js";
 import { CreateTaskOptions, TaskStore } from "@modelcontextprotocol/sdk/experimental/tasks/interfaces.js";

package/build/mcp.mjs

@@ -1,8 +1,8 @@
 import { t as browserMissing } from "./browser-CKRNbIDT.mjs";
-import { i as durationMs, r as VERSION } from "./version-ntbY-6wh.mjs";
-import { A as crawlSite, M as discoverSite, i as openCrawler } from "./open-C4fhXJgD.mjs";
-import { a as formatDiff, n as diffDocument } from "./diff-NQcQqKe9.mjs";
-import { r as extractDocument } from "./extract-ByFfc8Zk.mjs";
+import { i as durationMs, r as VERSION } from "./version-BPem-XZd.mjs";
+import { S as crawlSite, i as openCrawler, w as discoverSite } from "./open-CbpSCEZc.mjs";
+import { a as formatDiff, n as diffDocument } from "./diff-BUM91Rev.mjs";
+import { r as extractDocument } from "./extract-CefB5Vfn.mjs";
 import { n as urlHost, t as isLoopbackHost } from "./loopback-C_QYowSf.mjs";
 import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
 import { readFileSync, statSync } from "node:fs";

package/build/{open-C4fhXJgD.mjs → open-CbpSCEZc.mjs}

@@ -2,14 +2,15 @@ import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
 import { n as FileBlobStore, t as CaptureLog } from "./log-Dx-wXjJl.mjs";
 import { r as playwrightRenderer } from "./render-C1CUraWF.mjs";
 import { r as checkedRenderer } from "./browser-CKRNbIDT.mjs";
-import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-ntbY-6wh.mjs";
-import { A as snippetOf, C as fallbackReason, E as fold, L as sha256$1, M as Crawler, N as DEADLINE_NOTE, R as stableStringify, S as archivedOf, a as fetchPage, c as isHeldBack, f as readDocument, g as verifyQuote, i as fetchAndIndex, k as parseQuery, m as reindex, n as HELD_BACK, o as findDocuments, p as recordOverride, r as catchUpIndex, s as heldBackMeta, t as Extractors, y as archivedCopy } from "./runtime-JToRO63o.mjs";
+import { i as durationMs, n as DEFAULT_TOKEN, r as VERSION, t as BOT_PAGE } from "./version-BPem-XZd.mjs";
+import { A as snippetOf, E as fold, L as sha256$1, M as Crawler, N as DEADLINE_NOTE, R as stableStringify, S as archivedOf, a as fetchPage, c as isHeldBack, f as readDocument, g as verifyQuote, i as fetchAndIndex, k as parseQuery, m as reindex, o as findDocuments, r as catchUpIndex, t as Extractors } from "./runtime-D_r2SYge.mjs";
 import { MemoryHostStateStore, nextValidatorMemo, realClock } from "./politeness.mjs";
 import { matchesPattern } from "./robots.mjs";
 import { n as canonicalUrl, r as scopeTest, t as cacheKeyOf } from "./urls-BHFeSyEK.mjs";
 import { o as decodeBody, t as extractPage } from "./html-DkQUAE_d.mjs";
-import { n as diffDocument, r as diffTexts } from "./diff-NQcQqKe9.mjs";
-import { r as extractDocument } from "./extract-ByFfc8Zk.mjs";
+import { n as diffDocument, r as diffTexts } from "./diff-BUM91Rev.mjs";
+import { r as extractDocument } from "./extract-CefB5Vfn.mjs";
+import { o as registerRecipe } from "./recipe-GiFQojmo.mjs";
 import { createHash, randomUUID } from "node:crypto";
 import { dirname, join } from "node:path";
 import { existsSync, lstatSync, mkdirSync, realpathSync, statSync } from "node:fs";
@@ -847,7 +848,7 @@ const XML_ENTITIES = {
 };
 /** A tag keeps this number of attributes at most. The reader reads the others and keeps nothing of them. */
 const MAX_ATTRIBUTES = 64;
-const NAME$1 = /^[A-Za-z_][\w.:-]*$/;
+const NAME = /^[A-Za-z_][\w.:-]*$/;
 const W3C_DATE = /^\d{4}(?:-\d{2}(?:-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?)?)?$/;
 const MAIL_DATE = /^(?:[A-Za-z]{3},?\s+)?\d{1,2}\s+[A-Za-z]{3,9}\s+\d{2,4}\s+\d{1,2}:\d{2}(?::\d{2})?(?:\s*(?:[A-Za-z]{1,5}|[+-]\d{4}))?$/;
 const short = (name) => name === void 0 ? "nothing" : name.length > 40 ? `${name.slice(0, 40)}...` : name;
@@ -920,7 +921,7 @@ const tagAt = (text, from) => {
 		at += 1;
 	}
 	const name = text.slice(from, Math.min(at, from + 256));
-	if (at - from > 256 || !NAME$1.test(name)) return void 0;
+	if (at - from > 256 || !NAME.test(name)) return void 0;
 	const attributes = [];
 	for (;;) {
 		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
@@ -945,7 +946,7 @@ const tagAt = (text, from) => {
 			at += 1;
 		}
 		const key = text.slice(start, Math.min(at, start + 256));
-		if (at - start > 256 || !NAME$1.test(key)) return void 0;
+		if (at - start > 256 || !NAME.test(key)) return void 0;
 		while (at < n && isSpace(text.charCodeAt(at))) at += 1;
 		if (text.charCodeAt(at) !== 61) return void 0;
 		at += 1;
@@ -1339,7 +1340,7 @@ const DISCOVER_LIMITS = {
 	/** The most feeds that discovery reads from the links of the page. */
 	maxLinkedFeeds: 5
 };
-const PAGE_ACCEPT$1 = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
+const PAGE_ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
 const SITEMAP_ACCEPT = "application/xml,text/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5";
 const FEED_ACCEPT = "application/rss+xml,application/atom+xml,application/xml;q=0.9,text/xml;q=0.8,*/*;q=0.5";
 const FEED_KINDS = /* @__PURE__ */ new Set([
@@ -1534,7 +1535,7 @@ const discoverSite = async (runtime, site, options = {}) => {
 			url: start.href,
 			mode: "page",
 			recipe: "page",
-			accept: PAGE_ACCEPT$1,
+			accept: PAGE_ACCEPT,
 			maxAgeMs,
 			onBehalfOf: options.onBehalfOf,
 			signal,
@@ -3018,279 +3019,6 @@ async function* pagesOf(runtime) {
 	if (page.length > 0) yield page;
 }
 //#endregion
-//#region src/recipe.ts
-/**
-* This function reads a body as JSON, in the encoding that the response declares. A body that is
-* not JSON (a page about maintenance, or a login page) throws an error that names the URL and the
-* content type. In an extractor, the runtime then reports that error.
-*/
-const readJson = (body, capture) => {
-	const text = decodeBody(body, capture.contentType);
-	try {
-		return JSON.parse(text);
-	} catch {
-		throw new Error(`${capture.url} answered ${capture.contentType ?? "a body of no stated type"}, not JSON`);
-	}
-};
-/**
-* These recipe names belong to the crawler: the page reader, the archived fallback, robots.txt,
-* and the sitemaps and feeds of `discover`. A recipe cannot use them.
-*/
-const RESERVED_RECIPES = [
-	"page",
-	"archive-copy",
-	"archive-index",
-	"robots.txt",
-	"sitemap",
-	"feed"
-];
-/** These are the limits of a recipe. */
-const RECIPE_LIMITS = {
-	nameChars: 64,
-	docIdChars: 2048
-};
-const NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
-const PAGE_ACCEPT = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
-const API_ACCEPT = "application/json";
-const AI_INPUTS = [
-	"yes",
-	"no",
-	"unset"
-];
-/** This function tells what is wrong with one document of an extractor, or gives undefined. */
-const faultOf = (value, seen) => {
-	if (value === null || typeof value !== "object" || Array.isArray(value)) return "a document must be an object";
-	const doc = value;
-	const id = doc["doc"];
-	if (typeof id !== "string" || id.length === 0 || id.length > RECIPE_LIMITS.docIdChars) return `doc must be a text of 1 to ${String(RECIPE_LIMITS.docIdChars)} characters`;
-	if (id.startsWith("archive:")) return `the id ${id} starts with archive:, which is for the archived copies`;
-	if (seen.has(id)) return `two documents have the id ${id}`;
-	seen.add(id);
-	const kind = doc["kind"];
-	if (typeof kind !== "string" || kind.length === 0) return "kind must be a text that is not empty";
-	if (kind === "archived") return "the kind archived is for the archived copies";
-	if (typeof doc["title"] !== "string") return "title must be a text";
-	if (typeof doc["text"] !== "string") return "text must be a text";
-	if (doc["url"] !== null && typeof doc["url"] !== "string") return "url must be a text or null";
-	const meta = doc["meta"];
-	if (meta === null || typeof meta !== "object" || Array.isArray(meta)) return "meta must be an object";
-	try {
-		JSON.stringify(meta);
-	} catch {
-		return "meta must be JSON: no cycle and no BigInt";
-	}
-	const merge = doc["merge"];
-	if (merge !== void 0 && merge !== "latest" && merge !== "richest") return "merge must be \"latest\" or \"richest\"";
-	if (doc["full_text"] !== void 0 && typeof doc["full_text"] !== "string") return "full_text must be a text";
-	const aiInput = doc["ai_input"];
-	if (aiInput !== void 0 && !AI_INPUTS.includes(aiInput)) return "ai_input must be \"yes\", \"no\" or \"unset\"";
-};
-/** This function gives the documents of an extractor, or throws an error that names the first fault. */
-const checkedDocs = (name, docs) => {
-	if (!Array.isArray(docs)) throw new TypeError(`the recipe ${name} gave ${typeof docs}, not an array of documents`);
-	const seen = /* @__PURE__ */ new Set();
-	for (const [index, doc] of docs.entries()) {
-		const fault = faultOf(doc, seen);
-		if (fault !== void 0) throw new TypeError(`the recipe ${name} gave a document that is not valid (${String(index)}): ${fault}`);
-	}
-	return docs;
-};
-/**
-* This function declares a recipe. It checks the definition at once: a name that is not valid, a
-* reserved name or a duration that it cannot read throws an error. The extractor of the recipe
-* checks each document of `extract`. A document that is not valid makes the extractor throw, and
-* the runtime reports it as an error of the extractor.
-*/
-const defineRecipe = (definition) => {
-	const name = definition.name;
-	if (typeof name !== "string" || name.length > RECIPE_LIMITS.nameChars || !NAME.test(name)) throw new RangeError(`a recipe name is 1 to ${String(RECIPE_LIMITS.nameChars)} letters, digits and . _ : -, and it starts with a letter or a digit, not ${JSON.stringify(name) ?? "undefined"}`);
-	if (RESERVED_RECIPES.includes(name)) throw new RangeError(`the recipe name ${name} belongs to the crawler: give the recipe a name of its own`);
-	if (typeof definition.extract !== "function") throw new TypeError(`the recipe ${name} needs extract, a function that reads a body into documents`);
-	const mode = definition.mode ?? "page";
-	if (mode !== "page" && mode !== "api") throw new RangeError(`the mode of the recipe ${name} must be "page" or "api", not ${String(mode)}`);
-	const accept = definition.accept ?? (mode === "api" ? API_ACCEPT : PAGE_ACCEPT);
-	const minIntervalMs = definition.minInterval === void 0 ? void 0 : durationMs(definition.minInterval);
-	const maxAgeOf = (value) => value === "auto" ? "auto" : durationMs(value);
-	const maxAge = maxAgeOf(definition.maxAge ?? "24h");
-	const maxBytes = definition.maxBytes;
-	if (maxBytes !== void 0 && (!Number.isSafeInteger(maxBytes) || maxBytes < 1)) throw new RangeError(`maxBytes of the recipe ${name} must be a whole number of 1 or more`);
-	const extract = definition.extract;
-	const extractor = async (body, capture) => checkedDocs(name, await extract(body, {
-		url: capture.url,
-		contentType: capture.contentType
-	}));
-	const fetch = definition.fetch;
-	return {
-		name,
-		description: definition.description ?? null,
-		mode,
-		accept,
-		extractor,
-		request(input, options = {}) {
-			const asked = fetch === void 0 ? input : fetch(input);
-			const target = typeof asked === "string" ? { url: asked } : asked;
-			const url = target?.url;
-			let parsed;
-			try {
-				parsed = typeof url === "string" ? new URL(url) : void 0;
-			} catch {
-				parsed = void 0;
-			}
-			if (parsed === void 0 || parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new TypeError(`the recipe ${name} needs an http or https URL for its input, not ${JSON.stringify(url) ?? String(url)}`);
-			return {
-				url: parsed.href,
-				mode,
-				recipe: name,
-				accept: target?.accept ?? accept,
-				headers: target?.headers,
-				secretParams: target?.secretParams,
-				minIntervalMs,
-				maxAgeMs: options.maxAge === void 0 ? maxAge : maxAgeOf(options.maxAge),
-				maxBytes,
-				onBehalfOf: options.onBehalfOf,
-				job: options.job,
-				signal: options.signal,
-				scope: options.scope,
-				labels: options.labels,
-				deadline: options.deadline
-			};
-		}
-	};
-};
-/**
-* This function gives the extractor of a recipe to the extractors of a runtime. A second
-* registration of the same recipe does nothing. A different extractor with the same name throws
-* an error, because `reindex` must read each capture with one extractor.
-*/
-const registerRecipe = (extractors, recipe) => {
-	const known = extractors.get(recipe.name);
-	if (known === recipe.extractor) return;
-	if (known !== void 0) throw new Error(`the recipe ${recipe.name} has a different extractor in this runtime: give each recipe a name of its own`);
-	extractors.register(recipe.name, recipe.extractor);
-};
-/** This function gives the runtime of a target. */
-const runtimeOf = (target) => "crawler" in target ? target : target.runtime;
-/** This function gives the deadline of a budget, on the clock of the runtime. */
-const deadlineOf = (runtime, budget) => budget === void 0 ? void 0 : Date.parse(runtime.now()) + durationMs(budget);
-/** This function gives one document of a run, with the rule for a held-back document. */
-const recipeDoc = async (runtime, doc, capture, userProvided, action) => {
-	const current = runtime.index.get(doc.doc) ?? doc;
-	const version = (capture === void 0 ? void 0 : runtime.index.versionText(doc.doc, capture)) ?? current;
-	const aiInput = current.ai_input === "no" || version.ai_input === "no" ? "no" : version.ai_input;
-	const held = isHeldBack({ ai_input: aiInput }, userProvided);
-	if (aiInput === "no" && userProvided) await recordOverride(runtime, current, action);
-	const archived = archivedOf(version.meta);
-	return {
-		doc: doc.doc,
-		kind: version.kind,
-		title: held ? "" : version.title,
-		url: version.url,
-		meta: held ? heldBackMeta(version.meta) : version.meta,
-		chars: version.text.length,
-		capture: version.capture,
-		fetched_at: version.fetched_at,
-		held_back: held,
-		...archived === void 0 ? {} : { archived }
-	};
-};
-const noteOf = (parts) => {
-	const kept = parts.filter((part) => typeof part === "string" && part.length > 0);
-	return kept.length > 0 ? kept.join("; ") : null;
-};
-/**
-* This function starts a recipe for one input. It fetches the request through the crawler, and it
-* reads the capture with the extractor of the recipe. It gives the documents of that capture.
-* These are its rules:
-*
-* - The runtime registers the recipe the first time that it gets it. Give the recipe to
-*   `openCrawler` too, so that `reindex` reads its captures before the first fetch.
-* - A stored copy younger than `maxAge` answers with no request. When the extractor cannot read a
-*   stored copy, the crawler asks the source one more time.
-* - An extractor that throws gives no document, and `error` tells why. The fetch still succeeds.
-* - A document that asked not to be AI input shows no title and only its structural metadata,
-*   unless `userProvided`. The crawler logs each `userProvided` read of a held-back document.
-* - With `fallback: "archive"`, a page that the site refuses gets an archived copy, with
-*   `archived`. The page reader reads the copy. The extractor of the recipe reads only live
-*   captures. Thus its documents do not mix a live answer and an old copy of a third party.
-*/
-const runRecipe = async (target, recipe, input, options = {}) => {
-	const runtime = runtimeOf(target);
-	if (options.fallback === "archive" && recipe.mode !== "page") throw new RangeError(`the recipe ${recipe.name} reads an API: an archived copy is for a page, so give fallback only to a recipe of the mode page`);
-	registerRecipe(runtime.extractors, recipe);
-	const deadline = deadlineOf(runtime, options.budget);
-	const request = recipe.request(input, {
-		...options,
-		deadline
-	});
-	let indexed = await fetchAndIndex(runtime, request);
-	if (indexed.error !== void 0 && indexed.blocked === void 0 && indexed.fetched.fromCache) indexed = await fetchAndIndex(runtime, {
-		...request,
-		maxAgeMs: 0
-	});
-	const userProvided = options.userProvided ?? false;
-	const action = options.job ?? `recipe ${recipe.name}`;
-	const fetched = indexed.fetched;
-	const notes = [fetched.note, indexed.error];
-	if (indexed.docs.length === 0 && options.fallback === "archive") {
-		const reason = fallbackReason(indexed);
-		if (reason !== void 0) {
-			const page = {
-				render: "never",
-				maxAgeMs: request.maxAgeMs,
-				onBehalfOf: request.onBehalfOf,
-				job: request.job,
-				signal: request.signal,
-				scope: request.scope,
-				labels: request.labels,
-				deadline,
-				fallback: "archive",
-				archiveDespiteRobots: options.archiveDespiteRobots
-			};
-			const answer = await archivedCopy(runtime, request.url, reason, page);
-			if (answer.doc !== void 0 && answer.archived !== void 0) {
-				const doc = await recipeDoc(runtime, answer.doc, answer.doc.capture, userProvided, action);
-				return {
-					recipe: recipe.name,
-					url: fetched.url,
-					outcome: fetched.outcome,
-					status: fetched.status,
-					docs: [doc],
-					capture: doc.capture,
-					fetched_at: doc.fetched_at,
-					from_cache: answer.fromCache,
-					held_back: doc.held_back,
-					error: indexed.error ?? null,
-					archived: answer.archived,
-					note: noteOf([
-						...notes,
-						...answer.notes,
-						"the page reader read the archived copy, not the extractor of the recipe",
-						doc.held_back ? HELD_BACK : void 0
-					])
-				};
-			}
-			notes.push(...answer.notes);
-		}
-	}
-	const capture = fetched.capture;
-	const docs = [];
-	for (const doc of indexed.docs) docs.push(await recipeDoc(runtime, doc, capture, userProvided, action));
-	const held = docs.some((doc) => doc.held_back);
-	return {
-		recipe: recipe.name,
-		url: fetched.url,
-		outcome: fetched.outcome,
-		status: fetched.status,
-		docs,
-		capture: docs[0]?.capture ?? capture ?? null,
-		fetched_at: docs[0]?.fetched_at ?? fetched.fetchedAt ?? null,
-		from_cache: fetched.fromCache,
-		held_back: held,
-		error: indexed.error ?? null,
-		note: noteOf([...notes, held ? HELD_BACK : void 0])
-	};
-};
-//#endregion
 //#region src/vectors.ts
 /**
 * The vector store keeps the vectors of hybrid search in `vectors.sqlite`, beside the index. It is
@@ -4539,6 +4267,6 @@ const openCrawler = (options) => {
 	};
 };
 //#endregion
-export { crawlSite as A, chunkText as C, unitVector as D, fuseRanks as E, readSitemap as F, DocIndex as I, FAST_RANK_SHARE as L, discoverSite as M, SITEMAP_LIMITS as N, SqliteHostStateStore as O, feedDate as P, NO_EMBEDDER as R, HYBRID_DEFAULTS as S, estimateTokens as T, registerRecipe as _, open_exports as a, createHybridSearch as b, WatchStore as c, VectorStore as d, RECIPE_LIMITS as f, readJson as g, defineRecipe as h, openCrawler as i, DISCOVER_LIMITS as j, CRAWL_LIMITS as k, runWatch as l, deadlineOf as m, defaultUserAgent as n, WATCH_LIMITS as o, RESERVED_RECIPES as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, VECTORS_SCHEMA as u, runRecipe as v, chunkTokensOf as w, CHUNK_LIMITS as x, runtimeOf as y };
+export { NO_EMBEDDER as A, DISCOVER_LIMITS as C, readSitemap as D, feedDate as E, DocIndex as O, crawlSite as S, SITEMAP_LIMITS as T, estimateTokens as _, open_exports as a, SqliteHostStateStore as b, WatchStore as c, VectorStore as d, createHybridSearch as f, chunkTokensOf as g, chunkText as h, openCrawler as i, FAST_RANK_SHARE as k, runWatch as l, HYBRID_DEFAULTS as m, defaultUserAgent as n, WATCH_LIMITS as o, CHUNK_LIMITS as p, isMailbox as r, WATCH_SCHEMA as s, createRuntime as t, VECTORS_SCHEMA as u, fuseRanks as v, discoverSite as w, CRAWL_LIMITS as x, unitVector as y };
 
-//# sourceMappingURL=open-C4fhXJgD.mjs.map
+//# sourceMappingURL=open-CbpSCEZc.mjs.map