polite-crawler 0.1.1 → 0.1.3

package/build/diff-NQcQqKe9.mjs

@@ -0,0 +1,648 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { R as stableStringify, _ as versionAtTime, c as isHeldBack, l as isoTime, n as HELD_BACK, s as heldBackMeta } from "./runtime-JToRO63o.mjs";
+//#region src/diff.ts
+var diff_exports = /* @__PURE__ */ __exportAll({
+	DIFF_LIMITS: () => DIFF_LIMITS,
+	diffDocument: () => diffDocument,
+	diffTexts: () => diffTexts,
+	editScript: () => editScript,
+	formatDiff: () => formatDiff
+});
+/**
+* A diff compares two versions of a stored document. It gives the changes of the text as a line
+* diff, with a word diff inside each changed line. It gives the changes of the title, the URL and
+* the metadata (the JSON-LD too) as changes at a path.
+*
+* The line diff is the algorithm of Myers (1986). Its time is linear in the length of the texts
+* when the change is small. A budget of steps (`DIFF_LIMITS`) bounds the time of a large change.
+* After the budget, the diff gives the rest of the change as one removal and one addition. Then
+* the result says that it is not minimal. A huge text gives its first part, and the result says so.
+*
+* A held-back version gives no text and no title to a diff, and only its structural metadata.
+*/
+/** These are the limits of a diff. Each limit bounds the time or the size of a result. */
+const DIFF_LIMITS = {
+	/** The characters of each text that a diff compares. A longer text gives its first part. */
+	maxChars: 1e6,
+	/** The lines of each text that a diff compares. A longer text gives its first lines. */
+	maxLines: 5e4,
+	/** The steps of the line diff. After them, the rest of the change is one removal and one addition. */
+	maxSteps: 1e6,
+	/** The steps of the word diffs of all the changed lines together. */
+	maxWordSteps: 1e6,
+	/** The words of one line that a word diff compares. A longer line gets no word diff. */
+	maxLineWords: 5e3,
+	/** The lines of context before and after each change. */
+	context: 2,
+	/** The lines that the hunks of a result show. The summary counts each line. */
+	maxHunkLines: 2e3,
+	/** The changes of the metadata that a result shows. The summary counts each change. */
+	maxMetaChanges: 200,
+	/** The depth of the metadata that a diff compares. A deeper change is a change of the whole value. */
+	maxMetaDepth: 16,
+	/** The characters of the JSON of each value of a change of the metadata. */
+	maxValueChars: 300
+};
+/**
+* This function gives an edit script from `a` to `b`, with the algorithm of Myers. It first
+* removes the common start and the common end. When the budget ends, it keeps the furthest path
+* that it found, and the rest is one removal and one addition. `exact` is then false.
+*/
+const editScript = (a, b, budget) => {
+	const n = a.length;
+	const m = b.length;
+	let head = 0;
+	while (head < n && head < m && a[head] === b[head]) head += 1;
+	let endA = n;
+	let endB = m;
+	while (endA > head && endB > head && a[endA - 1] === b[endB - 1]) {
+		endA -= 1;
+		endB -= 1;
+	}
+	budget.steps += head + (n - endA) + 1;
+	const reversed = [];
+	const width = endA - head;
+	const height = endB - head;
+	let exact = true;
+	reversed.push([0, n - endA]);
+	if (width === 0 || height === 0) reversed.push([2, height], [1, width]);
+	else {
+		const at = (x) => a[head + x];
+		const bt = (y) => b[head + y];
+		const max = width + height;
+		const offset = max + 1;
+		const v = new Int32Array(2 * max + 3);
+		const trace = [];
+		let end;
+		rounds: for (let d = 0; d <= max; d += 1) {
+			for (let k = -d; k <= d; k += 2) {
+				budget.steps += 1;
+				if (budget.steps > budget.max) break rounds;
+				let x = k === -d || k !== d && v[offset + k - 1] < v[offset + k + 1] ? v[offset + k + 1] : v[offset + k - 1] + 1;
+				let y = x - k;
+				while (x < width && y < height && at(x) === bt(y)) {
+					x += 1;
+					y += 1;
+					budget.steps += 1;
+				}
+				v[offset + k] = x;
+				if (x >= width && y >= height) {
+					end = {
+						x,
+						y,
+						d
+					};
+					break rounds;
+				}
+			}
+			trace.push(v.slice(offset - d, offset + d + 1));
+		}
+		if (end === void 0) {
+			exact = false;
+			const d = trace.length - 1;
+			let x = 0;
+			let y = 0;
+			if (d >= 0) {
+				const band = trace[d];
+				for (let k = -d; k <= d; k += 2) {
+					const fx = band[k + d];
+					const fy = fx - k;
+					if (fx <= width && fy >= 0 && fy <= height && fx + fy > x + y) {
+						x = fx;
+						y = fy;
+					}
+				}
+			}
+			reversed.push([2, height - y], [1, width - x]);
+			end = {
+				x,
+				y,
+				d: Math.max(0, d)
+			};
+		}
+		let { x, y } = end;
+		for (let d = end.d; d > 0; d -= 1) {
+			const band = trace[d - 1];
+			const k = x - y;
+			const before = (kk) => band[kk + d - 1];
+			const down = k === -d || k !== d && before(k - 1) < before(k + 1);
+			const fromK = down ? k + 1 : k - 1;
+			const fromX = before(fromK);
+			const fromY = fromX - fromK;
+			const moved = down ? fromX : fromX + 1;
+			reversed.push([0, x - moved], down ? [2, 1] : [1, 1]);
+			x = fromX;
+			y = fromY;
+		}
+		reversed.push([0, x]);
+	}
+	reversed.push([0, head]);
+	const runs = [];
+	for (let at = reversed.length - 1; at >= 0; at -= 1) {
+		const [op, count] = reversed[at];
+		if (count <= 0) continue;
+		const last = runs.at(-1);
+		if (last !== void 0 && last.op === op) runs[runs.length - 1] = {
+			op,
+			count: last.count + count
+		};
+		else runs.push({
+			op,
+			count
+		});
+	}
+	return {
+		runs,
+		exact
+	};
+};
+/** This function gives a number for each value, equal numbers for equal values. */
+const numberer = () => {
+	const ids = /* @__PURE__ */ new Map();
+	return (value) => {
+		let id = ids.get(value);
+		if (id === void 0) {
+			id = ids.size;
+			ids.set(value, id);
+		}
+		return id;
+	};
+};
+/** A token is a word, a run of spaces, or one sign. */
+const TOKEN = /\s+|[\p{L}\p{N}\p{M}_]+|[^\s\p{L}\p{N}\p{M}_]/gu;
+const tokensOf = (line) => line.match(TOKEN) ?? [];
+/** This function counts the words of a text: each token that is not a space. */
+const wordsOf = (text) => {
+	let count = 0;
+	for (const token of tokensOf(text)) if (!/^\s+$/.test(token)) count += 1;
+	return count;
+};
+/** This function gives the word diff of a changed line, or null when the line is too long or the budget ended. */
+const wordDiff = (before, after, budget, limits) => {
+	if (budget.steps >= budget.max) return null;
+	const a = tokensOf(before);
+	const b = tokensOf(after);
+	if (a.length > limits.maxLineWords || b.length > limits.maxLineWords) return null;
+	const id = numberer();
+	const script = editScript(a.map((token) => id(token)), b.map((token) => id(token)), budget);
+	const parts = [];
+	const push = (op, text) => {
+		const last = parts.at(-1);
+		if (last !== void 0 && last.op === op) parts[parts.length - 1] = {
+			op,
+			text: last.text + text
+		};
+		else parts.push({
+			op,
+			text
+		});
+	};
+	let i = 0;
+	let j = 0;
+	for (const run of script.runs) if (run.op === 0) {
+		push("same", a.slice(i, i + run.count).join(""));
+		i += run.count;
+		j += run.count;
+	} else if (run.op === 1) {
+		push("removed", a.slice(i, i + run.count).join(""));
+		i += run.count;
+	} else {
+		push("added", b.slice(j, j + run.count).join(""));
+		j += run.count;
+	}
+	return {
+		parts,
+		exact: script.exact
+	};
+};
+const IDENTIFIER = /^[A-Za-z_$][\w$]*$/;
+const keyPath = (path, key) => IDENTIFIER.test(key) ? path === "" ? key : `${path}.${key}` : `${path}[${JSON.stringify(key)}]`;
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+/** This function gives the changes at paths between two values, to the limits. */
+const metaChanges = (before, after, limits) => {
+	const changes = [];
+	let count = 0;
+	const show = (value) => {
+		const text = stableStringify(value);
+		return text.length > limits.maxValueChars ? `${text.slice(0, limits.maxValueChars)}…` : text;
+	};
+	const note = (path, change, a, b) => {
+		count += 1;
+		if (changes.length < limits.maxMetaChanges) changes.push({
+			path,
+			change,
+			before: change === "added" ? null : show(a),
+			after: change === "removed" ? null : show(b)
+		});
+	};
+	const walk = (a, b, path, depth) => {
+		if (a === b) return;
+		if (depth >= limits.maxMetaDepth) {
+			if (stableStringify(a) !== stableStringify(b)) note(path, "changed", a, b);
+			return;
+		}
+		if (isRecord(a) && isRecord(b)) {
+			const keys = [.../* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)])].sort();
+			for (const key of keys) {
+				const inA = Object.hasOwn(a, key) && a[key] !== void 0;
+				const inB = Object.hasOwn(b, key) && b[key] !== void 0;
+				const at = keyPath(path, key);
+				if (inA && !inB) note(at, "removed", a[key], void 0);
+				else if (!inA && inB) note(at, "added", void 0, b[key]);
+				else if (inA && inB) walk(a[key], b[key], at, depth + 1);
+			}
+			return;
+		}
+		if (Array.isArray(a) && Array.isArray(b)) {
+			const id = numberer();
+			const script = editScript(a.map((item) => id(stableStringify(item))), b.map((item) => id(stableStringify(item))), {
+				steps: 0,
+				max: 1e5
+			});
+			let i = 0;
+			let j = 0;
+			const removed = [];
+			const added = [];
+			const flush = () => {
+				const pairs = Math.min(removed.length, added.length);
+				for (let at = 0; at < pairs; at += 1) walk(a[removed[at]], b[added[at]], `${path}[${String(added[at])}]`, depth + 1);
+				for (const at of removed.slice(pairs)) note(`${path}[${String(at)}]`, "removed", a[at], void 0);
+				for (const at of added.slice(pairs)) note(`${path}[${String(at)}]`, "added", void 0, b[at]);
+				removed.length = 0;
+				added.length = 0;
+			};
+			for (const run of script.runs) if (run.op === 0) {
+				flush();
+				i += run.count;
+				j += run.count;
+			} else if (run.op === 1) {
+				for (let at = 0; at < run.count; at += 1) removed.push(i + at);
+				i += run.count;
+			} else {
+				for (let at = 0; at < run.count; at += 1) added.push(j + at);
+				j += run.count;
+			}
+			flush();
+			return;
+		}
+		if (stableStringify(a) !== stableStringify(b)) note(path, "changed", a, b);
+	};
+	walk(before, after, "", 0);
+	return {
+		changes,
+		count
+	};
+};
+/** This function gives the limits of a diff, with the defaults of `DIFF_LIMITS` for the limits that it does not get. */
+const limitsOf = (given) => {
+	const limits = { ...DIFF_LIMITS };
+	for (const [name, value] of Object.entries(given)) {
+		if (value === void 0) continue;
+		if (!Number.isInteger(value) || value < 0) throw new RangeError(`the diff limit ${name} must be an integer of 0 or more`);
+		limits[name] = value;
+	}
+	return limits;
+};
+/**
+* This function compares two versions: the lines of the text, the words of each changed line, and
+* the fields (the title, the URL and the metadata) at their paths. It writes nothing, and its time
+* and its result stay inside the limits.
+*/
+const diffTexts = (before, after, given = {}) => {
+	const limits = limitsOf(given);
+	const notes = [];
+	let truncated = false;
+	const cut = (text, side) => {
+		let kept = text;
+		if (kept.length > limits.maxChars) {
+			kept = kept.slice(0, limits.maxChars);
+			truncated = true;
+			notes.push(`the ${side} text is longer than ${String(limits.maxChars)} characters, so the diff compares its first part`);
+		}
+		const lines = kept.length === 0 ? [] : kept.split("\n");
+		if (lines.length > limits.maxLines) {
+			truncated = true;
+			notes.push(`the ${side} text has more than ${String(limits.maxLines)} lines, so the diff compares its first lines`);
+			return lines.slice(0, limits.maxLines);
+		}
+		return lines;
+	};
+	const a = cut(before.text, "old");
+	const b = cut(after.text, "new");
+	const id = numberer();
+	const lineBudget = {
+		steps: 0,
+		max: limits.maxSteps
+	};
+	const script = editScript(a.map((line) => id(line)), b.map((line) => id(line)), lineBudget);
+	let exact = script.exact;
+	if (!script.exact) notes.push(`the change is too large for a minimal diff in ${String(limits.maxSteps)} steps, so the diff shows its last part as one removal and one addition`);
+	const wordBudget = {
+		steps: 0,
+		max: limits.maxWordSteps
+	};
+	const entries = [];
+	let linesAdded = 0;
+	let linesRemoved = 0;
+	let linesChanged = 0;
+	let wordsAdded = 0;
+	let wordsRemoved = 0;
+	let wordsCut = false;
+	let i = 0;
+	let j = 0;
+	const removed = [];
+	const added = [];
+	const flush = () => {
+		const pairs = Math.min(removed.length, added.length);
+		for (let at = 0; at < pairs; at += 1) {
+			const from = removed[at];
+			const to = added[at];
+			const words = wordDiff(a[from], b[to], wordBudget, limits);
+			if (words === null) {
+				wordsCut = true;
+				wordsRemoved += wordsOf(a[from]);
+				wordsAdded += wordsOf(b[to]);
+			} else {
+				if (!words.exact) exact = false;
+				for (const part of words.parts) if (part.op === "added") wordsAdded += wordsOf(part.text);
+				else if (part.op === "removed") wordsRemoved += wordsOf(part.text);
+			}
+			entries.push({
+				op: "changed",
+				a: from,
+				b: to,
+				words: words?.parts ?? null
+			});
+		}
+		for (const from of removed.slice(pairs)) {
+			wordsRemoved += wordsOf(a[from]);
+			entries.push({
+				op: "removed",
+				a: from
+			});
+		}
+		for (const to of added.slice(pairs)) {
+			wordsAdded += wordsOf(b[to]);
+			entries.push({
+				op: "added",
+				b: to
+			});
+		}
+		linesChanged += pairs;
+		removed.length = 0;
+		added.length = 0;
+	};
+	for (const run of script.runs) if (run.op === 0) {
+		flush();
+		for (let at = 0; at < run.count; at += 1) entries.push({
+			op: "same",
+			a: i + at,
+			b: j + at
+		});
+		i += run.count;
+		j += run.count;
+	} else if (run.op === 1) {
+		for (let at = 0; at < run.count; at += 1) removed.push(i + at);
+		linesRemoved += run.count;
+		i += run.count;
+	} else {
+		for (let at = 0; at < run.count; at += 1) added.push(j + at);
+		linesAdded += run.count;
+		j += run.count;
+	}
+	flush();
+	if (wordsCut) {
+		exact = false;
+		notes.push("some changed lines have no word diff: they are too long, or the word budget ended");
+	}
+	const hunks = [];
+	let shown = 0;
+	let current;
+	const lineOf = (entry) => {
+		switch (entry.op) {
+			case "same": return {
+				op: "same",
+				text: a[entry.a]
+			};
+			case "removed": return {
+				op: "removed",
+				text: a[entry.a]
+			};
+			case "added": return {
+				op: "added",
+				text: b[entry.b]
+			};
+			default: return {
+				op: "changed",
+				before: a[entry.a],
+				after: b[entry.b],
+				words: entry.words
+			};
+		}
+	};
+	const close = () => {
+		if (current === void 0) return;
+		hunks.push({
+			from_line: current.from + 1,
+			from_count: current.a,
+			to_line: current.to + 1,
+			to_count: current.b,
+			lines: current.lines
+		});
+		current = void 0;
+	};
+	let posA = 0;
+	let posB = 0;
+	const positions = [];
+	for (const entry of entries) {
+		positions.push([posA, posB]);
+		if (entry.op !== "added") posA += 1;
+		if (entry.op !== "removed") posB += 1;
+	}
+	const changeAt = [];
+	for (const [at, entry] of entries.entries()) if (entry.op !== "same") changeAt.push(at);
+	let hunkCut = false;
+	for (const at of changeAt) {
+		if (shown >= limits.maxHunkLines) {
+			hunkCut = true;
+			break;
+		}
+		const start = Math.max(0, at - limits.context);
+		const stop = Math.min(entries.length - 1, at + limits.context);
+		if (current !== void 0 && start > current.end + 1) close();
+		if (current === void 0) {
+			const [fromA, fromB] = positions[start];
+			current = {
+				lines: [],
+				from: fromA,
+				to: fromB,
+				a: 0,
+				b: 0,
+				end: start - 1
+			};
+		}
+		for (let line = current.end + 1; line <= stop; line += 1) {
+			if (shown >= limits.maxHunkLines) {
+				hunkCut = true;
+				break;
+			}
+			const entry = entries[line];
+			current.lines.push(lineOf(entry));
+			if (entry.op !== "added") current.a += 1;
+			if (entry.op !== "removed") current.b += 1;
+			current.end = line;
+			shown += 1;
+		}
+	}
+	close();
+	if (hunkCut) {
+		truncated = true;
+		notes.push(`the hunks show ${String(limits.maxHunkLines)} lines, and the summary counts each line`);
+	}
+	const fields = metaChanges(before.fields, after.fields, limits);
+	if (fields.count > fields.changes.length) {
+		truncated = true;
+		notes.push(`the result shows ${String(fields.changes.length)} of ${String(fields.count)} changes of the metadata`);
+	}
+	return {
+		summary: {
+			lines_added: linesAdded,
+			lines_removed: linesRemoved,
+			lines_changed: linesChanged,
+			words_added: wordsAdded,
+			words_removed: wordsRemoved,
+			meta_changes: fields.count
+		},
+		hunks,
+		meta: fields.changes,
+		same: linesAdded === 0 && linesRemoved === 0 && fields.count === 0 && before.text === after.text,
+		exact,
+		truncated,
+		steps: lineBudget.steps + wordBudget.steps,
+		notes
+	};
+};
+/** This function gives the version of a choice, or throws an error that names the problem. */
+const chosen = (runtime, id, choice) => {
+	if (typeof choice === "string") {
+		const version = runtime.index.versionText(id, choice);
+		if (version === void 0) throw new Error(`the document ${id} has no version from the capture ${choice}`);
+		return {
+			version,
+			at: null
+		};
+	}
+	if (!isRecord(choice) || choice.at === void 0) throw new RangeError("a version is a capture id, or an object with the time at");
+	const at = isoTime("at", choice.at);
+	const found = versionAtTime(runtime, id, at);
+	if (found.version === void 0) throw new Error(`the document ${id} has no version at ${at} or before${found.first === void 0 ? "" : `: its first capture is from ${found.first}`}`);
+	return {
+		version: found.version,
+		at
+	};
+};
+const sideOf = (version, at, held) => ({
+	capture: version.capture,
+	fetched_at: version.fetched_at,
+	chars: version.chars,
+	rendered: version.rendered,
+	held_back: held,
+	at
+});
+/**
+* This function compares two versions of a stored document, and writes nothing. It applies the
+* rule for a held-back document. When the document or a version is held back, the diff compares
+* only the structural metadata, and it shows no text and no title.
+*/
+const diffDocument = async (runtime, id, options = {}) => {
+	await Promise.resolve();
+	const doc = runtime.index.get(id);
+	if (doc === void 0) throw new Error(`no stored document ${id}: find one with find(), or fetch its page`);
+	let to;
+	if (options.to === void 0) {
+		const newest = runtime.index.versions(id, { limit: 1 })[0];
+		const version = newest === void 0 ? void 0 : runtime.index.versionText(id, newest.capture);
+		if (version === void 0) throw new Error(`the document ${id} has no version`);
+		to = {
+			version,
+			at: null
+		};
+	} else to = chosen(runtime, id, options.to);
+	let from;
+	if (options.from === void 0) {
+		const earlier = runtime.index.versions(id, { limit: 1e3 }).find((version) => version.capture !== to.version.capture && version.recipe === to.version.recipe && version.rendered === to.version.rendered && version.fetched_at <= to.version.fetched_at);
+		const version = earlier === void 0 ? void 0 : runtime.index.versionText(id, earlier.capture);
+		if (version !== void 0) from = {
+			version,
+			at: null
+		};
+	} else from = chosen(runtime, id, options.from);
+	const notes = [];
+	if (from === void 0) {
+		from = to;
+		notes.push(`the document ${id} has no earlier version of the same kind to compare`);
+	}
+	const docHeld = isHeldBack(doc);
+	const fromHeld = docHeld || isHeldBack(from.version);
+	const toHeld = docHeld || isHeldBack(to.version);
+	const held = fromHeld || toHeld;
+	const fields = (version) => held ? {
+		kind: version.kind,
+		url: version.url,
+		meta: heldBackMeta(version.meta)
+	} : {
+		title: version.title,
+		kind: version.kind,
+		url: version.url,
+		meta: version.meta
+	};
+	const diff = diffTexts({
+		text: held ? "" : from.version.text,
+		fields: fields(from.version)
+	}, {
+		text: held ? "" : to.version.text,
+		fields: fields(to.version)
+	}, options.limits ?? {});
+	if (held) notes.unshift(HELD_BACK);
+	notes.push(...diff.notes);
+	return {
+		doc: doc.doc,
+		url: to.version.url ?? doc.url,
+		from: sideOf(from.version, from.at, fromHeld),
+		to: sideOf(to.version, to.at, toHeld),
+		same: held ? null : diff.same,
+		summary: diff.summary,
+		hunks: diff.hunks,
+		meta: diff.meta,
+		held_back: held,
+		exact: diff.exact,
+		truncated: diff.truncated,
+		steps: diff.steps,
+		note: notes.length > 0 ? notes.join("; ") : null
+	};
+};
+/** This function gives the text of the word diff of a changed line: `[-removed-]` and `{+added+}` mark the words. */
+const wordsText = (line) => line.words === null ? `[-${line.before}-]{+${line.after}+}` : line.words.map((part) => part.op === "same" ? part.text : part.op === "added" ? `{+${part.text}+}` : `[-${part.text}-]`).join("");
+/**
+* This function gives the text of a diff for a person or an assistant. The text has the counts, the
+* hunks as a unified diff, and the changes of the metadata. A changed line marks its words with `[-removed-]`
+* and `{+added+}`. The text holds the text of the pages, so the MCP server puts it in an envelope.
+*/
+const formatDiff = (diff) => {
+	const { summary } = diff;
+	const lines = [`the text: ${String(summary.lines_added)} lines added, ${String(summary.lines_removed)} removed (${String(summary.lines_changed)} changed), ${String(summary.words_added)} words added, ${String(summary.words_removed)} removed`];
+	for (const hunk of diff.hunks) {
+		lines.push(`@@ -${String(hunk.from_line)},${String(hunk.from_count)} +${String(hunk.to_line)},${String(hunk.to_count)} @@`);
+		for (const line of hunk.lines) if (line.op === "changed") lines.push(`~ ${wordsText(line)}`);
+		else lines.push(`${line.op === "same" ? " " : line.op === "added" ? "+" : "-"} ${line.text}`);
+	}
+	lines.push(`the metadata: ${String(summary.meta_changes)} changes`);
+	for (const change of diff.meta) if (change.change === "added") lines.push(`+ ${change.path}: ${change.after ?? ""}`);
+	else if (change.change === "removed") lines.push(`- ${change.path}: ${change.before ?? ""}`);
+	else lines.push(`~ ${change.path}: ${change.before ?? ""} -> ${change.after ?? ""}`);
+	return lines.join("\n");
+};
+//#endregion
+export { formatDiff as a, diff_exports as i, diffDocument as n, diffTexts as r, DIFF_LIMITS as t };
+
+//# sourceMappingURL=diff-NQcQqKe9.mjs.map