pi-wsearch 0.1.0 → 0.2.0

package/src/scrape.ts

@@ -12,6 +12,7 @@ import {
 	isRawText,
 	isLowQualityOutput,
 	stripDataUriImages,
+	withTimeout,
 	DEFAULT_SECTION_CAP,
 	type RemoteDoc,
 	type Renderer,
@@ -25,6 +26,10 @@ export interface ScrapeResult {
 	sections: string[];
 	text: string;
 	truncated: boolean;
+	continued: number[];
+	publishedAt?: string;
+	paywalled?: boolean;
+	validators?: { etag?: string; lastModified?: string };
 	error?: string;
 }
 
@@ -146,11 +151,47 @@ export function validateUrl(raw: string, allowPrivate = false): URL {
 	return u;
 }
 
-export function extractHtml(html: string): { title: string; text: string } {
+function stripKatexSpans(h: string): string {
+	let out = '';
+	let i = 0;
+	for (;;) {
+		const start = /<span[^>]*class="[^"]*katex[^"]*"[^>]*>/i.exec(h.slice(i));
+		if (!start || start.index === undefined) {
+			out += h.slice(i);
+			break;
+		}
+		const abs = i + start.index;
+		out += h.slice(i, abs);
+		let depth = 1;
+		let j = abs + start[0].length;
+		while (depth > 0 && j < h.length) {
+			const next = /<span\b[^>]*>|<\/span>/gi;
+			next.lastIndex = j;
+			const tag = next.exec(h);
+			if (!tag) {
+				j = h.length;
+				break;
+			}
+			j = tag.index + tag[0].length;
+			depth += tag[0].startsWith('</') ? -1 : 1;
+		}
+		i = j;
+	}
+	return out;
+}
+
+export function extractHtml(html: string): { title: string; text: string; publishedAt?: string; paywalled: boolean } {
+	const paywalled =
+		/class="[^"]*(?:fc-ab-root|metering-modal|paywall|pw-overlay|subscription-overlay|regwall)[^"]*"/i.test(html) ||
+		/id="[^"]*(?:paywall|metering|pay-overlay)[^"]*"/i.test(html) ||
+		/<meta[^>]+(?:property|name)=["'](?:isAccessibleForFree|tunnel:[^"']*paywall)["'][^>]+content=["'](?:false|yes)["']/i.test(
+			html,
+		);
 	const title =
 		/<title[^>]*>([^<]*)<\/title>/i.exec(html)?.[1]?.trim()?.slice(0, 200) ||
 		/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i.exec(html)?.[1]?.slice(0, 200) ||
 		'';
+	const publishedAt = extractPublishedDate(html);
 	let h = html
 		.replace(/<script[\s\S]*?<\/script>/gi, ' ')
 		.replace(/<style[\s\S]*?<\/style>/gi, ' ')
@@ -161,6 +202,34 @@ export function extractHtml(html: string): { title: string; text: string } {
 		.replace(/<footer[\s\S]*?<\/footer>/gi, ' ')
 		.replace(/<aside[\s\S]*?<\/aside>/gi, ' ');
 	h = h.replace(/<pre[\s>][\s\S]*?<\/pre>/gi, (m) => m.replace(/\n/g, '\uE000'));
+	h = h.replace(/<table[\s>][\s\S]*?<\/table>/gi, (table) => {
+		const rows = [...table.matchAll(/<tr[\s>][\s\S]*?<\/tr>/gi)].map((r) =>
+			[...r[0].matchAll(/<t([hd])[\s>][\s\S]*?<\/t\1>/gi)].map((c) =>
+				c[0]
+					.replace(/<[^>]+>/g, ' ')
+					.replace(/\s+/g, ' ')
+					.replace(/\|/g, '\\|')
+					.trim(),
+			),
+		);
+		const grid = rows.filter((r) => r.length);
+		if (grid.length < 2) return table;
+		const cols = Math.max(...grid.map((g) => g.length));
+		if (cols < 2) return table;
+		const line = (g: string[]) => {
+			const cells = g.slice(0, cols);
+			while (cells.length < cols) cells.push(' ');
+			return `| ${cells.join(' | ')} |`;
+		};
+		const md = [line(grid[0]), `| ${Array.from({ length: cols }, () => '---').join(' | ')} |`];
+		for (const g of grid.slice(1)) md.push(line(g));
+		return `\n${md.join('\n')}\n`;
+	});
+	h = stripKatexSpans(h);
+	h = h.replace(/<math[\s>][\s\S]*?<\/math>/gi, (m) => {
+		const tex = /<annotation[^>]*encoding=["']application\/x-tex["'][^>]*>([\s\S]*?)<\/annotation>/i.exec(m)?.[1];
+		return tex?.trim() ? ` $${tex.trim()}$ ` : ' ';
+	});
 	h = h.replace(
 		/<h([1-6])[^>]*>(.*?)<\/h\1>/gis,
 		(_, n, inner) => `\n${'#'.repeat(Number(n))} ${inner.replace(/<[^>]+>/g, ' ').trim()}\n`,
@@ -178,7 +247,10 @@ export function extractHtml(html: string): { title: string; text: string } {
 		.split('\n')
 		.map((l) => l.trim())
 		.filter((l) => l.length > 0 && !/^•\s*$/.test(l))
-		.filter((l) => !(/^[-*•]\s/.test(l) && isNavBullet(l)));
+		.filter((l) => !(/^[-*•]\s/.test(l) && isNavBullet(l)))
+		.filter((l) => !/^#{1,6}\s*$/.test(l))
+		.map((l) => (/^[-*•]\s{2,}/.test(l) ? l.replace(/^[-*•]\s+/, '').replace(/\s{2,}/g, ' ') : l))
+		.filter((l) => !(l.length < 120 && isPromo(l)));
 	const out: string[] = [];
 	let blank = 0;
 	for (const l of lines) {
@@ -212,7 +284,22 @@ export function extractHtml(html: string): { title: string; text: string } {
 		out.push(l);
 		blank = 0;
 	}
-	return { title, text: out.join('\n').replace(/\n{3,}/g, '\n\n') };
+	return { title, publishedAt, paywalled, text: out.join('\n').replace(/\n{3,}/g, '\n\n') };
+}
+
+function extractPublishedDate(html: string): string | undefined {
+	const iso = (v: string | undefined) => {
+		if (!v) return undefined;
+		const t = Date.parse(v.trim());
+		return Number.isNaN(t) || t < 0 ? undefined : new Date(t).toISOString().slice(0, 10);
+	};
+	const ld = /"datePublished"\s*:\s*"([^"]+)"/.exec(html)?.[1];
+	const meta =
+		/<meta[^>]+(?:property|name)=["'](?:article:published_time|datePublished|publish-date|pubdate|parsely-pub-date|date)["'][^>]+content=["']([^"']+)["']/i.exec(
+			html,
+		)?.[1];
+	const timeEl = /<time[^>]+datetime=["']([^"']+)["']/i.exec(html)?.[1];
+	return iso(ld) ?? iso(meta) ?? iso(timeEl);
 }
 
 const NAV_LABEL_RE =
@@ -284,27 +371,122 @@ function isOutlineEcho(block: string, outline: string[]): boolean {
 	return raw.filter(hit).length / raw.length >= 0.6;
 }
 
+const PROMO_STRONG = [
+	/become a (?:premium )?member/i,
+	/unlock exclusive/i,
+	/go premium/i,
+	/upgrade to (?:the )?premium/i,
+	/your membership journey/i,
+	/stay on the cutting edge/i,
+	/enter your email below/i,
+	/by submitting your information/i,
+	/sign (?:up|you up) (?:for|to) (?:our )?(?:free )?newsletter/i,
+	/subscribe to (?:our )?newsletter/i,
+	/get the [a-z][a-z ]{2,40}newsletter/i,
+	/limited-?time (?:premium )?offer/i,
+	/choose how you want to join/i,
+	/explore (?:our )?(?:premium|pro)\b/i,
+	/^email newsletter\b/i,
+	/^your \(?[\w\s-]+\)? email$/i,
+	/^IE \d+ is not supported/i,
+	/for an optimal experience visit our site/i,
+	/^skip to (?:content|main)/i,
+	/^for subscribers$/i,
+	/^subscriber (?:only|exclusive)$/i,
+	/^[A-Z][A-Z0-9 &'’-]{5,50}$/,
+	/\|\s*[A-Z][\w'’&. ]{2,40}\s+\|\s*[A-Z][\w'’&. ]{2,40}\s+News\s*$/,
+	/^(?:[A-Z][\w'’.\- ]{2,30}\s+\/\s+)?(?:via )?(?:Getty Images(?:\s+file)?|AP(?: Photo)?|Reuters|Xinhua(?:[\s\w/]+)?|AFP|Bloomberg(?: Photo)?|The Associated Press|NBC(?: News)? via|CNN Wire|Sipa USA|AP Photo\/[^,]{2,40})[\s.]*$/i,
+	/^\d+\s*(?:s|sec|seconds?|m|min|minutes?|h|hr|hours?|d|day|days|w|weeks?)\s+ago[\s.]*$/i,
+	/^(?:updated|published)\s+(?:just\s+now|\d+\s*\w+\s+ago)$/i,
+	/^\d+\s+(?:hours?|minutes?)\s+ago$/i,
+	/get started with free access/i,
+	/premium (?:newsletter|articles?|membership|features|offer)/i,
+	/sign[- ]?up (?:was|is) successful/i,
+	/we use cookies[^.]*(?:to (?:improve|enhance|personalize|analyse|analyze))/i,
+	/^by continuing (?:to use|to browse)/i,
+	/^(?:accept|allow) (?:all|selected|necessary)/i,
+	/^(?:manage|customize|consent) (?:preferences|settings|choices)/i,
+	/^this (?:site|website) uses (?:cookies|essential cookies)/i,
+];
+const PROMO_CTA_RE =
+	/\b(?:newsletter|membership|club membership|premium member|sign ?up|subscribe|unlock|upgrade|cancel anytime|limited-?time|free trial|join now|join today)\b/gi;
+
+function isPromo(t: string): boolean {
+	if (PROMO_STRONG.some((r) => r.test(t))) return true;
+	return (t.match(PROMO_CTA_RE) ?? []).length >= 2;
+}
+
+function splitSection(s: string, sectionCap: number): string[] {
+	const hm = /^(#{1,6})\s+([^\n]+)/.exec(s);
+	const partHead = hm ? `${'#'.repeat(hm[1].length)} ${hm[2].trim()}\n\n` : '';
+	const floor = Math.min(partHead.length + 1, sectionCap);
+	const parts: string[] = [];
+	let rest = s;
+	for (;;) {
+		if (rest.length <= sectionCap) {
+			parts.push(rest);
+			return parts;
+		}
+		const cut = rest.slice(0, sectionCap);
+		let at = cut.lastIndexOf('\n\n');
+		if (at < floor) at = cut.lastIndexOf(' ');
+		if (at < floor) at = sectionCap;
+		const used = Math.min(at, rest.length - 1);
+		parts.push(rest.slice(0, used).replace(/\s+$/, ''));
+		rest = partHead + rest.slice(used).trimStart();
+	}
+}
+
 export function sliceSections(
 	text: string,
 	sectionCap = DEFAULT_SECTION_CAP,
-): { outline: string[]; sections: string[] } {
+	docTitle = '',
+): { outline: string[]; sections: string[]; continued: number[] } {
+	const titleEcho = docTitle ? normHeading(docTitle) : '';
 	const blocks = text.replace(/\n{3,}/g, '\n\n').split('\n\n');
 	const sections: string[] = [];
-	const outline: string[] = [];
+	const outlineNames: string[] = [];
+	const seenParas = new Set<string>();
 	let cur = '';
+	let lastBlock = '';
 	for (const b of blocks) {
-		if (!b.trim()) continue;
-		if (isBoilerplate(b)) continue;
-		if (isOutlineEcho(b, outline)) continue;
+		const tb = b.trim();
+		if (!tb || tb === lastBlock) continue;
+		lastBlock = tb;
+		if (!tb.startsWith('#')) {
+			const paraKey = tb.replace(/\s+/g, ' ').toLowerCase();
+			if (seenParas.has(paraKey)) continue;
+			seenParas.add(paraKey);
+		} else {
+			const hm = /^(#{1,6})\s+([^\n]+)([\s\S]*)$/.exec(tb);
+			const body = (hm?.[3] ?? '').trim();
+			if (body) {
+				const paraKey = body.replace(/\s+/g, ' ').toLowerCase();
+				if (seenParas.has(paraKey)) {
+					const name = hm![2].trim();
+					if (isBoilerplateHeading(name) || isPromo(name)) continue;
+					const depth = hm![1].length;
+					if (cur.trim()) sections.push(cur.trim());
+					cur = `${'#'.repeat(depth)} ${name}\n`;
+					if (outlineNames.length < OUTLINE_CAP && depth <= 4)
+						outlineNames.push(name.replace(/\s+/g, ' ').slice(0, 90));
+					continue;
+				}
+				seenParas.add(paraKey);
+			}
+		}
+		if (titleEcho && normHeading(tb) === titleEcho) continue;
+		if (isBoilerplate(b) || isPromo(b)) continue;
+		if (isOutlineEcho(b, outlineNames)) continue;
 		const hm = /^(#{1,6})\s+([^\n]+)([\s\S]*)$/.exec(b);
 		if (hm) {
 			const name = hm[2].trim();
-			if (/^(navigation|contents|related topics|quick search|sidebar|footer)$/i.test(name)) continue;
+			if (isBoilerplateHeading(name) || isPromo(name)) continue;
 			if (cur.trim()) sections.push(cur.trim());
 			const depth = hm[1].length;
 			const rest = (hm[3] ?? '').replace(/^[\n\s]+/, '');
 			cur = `${'#'.repeat(depth)} ${name}` + (rest ? `\n${rest}` : '') + '\n';
-			if (outline.length < OUTLINE_CAP && depth <= 4) outline.push(name.replace(/\s+/g, ' ').slice(0, 90));
+			if (outlineNames.length < OUTLINE_CAP && depth <= 4) outlineNames.push(name.replace(/\s+/g, ' ').slice(0, 90));
 			continue;
 		}
 		cur += scrubNavBullets(b) + '\n\n';
@@ -317,26 +499,36 @@ export function sliceSections(
 				.split('\n')
 				.filter((l) => !/^#{1,6} /.test(l))
 				.join('\n');
-			return !nonHeading.trim() || !isOutlineEcho(nonHeading, outline);
+			return !nonHeading.trim() || !isOutlineEcho(nonHeading, outlineNames);
 		});
 		return body.join('\n\n');
 	});
 	const capped: string[] = [];
-	for (let s of pruned) {
-		if (s.length > sectionCap) {
-			const cut = s.slice(0, sectionCap);
-			const at = cut.lastIndexOf('\n\n');
-			const note = `…[section truncated: capped at ${sectionCap} chars — the rest of this section is not retrievable]`;
-			if (at > sectionCap - CAP_BREAK_GRACE) s = cut.slice(0, at) + '\n' + note;
-			else s = cut + ' ' + note;
-		}
-		if (s.trim().length >= SECTION_MIN || /^#{1,6} /.test(s)) capped.push(s.trim());
+	const continued: number[] = [];
+	const outline: string[] = [];
+	const lead = sections.length > 0 && !/^#{1,6} /.test(sections[0]);
+	for (let i = 0; i < pruned.length; i++) {
+		const s = pruned[i].trim();
+		const headingOnly = /^#{1,6} [^\n]+$/.test(s);
+		if (s.length < SECTION_MIN && !/^#{1,6} /.test(s)) continue;
+		if (headingOnly && s.length < 120) continue;
+		const hm = /^(#{1,6})\s+([^\n]+)/.exec(s);
+		const own = hm ? hm[2].trim() : '(lead)';
+		const base = hm ? (outlineNames[i - (lead ? 1 : 0)] ?? own) : '(lead)';
+		const name = base.replace(/\s+/g, ' ').slice(0, 90);
+		const parts = s.length <= sectionCap ? [s] : splitSection(s, sectionCap);
+		const last = parts.length - 1;
+		parts.forEach((p, k) => {
+			outline.push(k === 0 ? name : `${name} (cont. ${k})`);
+			if (k < last) continued.push(capped.length);
+			const body =
+				parts.length === 1 || k === last
+					? p
+					: `${p.replace(/\s+$/, '')}\n\n…(continues at outline index ${capped.length + 1})`;
+			capped.push(body);
+		});
 	}
-	// A leading heading-less section (the page lead) shifts every outline index off by one;
-	// synthesizing its outline entry keeps section[i] and outline[i] aligned and the lead requestable.
-	const preambleIdx = capped.findIndex((s) => !/^#{1,6} /.test(s));
-	if (preambleIdx === 0 && outline.length > 0) outline.unshift('(lead)');
-	return { outline, sections: capped };
+	return { outline, sections: capped, continued };
 }
 
 export interface ScrapeOptions {
@@ -345,15 +537,15 @@ export interface ScrapeOptions {
 	render?: 'auto' | 'local' | 'firecrawl';
 	allowPrivate?: boolean;
 	signal?: AbortSignal;
+	validators?: { etag?: string; lastModified?: string };
 }
 
 const SHORT_LINE_JOIN = 28;
 const NAV_LABEL_MAX = 40;
 const NAV_BLOCK_MAX = 90;
-const OUTLINE_CAP = 14;
+const OUTLINE_CAP = 20;
 const OUTLINE_LINE_MAX = 90;
 const MIN_OUTLINE_FOR_ECHO = 3;
-const CAP_BREAK_GRACE = 400;
 const SECTION_MIN = 12;
 
 const LOCAL_MIN_TEXT = 240;
@@ -370,6 +562,7 @@ function abortResult(url: URL): ScrapeResult {
 		sections: [],
 		text: '',
 		truncated: false,
+		continued: [],
 		error: ABORT_ERROR,
 	};
 }
@@ -383,34 +576,107 @@ function errorResult(url: URL, error: string): ScrapeResult {
 		sections: [],
 		text: '',
 		truncated: false,
+		continued: [],
 		error,
 	};
 }
 
+const GITHUB_REPO_RE = /^\/([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)(?:\/?|\?.*)?$/;
+const GITHUB_BLOB_RE = /^\/([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)\/blob\/([^/]+)\/(.+)$/;
+
+function readmeCandidates(url: URL): string[] {
+	if (url.hostname !== 'github.com') return [];
+	const repo = GITHUB_REPO_RE.exec(url.pathname);
+	if (repo) {
+		return ['README.md', 'readme.md', 'README.rst', 'README.org'].map(
+			(name) => `https://raw.githubusercontent.com/${repo[1]}/${repo[2]}/HEAD/${name}`,
+		);
+	}
+	const blob = GITHUB_BLOB_RE.exec(url.pathname);
+	if (blob) {
+		return [`https://raw.githubusercontent.com/${blob[1]}/${blob[2]}/${blob[3]}/${blob[4]}`];
+	}
+	return [];
+}
+
+async function scrapeReadme(
+	origUrl: URL,
+	opts: ScrapeOptions,
+	sectionCap: number,
+	maxChars: number,
+): Promise<ScrapeResult | undefined> {
+	const isBlob = GITHUB_BLOB_RE.test(origUrl.pathname);
+	const minLen = isBlob ? 1 : 500;
+	for (const rawUrl of readmeCandidates(origUrl)) {
+		try {
+			const { body, contentType } = await localFetch(new URL(rawUrl), opts.signal);
+			if (!isRawText(contentType) || body.length < minLen) continue;
+			const text = stripToAscii(decodeHtmlEntities(body));
+			if (text.length < (isBlob ? 1 : LOCAL_MIN_TEXT)) continue;
+			const { outline, sections, continued } = sliceSections(text, sectionCap);
+			const { text: out, truncated } = joinSections(outline, sections, maxChars);
+			if (!out) continue;
+			const label = isBlob ? 'source (github.com)' : 'README (github.com)';
+			return {
+				url: origUrl.toString(),
+				title: `${origUrl.pathname.split('/')[2]} — ${label}`,
+				renderer: 'local',
+				outline,
+				sections,
+				continued,
+				text: out,
+				truncated,
+			};
+		} catch {
+			continue;
+		}
+	}
+	return undefined;
+}
+
 async function tryLocal(
 	url: URL,
 	render: 'auto' | 'local',
 	signal: AbortSignal | undefined,
 	sectionCap: number,
 	maxChars: number,
-): Promise<ScrapeResult | undefined> {
-	const { body, contentType } = await localFetch(url, signal);
+	validators?: { etag?: string; lastModified?: string },
+): Promise<(ScrapeResult & { validators?: { etag?: string; lastModified?: string } }) | undefined | 'not-modified'> {
+	const cond = await localFetch(url, signal, validators);
+	if (cond.notModified) return 'not-modified';
+	const { body, contentType } = cond;
 	let title = '';
 	let text: string;
+	let publishedAt: string | undefined;
+	let paywalled = false;
 	if (isRawText(contentType)) {
 		text = stripToAscii(decodeHtmlEntities(body));
 	} else {
 		const extracted = extractHtml(body);
 		title = extracted.title;
 		text = extracted.text;
+		publishedAt = extracted.publishedAt;
+		paywalled = extracted.paywalled;
 	}
-	if (render === 'auto' && text.length < LOCAL_MIN_TEXT) return undefined;
-	const { outline, sections } = sliceSections(text, sectionCap);
+	const { outline, sections, continued } = sliceSections(text, sectionCap, title);
 	const { text: out, truncated } = joinSections(outline, sections, maxChars);
+	if (render === 'auto' && text.length < LOCAL_MIN_TEXT) return undefined;
 	if (render === 'auto' && (sections.length === 0 || out.length < LOCAL_MIN_OUTPUT || isLowQualityOutput(out))) {
 		return undefined;
 	}
-	return { url: url.toString(), title, renderer: 'local', outline, sections, text: out, truncated };
+	return {
+		url: url.toString(),
+		title,
+		renderer: 'local',
+		outline,
+		sections,
+		continued,
+		text: out,
+		truncated,
+		publishedAt,
+		paywalled: paywalled || undefined,
+		validators: cond.validators,
+	};
 }
 
 function fromChain(
@@ -426,9 +692,72 @@ function fromChain(
 		markdown = extracted.text;
 		if (extracted.title) title = extracted.title;
 	}
-	const { outline, sections } = sliceSections(markdown, sectionCap);
+	const { outline, sections, continued } = sliceSections(markdown, sectionCap);
 	const { text, truncated } = joinSections(outline, sections, maxChars);
-	return { url: url.toString(), title, renderer: doc.renderer, outline, sections, text, truncated };
+	return { url: url.toString(), title, renderer: doc.renderer, outline, sections, continued, text, truncated };
+}
+
+const BOILERPLATE_HEADING =
+	/^(?:site |main |global |primary )?(?:navigation(?: menu| bar| links)?|menu|sidebar|footer|header|contents?|related topics|quick search|site index|breadcrumbs?|personal tools|namespaces|views|user links|wiki tools|languages|repository navigation|in other(?: languages| projects)|print\/export|navigation menu)$/i;
+
+function isBoilerplateHeading(name: string): boolean {
+	return BOILERPLATE_HEADING.test(name.trim());
+}
+
+// Archive fallback for DEAD pages only (doctrine line): 404/dead-host, never
+// 403-walls. Resurrecting lost content is a service; bypassing walls is not.
+async function archivedFetch(
+	url: URL,
+	signal: AbortSignal | undefined,
+	sectionCap: number,
+	maxChars: number,
+): Promise<ScrapeResult | undefined> {
+	if (signal?.aborted) return undefined;
+	try {
+		const { ctrl, done } = withTimeout('archive lookup', 10_000, signal);
+		let snapshotUrl: string | undefined;
+		try {
+			const res = await fetch(`https://archive.org/wayback/available?url=${encodeURIComponent(url.toString())}`, {
+				signal: ctrl.signal,
+				headers: { Accept: 'application/json' },
+			});
+			if (res.ok) {
+				const d = (await res.json()) as {
+					archived_snapshots?: { closest?: { url?: string; status?: string; timestamp?: string } };
+				};
+				const snap = d.archived_snapshots?.closest;
+				if (snap?.url && snap.status === '200') snapshotUrl = snap.url;
+				var snapTs = snap?.timestamp;
+			}
+		} finally {
+			done();
+		}
+		if (!snapshotUrl || signal?.aborted) return undefined;
+		const { body, contentType } = await localFetch(new URL(snapshotUrl), signal);
+		if (!/html|text/.test(contentType)) return undefined;
+		const extracted = extractHtml(body);
+		if (extracted.text.length < LOCAL_MIN_TEXT) return undefined;
+		const { outline, sections, continued } = sliceSections(extracted.text, sectionCap, extracted.title);
+		const { text: out, truncated } = joinSections(outline, sections, maxChars);
+		if (!out) return undefined;
+		const snapDate =
+			typeof snapTs === 'string' && snapTs.length >= 8
+				? `${snapTs.slice(0, 4)}-${snapTs.slice(4, 6)}-${snapTs.slice(6, 8)}`
+				: 'unknown date';
+		return {
+			url: url.toString(),
+			title: `${extracted.title || url.hostname} [ARCHIVED copy ${snapDate} — live page is gone]`,
+			renderer: 'local',
+			outline,
+			sections,
+			continued,
+			text: out,
+			truncated,
+			publishedAt: extracted.publishedAt,
+		};
+	} catch {
+		return undefined;
+	}
 }
 
 export async function scrape(urlRaw: string, opts: ScrapeOptions = {}): Promise<ScrapeResult> {
@@ -440,10 +769,16 @@ export async function scrape(urlRaw: string, opts: ScrapeOptions = {}): Promise<
 
 	if (opts.signal?.aborted) return abortResult(url);
 
+	if (render === 'auto' || render === 'local') {
+		const readme = await scrapeReadme(url, opts, sectionCap, maxChars);
+		if (readme) return readme;
+	}
+
 	if (render === 'local') {
 		try {
-			// undefined is auto-mode only (thin/empty gates); forced local always produces a result
-			return (await tryLocal(url, 'local', opts.signal, sectionCap, maxChars))!;
+			const forced = await tryLocal(url, 'local', opts.signal, sectionCap, maxChars, opts.validators);
+			if (forced === 'not-modified') return { ...abortResult(url), error: 'not-modified', renderer: 'local' };
+			return forced!;
 		} catch (e) {
 			return opts.signal?.aborted ? abortResult(url) : errorResult(url, sanitizeError(e));
 		}
@@ -455,14 +790,36 @@ export async function scrape(urlRaw: string, opts: ScrapeOptions = {}): Promise<
 		return sections.length > 0;
 	};
 
+	const isJinaProxy = url.hostname === 'r.jina.ai' || url.hostname === 's.jina.ai';
+
 	if (render === 'auto') {
-		let local: ScrapeResult | undefined;
+		let local: (ScrapeResult & { validators?: { etag?: string; lastModified?: string } }) | undefined | 'not-modified';
+		let localErr: string | undefined;
 		try {
-			local = await tryLocal(url, 'auto', opts.signal, sectionCap, maxChars);
+			local = await tryLocal(url, 'auto', opts.signal, sectionCap, maxChars, opts.validators);
 		} catch (e) {
 			if (opts.signal?.aborted) return abortResult(url);
+			localErr = sanitizeError(e);
+		}
+		if (local === 'not-modified') {
+			return { ...abortResult(url), error: 'not-modified', renderer: 'local' };
 		}
 		if (local) return local;
+		if (localErr && /HTTP 404|not a readable page|page too large|empty body/.test(localErr)) {
+			if (/HTTP 404/.test(localErr)) {
+				const archived = await archivedFetch(url, opts.signal, sectionCap, maxChars);
+				if (archived) return archived;
+			}
+			return errorResult(url, localErr);
+		}
+		if (isJinaProxy) {
+			return errorResult(
+				url,
+				localErr
+					? `${localErr} (jina proxy; target walled or unavailable)`
+					: 'jina reader returned no usable content for the target (walled or unavailable)',
+			);
+		}
 	}
 
 	const chainResult = await remoteChain(url, opts.signal, usable);