pi-wsearch 0.2.1 → 0.3.1

package/README.md

@@ -7,9 +7,11 @@
 **One search, many judges.** Most web tools hand you a single engine's
 ranking. `web.search` runs every engine you've configured in parallel and
 merges their results by reciprocal rank fusion (RRF). A hit only earns its
-place by surviving several independent judgments. The response shows how
-many: `← 3 engines agree` means three separate indexes ranked that page.
-Agreement is the spam defense; generative-SEO can stuff one index, not three.
+place by surviving several independent judgments. The response shows the
+corroboration in detail: `⇐ serper#1, exa#2, you#4` means three separate
+indexes ranked that page — and where each one placed it. Single-engine hits
+are labeled `single-engine: unverified`. Agreement is the spam defense;
+generative-SEO can stuff one index, not three.
 
 **Pick your engines.** Nine are supported; three or four is plenty. Use
 whatever keys you have, even one, because what follows doesn't depend on how
@@ -42,7 +44,11 @@ Every cut is marked — nothing is silently lost.
 **Reads are free.** Page parsing is built in, nothing billed per read, no key
 required. Real headings become the map, TOC echoes get dropped, so reads come
 back as clean as a paid one at zero cost. Paid readers only step in for
-JavaScript heavy pages or when the local read fails.
+JavaScript heavy pages or when the local read fails. Reader output is
+laundered too: standalone photo credits (REUTERS/, AP Photo/, Getty), video
+durations, carousel markers and relative-time chips are stripped from news
+pages (whole-line rules only — prose is untouched), and a reader that returns
+a 404-page template is skipped so the chain moves on to the next backend.
 
 **Tracking links cache as one page.** `?utm_source=…` variants of the same URL
 share one 24 hour cache entry. Faster repeats, smaller disk.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-wsearch",
-  "version": "0.2.1",
+  "version": "0.3.1",
   "description": "Pi extension: web.search + web.fetch",
   "license": "MIT",
   "type": "module",

package/src/cache.ts

@@ -44,9 +44,11 @@ export class TtlCache {
 			/* cache writes are best-effort */
 		}
 	}
+	/** Entry valid while unexpired; non-numeric/NaN exp (corrupt store) must NOT
+	 * count as "never expires" — treat it as expired so a poisoned entry dies. */
 	get(key: string): unknown | undefined {
 		const e = this.store.entries[key];
-		if (!e || e.exp < Date.now()) return undefined;
+		if (!e || typeof e.exp !== 'number' || !Number.isFinite(e.exp) || e.exp < Date.now()) return undefined;
 		const i = this.store.order.indexOf(key);
 		if (i >= 0) this.store.order.splice(i, 1);
 		this.store.order.push(key);
@@ -56,7 +58,7 @@ export class TtlCache {
 	/** Expired copy WITHOUT deleting it (stale-if-error reserve). */
 	peekStale(key: string): { value: unknown; ageHours: number } | undefined {
 		const e = this.store.entries[key];
-		if (!e) return undefined;
+		if (!e || typeof e.exp !== 'number' || !Number.isFinite(e.exp)) return undefined;
 		const ageHours = (Date.now() - e.exp) / 3600_000;
 		if (ageHours <= 0) return undefined;
 		return { value: e.value, ageHours };

package/src/engines.ts

@@ -395,12 +395,18 @@ function selectEngines(opts: SearchOptions): EngineDef[] {
 
 export async function runEngines(opts: SearchOptions): Promise<EngineOutcome[]> {
 	const cfg = getConfig();
-	const chosen = selectEngines(opts).filter((def) => !deadEngines.has(def.id));
+	const selected = selectEngines(opts);
+	const chosen = selected.filter((def) => !deadEngines.has(def.id));
 	if (chosen.length === 0) {
 		const wanted = opts.engines ?? parseEngineList(getConfig().searchEngines);
-		const error = wanted?.length
-			? `no configured engines match the requested set (${wanted.join(', ')})`
-			: 'no search engine keys configured; run /websearch to add one';
+		// selected-but-dead means KEYS EXIST and the engines were temp-banned
+		// (stable HTTP 401/402/403): the old wording lied about the cause.
+		const error =
+			selected.length > 0
+				? 'all configured engines are temporarily unavailable (HTTP 401/402/403); try again later'
+				: wanted?.length
+					? `no configured engines match the requested set (${wanted.join(', ')})`
+					: 'no search engine keys configured; run /websearch to add one';
 		return [{ engine: 'none', hits: [], error, latencyMs: 0 }];
 	}
 	return Promise.all(

package/src/fuse.ts

@@ -3,6 +3,8 @@ import { isJunk, normalizeUrl, hostOf } from './urls';
 
 export interface FusedHit extends SearchHit {
 	engines: string[];
+	/** Per-engine rank of this hit: {serper: 1} = serper ranked it #1. */
+	engineRanks: { engine: string; rank: number }[];
 	rrfScore: number;
 	finalScore: number;
 	junk: boolean;
@@ -207,6 +209,12 @@ function collapseNearDuplicates(fused: FusedHit[]): FusedHit[] {
 		const twin = kept.find((k) => isNearDuplicate(k, h));
 		if (twin) {
 			twin.engines = [...new Set([...twin.engines, ...h.engines])];
+			const best = new Map(twin.engineRanks.map((r) => [r.engine, r]));
+			for (const r of h.engineRanks) {
+				const cur = best.get(r.engine);
+				if (!cur || r.rank < cur.rank) best.set(r.engine, r);
+			}
+			twin.engineRanks = [...best.values()];
 			if (h.snippet.length > twin.snippet.length) twin.snippet = h.snippet;
 			if (h.title.length > twin.title.length && !twin.title.includes(h.title)) twin.title = h.title;
 			continue;
@@ -250,6 +258,7 @@ export function fuse(outcomes: EngineOutcome[], opts: FuseOptions = {}): FusedHi
 		fused.push({
 			...hit,
 			engines: ranks.map((r) => r.engine),
+			engineRanks: [...ranks].toSorted((a, b) => a.rank - b.rank),
 			rrfScore: rrf,
 			finalScore: Math.round(score * 1e5) / 1e5,
 			junk,

package/src/scrape.ts

@@ -5,6 +5,7 @@
  */
 import { getConfig, sanitizeError } from './config';
 import { ABORT_ERROR } from './constants';
+import { isPrivateHost, validateUrl } from './urls';
 import {
 	localFetch,
 	looksLikeHtml,
@@ -33,6 +34,8 @@ export interface ScrapeResult {
 	error?: string;
 }
 
+export { isPrivateHost, validateUrl } from './urls'; // public surface unchanged; the gates live in the URL leaf
+
 const DEFAULT_MAX_CHARS = 12000;
 
 function stripToAscii(s: string): string {
@@ -91,66 +94,6 @@ export function decodeHtmlEntities(s: string): string {
 	});
 }
 
-function isPrivateIpv4(h: string): boolean {
-	const ipv4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(h);
-	if (!ipv4) return false;
-	const [a, b] = [Number(ipv4[1]), Number(ipv4[2])];
-	if (a === 10) return true;
-	if (a === 127) return true;
-	if (a === 0) return true;
-	if (a === 169 && b === 254) return true;
-	if (a === 172 && b >= 16 && b <= 31) return true;
-	if (a === 192 && b === 168) return true;
-	return false;
-}
-
-function numericIpv4(h: string): string | null {
-	if (!/^[0-9.]+$/.test(h) && !/^0[xX][0-9a-fA-F.]+$/.test(h)) return null;
-	const fields = h.split('.');
-	if (fields.length > 4) return null;
-	const parts: number[] = [];
-	for (const f of fields) {
-		if (/^0[xX][0-9a-fA-F]+$/.test(f)) parts.push(parseInt(f, 16));
-		else if (/^0[0-7]+$/.test(f)) parts.push(parseInt(f, 8));
-		else if (/^\d+$/.test(f)) parts.push(Number(f));
-		else return null;
-	}
-	const last = parts[parts.length - 1];
-	if (parts.length === 1) {
-		if (last >= 2 ** 32) return null;
-		return [(last >>> 24) & 0xff, (last >>> 16) & 0xff, (last >>> 8) & 0xff, last & 0xff].join('.');
-	}
-	if (parts.slice(0, -1).some((n) => n > 255)) return null;
-	if (last >= 2 ** (8 * (4 - parts.length + 1))) return null;
-	if (parts.length === 2) return [parts[0], 0, 0, last].join('.');
-	if (parts.length === 3) return [parts[0], parts[1], (last >>> 8) & 0xff, last & 0xff].join('.');
-	return parts.join('.');
-}
-
-export function isPrivateHost(host: string): boolean {
-	const h = host.toLowerCase().replace(/^\[(.*)\]$/, '$1');
-	if (h === 'localhost' || h.endsWith('.local') || h.endsWith('.localhost')) return true;
-	const mapped = /:ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i.exec(h);
-	if (mapped) {
-		const [a, b] = [parseInt(mapped[1], 16), parseInt(mapped[2], 16)];
-		if (isPrivateIpv4(`${(a >> 8) & 0xff}.${a & 0xff}.${(b >> 8) & 0xff}.${b & 0xff}`)) return true;
-	}
-	const ipv4 = numericIpv4(h);
-	if (ipv4 && isPrivateIpv4(ipv4)) return true;
-	if (/^[0-9a-f:]+$/.test(h) && h.includes(':')) {
-		if (h.startsWith('::1') || h === '0:0:0:0:0:0:0:1') return true;
-		if (h.startsWith('fe80:') || h.startsWith('fc') || h.startsWith('fd')) return true;
-	}
-	return false;
-}
-
-export function validateUrl(raw: string, allowPrivate = false): URL {
-	const u = new URL(raw);
-	if (u.protocol !== 'http:' && u.protocol !== 'https:') throw new Error(`unsupported protocol: ${u.protocol}`);
-	if (!allowPrivate && isPrivateHost(u.hostname)) throw new Error('private/loopback address rejected');
-	return u;
-}
-
 function stripKatexSpans(h: string): string {
 	let out = '';
 	let i = 0;
@@ -468,8 +411,7 @@ export function sliceSections(
 					const depth = hm![1].length;
 					if (cur.trim()) sections.push(cur.trim());
 					cur = `${'#'.repeat(depth)} ${name}\n`;
-					if (outlineNames.length < OUTLINE_CAP && depth <= 4)
-						outlineNames.push(name.replace(/\s+/g, ' ').slice(0, 90));
+					if (outlineNames.length < OUTLINE_CAP) outlineNames.push(name.replace(/\s+/g, ' ').slice(0, 90));
 					continue;
 				}
 				seenParas.add(paraKey);
@@ -486,7 +428,7 @@ export function sliceSections(
 			const depth = hm[1].length;
 			const rest = (hm[3] ?? '').replace(/^[\n\s]+/, '');
 			cur = `${'#'.repeat(depth)} ${name}` + (rest ? `\n${rest}` : '') + '\n';
-			if (outlineNames.length < OUTLINE_CAP && depth <= 4) outlineNames.push(name.replace(/\s+/g, ' ').slice(0, 90));
+			if (outlineNames.length < OUTLINE_CAP) outlineNames.push(name.replace(/\s+/g, ' ').slice(0, 90));
 			continue;
 		}
 		cur += scrubNavBullets(b) + '\n\n';
@@ -685,7 +627,7 @@ function fromChain(
 	sectionCap: number,
 	maxChars: number,
 ): ScrapeResult {
-	let markdown = stripDataUriImages(doc.markdown);
+	let markdown = cleanReaderMarkdown(stripDataUriImages(doc.markdown));
 	let title = doc.title;
 	if (looksLikeHtml(markdown)) {
 		const extracted = extractHtml(markdown);
@@ -713,25 +655,27 @@ async function archivedFetch(
 	maxChars: number,
 ): Promise<ScrapeResult | undefined> {
 	if (signal?.aborted) return undefined;
+	if (isPrivateHost(url.hostname)) return undefined;
+	let snapshotUrl: string | undefined;
+	let snapTs: string | undefined;
+	const { ctrl, done } = withTimeout('archive lookup', 10_000, signal);
 	try {
-		const { ctrl, done } = withTimeout('archive lookup', 10_000, signal);
-		let snapshotUrl: string | undefined;
-		try {
-			const res = await fetch(`https://archive.org/wayback/available?url=${encodeURIComponent(url.toString())}`, {
-				signal: ctrl.signal,
-				headers: { Accept: 'application/json' },
-			});
-			if (res.ok) {
-				const d = (await res.json()) as {
-					archived_snapshots?: { closest?: { url?: string; status?: string; timestamp?: string } };
-				};
-				const snap = d.archived_snapshots?.closest;
-				if (snap?.url && snap.status === '200') snapshotUrl = snap.url;
-				var snapTs = snap?.timestamp;
-			}
-		} finally {
-			done();
+		const res = await fetch(`https://archive.org/wayback/available?url=${encodeURIComponent(url.toString())}`, {
+			signal: ctrl.signal,
+			headers: { Accept: 'application/json' },
+		});
+		if (res.ok) {
+			const d = (await res.json()) as {
+				archived_snapshots?: { closest?: { url?: string; status?: string; timestamp?: string } };
+			};
+			const snap = d.archived_snapshots?.closest;
+			if (snap?.url && snap.status === '200') snapshotUrl = snap.url;
+			snapTs = snap?.timestamp;
 		}
+	} finally {
+		done();
+	}
+	try {
 		if (!snapshotUrl || signal?.aborted) return undefined;
 		const { body, contentType } = await localFetch(new URL(snapshotUrl), signal);
 		if (!/html|text/.test(contentType)) return undefined;
@@ -785,8 +729,10 @@ export async function scrape(urlRaw: string, opts: ScrapeOptions = {}): Promise<
 	}
 
 	const usable = (markdown: string): boolean => {
-		if (markdown.replace(/\s+/g, '').length < 100) return false;
-		const { sections } = sliceSections(markdown, DEFAULT_SECTION_CAP);
+		const md = cleanReaderMarkdown(markdown);
+		if (md.replace(/\s+/g, '').length < 100) return false;
+		if (notFoundDoc(md)) return false;
+		const { sections } = sliceSections(md, DEFAULT_SECTION_CAP);
 		return sections.length > 0;
 	};
 
@@ -829,6 +775,43 @@ export async function scrape(urlRaw: string, opts: ScrapeOptions = {}): Promise<
 	return fromChain(url, chainResult.doc, sectionCap, maxChars);
 }
 
+/** Whole-line news junk in reader output: photo credits, carousel markers,
+ * video durations, relative-time chips. Prose lines are untouched. */
+const READER_JUNK_LINE_RE = [
+	// bare credits: "REUTERS/Sofiia Gatilova", "AP Photo/John Smith"
+	/^(?:reuters|ap photo|getty images|afp|epa)\/(?:[a-z0-9][\w.' -]{1,80})$/i,
+	// handout/file photo tags at line end: "…REUTERS/Rachel Wisniewski/File Photo"
+	/\/(?:file|handout)\s+photo\s*\.*$/i,
+	// carousel markers: "Item 1 of 15"
+	/^item\s+\d{1,3}\s+of\s+\d{1,3}\s*$/i,
+	// video durations: "0:48"
+	/^\d{1,3}:\d{2}$/,
+	// relative-time chips: "* 1 hour ago", "just now", "yesterday"
+	/^[*\-–—]?\s*(?:just now|today|yesterday|\d+\s*(?:m|min(?:ute)?s?|hrs?|h|hours?|d|days?|w|weeks?)\s*ago\.?)\s*$/i,
+];
+
+const NOT_FOUND_DOC_RE =
+	/(?:page|url|requested page|webpage)[^.]{0,40}(?:not found|not be found|cannot be found|does not exist|doesn't exist|no longer exists|error 404|404 error|404 not found)|we (?:couldn't|could not|can't) find(?: the)? (?:page|requested|article|that)|it looks like there isn't a page/i;
+
+/** Best-effort scrub of reader markdown (news sites leak credits/captions). */
+export function cleanReaderMarkdown(markdown: string): string {
+	const out: string[] = [];
+	let inFence = false;
+	for (const line of markdown.split('\n')) {
+		if (/^\s*(?:```|~~~)/.test(line)) inFence = !inFence;
+		const t = line.trim();
+		if (!inFence && t && t.length <= 200 && READER_JUNK_LINE_RE.some((re) => re.test(t))) continue;
+		out.push(line);
+	}
+	return out.join('\n').replace(/\n{3,}/g, '\n\n');
+}
+
+/** True for a 404-style reader stub; the chain skips it and tries the next backend. */
+export function notFoundDoc(markdown: string): boolean {
+	if (markdown.replace(/\s+/g, '').length > 500) return false;
+	return NOT_FOUND_DOC_RE.test(markdown.slice(0, 300));
+}
+
 /** Compose outline + sections; the cap is honest (marker included). */
 export function joinSections(
 	outline: string[],

package/src/tools.ts

@@ -21,7 +21,7 @@ import {
 import { ABORT_ERROR, DEFAULT_MAX_RESULTS } from './constants';
 import { cacheKey, openCache, CACHE_TTL_HOURS } from './cache';
 import { clearDeadEngines, runEngines } from './engines';
-import { fuse, diversifyByHost } from './fuse';
+import { fuse, diversifyByHost, type FusedHit } from './fuse';
 import { hostOf, normalizeUrl } from './urls';
 import { scrape } from './scrape';
 
@@ -83,8 +83,6 @@ function fmtSnippet(s: string, max: number): string {
 	return t.length > max ? t.slice(0, max) + '…' : t;
 }
 
-const ABORT_ERROR_LOCAL = ABORT_ERROR;
-
 function dropRarestTerm(query: string): string {
 	const words = query.split(/\s+/).filter((w) => w.length > 2 && !/^(?:site|inurl|intitle|filetype):/i.test(w));
 	if (words.length < 2) return '';
@@ -115,19 +113,18 @@ function normalizeHitDate(raw: string | undefined): string | undefined {
 	if (rel) {
 		const n = parseInt(rel[1], 10);
 		const unit = rel[2].toLowerCase();
-		const ms =
-			unit.startsWith('m') && unit !== 'mo'
-				? 60_000
-				: unit.startsWith('h')
-					? 3_600_000
+		// 'mo' must win over 'm' BEFORE the minutes branch: the regex captures
+		// 'months' verbatim, so a bare startsWith('m') misclassifies it ("3
+		// months ago" used to render as ~today). Divide by unit, in months>…>ms.
+		const ms = unit.startsWith('h')
+			? 3_600_000
+			: unit.startsWith('mo')
+				? 2_592_000_000
+				: unit.startsWith('m')
+					? 60_000
 					: unit.startsWith('d')
 						? 86_400_000
-						: unit.startsWith('w')
-							? 604_800_000
-							: unit.startsWith('mo')
-								? 2_592_000_000
-								: 604_800_000;
-		void ms;
+						: 604_800_000;
 		return new Date(Date.now() - n * ms).toISOString().slice(0, 10);
 	}
 	if (/^yesterday$/i.test(raw.trim())) return new Date(Date.now() - 86_400_000).toISOString().slice(0, 10);
@@ -231,7 +228,7 @@ export function registerWebTools(pi: ExtensionAPI) {
 				});
 				const title = fmtSnippet(stripSiteSuffix(cleanSnippet(r.title), hostOf(r.url)), 100) || r.url;
 				const snippet = fmtSnippet(cleanSnippet(r.snippet), snip);
-				const via = r.engines.length > 1 ? ` ← ${r.engines.length} engines agree` : ` ← ${r.engines[0]}`;
+				const via = formatEvidence(r);
 				const when = normalizeHitDate(r.date);
 				text += `${i + 1}. [${title}](${r.url})${when ? ` — ${when}` : ''}${via}\n`;
 				if (snippet) text += `   ${snippet}\n`;
@@ -252,6 +249,7 @@ export function registerWebTools(pi: ExtensionAPI) {
 						url: r.url,
 						title: r.title,
 						engines: r.engines,
+						ranks: r.engineRanks,
 						score: r.finalScore,
 					})),
 				},
@@ -286,6 +284,9 @@ export function registerWebTools(pi: ExtensionAPI) {
 		},
 
 		async execute(_id, params, signal) {
+			// Abort = throw: a stale signal must never START a fetch, and an abort
+			// mid-flight must never be masked by the stale-if-error reserve below.
+			if (signal?.aborted) throw new Error(ABORT_ERROR);
 			const cfg = getConfig();
 			const key = cacheKey(['fetch', 'v3', normalizeUrl(params.url)]);
 			const ranked = lastSearch.urlRank.get(normalizeUrl(params.url));
@@ -313,6 +314,7 @@ export function registerWebTools(pi: ExtensionAPI) {
 				signal,
 				validators: stale?.value?.validators,
 			});
+			if (signal?.aborted) throw new Error(ABORT_ERROR); // mid-flight abort: never a payload, never stale
 			if (res.error === 'not-modified' && stale?.value) {
 				cache.set(key, stale.value, CACHE_TTL_HOURS * 3600_000);
 				return makeFetchResult(stale.value, params);
@@ -333,9 +335,10 @@ export function registerWebTools(pi: ExtensionAPI) {
 			return makeFetchResult(res, params);
 		},
 		renderResult(result, options, theme) {
-			const lines = resultText(result).split('\n');
+			const raw = italicizeIndices(resultText(result), theme);
+			const lines = raw.split('\n');
 			if (!options.expanded) return new Text(previewWithHint(lines, theme), 0, 0);
-			return new Text(colorizedResult(resultText(result), theme), 0, 0);
+			return new Text(colorizedResult(raw, theme), 0, 0);
 		},
 	});
 }
@@ -365,12 +368,16 @@ interface FetchResult {
 	isError?: boolean;
 }
 
-function fmtSize(chars: number): string {
-	return chars >= 1024 ? `${(chars / 1024).toFixed(1)} KB` : `${chars} chars`;
-}
-
 const PREVIEW_LINES = 10;
 
+/** Display-only: the model-facing markers (`indices …`, `- *N*`) render as
+ * genuine ANSI italics in the TUI; the content text itself is untouched. */
+function italicizeIndices(text: string, theme: Theme): string {
+	return text
+		.replace(/^(indices - 0 - \d+ sections?)$/gm, (m) => theme.italic(m))
+		.replace(/^- \*(\d+)\* /gm, (_, n: string) => `- ${theme.italic(n)} `);
+}
+
 function colorizedResult(text: string, theme: Theme): string {
 	return text
 		.split('\n')
@@ -391,6 +398,18 @@ function colorizedResult(text: string, theme: Theme): string {
 		.join('\n');
 }
 
+/** Per-hit evidence line: "⇐ serper#1, exa#2"; singles are unverified. */
+function formatEvidence(r: FusedHit): string {
+	if (!r.engineRanks?.length) return r.engines.length > 1 ? ` ← ${r.engines.length} engines` : '';
+	if (r.engineRanks.length === 1) return ` ⇐ ${r.engineRanks[0].engine} only (single-engine: unverified)`;
+	const shown = r.engineRanks
+		.slice(0, 6)
+		.map((e) => `${e.engine}#${e.rank}`)
+		.join(', ');
+	const rest = r.engineRanks.length > 6 ? `, +${r.engineRanks.length - 6} more` : '';
+	return ` ⇐ ${shown}${rest}`;
+}
+
 function resultText(result: { content: { type: string; text?: string }[] }): string {
 	return (result.content[0] as { type: string; text: string } | undefined)?.text ?? '';
 }
@@ -421,7 +440,9 @@ function fetchErrorResult(details: FetchDetails): FetchResult {
 function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: number[]): FetchResult {
 	const want = [...new Set(sections.map((i) => Math.max(0, Math.min(res.sections.length - 1, i))))];
 	const chosen = want.map((i) => res.sections[i]).filter(Boolean);
-	const body = res.outline.length ? res.outline.map((h) => `- ${h}`).join('\n') + '\n\n' : '';
+	const body = res.outline.length
+		? `indices - 0 - ${res.outline.length - 1} section${res.outline.length === 1 ? '' : 's'}\n${res.outline.map((h, i) => `- *${i}* ${h}`).join('\n')}\n\n`
+		: '';
 	const parts: string[] = [];
 	want.forEach((idx) => {
 		const s = res.sections[idx];

package/src/transport.ts

@@ -1,9 +1,12 @@
 /**
  * Transport for web.fetch: UA-rotating local HTTP + the key-gated reader
- * backends. Text processing lives in scrape.ts (one-way import).
+ * backends. Text processing lives in scrape.ts (one-way import). SSRF: every
+ * redirect hop is re-validated against the private/loopback gates on the
+ * local path (see urls.isPrivateHost for the deliberate scope).
  */
 import { getConfig, sanitizeError } from './config';
 import { ABORT_ERROR } from './constants';
+import { isPrivateHost } from './urls';
 
 export type Renderer = 'local' | 'firecrawl' | 'tavily' | 'exa' | 'jina' | 'kagi' | 'tinyfish' | 'serper' | 'you';
 
@@ -24,6 +27,7 @@ const USER_AGENTS = [
 const MAX_HTML_BYTES = 5 * 1024 * 1024;
 const LOCAL_TIMEOUT_MS = 15_000;
 const RETRY_AFTER_MAX_MS = 10_000;
+const MAX_REDIRECTS = 8;
 const READER_TIMEOUTS: Record<RemoteRenderer, number> = {
 	firecrawl: 45_000,
 	tavily: 30_000,
@@ -241,24 +245,44 @@ export async function localFetch(
 		if (signal?.aborted) throw new Error(ABORT_ERROR);
 		const { ctrl, done } = withTimeout('local fetch', LOCAL_TIMEOUT_MS, signal);
 		try {
+			const headers: Record<string, string> = {
+				'User-Agent': USER_AGENTS[attempt],
+				Accept: 'text/markdown,text/html,application/xhtml+xml,text/plain;q=0.8,*/*;q=0.5',
+				'Accept-Language': 'en-US,en;q=0.5',
+			};
 			const condHeaders: Record<string, string> = {};
 			if (validators?.etag) condHeaders['If-None-Match'] = validators.etag;
 			if (validators?.lastModified) condHeaders['If-Modified-Since'] = validators.lastModified;
-			const res = await fetch(url, {
-				headers: {
-					'User-Agent': USER_AGENTS[attempt],
-					Accept: 'text/markdown,text/html,application/xhtml+xml,text/plain;q=0.8,*/*;q=0.5',
-					'Accept-Language': 'en-US,en;q=0.5',
-					...condHeaders,
-				},
-				redirect: 'follow',
-				signal: ctrl.signal,
-			});
-			if (res.status === 304 && attempt === 0) {
+			// 'manual' + explicit per-hop re-validation: a hostile page must never
+			// aim the agent at its own machine via a Location header. Conditional
+			// headers describe the FIRST resource only, so they drop after hop 0.
+			let current = url;
+			let res: Response | undefined;
+			for (let hop = 0; ; hop++) {
+				if (signal?.aborted) throw new Error(ABORT_ERROR);
+				res = await fetch(current, {
+					redirect: 'manual',
+					headers: hop === 0 ? { ...headers, ...condHeaders } : headers,
+					signal: ctrl.signal,
+				});
+				if (!isRedirect(res.status)) break;
+				if (hop >= MAX_REDIRECTS) throw new Error(`too many redirects (HTTP ${res.status})`);
+				const loc = res.headers.get('location');
+				await res.body?.cancel().catch(() => {});
+				if (!loc) throw new Error(`redirect without Location (HTTP ${res.status})`);
+				const next = new URL(loc, current);
+				if (next.protocol !== 'http:' && next.protocol !== 'https:') {
+					throw new Error(`unsupported redirect protocol: ${next.protocol}`);
+				}
+				if (isPrivateHost(next.hostname)) throw new Error('private/loopback address rejected (redirect)');
+				current = next;
+			}
+			const final = res!;
+			if (final.status === 304 && attempt === 0) {
 				done();
 				return { body: '', contentType: '', validators: validators ?? {}, notModified: true };
 			}
-			const ct = res.headers.get('content-type') ?? '';
+			const ct = final.headers.get('content-type') ?? '';
 			const mime = ct.split(';')[0]?.trim().toLowerCase() ?? '';
 			if (
 				!/html|xml|text/.test(mime) &&
@@ -267,25 +291,25 @@ export async function localFetch(
 			) {
 				throw new Error(`not a readable page (${mime || '?'})`);
 			}
-			if (res.status === 429 && !retried429) {
+			if (final.status === 429 && !retried429) {
 				retried429 = true;
-				await waitAbortable(parseRetryAfter(res.headers.get('retry-after')), signal);
+				await waitAbortable(parseRetryAfter(final.headers.get('retry-after')), signal);
 				attempt--;
 				continue;
 			}
-			if (!res.ok) {
-				const sample = await res.text().catch(() => '');
-				if (isBotBlocked(res.status, sample) && attempt < USER_AGENTS.length - 1) continue;
-				throw new Error(`HTTP ${res.status}`);
+			if (!final.ok) {
+				const sample = await final.text().catch(() => '');
+				if (isBotBlocked(final.status, sample) && attempt < USER_AGENTS.length - 1) continue;
+				throw new Error(`HTTP ${final.status}`);
 			}
-			const bytes = await readBodyBounded(res, MAX_HTML_BYTES);
+			const bytes = await readBodyBounded(final, MAX_HTML_BYTES);
 			if (bytes.byteLength === 0) throw new Error('empty body');
 			return {
 				body: decodeBody(bytes, ct),
 				contentType: ct,
 				validators: {
-					etag: res.headers.get('etag') ?? undefined,
-					lastModified: res.headers.get('last-modified') ?? undefined,
+					etag: final.headers.get('etag') ?? undefined,
+					lastModified: final.headers.get('last-modified') ?? undefined,
 				},
 				notModified: false,
 			};
@@ -306,6 +330,11 @@ export async function localFetch(
 	throw new Error('local fetch failed');
 }
 
+// 3xx redirect statuses; manual following must not treat these as content.
+function isRedirect(status: number): boolean {
+	return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
+}
+
 // Per-attempt timeout + pre-abort: an already-aborted signal never fires
 // abort listeners, so the request must reject immediately on its own.
 export function withTimeout(

package/src/urls.ts

@@ -1,4 +1,5 @@
-/** URL facts leaf: normalization (dedupe + cache keys), junk classification. */
+/** URL facts leaf: normalization (dedupe + cache keys), junk classification,
+ * and the SSRF private-host gates (shared by scrape + transport). */
 
 const JUNK_HOSTS = [
 	'quora.com',
@@ -19,6 +20,7 @@ const JUNK_HOSTS = [
 	'threadreaderapp.com',
 	'nitter.net',
 	'upd.dev',
+	'fb.watch',
 ];
 const JUNK_TLDS = new Set(['xyz', 'top', 'loan', 'click', 'work', 'gq', 'icu', 'rest', 'cyou']);
 
@@ -47,28 +49,45 @@ export function hostOf(raw: string): string {
 	}
 }
 
-/** Keeps scheme+port deliberately (http vs https twins are different pages). */
+/** Keeps scheme+port deliberately (http vs https twins are different pages).
+ * Path CASE and query ENCODING are preserved verbatim: on case-sensitive hosts
+ * `/README` and `/readme` are different resources, and `?q=a%2Fb` is not
+ * `?q=a/b`. Folding them would collide fetch cache keys and merge distinct
+ * hits in fuse, so the raw query pairs are only filtered and sorted, never
+ * re-serialized (URLSearchParams would decode the encoding away). */
 export function normalizeUrl(raw: string): string {
 	try {
 		const u = new URL(raw);
 		const host = u.hostname.toLowerCase().replace(/^www\./, '');
-		for (const p of Array.from(u.searchParams.keys())) {
-			if (TRACKING_PARAMS.has(p.toLowerCase())) u.searchParams.delete(p);
-		}
 		u.hash = '';
-		const pairs = Array.from(u.searchParams.entries()).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
-		const q = pairs.map(([k, v]) => `${k}=${v}`).join('&');
+		let q = '';
+		if (u.search) {
+			const kept = u.search
+				.slice(1)
+				.split('&')
+				.filter(Boolean)
+				.filter((pair) => !TRACKING_PARAMS.has(decodeKey(pair)));
+			kept.sort((a, b) => (decodeKey(a) < decodeKey(b) ? -1 : decodeKey(a) > decodeKey(b) ? 1 : 0));
+			q = kept.join('&');
+		}
 		const port = u.port ? `:${u.port}` : '';
-		const path = u.pathname
-			.replace(/\/{2,}/g, '/')
-			.replace(/\/+$/, '')
-			.toLowerCase();
+		const path = u.pathname.replace(/\/{2,}/g, '/').replace(/\/+$/, '');
 		return `${u.protocol}//${host}${port}${path}${q ? '?' + q : ''}`;
 	} catch {
 		return raw.toLowerCase();
 	}
 }
 
+/** Decoded query KEY (for tracking-param matching); malformed stays raw. */
+function decodeKey(pair: string): string {
+	const key = (pair.split('=')[0] ?? '').trim();
+	try {
+		return decodeURIComponent(key).toLowerCase();
+	} catch {
+		return key.toLowerCase();
+	}
+}
+
 export function isJunk(raw: string): boolean {
 	const host = hostOf(raw);
 	if (JUNK_HOSTS.some((j) => host === j || host.endsWith('.' + j))) return true;
@@ -85,3 +104,77 @@ function pathOf(raw: string): string {
 		return '';
 	}
 }
+
+// ── SSRF gates: shared by scrape.ts and transport.ts; this leaf imports ──────
+// ── nothing, so both consumers stay cycle-free. ───────────────────────────────
+
+function isPrivateIpv4(ipv4: string): boolean {
+	const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(ipv4);
+	if (!m) return false;
+	const [a, b] = [Number(m[1]), Number(m[2])];
+	return (
+		a === 0 ||
+		a === 10 ||
+		a === 127 ||
+		(a === 169 && b === 254) ||
+		(a === 172 && b >= 16 && b <= 31) ||
+		(a === 192 && b === 168)
+	);
+}
+
+function numericIpv4(host: string): string | null {
+	if (!/^[0-9.]+$/.test(host) && !/^0[xX][0-9a-fA-F.]+$/.test(host)) return null;
+	const fields = host.split('.');
+	if (fields.length > 4) return null;
+	const parts: number[] = [];
+	for (const f of fields) {
+		if (/^0[xX][0-9a-fA-F]+$/.test(f)) parts.push(parseInt(f, 16));
+		else if (/^0[0-7]+$/.test(f)) parts.push(parseInt(f, 8));
+		else if (/^\d+$/.test(f)) parts.push(Number(f));
+		else return null;
+	}
+	const last = parts[parts.length - 1];
+	if (parts.length === 1) {
+		if (last >= 2 ** 32) return null;
+		return [(last >>> 24) & 0xff, (last >>> 16) & 0xff, (last >>> 8) & 0xff, last & 0xff].join('.');
+	}
+	if (parts.slice(0, -1).some((n) => n > 255)) return null;
+	if (last >= 2 ** (8 * (4 - parts.length + 1))) return null;
+	if (parts.length === 2) return [parts[0], 0, 0, last].join('.');
+	if (parts.length === 3) return [parts[0], parts[1], (last >>> 8) & 0xff, last & 0xff].join('.');
+	return parts.join('.');
+}
+
+/** True for loopback/private/link-local numeric literals AND IPv6 forms that
+ * embed them (`::ffff:a.b.c.d`, and the `::aabb:ccdd` IPv4-compatible
+ * shorthand WHATWG serializes `::127.0.0.1` into). Scope is deliberate:
+ * hostnames that merely RESOLVE into private space are out of the threat
+ * model — the hostile web points the agent at literals, not at their DNS. */
+export function isPrivateHost(host: string): boolean {
+	const h = host.toLowerCase().replace(/^\[(.*)\]$/, '$1');
+	if (h === 'localhost' || h.endsWith('.local') || h.endsWith('.localhost')) return true;
+	const mapped = /:ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i.exec(h);
+	if (mapped) {
+		const [a, b] = [parseInt(mapped[1], 16), parseInt(mapped[2], 16)];
+		if (isPrivateIpv4(`${(a >> 8) & 0xff}.${a & 0xff}.${(b >> 8) & 0xff}.${b & 0xff}`)) return true;
+	}
+	const compat = /^::([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i.exec(h);
+	if (compat) {
+		const [a, b] = [parseInt(compat[1], 16), parseInt(compat[2], 16)];
+		if (isPrivateIpv4(`${(a >> 8) & 0xff}.${a & 0xff}.${(b >> 8) & 0xff}.${b & 0xff}`)) return true;
+	}
+	const ipv4 = numericIpv4(h);
+	if (ipv4 && isPrivateIpv4(ipv4)) return true;
+	if (/^[0-9a-f:]+$/.test(h) && h.includes(':')) {
+		if (h.startsWith('::1') || h === '0:0:0:0:0:0:0:1') return true;
+		if (h.startsWith('fe80:') || h.startsWith('fc') || h.startsWith('fd')) return true;
+	}
+	return false;
+}
+
+export function validateUrl(raw: string, allowPrivate = false): URL {
+	const u = new URL(raw);
+	if (u.protocol !== 'http:' && u.protocol !== 'https:') throw new Error(`unsupported protocol: ${u.protocol}`);
+	if (!allowPrivate && isPrivateHost(u.hostname)) throw new Error('private/loopback address rejected');
+	return u;
+}