pi-wsearch 0.1.0 → 0.2.0

package/src/tools.ts

@@ -6,8 +6,8 @@ import { Type } from 'typebox';
 import type { ExtensionAPI, ExtensionCommandContext, Theme } from '@earendil-works/pi-coding-agent';
 import { keyHint } from '@earendil-works/pi-coding-agent';
 import { Text } from '@earendil-works/pi-tui';
-import { mkdirSync, rmSync, writeFileSync, existsSync } from 'node:fs';
-import { dirname } from 'node:path';
+import { appendFileSync, mkdirSync, rmSync, writeFileSync, existsSync } from 'node:fs';
+import { dirname, join } from 'node:path';
 import {
 	getConfig,
 	PROVIDERS,
@@ -15,13 +15,14 @@ import {
 	resetConfigCache,
 	keyOrigins,
 	disabledFlagPath,
+	wsearchDir,
 	type ProviderInfo,
 } from './config';
 import { ABORT_ERROR, DEFAULT_MAX_RESULTS } from './constants';
 import { cacheKey, openCache, CACHE_TTL_HOURS } from './cache';
-import { runEngines } from './engines';
+import { clearDeadEngines, runEngines } from './engines';
 import { fuse, diversifyByHost } from './fuse';
-import { normalizeUrl } from './urls';
+import { hostOf, normalizeUrl } from './urls';
 import { scrape } from './scrape';
 
 const SEARCH_PARAMS = Type.Object({
@@ -38,6 +39,7 @@ const SEARCH_PARAMS = Type.Object({
 const SEARCH_OUTPUT = Type.Object({
 	query: Type.String(),
 	engines_used: Type.Array(Type.String()),
+	engines_errored: Type.Optional(Type.Array(Type.Object({ engine: Type.String(), error: Type.String() }))),
 	results: Type.Array(
 		Type.Object({
 			url: Type.String(),
@@ -81,6 +83,57 @@ function fmtSnippet(s: string, max: number): string {
 	return t.length > max ? t.slice(0, max) + '…' : t;
 }
 
+const ABORT_ERROR_LOCAL = ABORT_ERROR;
+
+function dropRarestTerm(query: string): string {
+	const words = query.split(/\s+/).filter((w) => w.length > 2 && !/^(?:site|inurl|intitle|filetype):/i.test(w));
+	if (words.length < 2) return '';
+	const rarest = words.reduce((a, b) => (b.length > a.length ? b : a));
+	return query
+		.replace(rarest, ' ')
+		.replace(/\s{2,}/g, ' ')
+		.trim();
+}
+
+function stripSiteSuffix(title: string, host: string): string {
+	const m = /^(.{3,})\s+[|–—-]\s+([^|–—-]{2,40})$/.exec(title.trim());
+	if (!m) return title;
+	const suffix = m[2].trim();
+	const head = m[1].trim();
+	const hostBase = host.replace(/^www\./, '').split('.')[0];
+	if (host && (suffix.toLowerCase().includes(hostBase) || suffix.length <= 25)) return head;
+	return title;
+}
+
+function normalizeHitDate(raw: string | undefined): string | undefined {
+	if (!raw) return undefined;
+	const t = Date.parse(raw);
+	if (!Number.isNaN(t)) return new Date(t).toISOString().slice(0, 10);
+	const rel = /^(\d+)\s*(m|min|minute|minutes|h|hour|hours|d|day|days|w|week|weeks|mo|month|months)\b/i.exec(
+		raw.trim(),
+	);
+	if (rel) {
+		const n = parseInt(rel[1], 10);
+		const unit = rel[2].toLowerCase();
+		const ms =
+			unit.startsWith('m') && unit !== 'mo'
+				? 60_000
+				: unit.startsWith('h')
+					? 3_600_000
+					: unit.startsWith('d')
+						? 86_400_000
+						: unit.startsWith('w')
+							? 604_800_000
+							: unit.startsWith('mo')
+								? 2_592_000_000
+								: 604_800_000;
+		void ms;
+		return new Date(Date.now() - n * ms).toISOString().slice(0, 10);
+	}
+	if (/^yesterday$/i.test(raw.trim())) return new Date(Date.now() - 86_400_000).toISOString().slice(0, 10);
+	return undefined;
+}
+
 function cleanSnippet(s: string): string {
 	return s
 		.replace(/\\([_*`[\]()#>~.-])/g, '$1')
@@ -94,6 +147,8 @@ function cleanSnippet(s: string): string {
 export function registerWebTools(pi: ExtensionAPI) {
 	const cache = openCache(getConfig().cacheDir + '/cache.json');
 	const exposure = existsSync(disabledFlagPath()) ? 'hidden' : 'direct';
+	const lastSearch = { urlRank: new Map<string, { rank: number; engines: string[]; query: string }>() };
+	const tracePath = join(wsearchDir(), 'fetchtrace.jsonl');
 
 	const namespace = {
 		name: 'web',
@@ -121,18 +176,47 @@ export function registerWebTools(pi: ExtensionAPI) {
 				maxResults: params.max_results ?? DEFAULT_MAX_RESULTS,
 				signal,
 			});
+			const errored = outcomes
+				.filter((o) => o.error && o.engine !== 'none')
+				.map((o) => ({ engine: o.engine, error: o.error! }));
 			// signal may have fired mid-round: still an abort, never a results payload
 			if (signal?.aborted) throw new Error(ABORT_ERROR);
-			const fused = fuse(outcomes, { query: params.query });
-			const candidates = diversifyByHost(
+			let fused = fuse(outcomes, { query: params.query });
+			let candidates = diversifyByHost(
 				fused.filter((r) => !r.junk),
 				2,
 			).slice(0, Math.max((params.max_results ?? DEFAULT_MAX_RESULTS) * 2, DEFAULT_MAX_RESULTS));
+			let retryNote = '';
+			if (!candidates.length && !signal?.aborted) {
+				const hasOps = /(?:^|\s)(?:site:|intitle:|inurl:|filetype:|-"|"[^"]+"|(?:^|\s)-\w+)/i.test(params.query);
+				const stripped = hasOps
+					? params.query
+							.replace(/(?:^|\s)(?:site:\S+|intitle:\S+|inurl:\S+|filetype:\S+)|"[^"]*"|(?:^|\s)-\w+/gi, ' ')
+							.replace(/\s{2,}/g, ' ')
+							.trim()
+					: '';
+				const relaxed = stripped && stripped !== params.query ? stripped : dropRarestTerm(params.query);
+				if (relaxed && relaxed !== params.query) {
+					const retryOutcomes = await runEngines({
+						query: relaxed,
+						maxResults: params.max_results ?? DEFAULT_MAX_RESULTS,
+						signal,
+					});
+					if (!signal?.aborted) {
+						fused = fuse(retryOutcomes, { query: relaxed });
+						candidates = diversifyByHost(
+							fused.filter((r) => !r.junk),
+							2,
+						).slice(0, Math.max((params.max_results ?? DEFAULT_MAX_RESULTS) * 2, DEFAULT_MAX_RESULTS));
+						if (candidates.length) retryNote = ` (retried as "${relaxed}" after zero results)`;
+					}
+				}
+			}
 			const visible = candidates.slice(0, params.max_results ?? DEFAULT_MAX_RESULTS);
 			const enginesUsed = [...new Set(fused.flatMap((r) => r.engines))];
 			const snip = 240;
 
-			let text = `${visible.length} result${visible.length === 1 ? '' : 's'} for "${params.query}": ${enginesUsed.length} engine${enginesUsed.length === 1 ? '' : 's'} contributed (${enginesUsed.join(', ')})\n\n`;
+			let text = `${visible.length} result${visible.length === 1 ? '' : 's'} for "${params.query}": ${enginesUsed.length} engine${enginesUsed.length === 1 ? '' : 's'} contributed (${enginesUsed.join(', ')})${retryNote}\n\n`;
 			if (!visible.length) {
 				const fatal = outcomes.find((o) => o.engine === 'none' && o.error);
 				text = fatal
@@ -140,13 +224,22 @@ export function registerWebTools(pi: ExtensionAPI) {
 					: `No results for "${params.query}". Broaden the query.\n`;
 			}
 			visible.forEach((r, i) => {
-				const title = fmtSnippet(cleanSnippet(r.title), 100) || r.url;
+				lastSearch.urlRank.set(normalizeUrl(r.url), {
+					rank: i + 1,
+					engines: r.engines,
+					query: params.query,
+				});
+				const title = fmtSnippet(stripSiteSuffix(cleanSnippet(r.title), hostOf(r.url)), 100) || r.url;
 				const snippet = fmtSnippet(cleanSnippet(r.snippet), snip);
 				const via = r.engines.length > 1 ? ` ← ${r.engines.length} engines agree` : ` ← ${r.engines[0]}`;
-				text += `${i + 1}. [${title}](${r.url})${via}\n`;
+				const when = normalizeHitDate(r.date);
+				text += `${i + 1}. [${title}](${r.url})${when ? ` — ${when}` : ''}${via}\n`;
 				if (snippet) text += `   ${snippet}\n`;
 				text += '\n';
 			});
+			if (errored.length) {
+				text += `⚠ engines: ${errored.map((e) => `${e.engine} (${e.error.slice(0, 80)})`).join('; ')}\n`;
+			}
 
 			return {
 				content: [{ type: 'text', text }],
@@ -154,6 +247,7 @@ export function registerWebTools(pi: ExtensionAPI) {
 				structuredContent: {
 					query: params.query,
 					engines_used: enginesUsed,
+					engines_errored: errored,
 					results: visible.map((r) => ({
 						url: r.url,
 						title: r.title,
@@ -193,14 +287,49 @@ export function registerWebTools(pi: ExtensionAPI) {
 
 		async execute(_id, params, signal) {
 			const cfg = getConfig();
-			const key = cacheKey(['fetch', normalizeUrl(params.url)]);
+			const key = cacheKey(['fetch', 'v3', normalizeUrl(params.url)]);
+			const ranked = lastSearch.urlRank.get(normalizeUrl(params.url));
+			if (ranked && !cache.get(key)) {
+				try {
+					mkdirSync(dirname(tracePath), { recursive: true });
+					appendFileSync(
+						tracePath,
+						`${JSON.stringify({
+							ts: Date.now(),
+							q: ranked.query,
+							rank: ranked.rank,
+							engines: ranked.engines,
+							url: normalizeUrl(params.url),
+						})}\n`,
+						{ flag: 'a', mode: 0o600 },
+					);
+				} catch {}
+			}
 			const cached = cache.get(key) as Awaited<ReturnType<typeof scrape>> | undefined;
 			if (cached) return makeFetchResult(cached, params);
+			const stale = cache.peekStale(key) as { value: Awaited<ReturnType<typeof scrape>>; ageHours: number } | undefined;
 			const res = await scrape(params.url, {
 				allowPrivate: cfg.allowPrivate,
 				signal,
+				validators: stale?.value?.validators,
 			});
-			if (!res.error) cache.set(key, res, CACHE_TTL_HOURS * 3600_000);
+			if (res.error === 'not-modified' && stale?.value) {
+				cache.set(key, stale.value, CACHE_TTL_HOURS * 3600_000);
+				return makeFetchResult(stale.value, params);
+			}
+			if (!res.error) {
+				cache.set(key, res.validators ? { ...res, validators: res.validators } : res, CACHE_TTL_HOURS * 3600_000);
+				return makeFetchResult(res, params);
+			}
+			if (stale?.value && !stale.value.error) {
+				return makeFetchResult(
+					{
+						...stale.value,
+						title: `${stale.value.title} [STALE — ~${Math.round(stale.ageHours)}h old, live fetch failed]`,
+					},
+					params,
+				);
+			}
 			return makeFetchResult(res, params);
 		},
 		renderResult(result, options, theme) {
@@ -246,7 +375,12 @@ function colorizedResult(text: string, theme: Theme): string {
 	return text
 		.split('\n')
 		.map((l) => {
-			if (l.includes('…[section truncated') || l.startsWith('…[clipped by tool]') || l.startsWith('…[truncated]'))
+			if (
+				l.includes('…[section truncated') ||
+				l.includes('…(continues at outline index') ||
+				l.startsWith('…[clipped by tool]') ||
+				l.startsWith('…[truncated]')
+			)
 				return theme.fg('warning', l);
 			if (l.startsWith('## [')) return theme.fg('accent', l);
 			if (l.startsWith('- ')) return theme.fg('dim', l);
@@ -316,7 +450,7 @@ function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: num
 	if (res.renderer && res.renderer !== 'local') text += `\n\n[rendered via ${res.renderer}]`;
 	const clipped = text.length > MAX_FETCH_TEXT ? text.slice(0, MAX_FETCH_TEXT) + '\n…[clipped by tool]' : text;
 	const missing = sections.filter((i) => i >= res.sections.length);
-	const truncatedCount = want.filter((idx) => res.sections[idx]?.includes('…[section truncated]')).length;
+	const truncatedCount = want.filter((idx) => res.continued?.includes(idx)).length;
 	const details = {
 		url: res.url,
 		title: res.title,
@@ -327,6 +461,8 @@ function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: num
 		sections_requested: [...new Set(sections)],
 		sections_missing: [...new Set(missing)],
 		sections_truncated: truncatedCount,
+		...(res.publishedAt ? { publishedAt: res.publishedAt } : {}),
+		...(res.paywalled ? { paywalled: true } : {}),
 		...(res.error ? { error: res.error } : {}),
 	};
 	if (res.error) return fetchErrorResult(details);
@@ -407,6 +543,7 @@ async function handleLogin(ctx: ExtensionCommandContext, name?: string): Promise
 	}
 	writeKey(provider, trimmed);
 	resetConfigCache();
+	clearDeadEngines();
 	ctx.ui.notify(`${provider.id} ✓ saved to wsearch/env (${isCmd ? 'command' : 'key'})`, 'info');
 }
 
@@ -453,9 +590,12 @@ async function handleLogout(ctx: ExtensionCommandContext, name?: string): Promis
 	if (!ok) return;
 	removeKey(provider);
 	resetConfigCache();
+	clearDeadEngines();
 	ctx.ui.notify(`${provider.id} removed from wsearch/env`, 'info');
 }
 
+const SEARCH_TOOL_NAMES = ['web.search', 'web.fetch'] as const;
+
 function toggleSearchTools(pi: ExtensionAPI, disabled: boolean): void {
 	const flag = disabledFlagPath();
 	if (disabled) {
@@ -465,6 +605,12 @@ function toggleSearchTools(pi: ExtensionAPI, disabled: boolean): void {
 		rmSync(flag, { force: true });
 	}
 	registerWebTools(pi);
+	const active = new Set(pi.getActiveTools());
+	for (const name of SEARCH_TOOL_NAMES) {
+		if (disabled) active.delete(name);
+		else active.add(name);
+	}
+	pi.setActiveTools([...active]);
 }
 
 export function registerWebCommand(pi: ExtensionAPI) {

package/src/transport.ts

@@ -5,7 +5,7 @@
 import { getConfig, sanitizeError } from './config';
 import { ABORT_ERROR } from './constants';
 
-export type Renderer = 'local' | 'firecrawl' | 'tavily' | 'exa' | 'jina';
+export type Renderer = 'local' | 'firecrawl' | 'tavily' | 'exa' | 'jina' | 'kagi' | 'tinyfish' | 'serper' | 'you';
 
 export type RemoteRenderer = Exclude<Renderer, 'local'>;
 
@@ -29,10 +29,14 @@ const READER_TIMEOUTS: Record<RemoteRenderer, number> = {
 	tavily: 30_000,
 	exa: 20_000,
 	jina: 20_000,
+	kagi: 30_000,
+	tinyfish: 25_000,
+	serper: 25_000,
+	you: 25_000,
 };
 
 function isBotBlocked(status: number, content: string): boolean {
-	if (status !== 403 && status !== 503) return false;
+	if (status !== 403 && status !== 503 && status !== 402) return false;
 	const lower = content.toLowerCase();
 	return (
 		lower.includes('cloudflare') ||
@@ -58,6 +62,11 @@ export function stripDataUriImages(markdown: string): string {
 
 export function isLowQualityOutput(content: string): boolean {
 	const lower = content.toLowerCase();
+	if (
+		/\b(?:experiencing an access issue|unusual traffic|confirm you are (?:a )?(?:human|not a bot))\b/.test(lower) ||
+		(lower.includes('icanhazip.com') && lower.length < 2048)
+	)
+		return true;
 	const jsGated = [
 		'enable javascript',
 		'javascript required',
@@ -67,8 +76,25 @@ export function isLowQualityOutput(content: string): boolean {
 	];
 	if (content.length < 1024 && jsGated.some((t) => lower.includes(t))) return true;
 	const lines = content.split('\n').filter((l) => l.trim());
-	const shortLines = lines.filter((l) => l.trim().length < 40);
-	if (lines.length > 10 && shortLines.length / lines.length > 0.7) return true;
+	if (lines.length > 30) {
+		const seen = new Map<string, number>();
+		for (const l of lines) seen.set(l, (seen.get(l) ?? 0) + 1);
+		const dupChars = [...seen.entries()].reduce((n, [, c]) => (c > 1 ? n + c : n), 0);
+		if (dupChars / lines.length > 0.3) return true;
+	}
+	const words = lower.split(/\s+/).filter(Boolean);
+	if (words.length > 150) {
+		const grams = new Map<string, number>();
+		for (let i = 0; i <= words.length - 5; i++) {
+			const g = words.slice(i, i + 5).join(' ');
+			grams.set(g, (grams.get(g) ?? 0) + 1);
+		}
+		const repeatedGrams = [...grams.values()].filter((c) => c > 1).length;
+		if (repeatedGrams / grams.size > 0.2) return true;
+	}
+	const totalChars = lines.reduce((n, l) => n + l.trim().length, 0);
+	const shortChars = lines.reduce((n, l) => (l.trim().length < 40 ? n + l.trim().length : n), 0);
+	if (lines.length > 10 && totalChars > 0 && shortChars / totalChars > 0.6) return true;
 	return false;
 }
 
@@ -101,16 +127,76 @@ function waitAbortable(ms: number, signal?: AbortSignal): Promise<void> {
 	});
 }
 
+const MOJIBAKE_RE = /[Ó†’éüï¬Åˆ]{2,}|â€[˜›”˜]/g;
+
+function looksMojibake(text: string): boolean {
+	if (!text.includes('Ã') && !text.includes('â€') && !text.includes('Â')) return false;
+	const hits = text.match(MOJIBAKE_RE);
+	return !!hits && hits.length >= 2 && hits.length * 12 > text.length * 0.001;
+}
+
+const CP1252_REV = new Map([
+	[0x20ac, 0x80],
+	[0x201a, 0x82],
+	[0x192, 0x83],
+	[0x201e, 0x84],
+	[0x2026, 0x85],
+	[0x2020, 0x86],
+	[0x2021, 0x87],
+	[0x2c6, 0x88],
+	[0x2030, 0x89],
+	[0x160, 0x8a],
+	[0x2039, 0x8b],
+	[0x152, 0x8c],
+	[0x17d, 0x8e],
+	[0x2018, 0x91],
+	[0x2019, 0x92],
+	[0x201c, 0x93],
+	[0x201d, 0x94],
+	[0x2022, 0x95],
+	[0x2013, 0x96],
+	[0x2014, 0x97],
+	[0x2dc, 0x98],
+	[0x2122, 0x99],
+	[0x161, 0x9a],
+	[0x203a, 0x9b],
+	[0x152, 0x9c],
+	[0x17e, 0x9e],
+	[0x178, 0x9f],
+]);
+
+function repairMojibake(text: string): string {
+	try {
+		const bytes = new Uint8Array(
+			[...text].map((c) => {
+				const cp = c.codePointAt(0)!;
+				return cp < 256 ? cp : (CP1252_REV.get(cp) ?? 0x3f);
+			}),
+		);
+		const repaired = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+		const before = (text.match(MOJIBAKE_RE) ?? []).length;
+		const after = (repaired.match(MOJIBAKE_RE) ?? []).length;
+		return after < before ? repaired : text;
+	} catch {
+		return text;
+	}
+}
+
 function decodeBody(bytes: Uint8Array, contentType: string): string {
 	const label =
 		/charset\s*=\s*"?([\w-]+)"?/i.exec(contentType)?.[1] ??
 		/<meta[^>]+charset\s*=\s*["']?([\w-]+)/i.exec(new TextDecoder('latin1').decode(bytes.subarray(0, 2048)))?.[1];
+	let text: string;
 	if (label && !/^utf-?8$/i.test(label)) {
 		try {
-			return new TextDecoder(label).decode(bytes);
-		} catch {}
+			text = new TextDecoder(label).decode(bytes);
+		} catch {
+			text = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+		}
+	} else {
+		text = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
 	}
-	return new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+	return looksMojibake(text) ? repairMojibake(text) : text;
 }
 
 async function readBodyBounded(res: Response, maxBytes: number): Promise<Uint8Array> {
@@ -140,21 +226,38 @@ async function readBodyBounded(res: Response, maxBytes: number): Promise<Uint8Ar
 	return out;
 }
 
-export async function localFetch(url: URL, signal?: AbortSignal): Promise<{ body: string; contentType: string }> {
+export interface ConditionalValidators {
+	etag?: string;
+	lastModified?: string;
+}
+
+export async function localFetch(
+	url: URL,
+	signal?: AbortSignal,
+	validators?: ConditionalValidators,
+): Promise<{ body: string; contentType: string; validators: ConditionalValidators; notModified: boolean }> {
 	let retried429 = false;
 	for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) {
 		if (signal?.aborted) throw new Error(ABORT_ERROR);
 		const { ctrl, done } = withTimeout('local fetch', LOCAL_TIMEOUT_MS, signal);
 		try {
+			const condHeaders: Record<string, string> = {};
+			if (validators?.etag) condHeaders['If-None-Match'] = validators.etag;
+			if (validators?.lastModified) condHeaders['If-Modified-Since'] = validators.lastModified;
 			const res = await fetch(url, {
 				headers: {
 					'User-Agent': USER_AGENTS[attempt],
-					Accept: 'text/html,application/xhtml+xml,text/plain,text/markdown,*/*;q=0.8',
+					Accept: 'text/markdown,text/html,application/xhtml+xml,text/plain;q=0.8,*/*;q=0.5',
 					'Accept-Language': 'en-US,en;q=0.5',
+					...condHeaders,
 				},
 				redirect: 'follow',
 				signal: ctrl.signal,
 			});
+			if (res.status === 304 && attempt === 0) {
+				done();
+				return { body: '', contentType: '', validators: validators ?? {}, notModified: true };
+			}
 			const ct = res.headers.get('content-type') ?? '';
 			const mime = ct.split(';')[0]?.trim().toLowerCase() ?? '';
 			if (
@@ -177,7 +280,15 @@ export async function localFetch(url: URL, signal?: AbortSignal): Promise<{ body
 			}
 			const bytes = await readBodyBounded(res, MAX_HTML_BYTES);
 			if (bytes.byteLength === 0) throw new Error('empty body');
-			return { body: decodeBody(bytes, ct), contentType: ct };
+			return {
+				body: decodeBody(bytes, ct),
+				contentType: ct,
+				validators: {
+					etag: res.headers.get('etag') ?? undefined,
+					lastModified: res.headers.get('last-modified') ?? undefined,
+				},
+				notModified: false,
+			};
 		} catch (e) {
 			if (signal?.aborted) throw new Error(ABORT_ERROR, { cause: e });
 			const clean = sanitizeError(e);
@@ -197,7 +308,11 @@ export async function localFetch(url: URL, signal?: AbortSignal): Promise<{ body
 
 // Per-attempt timeout + pre-abort: an already-aborted signal never fires
 // abort listeners, so the request must reject immediately on its own.
-function withTimeout(label: string, ms: number, signal?: AbortSignal): { ctrl: AbortController; done: () => void } {
+export function withTimeout(
+	label: string,
+	ms: number,
+	signal?: AbortSignal,
+): { ctrl: AbortController; done: () => void } {
 	const ctrl = new AbortController();
 	const t = setTimeout(() => ctrl.abort(new Error(`${label} timeout`)), ms);
 	const onAbort = () => ctrl.abort(signal?.reason ?? new Error(ABORT_ERROR));
@@ -320,6 +435,97 @@ async function jinaRead(url: URL, signal?: AbortSignal): Promise<RemoteDoc> {
 		done();
 	}
 }
+async function kagiRead(url: URL, signal?: AbortSignal): Promise<RemoteDoc> {
+	const key = getConfig().kagiKey!;
+	const { ctrl, done } = withTimeout('kagi', READER_TIMEOUTS.kagi, signal);
+	try {
+		const d = await readJson<{
+			data?: { output?: string; references?: { title?: string; url?: string }[] };
+		}>(
+			'https://kagi.com/api/v0/summarize',
+			{
+				method: 'POST',
+				headers: { Authorization: `Bot ${key}`, 'Content-Type': 'application/json' },
+				body: JSON.stringify({ url: url.toString(), summary_type: 'takeaway' }),
+			},
+			ctrl.signal,
+		);
+		const output = d.data?.output?.trim() ?? '';
+		if (output.length < 100) throw new Error('kagi summarizer returned no usable summary');
+		const refs = (d.data?.references ?? []).filter((r) => r?.url);
+		const markdown = refs.length
+			? `${output}\n\nSources:\n${refs.map((r) => `- [${r.title ?? r.url}](${r.url})`).join('\n')}`
+			: output;
+		return { title: '', markdown };
+	} finally {
+		done();
+	}
+}
+async function tinyfishRead(url: URL, signal?: AbortSignal): Promise<RemoteDoc> {
+	const key = getConfig().tinyfishKey!;
+	const { ctrl, done } = withTimeout('tinyfish', READER_TIMEOUTS.tinyfish, signal);
+	try {
+		const d = await readJson<{ results?: { url?: string; title?: string; text?: string; markdown?: string }[] }>(
+			'https://api.fetch.tinyfish.ai',
+			{
+				method: 'POST',
+				headers: { 'X-API-Key': key, 'Content-Type': 'application/json' },
+				body: JSON.stringify({ urls: [url.toString()] }),
+			},
+			ctrl.signal,
+		);
+		const doc = (d.results ?? []).find((r) => r && !!r.text);
+		if (!doc?.text?.trim()) throw new Error('tinyfish fetch returned no content');
+		return { title: doc.title ?? '', markdown: doc.markdown ?? doc.text };
+	} finally {
+		done();
+	}
+}
+async function serperRead(url: URL, signal?: AbortSignal): Promise<RemoteDoc> {
+	const key = getConfig().serperKey!;
+	const { ctrl, done } = withTimeout('serper', READER_TIMEOUTS.serper, signal);
+	try {
+		const d = await readJson<Record<string, unknown>>(
+			'https://scrape.serper.dev/',
+			{
+				method: 'POST',
+				headers: { 'X-API-KEY': key, 'Content-Type': 'application/json' },
+				body: JSON.stringify({ url: url.toString(), includeMarkdown: true }),
+			},
+			ctrl.signal,
+		);
+		const data = (typeof d.data === 'object' && d.data ? (d.data as Record<string, unknown>) : {}) as Record<
+			string,
+			unknown
+		>;
+		const md = (typeof data.markdown === 'string' ? data.markdown : (d.markdown ?? d.text ?? '')) as string | undefined;
+		const markdown = (md ?? '').trim();
+		if (markdown.length < 100) throw new Error('serper scrape returned no usable content');
+		return { title: typeof d.title === 'string' ? d.title : '', markdown };
+	} finally {
+		done();
+	}
+}
+async function youRead(url: URL, signal?: AbortSignal): Promise<RemoteDoc> {
+	const key = getConfig().youKey!;
+	const { ctrl, done } = withTimeout('you', READER_TIMEOUTS.you, signal);
+	try {
+		const rows = await readJson<{ url?: string; title?: string; markdown?: string }[]>(
+			'https://ydc-index.io/v1/contents',
+			{
+				method: 'POST',
+				headers: { 'X-API-Key': key, 'Content-Type': 'application/json' },
+				body: JSON.stringify({ urls: [url.toString()], formats: ['markdown'] }),
+			},
+			ctrl.signal,
+		);
+		const doc = (rows ?? []).find((r) => r && !!r.markdown);
+		if (!doc?.markdown?.trim()) throw new Error('you contents returned no markdown');
+		return { title: doc.title ?? '', markdown: doc.markdown };
+	} finally {
+		done();
+	}
+}
 
 interface Reader {
 	available(): boolean;
@@ -330,10 +536,17 @@ const READERS: Record<RemoteRenderer, Reader> = {
 	firecrawl: { available: () => !!getConfig().firecrawlKey, read: firecrawlRead },
 	tavily: { available: () => !!getConfig().tavilyKey, read: tavilyRead },
 	exa: { available: () => !!getConfig().exaKey, read: exaRead },
-	jina: { available: () => !!getConfig().jinaKey, read: jinaRead },
+	jina: {
+		available: () => !!getConfig().jinaKey || (getConfig().fetchChain ?? '').split(',').includes('jina'),
+		read: jinaRead,
+	},
+	kagi: { available: () => !!getConfig().kagiKey, read: kagiRead },
+	tinyfish: { available: () => !!getConfig().tinyfishKey, read: tinyfishRead },
+	serper: { available: () => !!getConfig().serperKey, read: serperRead },
+	you: { available: () => !!getConfig().youKey, read: youRead },
 };
 
-const REMOTE_ORDER: RemoteRenderer[] = ['firecrawl', 'tavily', 'exa', 'jina'];
+const REMOTE_ORDER: RemoteRenderer[] = ['jina', 'tinyfish', 'firecrawl', 'tavily', 'exa', 'serper', 'you', 'kagi'];
 
 function readerOrder(): RemoteRenderer[] {
 	const raw = getConfig().fetchChain;
@@ -367,7 +580,7 @@ export async function remoteChain(
 	usable: (markdown: string) => boolean,
 ): Promise<ChainResult> {
 	const order = readerOrder();
-	let lastError: string | undefined;
+	const errors: string[] = [];
 	for (const id of order) {
 		if (signal?.aborted) return { ok: false, aborted: true, error: ABORT_ERROR };
 		const reader = READERS[id];
@@ -378,12 +591,12 @@ export async function remoteChain(
 			return { ok: true, doc: { ...doc, renderer: id } };
 		} catch (e) {
 			if (signal?.aborted) return { ok: false, aborted: true, error: ABORT_ERROR };
-			lastError = sanitizeError(e);
+			errors.push(`${id}: ${sanitizeError(e)}`);
 		}
 	}
 	return {
 		ok: false,
 		aborted: false,
-		error: lastError ? `reader chain exhausted: ${lastError}` : 'no reader backend configured',
+		error: errors.length ? `reader chain exhausted: ${errors.join('; ')}` : 'no reader backend configured',
 	};
 }