pi-wsearch 0.5.1 → 0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-wsearch",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "Pi extension: WebSearch + WebFetch",
   "license": "MIT",
   "type": "module",

package/src/fetch.ts

@@ -0,0 +1,122 @@
+import { scrape } from './scrape';
+
+const MAX_FETCH_TEXT = 16000;
+
+type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
+
+type FetchDetails = {
+	url: string;
+	title: string;
+	renderer: string;
+	outline: string[];
+	sections: { index: number; heading: string; text: string }[];
+	section_count: number;
+	chars?: number;
+	sections_requested?: number[];
+	sections_missing?: number[];
+	sections_truncated?: number;
+	sections_clipped?: number[];
+	error?: string;
+};
+
+export interface FetchResult {
+	content: { type: 'text'; text: string }[];
+	details: FetchDetails;
+	structuredContent: JsonValue;
+	isError?: boolean;
+}
+
+function fetchErrorResult(details: FetchDetails): FetchResult {
+	const error = String(details.error ?? 'unknown error');
+	return {
+		content: [{ type: 'text', text: `WebFetch failed: ${error}` }],
+		details,
+		structuredContent: { ...details },
+		isError: true,
+	};
+}
+
+function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: number[]): FetchResult {
+	const want = [...new Set(sections.map((i) => Math.max(0, Math.min(res.sections.length - 1, i))))];
+	const body = res.outline.length
+		? `indices - 0 - ${res.outline.length - 1} section${res.outline.length === 1 ? '' : 's'}\n${res.outline.map((h, i) => `- *${i}* ${h}`).join('\n')}\n\n`
+		: '';
+	const parts: string[] = [];
+	want.forEach((idx) => {
+		const s = res.sections[idx];
+		if (s === undefined) return;
+		const head = res.outline[idx] ? `## [${idx}] ${res.outline[idx]}` : `## [${idx}]`;
+		parts.push(`${head}\n\n${s}`);
+	});
+	let text =
+		(res.title ? `# ${res.title}\n\n` : '') + body + parts.join('\n\n') + (res.truncated ? '\n…[truncated]' : '');
+	text = text.replace(/\n{3,}/g, '\n\n');
+	if (res.title) {
+		const t = res.title.trim();
+		const lines = text.split('\n');
+		const headIdx = lines.findIndex((l) => l.startsWith('# '));
+		if (headIdx >= 0) {
+			for (let i = headIdx + 1; i < Math.min(lines.length, headIdx + 6); i++) {
+				const l = lines[i].replace(/^#+\s*/, '').trim();
+				if (l === t) {
+					lines.splice(i, 1);
+					break;
+				}
+			}
+		}
+		text = lines.join('\n').replace(/\n{3,}/g, '\n\n');
+	}
+	if (res.renderer && res.renderer !== 'local') text += `\n\n[rendered via ${res.renderer}]`;
+	const clipped = text.length > MAX_FETCH_TEXT ? text.slice(0, MAX_FETCH_TEXT) + '\n…[clipped by tool]' : text;
+	const missing = sections.filter((i) => i >= res.sections.length);
+	const truncatedCount = want.filter((idx) => res.continued?.includes(idx)).length;
+
+	let budget = MAX_FETCH_TEXT;
+	const structuredSections: { index: number; heading: string; text: string }[] = [];
+	const clippedSections: number[] = [];
+	for (const idx of want) {
+		if (budget <= 0) {
+			clippedSections.push(idx);
+			continue;
+		}
+		const s = res.sections[idx];
+		if (s === undefined) continue;
+		const heading = res.outline[idx] ?? '';
+		const fullCost = heading.length + s.length;
+		if (fullCost <= budget) {
+			structuredSections.push({ index: idx, heading, text: s });
+			budget -= fullCost;
+			continue;
+		}
+		if (heading.length >= budget) {
+			structuredSections.push({ index: idx, heading: heading.slice(0, Math.max(0, budget - 1)) + '…', text: '' });
+		} else {
+			const textLen = Math.max(0, budget - heading.length - 1);
+			structuredSections.push({ index: idx, heading, text: s.slice(0, textLen) + (textLen < s.length ? '…' : '') });
+		}
+		clippedSections.push(idx);
+		budget = 0;
+	}
+	const details = {
+		url: res.url,
+		title: res.title,
+		renderer: res.renderer,
+		outline: res.outline,
+		sections: structuredSections,
+		section_count: structuredSections.length,
+		chars: clipped.length,
+		sections_requested: [...new Set(sections)],
+		sections_missing: [...new Set(missing)],
+		sections_truncated: truncatedCount,
+		sections_clipped: clippedSections,
+		...(res.publishedAt ? { publishedAt: res.publishedAt } : {}),
+		...(res.paywalled ? { paywalled: true } : {}),
+		...(res.error ? { error: res.error } : {}),
+	};
+	if (res.error) return fetchErrorResult(details);
+	return { content: [{ type: 'text', text: clipped }], details, structuredContent: { ...details } };
+}
+
+export function makeFetchResult(res: Awaited<ReturnType<typeof scrape>>, params: { sections: number[] }): FetchResult {
+	return buildFetchResult(res, params.sections);
+}

package/src/fuse.ts

@@ -5,6 +5,7 @@ export interface FusedHit extends SearchHit {
 	engines: string[];
 	/** Per-engine rank of this hit: {serper: 1} = serper ranked it #1. */
 	engineRanks: { engine: string; rank: number }[];
+	twinHosts?: number;
 	rrfScore: number;
 	finalScore: number;
 	junk: boolean;
@@ -18,6 +19,8 @@ const RRF_K = 60;
 // Tiny on purpose: RRF already rewards multi-engine agreement, and the bonus
 // must never scale with how many engines happen to be configured.
 const CONSENSUS_TIEBREAK = 0.003;
+const TWIN_HOST_PENALTY = 0.15;
+const TWIN_HOST_CAP = 2;
 
 // Dense/semantic ranking dilutes rare technical tokens (CVE-2026-…, pgvector,
 // error codes): a verbatim title/snippet hit gets a bounded nudge, never total.
@@ -189,7 +192,7 @@ function normTitle(s: string): string {
 		.trim();
 }
 
-function isNearDuplicate(a: FusedHit, b: FusedHit): boolean {
+function isNearDuplicate(a: { title: string }, b: { title: string }): boolean {
 	const A = normTitle(a.title);
 	const B = normTitle(b.title);
 	// dedupe both sides so a repeated word can't inflate the ratio past 1.0
@@ -203,11 +206,16 @@ function isNearDuplicate(a: FusedHit, b: FusedHit): boolean {
 	return (A.startsWith(B) || B.startsWith(A)) && Math.abs(ta.length - tb.length) <= 1;
 }
 
-function collapseNearDuplicates(fused: FusedHit[]): FusedHit[] {
+function collapseNearDuplicates(fused: FusedHit[]): { kept: FusedHit[]; twins: number[] } {
 	const kept: FusedHit[] = [];
+	const twins: number[] = [];
 	for (const h of fused) {
 		const twin = kept.find((k) => isNearDuplicate(k, h));
 		if (twin) {
+			if (hostOf(h.url) !== hostOf(twin.url)) {
+				const idx = kept.indexOf(twin);
+				twins[idx] = (twins[idx] ?? 0) + 1;
+			}
 			twin.engines = [...new Set([...twin.engines, ...h.engines])];
 			const best = new Map(twin.engineRanks.map((r) => [r.engine, r]));
 			for (const r of h.engineRanks) {
@@ -221,7 +229,7 @@ function collapseNearDuplicates(fused: FusedHit[]): FusedHit[] {
 		}
 		kept.push(h);
 	}
-	return kept;
+	return { kept, twins };
 }
 
 export function queryTermCoverage(hits: { title: string; snippet: string }[], query: string): number {
@@ -235,6 +243,15 @@ export function queryTermCoverage(hits: { title: string; snippet: string }[], qu
 	return matched / terms.length;
 }
 
+const COVERAGE_LOW_THRESHOLD = 0.5;
+
+export function coverageLow(hits: { title: string; snippet: string }[], query: string): boolean[] {
+	if (!query.trim()) return [];
+	const setCoverage = queryTermCoverage(hits, query);
+	if (setCoverage < COVERAGE_LOW_THRESHOLD) return hits.map(() => false);
+	return hits.map((h) => queryTermCoverage([h], query) < COVERAGE_LOW_THRESHOLD);
+}
+
 export function fuse(outcomes: EngineOutcome[], opts: FuseOptions = {}): FusedHit[] {
 	const { query } = opts;
 	const terms = query ? queryTerms(query) : [];
@@ -276,7 +293,13 @@ export function fuse(outcomes: EngineOutcome[], opts: FuseOptions = {}): FusedHi
 		});
 	}
 	const sorted = fused.toSorted((a, b) => b.finalScore - a.finalScore);
-	return collapseNearDuplicates(sorted);
+	const { kept, twins } = collapseNearDuplicates(sorted);
+	const penalized = kept.map((h, i) => ({
+		...h,
+		twinHosts: twins[i] ?? 0,
+		finalScore: Math.round(h.finalScore * (1 - TWIN_HOST_PENALTY * Math.min(twins[i] ?? 0, TWIN_HOST_CAP)) * 1e5) / 1e5,
+	}));
+	return penalized.toSorted((a, b) => b.finalScore - a.finalScore);
 }
 
 export function diversifyByHost(hits: FusedHit[], maxPerHost = 2): FusedHit[] {

package/src/keys.ts

@@ -16,8 +16,6 @@ import {
 	type ProviderInfo,
 } from './config';
 
-const PROVIDER_IDS = PROVIDER_LIST.map((p) => p.id);
-
 function providerNames(p: ProviderInfo): string[] {
 	return [p.envName, ...p.aliases];
 }
@@ -70,38 +68,13 @@ export function removeKey(p: ProviderInfo): void {
 	mutateFile(allNamesRemoved(p));
 }
 
-export function statusText(): string {
-	const st = keyStatus();
-	const origins = keyOrigins();
-	const armed = PROVIDER_IDS.filter((id) => st[id]);
-	const dormant = PROVIDER_IDS.filter((id) => !st[id]);
-	const cfg = getConfig();
-	const lines: string[] = [];
-	lines.push(`web tools: ${existsSync(disabledFlagPath()) ? 'OFF' : 'ON'}`);
-	lines.push(`web search: ${PROVIDER_IDS.length} pockets`);
-	lines.push(
-		armed.length ? `  armed:   ${armed.map((id) => `${id} ✓(${origins[id] ?? '?'})`).join(', ')}` : '  armed:   (none)',
-	);
-	lines.push(`  dormant: ${dormant.length ? dormant.join(', ') : '(none)'}`);
-	const chain = cfg.fetchChain
-		? cfg.fetchChain.replace(/,/g, ', ')
-		: 'jina, tinyfish, firecrawl, tavily, exa, serper, you, kagi';
-	lines.push(`  fetch:   local → ${chain}`);
-	const knobs = [];
-	if (cfg.searchEngines) knobs.push(`WSEARCH_ENGINES=${cfg.searchEngines}`);
-	if (cfg.fetchChain) knobs.push(`WSEARCH_FETCH_CHAIN=${cfg.fetchChain}`);
-	if (knobs.length) lines.push(`  knobs:   ${knobs.join(' · ')}`);
-	if (armed.length === 0) {
-		lines.push('');
-		lines.push('Getting started (30 seconds):');
-		lines.push('  1. get one free key; cheapest paths:');
-		lines.push('     · tavily    → tavily.com    (1,000 free searches/mo)');
-		lines.push('     · exa       → exa.ai        (1,000 free/mo)');
-		lines.push('     · firecrawl → firecrawl.dev (1,000 free credits/mo)');
-		lines.push('     · tinyfish  → tinyfish.ai   (free at any balance)');
-		lines.push('     · jina      → jina.ai       (free tier, search + reader)');
-		lines.push('  2. /websearch login, saved to wsearch/env');
-		lines.push('  3. run /websearch again; it should appear under "armed"');
-	}
-	return lines.join('\n');
+function originLabel(id: string, origin?: string): string {
+	if (origin === 'wsearch') return 'wsearch env';
+	if (origin === 'env') return `env: ${envNameFor(id)}`;
+	return 'shell';
+}
+
+function envNameFor(id: string): string {
+	const spec = PROVIDER_LIST.find((p) => p.id === id);
+	return spec?.envName ?? id;
 }

package/src/login.ts

@@ -0,0 +1,258 @@
+import { existsSync } from 'node:fs';
+import type { ExtensionCommandContext, Theme } from '@earendil-works/pi-coding-agent';
+import { Input, Key, fuzzyFilter, matchesKey, type Component } from '@earendil-works/pi-tui';
+import {
+	keyFilePath,
+	keyOrigins,
+	keyStatus,
+	resetConfigCache,
+	PROVIDERS,
+	disabledFlagPath,
+	type ProviderInfo,
+} from './config';
+import { clearDeadEngines } from './engines';
+import { writeKey, removeKey } from './keys';
+
+interface ProviderPick {
+	id: string;
+	line: string;
+}
+
+function providerById(id: string | undefined): ProviderInfo | undefined {
+	if (!id) return undefined;
+	const n = id.trim().toLowerCase();
+	return PROVIDERS.find((p) => p.id === n || p.label.toLowerCase().includes(n));
+}
+
+export function websearchStatusLine(): string {
+	if (existsSync(disabledFlagPath())) return 'websearch is off · /websearch on to re-enable';
+	const st = keyStatus();
+	const origins = keyOrigins();
+	const armed = PROVIDERS.filter((p) => st[p.id]).map((p) => `${p.id} ✓${origins[p.id] ?? '?'}`);
+	if (armed.length === 0) return 'websearch is on · no providers · /websearch login';
+	return `websearch is on · ${armed.join(', ')}`;
+}
+
+function providerPick(p: ProviderInfo, armed: boolean, origin?: string): ProviderPick {
+	const status = armed ? `✓ ${origin === 'wsearch' ? 'wsearch env' : `env: ${p.envName}`}` : 'not configured';
+	return { id: p.id, line: `${p.label} · ${status}` };
+}
+
+function sortProviders(list: ProviderInfo[]): ProviderInfo[] {
+	return [...list].toSorted((a, b) => a.label.localeCompare(b.label));
+}
+
+class ProviderPicker implements Component {
+	private filtered: ProviderPick[];
+	private selectedIndex = 0;
+	private searchInput = new Input({ placeholder: 'Type to filter' });
+	private cached: string[] | undefined;
+
+	constructor(
+		private tui: { requestRender(): void },
+		private theme: Theme,
+		private all: ProviderPick[],
+		private title: string,
+		private onPick: (id: string) => void,
+		private onClose: () => void,
+	) {
+		this.filtered = all;
+		this.searchInput.onSubmit = () => {
+			const picked = this.filtered[this.selectedIndex];
+			if (picked) this.onPick(picked.id);
+		};
+	}
+
+	render(width: number): string[] {
+		if (this.cached) return this.cached;
+		const lines: string[] = [];
+		const bar = '─'.repeat(Math.max(10, width));
+		lines.push(this.theme.fg('accent', bar));
+		lines.push('');
+		lines.push(this.theme.fg('accent', this.theme.bold(this.title)));
+		lines.push('');
+		for (const l of this.searchInput.render(width)) lines.push(l);
+		lines.push('');
+		const max = 7;
+		const n = this.filtered.length;
+		const start = n <= max ? 0 : Math.max(0, Math.min(this.selectedIndex - 3, n - max));
+		const end = Math.min(start + max, n);
+		for (let i = start; i < end; i++) {
+			const item = this.filtered[i];
+			if (!item) continue;
+			const selected = i === this.selectedIndex;
+			const sep = item.line.indexOf(' · ');
+			const name = sep >= 0 ? item.line.slice(0, sep).trim() : item.line.trim();
+			const status = sep >= 0 ? item.line.slice(sep + 3).trim() : '';
+			const configured = !status.startsWith('not configured');
+			const statusPart = configured
+				? this.theme.fg('success', '✓ ') + this.theme.fg('muted', status.replace(/^✓\s*/, ''))
+				: this.theme.fg('dim', 'not configured');
+			if (selected) {
+				lines.push(this.theme.fg('accent', '→ ') + this.theme.fg('accent', this.theme.bold(name)) + ' ' + statusPart);
+			} else {
+				lines.push('  ' + this.theme.fg('text', name) + ' ' + statusPart);
+			}
+		}
+		if (n === 0) lines.push(this.theme.fg('muted', '  No matching providers'));
+		if (start > 0 || end < n) lines.push(this.theme.fg('muted', `  (${this.selectedIndex + 1}/${n})`));
+		lines.push('');
+		lines.push(this.theme.fg('dim', '  ↑↓ navigate · Type to filter · Enter to select · Esc to cancel'));
+		lines.push(this.theme.fg('accent', bar));
+		this.cached = lines;
+		return lines;
+	}
+
+	invalidate(): void {
+		this.cached = undefined;
+	}
+
+	handleInput(data: string): void {
+		this.cached = undefined;
+		if (matchesKey(data, Key.up)) {
+			if (this.filtered.length) {
+				this.selectedIndex = (this.selectedIndex - 1 + this.filtered.length) % this.filtered.length;
+				this.tui.requestRender();
+			}
+			return;
+		}
+		if (matchesKey(data, Key.down)) {
+			if (this.filtered.length) {
+				this.selectedIndex = (this.selectedIndex + 1) % this.filtered.length;
+				this.tui.requestRender();
+			}
+			return;
+		}
+		if (matchesKey(data, Key.enter)) {
+			const picked = this.filtered[this.selectedIndex];
+			if (picked) this.onPick(picked.id);
+			return;
+		}
+		if (matchesKey(data, Key.escape)) {
+			this.onClose();
+			return;
+		}
+		this.searchInput.handleInput(data);
+		const q = this.searchInput.getValue();
+		this.filtered = q ? fuzzyFilter(this.all, q, (r) => r.line) : this.all;
+		this.selectedIndex = Math.max(0, Math.min(this.selectedIndex, this.filtered.length - 1));
+		this.tui.requestRender();
+	}
+}
+
+async function pickProvider(
+	ctx: ExtensionCommandContext,
+	title: string,
+	picks: ProviderPick[],
+): Promise<string | undefined> {
+	return ctx.ui.custom<string | undefined>(
+		(tui, theme, _kb, done) =>
+			new ProviderPicker(
+				tui as any,
+				theme as any,
+				picks,
+				title,
+				(id) => done(id),
+				() => done(undefined),
+			),
+	);
+}
+
+export async function handleLogin(ctx: ExtensionCommandContext, name?: string): Promise<void> {
+	if (!ctx.hasUI || ctx.mode === 'print') {
+		ctx.ui.notify('/websearch login is interactive-only', 'warning');
+		return;
+	}
+	let provider = providerById(name);
+	if (name && !provider) {
+		const fuzzy = PROVIDERS.filter((p) => p.id.includes(name.toLowerCase()) || name.toLowerCase().includes(p.id));
+		ctx.ui.notify(
+			fuzzy.length
+				? `unknown provider "${name}": did you mean ${fuzzy.map((p) => p.id).join(', ')}?`
+				: `unknown provider "${name}". Run /websearch to see the ${PROVIDERS.length} supported pockets.`,
+			'warning',
+		);
+		return;
+	}
+	if (!provider) {
+		const st = keyStatus();
+		const rowOrigins = keyOrigins();
+		const picks = sortProviders(PROVIDERS).map((p) => providerPick(p, !!st[p.id], rowOrigins[p.id]));
+		const pickedId = await pickProvider(ctx, 'Select provider to configure:', picks);
+		provider = pickedId ? providerById(pickedId) : undefined;
+		if (!provider) {
+			ctx.ui.notify('cancelled', 'info');
+			return;
+		}
+	}
+	const value = await ctx.ui.input(`Paste your ${provider.id} key`, 'key, or !command (Keychain/1Password)');
+	if (!value) {
+		ctx.ui.notify('cancelled', 'info');
+		return;
+	}
+	const trimmed = value.trim();
+	if (trimmed.length < 8) {
+		ctx.ui.notify('value too short to be a key', 'warning');
+		return;
+	}
+	const isCmd = trimmed.startsWith('!');
+	const ok = await ctx.ui.confirm(
+		`Write ${provider.id} to ${keyFilePath()}?`,
+		isCmd
+			? 'Stored as a !command: the secret itself never touches disk.'
+			: 'Stored in the dedicated wsearch env file (0600).',
+	);
+	if (!ok) {
+		ctx.ui.notify('cancelled', 'info');
+		return;
+	}
+	writeKey(provider, trimmed);
+	resetConfigCache();
+	clearDeadEngines();
+	ctx.ui.notify(`${provider.id} saved ✓`, 'info');
+}
+
+export async function handleLogout(ctx: ExtensionCommandContext, name?: string): Promise<void> {
+	const interactive = (): boolean => !!(ctx.hasUI && ctx.mode !== 'print');
+	const origins = keyOrigins();
+	const armed = PROVIDERS.filter((p) => origins[p.id]);
+	let provider = providerById(name);
+	if (name && !provider) {
+		ctx.ui.notify(`unknown provider "${name}". Run /websearch to see the supported pockets.`, 'warning');
+		return;
+	}
+	if (!provider) {
+		if (armed.length === 0) {
+			ctx.ui.notify('no providers are configured: nothing to remove.', 'info');
+			return;
+		}
+		if (!interactive()) {
+			ctx.ui.notify('/websearch logout is interactive-only', 'warning');
+			return;
+		}
+		const logoutPicks = sortProviders(armed).map((p) => providerPick(p, true, origins[p.id]));
+		const pickedId = await pickProvider(ctx, 'Select provider to logout:', logoutPicks);
+		provider = pickedId ? providerById(pickedId) : undefined;
+	}
+	if (!provider) {
+		ctx.ui.notify('cancelled', 'info');
+		return;
+	}
+	if (origins[provider.id] !== 'wsearch') {
+		ctx.ui.notify(
+			`${provider.id} is configured via ${origins[provider.id]}, not the wsearch env file: nothing to remove here.`,
+			'info',
+		);
+		return;
+	}
+	if (!interactive()) {
+		ctx.ui.notify('/websearch logout is interactive-only', 'warning');
+		return;
+	}
+	const ok = await ctx.ui.confirm(`Remove ${provider.id} from ${keyFilePath()}?`, '');
+	if (!ok) return;
+	removeKey(provider);
+	resetConfigCache();
+	clearDeadEngines();
+	ctx.ui.notify(`${provider.id} removed`, 'info');
+}

package/src/render.ts

@@ -0,0 +1,110 @@
+import type { Theme } from '@earendil-works/pi-coding-agent';
+import { keyHint, keyText, rawKeyHint } from '@earendil-works/pi-coding-agent';
+import { type Component } from '@earendil-works/pi-tui';
+import { READ_FIRST_TAG, wrappedLineKind, wrapAll } from './view';
+
+const PREVIEW_LINES = 7;
+
+export function italicizeIndices(text: string, theme: Theme): string {
+	return text
+		.replace(/^(indices - 0 - \d+ sections?)$/gm, (m) => theme.italic(m))
+		.replace(/^- \*(\d+)\* /gm, (_, n: string) => `- ${theme.italic(n)} `);
+}
+
+function italicizeRanks(text: string, theme: Theme): string {
+	return text.replace(/^(\s*)(\d+)\.(?=\s)/gm, (m, pad: string, n: string) => `${pad}${theme.italic(n)}.`);
+}
+
+const ANSI_SGR = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g');
+
+const DEFAULT_WRAP_COLS = 90;
+const MAX_WRAP_COLS = 240;
+
+export function terminalWrapWidth(): number {
+	const cols = process.stdout.columns ?? DEFAULT_WRAP_COLS;
+	return Math.min(MAX_WRAP_COLS, Math.max(60, cols - 4));
+}
+
+export class WrappedSearchText implements Component {
+	private raw: string;
+	private theme: Theme;
+	private expanded: boolean;
+	constructor(raw: string, theme: Theme, expanded: boolean) {
+		this.raw = raw;
+		this.theme = theme;
+		this.expanded = expanded;
+	}
+	invalidate(): void {}
+	render(width: number): string[] {
+		const wrapped = wrapAll(this.raw, Math.max(40, width));
+		const lines = wrapped.split('\n');
+		const expanded = this.expanded || lines.length <= PREVIEW_LINES;
+		const shown = expanded ? lines : lines.slice(0, PREVIEW_LINES);
+		const colored = italicizeRanks(shown.join('\n'), this.theme)
+			.split('\n')
+			.map((l) => (this.expanded ? colorLine(l, this.theme) : this.theme.fg('dim', l)))
+			.join('\n');
+		const out = replaceTabs(colored).split('\n');
+		if (!expanded) {
+			const hint = moreLinesHint(shown.length, lines.length, this.theme);
+			if (hint) out.push(hint);
+		}
+		return out;
+	}
+}
+
+function colorLine(l: string, theme: Theme): string {
+	const plain = l.replace(ANSI_SGR, '');
+	if (
+		plain.includes('…[section truncated') ||
+		plain.includes('…(continues at outline index') ||
+		plain.startsWith('…[clipped by tool]') ||
+		plain.startsWith('…[truncated]')
+	)
+		return theme.fg('warning', l);
+	if (plain.startsWith('## ') && plain.endsWith(READ_FIRST_TAG))
+		return theme.fg('accent', l.slice(0, l.length - READ_FIRST_TAG.length)) + theme.bold(READ_FIRST_TAG);
+	if (/^\s*· recommended first read$/.test(plain)) return theme.bold(l.trimEnd());
+	const wrapped = wrappedLineKind(plain);
+	if (wrapped === 'index') return theme.fg('dim', l);
+	if (wrapped === 'chapter') return theme.fg('accent', l);
+	if (plain.startsWith('## ')) return theme.fg('accent', l);
+	if (plain.startsWith('- ')) return theme.fg('dim', l);
+	if (plain.startsWith('[rendered via')) return theme.fg('muted', l);
+	if (/^\s*\d+\. /.test(plain)) return theme.fg('dim', l);
+	return l;
+}
+
+export function colorizedResult(text: string, theme: Theme): string {
+	return text
+		.split('\n')
+		.map((l) => colorLine(l, theme))
+		.join('\n');
+}
+
+export function resultText(result: { content: { type: string; text?: string }[] }): string {
+	return (result.content[0] as { type: string; text: string } | undefined)?.text ?? '';
+}
+
+function moreLinesHint(shown: number, total: number, theme: Theme): string {
+	const remaining = total - shown;
+	if (remaining <= 0) return '';
+	let hint: string;
+	try {
+		hint = keyText('app.tools.expand') ? keyHint('app.tools.expand', 'to expand') : rawKeyHint('ctrl+o', 'to expand');
+	} catch {
+		hint = rawKeyHint('ctrl+o', 'to expand');
+	}
+	return theme.fg('muted', `... (${remaining} more lines,`) + ' ' + hint + theme.fg('muted', ')');
+}
+
+export function previewWithHint(lines: string[], theme: Theme, maxLines = PREVIEW_LINES): string {
+	const shown = lines.slice(0, maxLines);
+	const text = shown.map((l) => theme.fg('dim', replaceTabs(l))).join('\n');
+	const hint = moreLinesHint(shown.length, lines.length, theme);
+	return hint ? `${text}\n${hint}` : text;
+}
+
+function replaceTabs(text: string): string {
+	return text.replace(/\t/g, '   ');
+}

package/src/tools.ts

@@ -3,9 +3,8 @@
  * evicted on 2026-10-05; do not add a research surface or nested-LLM path.
  */
 import { Type } from 'typebox';
-import type { ExtensionAPI, ExtensionCommandContext, Theme } from '@earendil-works/pi-coding-agent';
-import { keyHint, keyText, rawKeyHint } from '@earendil-works/pi-coding-agent';
-import { Text, type Component } from '@earendil-works/pi-tui';
+import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
+import { Text } from '@earendil-works/pi-tui';
 import { appendFileSync, mkdirSync, rmSync, writeFileSync, existsSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import {
@@ -21,10 +20,20 @@ import {
 import { ABORT_ERROR, DEFAULT_MAX_RESULTS } from './constants';
 import { cacheKey, openCache, CACHE_TTL_HOURS } from './cache';
 import { clearDeadEngines, runEngines, type EngineOutcome } from './engines';
-import { fuse, diversifyByHost, queryTermCoverage } from './fuse';
+import { fuse, diversifyByHost, queryTermCoverage, coverageLow } from './fuse';
 import { normalizeUrl } from './urls';
-import { buildSearchText, READ_FIRST_TAG, wrappedLineKind, wrapAll } from './view';
+import { buildSearchText } from './view';
 import { scrape } from './scrape';
+import {
+	colorizedResult,
+	italicizeIndices,
+	previewWithHint,
+	resultText,
+	terminalWrapWidth,
+	WrappedSearchText,
+} from './render';
+import { makeFetchResult } from './fetch';
+import { handleLogin, handleLogout, websearchStatusLine } from './login';
 
 const SEARCH_PARAMS = Type.Object({
 	query: Type.String({
@@ -54,6 +63,12 @@ const SEARCH_OUTPUT = Type.Object({
 			title: Type.String(),
 			engines: Type.Array(Type.String()),
 			score: Type.Number(),
+			coverage_low: Type.Optional(
+				Type.Boolean({
+					description:
+						'The hit matched only a few query terms while the result set matched most. Stuffing signature; distrust.',
+				}),
+			),
 		}),
 	),
 });
@@ -184,6 +199,7 @@ export function registerWebTools(pi: ExtensionAPI) {
 			const visible = candidates.slice(0, params.max_results ?? DEFAULT_MAX_RESULTS);
 			const enginesUsed = [...new Set(fused.flatMap((r) => r.engines))];
 			const offtopic = visible.length > 0 && queryTermCoverage(visible, params.query) < 0.5;
+			const lowCoverage = coverageLow(visible, params.query);
 			visible.forEach((r, i) => {
 				lastSearch.urlRank.set(normalizeUrl(r.url), {
 					rank: i + 1,
@@ -213,12 +229,13 @@ export function registerWebTools(pi: ExtensionAPI) {
 					engines_used: enginesUsed,
 					engines_errored: errored,
 					...(offtopic ? { results_look_offtopic: true } : {}),
-					results: visible.map((r) => ({
+					results: visible.map((r, i) => ({
 						url: r.url,
 						title: r.title,
 						engines: r.engines,
 						ranks: r.engineRanks,
 						score: r.finalScore,
+						...(lowCoverage[i] ? { coverage_low: true } : {}),
 					})),
 				},
 			};
@@ -307,360 +324,6 @@ export function registerWebTools(pi: ExtensionAPI) {
 	});
 }
 
-/** Response caps. The composed fetch view is bounded to what a turn needs;
- * structuredContent mirrors the same content trimmed for schema. */
-const MAX_FETCH_TEXT = 16000;
-type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
-
-type FetchDetails = {
-	url: string;
-	title: string;
-	renderer: string;
-	outline: string[];
-	sections: { index: number; heading: string; text: string }[];
-	section_count: number;
-	chars?: number;
-	sections_requested?: number[];
-	sections_missing?: number[];
-	sections_truncated?: number;
-	sections_clipped?: number[];
-	error?: string;
-};
-
-interface FetchResult {
-	content: { type: 'text'; text: string }[];
-	details: FetchDetails;
-	structuredContent: JsonValue;
-	isError?: boolean;
-}
-
-const PREVIEW_LINES = 10;
-const SEARCH_PREVIEW_LINES = 8;
-
-/** Display-only: the model-facing markers (`indices …`, `- *N*`) render as
- * genuine ANSI italics in the TUI; the content text itself is untouched. */
-function italicizeIndices(text: string, theme: Theme): string {
-	return text
-		.replace(/^(indices - 0 - \d+ sections?)$/gm, (m) => theme.italic(m))
-		.replace(/^- \*(\d+)\* /gm, (_, n: string) => `- ${theme.italic(n)} `);
-}
-
-function italicizeRanks(text: string, theme: Theme): string {
-	return text.replace(/^(\s*)(\d+)\.(?=\s)/gm, (m, pad: string, n: string) => `${pad}${theme.italic(n)}.`);
-}
-
-const ANSI_SGR = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g');
-
-const DEFAULT_WRAP_COLS = 90;
-const MAX_WRAP_COLS = 240;
-
-function terminalWrapWidth(): number {
-	const cols = process.stdout.columns ?? DEFAULT_WRAP_COLS;
-	return Math.min(MAX_WRAP_COLS, Math.max(60, cols - 4));
-}
-
-class WrappedSearchText implements Component {
-	private raw: string;
-	private theme: Theme;
-	private expanded: boolean;
-	constructor(raw: string, theme: Theme, expanded: boolean) {
-		this.raw = raw;
-		this.theme = theme;
-		this.expanded = expanded;
-	}
-	invalidate(): void {}
-	render(width: number): string[] {
-		const wrapped = wrapAll(this.raw, Math.max(40, width));
-		const lines = wrapped.split('\n');
-		const expanded = this.expanded || lines.length <= SEARCH_PREVIEW_LINES;
-		const shown = expanded ? lines : lines.slice(0, SEARCH_PREVIEW_LINES);
-		const colored = italicizeRanks(shown.join('\n'), this.theme)
-			.split('\n')
-			.map((l) => (this.expanded ? colorLine(l, this.theme) : this.theme.fg('dim', l)))
-			.join('\n');
-		const out = replaceTabs(colored).split('\n');
-		if (!expanded) {
-			const hint = moreLinesHint(shown.length, lines.length, this.theme);
-			if (hint) out.push(hint);
-		}
-		return out;
-	}
-}
-
-function colorLine(l: string, theme: Theme): string {
-	const plain = l.replace(ANSI_SGR, '');
-	if (
-		plain.includes('…[section truncated') ||
-		plain.includes('…(continues at outline index') ||
-		plain.startsWith('…[clipped by tool]') ||
-		plain.startsWith('…[truncated]')
-	)
-		return theme.fg('warning', l);
-	if (plain.startsWith('## ') && plain.endsWith(READ_FIRST_TAG))
-		return theme.fg('accent', l.slice(0, l.length - READ_FIRST_TAG.length)) + theme.bold(READ_FIRST_TAG);
-	if (/^\s*· recommended first read$/.test(plain)) return theme.bold(l.trimEnd());
-	const wrapped = wrappedLineKind(plain);
-	if (wrapped === 'index') return theme.fg('dim', l);
-	if (wrapped === 'chapter') return theme.fg('accent', l);
-	if (plain.startsWith('## ')) return theme.fg('accent', l);
-	if (plain.startsWith('- ')) return theme.fg('dim', l);
-	if (plain.startsWith('[rendered via')) return theme.fg('muted', l);
-	if (/^\s*\d+\. /.test(plain)) return theme.fg('dim', l);
-	return l;
-}
-
-function colorizedResult(text: string, theme: Theme): string {
-	return text
-		.split('\n')
-		.map((l) => colorLine(l, theme))
-		.join('\n');
-}
-
-function resultText(result: { content: { type: string; text?: string }[] }): string {
-	return (result.content[0] as { type: string; text: string } | undefined)?.text ?? '';
-}
-
-function moreLinesHint(shown: number, total: number, theme: Theme): string {
-	const remaining = total - shown;
-	if (remaining <= 0) return '';
-	let hint: string;
-	try {
-		hint = keyText('app.tools.expand') ? keyHint('app.tools.expand', 'to expand') : rawKeyHint('ctrl+o', 'to expand');
-	} catch {
-		hint = rawKeyHint('ctrl+o', 'to expand');
-	}
-	return theme.fg('muted', `… (${remaining} more lines,`) + ' ' + hint + theme.fg('muted', ')');
-}
-
-function previewWithHint(lines: string[], theme: Theme, maxLines = PREVIEW_LINES): string {
-	const shown = lines.slice(0, maxLines);
-	const text = shown.map((l) => theme.fg('dim', replaceTabs(l))).join('\n');
-	const hint = moreLinesHint(shown.length, lines.length, theme);
-	return hint ? `${text}\n${hint}` : text;
-}
-
-function replaceTabs(text: string): string {
-	return text.replace(/\t/g, '   ');
-}
-
-function fetchErrorResult(details: FetchDetails): FetchResult {
-	const error = String(details.error ?? 'unknown error');
-	return {
-		content: [{ type: 'text', text: `WebFetch failed: ${error}` }],
-		details,
-		structuredContent: { ...details },
-		isError: true,
-	};
-}
-
-function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: number[]): FetchResult {
-	const want = [...new Set(sections.map((i) => Math.max(0, Math.min(res.sections.length - 1, i))))];
-	const body = res.outline.length
-		? `indices - 0 - ${res.outline.length - 1} section${res.outline.length === 1 ? '' : 's'}\n${res.outline.map((h, i) => `- *${i}* ${h}`).join('\n')}\n\n`
-		: '';
-	const parts: string[] = [];
-	want.forEach((idx) => {
-		const s = res.sections[idx];
-		if (s === undefined) return;
-		const head = res.outline[idx] ? `## [${idx}] ${res.outline[idx]}` : `## [${idx}]`;
-		parts.push(`${head}\n\n${s}`);
-	});
-	let text =
-		(res.title ? `# ${res.title}\n\n` : '') + body + parts.join('\n\n') + (res.truncated ? '\n…[truncated]' : '');
-	text = text.replace(/\n{3,}/g, '\n\n');
-	if (res.title) {
-		const t = res.title.trim();
-		const lines = text.split('\n');
-		const headIdx = lines.findIndex((l) => l.startsWith('# '));
-		if (headIdx >= 0) {
-			for (let i = headIdx + 1; i < Math.min(lines.length, headIdx + 6); i++) {
-				const l = lines[i].replace(/^#+\s*/, '').trim();
-				if (l === t) {
-					lines.splice(i, 1);
-					break;
-				}
-			}
-		}
-		text = lines.join('\n').replace(/\n{3,}/g, '\n\n');
-	}
-	if (res.renderer && res.renderer !== 'local') text += `\n\n[rendered via ${res.renderer}]`;
-	const clipped = text.length > MAX_FETCH_TEXT ? text.slice(0, MAX_FETCH_TEXT) + '\n…[clipped by tool]' : text;
-	const missing = sections.filter((i) => i >= res.sections.length);
-	// continuation cuts (res.continued) vs local budget clips (clippedSections)
-	const truncatedCount = want.filter((idx) => res.continued?.includes(idx)).length;
-
-	// same budget as the text view; headings and the '…' marker count against it
-	let budget = MAX_FETCH_TEXT;
-	const structuredSections: { index: number; heading: string; text: string }[] = [];
-	const clippedSections: number[] = [];
-	for (const idx of want) {
-		if (budget <= 0) {
-			clippedSections.push(idx);
-			continue;
-		}
-		const s = res.sections[idx];
-		if (s === undefined) continue;
-		const heading = res.outline[idx] ?? '';
-		const fullCost = heading.length + s.length;
-		if (fullCost <= budget) {
-			structuredSections.push({ index: idx, heading, text: s });
-			budget -= fullCost;
-			continue;
-		}
-		if (heading.length >= budget) {
-			structuredSections.push({ index: idx, heading: heading.slice(0, Math.max(0, budget - 1)) + '…', text: '' });
-		} else {
-			const textLen = Math.max(0, budget - heading.length - 1);
-			structuredSections.push({ index: idx, heading, text: s.slice(0, textLen) + (textLen < s.length ? '…' : '') });
-		}
-		clippedSections.push(idx);
-		budget = 0;
-	}
-	const details = {
-		url: res.url,
-		title: res.title,
-		renderer: res.renderer,
-		outline: res.outline,
-		sections: structuredSections,
-		section_count: structuredSections.length,
-		chars: clipped.length,
-		sections_requested: [...new Set(sections)],
-		sections_missing: [...new Set(missing)],
-		sections_truncated: truncatedCount,
-		sections_clipped: clippedSections,
-		...(res.publishedAt ? { publishedAt: res.publishedAt } : {}),
-		...(res.paywalled ? { paywalled: true } : {}),
-		...(res.error ? { error: res.error } : {}),
-	};
-	if (res.error) return fetchErrorResult(details);
-	return { content: [{ type: 'text', text: clipped }], details, structuredContent: { ...details } };
-}
-
-function makeFetchResult(res: Awaited<ReturnType<typeof scrape>>, params: { sections: number[] }): FetchResult {
-	return buildFetchResult(res, params.sections);
-}
-
-import { statusText, writeKey, removeKey } from './keys';
-
-function providerById(id: string | undefined): ProviderInfo | undefined {
-	if (!id) return undefined;
-	const n = id.trim().toLowerCase();
-	return PROVIDERS.find((p) => p.id === n || p.label.toLowerCase().includes(n));
-}
-
-async function promptProvider(
-	ctx: ExtensionCommandContext,
-	title: string,
-	options: string[],
-): Promise<ProviderInfo | undefined> {
-	const picked = await ctx.ui.select(title, options);
-	if (!picked) return undefined;
-	const idToken = picked.trim().split(/\s+/)[0];
-	return providerById(idToken);
-}
-
-async function handleLogin(ctx: ExtensionCommandContext, name?: string): Promise<void> {
-	// gate before ANY prompt: print mode must fail without side effects
-	if (!ctx.hasUI || ctx.mode === 'print') {
-		ctx.ui.notify('/websearch login is interactive-only', 'warning');
-		return;
-	}
-	let provider = providerById(name);
-	if (name && !provider) {
-		const fuzzy = PROVIDERS.filter((p) => p.id.includes(name.toLowerCase()) || name.toLowerCase().includes(p.id));
-		ctx.ui.notify(
-			fuzzy.length
-				? `unknown provider "${name}": did you mean ${fuzzy.map((p) => p.id).join(', ')}?`
-				: `unknown provider "${name}". Run /websearch to see the ${PROVIDERS.length} supported pockets.`,
-			'warning',
-		);
-		return;
-	}
-	if (!provider) {
-		provider = await promptProvider(
-			ctx,
-			'login: choose a provider',
-			PROVIDERS.map((p) => `${p.id}${p.label !== p.id ? `: ${p.label}` : ''}`),
-		);
-		if (!provider) {
-			ctx.ui.notify('login cancelled', 'info');
-			return;
-		}
-	}
-	const value = await ctx.ui.input(`Paste your ${provider.id} key`, 'key, or !command (Keychain/1Password)');
-	if (!value) {
-		ctx.ui.notify('login cancelled', 'info');
-		return;
-	}
-	const trimmed = value.trim();
-	if (trimmed.length < 8) {
-		ctx.ui.notify('value too short to be a key', 'warning');
-		return;
-	}
-	const isCmd = trimmed.startsWith('!');
-	const ok = await ctx.ui.confirm(
-		`Write ${provider.id} to ${keyFilePath()}?`,
-		isCmd
-			? 'Stored as a !command: the secret itself never touches disk.'
-			: 'Stored in the dedicated wsearch env file (0600).',
-	);
-	if (!ok) {
-		ctx.ui.notify('login cancelled', 'info');
-		return;
-	}
-	writeKey(provider, trimmed);
-	resetConfigCache();
-	clearDeadEngines();
-	ctx.ui.notify(`${provider.id} ✓ saved to wsearch/env (${isCmd ? 'command' : 'key'})`, 'info');
-}
-
-async function handleLogout(ctx: ExtensionCommandContext, name?: string): Promise<void> {
-	const interactive = (): boolean => !!(ctx.hasUI && ctx.mode !== 'print');
-	const origins = keyOrigins();
-	const armed = PROVIDERS.filter((p) => origins[p.id]);
-	let provider = providerById(name);
-	if (name && !provider) {
-		ctx.ui.notify(`unknown provider "${name}". Run /websearch to see the supported pockets.`, 'warning');
-		return;
-	}
-	if (!provider) {
-		if (armed.length === 0) {
-			ctx.ui.notify('no providers are configured: nothing to remove.', 'info');
-			return;
-		}
-		if (!interactive()) {
-			ctx.ui.notify('/websearch logout is interactive-only', 'warning');
-			return;
-		}
-		provider = await promptProvider(
-			ctx,
-			'logout: choose a provider (wsearch = removable)',
-			armed.map((p) => `${p.id}: ${origins[p.id]}${origins[p.id] === 'wsearch' ? ' (removable)' : ' (set in config)'}`),
-		);
-	}
-	if (!provider) {
-		ctx.ui.notify('logout cancelled', 'info');
-		return;
-	}
-	if (origins[provider.id] !== 'wsearch') {
-		ctx.ui.notify(
-			`${provider.id} is configured via ${origins[provider.id]}, not the wsearch env file: nothing to remove here.`,
-			'info',
-		);
-		return;
-	}
-	if (!interactive()) {
-		ctx.ui.notify('/websearch logout is interactive-only', 'warning');
-		return;
-	}
-	const ok = await ctx.ui.confirm(`Remove ${provider.id} from ${keyFilePath()}?`, '');
-	if (!ok) return;
-	removeKey(provider);
-	resetConfigCache();
-	clearDeadEngines();
-	ctx.ui.notify(`${provider.id} removed from wsearch/env`, 'info');
-}
-
 const SEARCH_TOOL_NAMES = ['WebSearch', 'WebFetch'] as const;
 
 function toggleSearchTools(pi: ExtensionAPI, disabled: boolean): void {
@@ -705,7 +368,7 @@ export function registerWebCommand(pi: ExtensionAPI) {
 			}
 			switch (action ?? 'status') {
 				case 'status': {
-					ctx.ui.notify(statusText(), 'info');
+					ctx.ui.notify(websearchStatusLine(), 'info');
 					return;
 				}
 				case 'login':
@@ -720,7 +383,7 @@ export function registerWebCommand(pi: ExtensionAPI) {
 						return;
 					}
 					toggleSearchTools(pi, true);
-					ctx.ui.notify('WebSearch + WebFetch OFF — /websearch on re-enables', 'info');
+					ctx.ui.notify('websearch is off · /websearch on to re-enable', 'info');
 					return;
 				}
 				case 'on': {
@@ -729,7 +392,7 @@ export function registerWebCommand(pi: ExtensionAPI) {
 						return;
 					}
 					toggleSearchTools(pi, false);
-					ctx.ui.notify('WebSearch + WebFetch ON', 'info');
+					ctx.ui.notify('websearch is on', 'info');
 					return;
 				}
 				default: