pi-wsearch 0.3.1 → 0.4.0

package/README.md

@@ -39,7 +39,10 @@ and reads only those. No
 full reads, no 40 KB dumps. Sections over 3,000 characters split into
 addressable continuation entries (`Section (cont. 1)`), so no tail is ever
 unreachable; pages that overflow are capped; the final response is clipped.
-Every cut is marked — nothing is silently lost.
+Every cut is marked — nothing is silently lost. Codemode consumers get the same
+view via structuredContent: per-section `{index, heading, text}`, continuation
+cuts counted in `sections_truncated`, local budget cuts listed in
+`sections_clipped`.
 
 **Reads are free.** Page parsing is built in, nothing billed per read, no key
 required. Real headings become the map, TOC echoes get dropped, so reads come

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-wsearch",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "Pi extension: web.search + web.fetch",
   "license": "MIT",
   "type": "module",
@@ -36,7 +36,7 @@
     "check": "oxlint && oxfmt --check && tsc --noEmit && knip && bun test --jobs 1 test/",
     "test": "bun test --jobs 1 test/",
     "typecheck": "tsc --noEmit",
-    "knip": "knip",
+    "knip": "bun scripts/knip.mjs",
     "coverage": "bun test --jobs 1 test/ --coverage"
   },
   "pi": {
@@ -45,10 +45,10 @@
     ]
   },
   "peerDependencies": {
-    "@earendil-works/pi-ai": "*",
-    "@earendil-works/pi-agent-core": "*",
-    "@earendil-works/pi-coding-agent": "*",
-    "@earendil-works/pi-tui": "*",
+    "@earendil-works/pi-ai": "^1.3.0",
+    "@earendil-works/pi-agent-core": "^1.3.0",
+    "@earendil-works/pi-coding-agent": "^1.3.0",
+    "@earendil-works/pi-tui": "^1.3.0",
     "typebox": "*"
   },
   "devDependencies": {
@@ -57,9 +57,13 @@
     "knip": "^6.39.0",
     "oxfmt": "^0.71.0",
     "oxlint": "^1.86.0",
-    "typescript": "^7.0.2"
+    "typescript": "^5.9.3"
+  },
+  "dependencies": {
+    "htmlparser2": "^12.0.0",
+    "parse5": "^8.0.1",
+    "parse5-htmlparser2-tree-adapter": "^8.0.1"
   },
-  "dependencies": {},
   "main": "index.ts",
   "files": [
     "index.ts",

package/src/dom.ts

@@ -0,0 +1,328 @@
+import { parse } from 'parse5';
+import { adapter } from 'parse5-htmlparser2-tree-adapter';
+import { parseDocument } from 'htmlparser2';
+
+interface DomNode {
+	type: string;
+	name?: string;
+	attribs?: Record<string, string>;
+	data?: string;
+	children?: DomNode[];
+}
+
+const SKIP = new Set(['script', 'style', 'noscript', 'svg', 'nav', 'footer', 'aside', 'head', 'title']);
+
+const BLOCK = new Set(['p', 'div', 'section', 'article', 'blockquote', 'pre']);
+
+function isKatex(n: DomNode): boolean {
+	return (n.attribs?.['class'] ?? '').split(/\s+/).includes('katex');
+}
+
+function toDom(html: string): DomNode {
+	return parse(html, { treeAdapter: adapter }) as unknown as DomNode;
+}
+
+function inlineOf(n: DomNode): string {
+	const t = n.type;
+	if (t === 'text') return n.data ?? '';
+	if (t === 'comment' || t === 'directive') return '';
+	const name = n.name ?? '';
+	if (!name || SKIP.has(name) || isKatex(n)) return '';
+	if (name === 'math') {
+		const tex = annotationTex(n);
+		return tex ? `$${tex}$` : '';
+	}
+	if (name === 'br' || name === 'hr' || name === 'li') return ' ';
+	return (n.children ?? []).map(inlineOf).join('');
+}
+
+function annotationTex(n: DomNode): string | undefined {
+	const found: DomNode[] = [];
+	allDescendants(n, 'annotation', found);
+	for (const c of found) {
+		const enc = (c.attribs?.['encoding'] ?? '').toLowerCase();
+		if (enc === 'application/x-tex' || enc.includes('x-tex')) return inlineOf(c).trim();
+	}
+	return undefined;
+}
+
+function allDescendants(n: DomNode, name: string, out: DomNode[]): void {
+	for (const c of n.children ?? []) {
+		if (c.type === 'tag') {
+			if (c.name === name) out.push(c);
+			allDescendants(c, name, out);
+		}
+	}
+}
+
+function listText(n: DomNode, out: string[]): void {
+	for (const c of n.children ?? []) {
+		if (c.type === 'tag' && c.name === 'li') {
+			newline(out);
+			out.push('• ');
+			liText(c, out);
+		}
+	}
+}
+
+function liText(n: DomNode, out: string[]): void {
+	for (const c of n.children ?? []) {
+		const t = c.type;
+		if (t === 'text') {
+			out.push((c.data ?? '').replace(/\s+/g, ' '));
+			continue;
+		}
+		if (t === 'comment' || t === 'directive') continue;
+		const name = c.name ?? '';
+		if (SKIP.has(name) || isKatex(c)) continue;
+		if (name === 'li') {
+			newline(out);
+			out.push('• ');
+			liText(c, out);
+			continue;
+		}
+		if (name === 'ul' || name === 'ol') {
+			listText(c, out);
+			continue;
+		}
+		if (name === 'br' || name === 'hr') {
+			out.push('\n');
+			continue;
+		}
+		if (name === 'pre') {
+			out.push('\n');
+			out.push(inlineOf(c).replace(/\n/g, '\uE000'));
+			continue;
+		}
+		if (name === 'math') {
+			const tex = annotationTex(c);
+			if (tex) out.push(`$${tex}$`);
+			continue;
+		}
+		if (BLOCK.has(name) || name === 'table') {
+			newline(out);
+			liText(c, out);
+			continue;
+		}
+		out.push(inlineOf(c));
+	}
+}
+
+function renderTable(n: DomNode, out: string[]): void {
+	const rows: string[][] = [];
+	for (const tr of (() => {
+		const found: DomNode[] = [];
+		allDescendants(n, 'tr', found);
+		return found;
+	})()) {
+		const cells: string[] = [];
+		for (const cell of tr.children ?? []) {
+			if (cell.type === 'tag' && (cell.name === 'td' || cell.name === 'th')) {
+				cells.push(inlineOf(cell).replace(/\s+/g, ' ').trim().replace(/\|/g, '\\|'));
+			}
+		}
+		if (cells.length) rows.push(cells);
+	}
+	newline(out);
+	if (rows.length < 2 || Math.max(...rows.map((r) => r.length)) < 2) {
+		out.push(rows.map((r) => r.join(' ')).join('\n'));
+		out.push('\n');
+		return;
+	}
+	const cols = Math.max(...rows.map((r) => r.length));
+	const line = (r: string[]) => `| ${r.concat(Array(Math.max(0, cols - r.length)).fill(' ')).join(' | ')} |`;
+	out.push(
+		[line(rows[0]), `| ${Array.from({ length: cols }, () => '---').join(' | ')} |`, ...rows.slice(1).map(line)].join(
+			'\n',
+		),
+	);
+	out.push('\n');
+}
+
+function newline(out: string[]): void {
+	if (out.length > 0 && !out[out.length - 1].endsWith('\n')) out.push('\n');
+}
+
+export function treeToMarkdownish(html: string): string {
+	const doc = toDom(html);
+	const out: string[] = [];
+	renderBlocks(doc.children ?? [], out);
+	return out.join('');
+}
+
+function renderBlocks(nodes: DomNode[], out: string[]): void {
+	for (const n of nodes) renderBlock(n, out);
+}
+
+function renderBlock(n: DomNode, out: string[]): void {
+	const t = n.type;
+	if (t === 'text') {
+		out.push(n.data ?? '');
+		return;
+	}
+	if (t === 'comment' || t === 'directive' || t === 'style' || t === 'script') return;
+	const name = n.name ?? '';
+	if (!name || SKIP.has(name) || isKatex(n)) return;
+	switch (name) {
+		case 'h1':
+		case 'h2':
+		case 'h3':
+		case 'h4':
+		case 'h5':
+		case 'h6':
+			newline(out);
+			out.push(`${'#'.repeat(Number(name[1]))} ${inlineOf(n).trim()}\n`);
+			return;
+		case 'li':
+			newline(out);
+			out.push('• ');
+			liText(n, out);
+			return;
+		case 'ul':
+		case 'ol':
+			newline(out);
+			listText(n, out);
+			newline(out);
+			return;
+		case 'pre':
+			newline(out);
+			out.push(inlineOf(n).replace(/\n/g, '\uE000'));
+			newline(out);
+			return;
+		case 'math': {
+			const tex = annotationTex(n);
+			if (tex) out.push(` $${tex}$ `);
+			return;
+		}
+		case 'br':
+		case 'hr':
+			newline(out);
+			return;
+		case 'table':
+			renderTable(n, out);
+			return;
+		case 'p':
+		case 'div':
+		case 'section':
+		case 'article':
+		case 'blockquote':
+			newline(out);
+			renderBlocks(n.children ?? [], out);
+			newline(out);
+			return;
+		default:
+			renderBlocks(n.children ?? [], out);
+	}
+}
+
+const DATE_KEYWORDS = ['datePublished', 'dateCreated', 'uploadDate', 'publishedAt'];
+const DATE_META_NAMES = [
+	'article:published_time',
+	'datePublished',
+	'publish-date',
+	'pubdate',
+	'parsely-pub-date',
+	'date',
+];
+
+function toIso(v: string | undefined): string | undefined {
+	if (!v) return undefined;
+	const ms = Date.parse(v.trim());
+	if (Number.isNaN(ms) || ms < 0) return undefined;
+	return new Date(ms).toISOString().slice(0, 10);
+}
+
+function jsonLdDate(nodes: DomNode[]): string | undefined {
+	const pick = (value: unknown): string | undefined => {
+		if (typeof value === 'string') {
+			const iso = toIso(value);
+			if (iso) return iso;
+		}
+		return undefined;
+	};
+	const walk = (value: unknown): string | undefined => {
+		if (Array.isArray(value)) {
+			for (const v of value) {
+				const hit = walk(v);
+				if (hit) return hit;
+			}
+			return undefined;
+		}
+		if (typeof value === 'object' && value !== null) {
+			for (const key of DATE_KEYWORDS) {
+				const hit = pick((value as Record<string, unknown>)[key]);
+				if (hit) return hit;
+			}
+			for (const v of Object.values(value as Record<string, unknown>)) {
+				const hit = walk(v);
+				if (hit) return hit;
+			}
+		}
+		return undefined;
+	};
+	const visit = (n: DomNode): string | undefined => {
+		if (n.type === 'tag' && n.name === 'script') {
+			const type = (n.attribs?.['type'] ?? '').toLowerCase().trim();
+			if (type === 'application/ld+json' || type === 'application/json' || type === 'ld+json') {
+				try {
+					const hit = walk(JSON.parse(n.data ?? ''));
+					if (hit) return hit;
+				} catch {}
+			}
+		}
+		for (const c of n.children ?? []) {
+			const hit = visit(c);
+			if (hit) return hit;
+		}
+		return undefined;
+	};
+	for (const n of nodes) {
+		const hit = visit(n);
+		if (hit) return hit;
+	}
+	return undefined;
+}
+
+export interface HeadScan {
+	title: string;
+	publishedAt?: string;
+}
+
+export function headScan(html: string): HeadScan {
+	let doc: DomNode;
+	try {
+		doc = parseDocument(html) as unknown as DomNode;
+	} catch {
+		const title = /<title[^>]*>([^<]*)<\/title>/i.exec(html)?.[1]?.trim()?.slice(0, 200) ?? '';
+		return { title };
+	}
+	const nodes = doc.children ?? [];
+	let title = '';
+	let ogTitle = '';
+	let metaDate: string | undefined;
+	let timeDate: string | undefined;
+
+	const visit = (n: DomNode): void => {
+		if (n.type !== 'tag') return;
+		const name = n.name ?? '';
+		const attribs = n.attribs ?? {};
+		if (name === 'title' && !title) {
+			title = (n.children ?? [])
+				.filter((c) => c.type === 'text')
+				.map((c) => c.data ?? '')
+				.join('')
+				.trim()
+				.slice(0, 200);
+		}
+		if (name === 'meta') {
+			const key = (attribs['property'] ?? attribs['name'] ?? '').toLowerCase().trim();
+			if (!ogTitle && key === 'og:title') ogTitle = (attribs['content'] ?? '').trim().slice(0, 200);
+			if (!metaDate && DATE_META_NAMES.includes(key)) metaDate = toIso(attribs['content']);
+		}
+		if (name === 'time' && !timeDate && attribs['datetime']) timeDate = toIso(attribs['datetime']);
+		for (const c of n.children ?? []) visit(c);
+	};
+	for (const n of nodes) visit(n);
+
+	return { title: title || ogTitle, publishedAt: jsonLdDate(nodes) ?? metaDate ?? timeDate };
+}

package/src/scrape.ts

@@ -18,6 +18,7 @@ import {
 	type RemoteDoc,
 	type Renderer,
 } from './transport';
+import { headScan, treeToMarkdownish } from './dom';
 
 export interface ScrapeResult {
 	url: string;
@@ -94,35 +95,6 @@ export function decodeHtmlEntities(s: string): string {
 	});
 }
 
-function stripKatexSpans(h: string): string {
-	let out = '';
-	let i = 0;
-	for (;;) {
-		const start = /<span[^>]*class="[^"]*katex[^"]*"[^>]*>/i.exec(h.slice(i));
-		if (!start || start.index === undefined) {
-			out += h.slice(i);
-			break;
-		}
-		const abs = i + start.index;
-		out += h.slice(i, abs);
-		let depth = 1;
-		let j = abs + start[0].length;
-		while (depth > 0 && j < h.length) {
-			const next = /<span\b[^>]*>|<\/span>/gi;
-			next.lastIndex = j;
-			const tag = next.exec(h);
-			if (!tag) {
-				j = h.length;
-				break;
-			}
-			j = tag.index + tag[0].length;
-			depth += tag[0].startsWith('</') ? -1 : 1;
-		}
-		i = j;
-	}
-	return out;
-}
-
 export function extractHtml(html: string): { title: string; text: string; publishedAt?: string; paywalled: boolean } {
 	const paywalled =
 		/class="[^"]*(?:fc-ab-root|metering-modal|paywall|pw-overlay|subscription-overlay|regwall)[^"]*"/i.test(html) ||
@@ -130,61 +102,8 @@ export function extractHtml(html: string): { title: string; text: string; publis
 		/<meta[^>]+(?:property|name)=["'](?:isAccessibleForFree|tunnel:[^"']*paywall)["'][^>]+content=["'](?:false|yes)["']/i.test(
 			html,
 		);
-	const title =
-		/<title[^>]*>([^<]*)<\/title>/i.exec(html)?.[1]?.trim()?.slice(0, 200) ||
-		/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i.exec(html)?.[1]?.slice(0, 200) ||
-		'';
-	const publishedAt = extractPublishedDate(html);
-	let h = html
-		.replace(/<script[\s\S]*?<\/script>/gi, ' ')
-		.replace(/<style[\s\S]*?<\/style>/gi, ' ')
-		.replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
-		.replace(/<svg[\s\S]*?<\/svg>/gi, ' ')
-		.replace(/<!--[\s\S]*?-->/g, ' ')
-		.replace(/<nav[\s\S]*?<\/nav>/gi, ' ')
-		.replace(/<footer[\s\S]*?<\/footer>/gi, ' ')
-		.replace(/<aside[\s\S]*?<\/aside>/gi, ' ');
-	h = h.replace(/<pre[\s>][\s\S]*?<\/pre>/gi, (m) => m.replace(/\n/g, '\uE000'));
-	h = h.replace(/<table[\s>][\s\S]*?<\/table>/gi, (table) => {
-		const rows = [...table.matchAll(/<tr[\s>][\s\S]*?<\/tr>/gi)].map((r) =>
-			[...r[0].matchAll(/<t([hd])[\s>][\s\S]*?<\/t\1>/gi)].map((c) =>
-				c[0]
-					.replace(/<[^>]+>/g, ' ')
-					.replace(/\s+/g, ' ')
-					.replace(/\|/g, '\\|')
-					.trim(),
-			),
-		);
-		const grid = rows.filter((r) => r.length);
-		if (grid.length < 2) return table;
-		const cols = Math.max(...grid.map((g) => g.length));
-		if (cols < 2) return table;
-		const line = (g: string[]) => {
-			const cells = g.slice(0, cols);
-			while (cells.length < cols) cells.push(' ');
-			return `| ${cells.join(' | ')} |`;
-		};
-		const md = [line(grid[0]), `| ${Array.from({ length: cols }, () => '---').join(' | ')} |`];
-		for (const g of grid.slice(1)) md.push(line(g));
-		return `\n${md.join('\n')}\n`;
-	});
-	h = stripKatexSpans(h);
-	h = h.replace(/<math[\s>][\s\S]*?<\/math>/gi, (m) => {
-		const tex = /<annotation[^>]*encoding=["']application\/x-tex["'][^>]*>([\s\S]*?)<\/annotation>/i.exec(m)?.[1];
-		return tex?.trim() ? ` $${tex.trim()}$ ` : ' ';
-	});
-	h = h.replace(
-		/<h([1-6])[^>]*>(.*?)<\/h\1>/gis,
-		(_, n, inner) => `\n${'#'.repeat(Number(n))} ${inner.replace(/<[^>]+>/g, ' ').trim()}\n`,
-	);
-	h = h
-		.replace(/<li[^>]*>/gi, '\n• ')
-		.replace(/<\/(p|div|li|tr|section|article|blockquote|table|ul|ol|pre)>/gi, '\n')
-		.replace(/<(br|hr)[^>]*>/gi, '\n');
-	// strip tags first, THEN decode entities: escaped markup shown as literal
-	// text (`&lt;limit...&gt;`) must survive as text, not be stripped as a tag
-	h = h.replace(/<[^>]+>/g, ' ');
-	h = h.replace(/\uE000/g, '\n');
+	const { title, publishedAt } = headScan(html);
+	let h = treeToMarkdownish(html).replace(/\uE000/g, '\n');
 	h = stripToAscii(decodeHtmlEntities(h));
 	const lines = h
 		.split('\n')
@@ -230,21 +149,6 @@ export function extractHtml(html: string): { title: string; text: string; publis
 	return { title, publishedAt, paywalled, text: out.join('\n').replace(/\n{3,}/g, '\n\n') };
 }
 
-function extractPublishedDate(html: string): string | undefined {
-	const iso = (v: string | undefined) => {
-		if (!v) return undefined;
-		const t = Date.parse(v.trim());
-		return Number.isNaN(t) || t < 0 ? undefined : new Date(t).toISOString().slice(0, 10);
-	};
-	const ld = /"datePublished"\s*:\s*"([^"]+)"/.exec(html)?.[1];
-	const meta =
-		/<meta[^>]+(?:property|name)=["'](?:article:published_time|datePublished|publish-date|pubdate|parsely-pub-date|date)["'][^>]+content=["']([^"']+)["']/i.exec(
-			html,
-		)?.[1];
-	const timeEl = /<time[^>]+datetime=["']([^"']+)["']/i.exec(html)?.[1];
-	return iso(ld) ?? iso(meta) ?? iso(timeEl);
-}
-
 const NAV_LABEL_RE =
 	/^(home|menu|about( us)?|docs?|documentation|downloads?|license|licence|support|purchase|buy|pricing|search|login|log ?in|sign ?(in|up|out)|register|account|blog|news|contact|careers?|jobs|store|shop|forum|community|tutorials?|api|guide|guides|reference|examples?|faq|help|terms|privacy|cookies?|sitemap|rss|changelog|releases?|twitter|facebook|github|discord|youtube|linkedin|mastodon|x)$/i;
 

package/src/tools.ts

@@ -4,8 +4,8 @@
  */
 import { Type } from 'typebox';
 import type { ExtensionAPI, ExtensionCommandContext, Theme } from '@earendil-works/pi-coding-agent';
-import { keyHint } from '@earendil-works/pi-coding-agent';
-import { Text } from '@earendil-works/pi-tui';
+import { keyHint, keyText, rawKeyHint } from '@earendil-works/pi-coding-agent';
+import { Text, type Component } from '@earendil-works/pi-tui';
 import { appendFileSync, mkdirSync, rmSync, writeFileSync, existsSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import {
@@ -20,9 +20,10 @@ import {
 } from './config';
 import { ABORT_ERROR, DEFAULT_MAX_RESULTS } from './constants';
 import { cacheKey, openCache, CACHE_TTL_HOURS } from './cache';
-import { clearDeadEngines, runEngines } from './engines';
-import { fuse, diversifyByHost, type FusedHit } from './fuse';
-import { hostOf, normalizeUrl } from './urls';
+import { clearDeadEngines, runEngines, type EngineOutcome } from './engines';
+import { fuse, diversifyByHost } from './fuse';
+import { normalizeUrl } from './urls';
+import { buildSearchText, READ_FIRST_TAG, wrapAll } from './view';
 import { scrape } from './scrape';
 
 const SEARCH_PARAMS = Type.Object({
@@ -70,19 +71,22 @@ const FETCH_OUTPUT = Type.Object({
 	title: Type.String(),
 	renderer: Type.String(),
 	outline: Type.Array(Type.String()),
+	sections: Type.Array(
+		Type.Object({
+			index: Type.Integer({ minimum: 0 }),
+			heading: Type.String(),
+			text: Type.String(),
+		}),
+	),
 	section_count: Type.Number(),
 	chars: Type.Number(),
 	sections_requested: Type.Optional(Type.Array(Type.Number())),
 	sections_missing: Type.Optional(Type.Array(Type.Number())),
 	sections_truncated: Type.Optional(Type.Number()),
+	sections_clipped: Type.Optional(Type.Array(Type.Integer({ minimum: 0 }))),
 	error: Type.Optional(Type.String()),
 });
 
-function fmtSnippet(s: string, max: number): string {
-	const t = s.replace(/\s+/g, ' ').trim();
-	return t.length > max ? t.slice(0, max) + '…' : t;
-}
-
 function dropRarestTerm(query: string): string {
 	const words = query.split(/\s+/).filter((w) => w.length > 2 && !/^(?:site|inurl|intitle|filetype):/i.test(w));
 	if (words.length < 2) return '';
@@ -93,54 +97,6 @@ function dropRarestTerm(query: string): string {
 		.trim();
 }
 
-function stripSiteSuffix(title: string, host: string): string {
-	const m = /^(.{3,})\s+[|–—-]\s+([^|–—-]{2,40})$/.exec(title.trim());
-	if (!m) return title;
-	const suffix = m[2].trim();
-	const head = m[1].trim();
-	const hostBase = host.replace(/^www\./, '').split('.')[0];
-	if (host && (suffix.toLowerCase().includes(hostBase) || suffix.length <= 25)) return head;
-	return title;
-}
-
-function normalizeHitDate(raw: string | undefined): string | undefined {
-	if (!raw) return undefined;
-	const t = Date.parse(raw);
-	if (!Number.isNaN(t)) return new Date(t).toISOString().slice(0, 10);
-	const rel = /^(\d+)\s*(m|min|minute|minutes|h|hour|hours|d|day|days|w|week|weeks|mo|month|months)\b/i.exec(
-		raw.trim(),
-	);
-	if (rel) {
-		const n = parseInt(rel[1], 10);
-		const unit = rel[2].toLowerCase();
-		// 'mo' must win over 'm' BEFORE the minutes branch: the regex captures
-		// 'months' verbatim, so a bare startsWith('m') misclassifies it ("3
-		// months ago" used to render as ~today). Divide by unit, in months>…>ms.
-		const ms = unit.startsWith('h')
-			? 3_600_000
-			: unit.startsWith('mo')
-				? 2_592_000_000
-				: unit.startsWith('m')
-					? 60_000
-					: unit.startsWith('d')
-						? 86_400_000
-						: 604_800_000;
-		return new Date(Date.now() - n * ms).toISOString().slice(0, 10);
-	}
-	if (/^yesterday$/i.test(raw.trim())) return new Date(Date.now() - 86_400_000).toISOString().slice(0, 10);
-	return undefined;
-}
-
-function cleanSnippet(s: string): string {
-	return s
-		.replace(/\\([_*`[\]()#>~.-])/g, '$1')
-		.replace(/!\[[^\]]*\]\([^)]*\)/g, '')
-		.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
-		.replace(/(^|\s)([*]{1,3}|_{1,2}|`)(\S(?:[^*`_]*\S)?)\2(?=\s|$|[.,;:!?)]|\b)/g, '$1$3')
-		.replace(/\s+/g, ' ')
-		.trim();
-}
-
 export function registerWebTools(pi: ExtensionAPI) {
 	const cache = openCache(getConfig().cacheDir + '/cache.json');
 	const exposure = existsSync(disabledFlagPath()) ? 'hidden' : 'direct';
@@ -166,6 +122,13 @@ export function registerWebTools(pi: ExtensionAPI) {
 		parameters: SEARCH_PARAMS,
 		outputSchema: SEARCH_OUTPUT,
 
+		renderCall(args, theme) {
+			const header = theme.fg('toolTitle', theme.bold('web.search'));
+			const q = typeof args.query === 'string' ? args.query : '…';
+			const tail = typeof args.max_results === 'number' ? theme.fg('muted', ` max=${args.max_results}`) : '';
+			return new Text(`${header} ${theme.fg('accent', q)}${tail}`, 0, 0);
+		},
+
 		async execute(_id, params, signal, _onUpdate, _ctx) {
 			if (signal?.aborted) throw new Error(ABORT_ERROR);
 			const outcomes = await runEngines({
@@ -205,38 +168,32 @@ export function registerWebTools(pi: ExtensionAPI) {
 							fused.filter((r) => !r.junk),
 							2,
 						).slice(0, Math.max((params.max_results ?? DEFAULT_MAX_RESULTS) * 2, DEFAULT_MAX_RESULTS));
-						if (candidates.length) retryNote = ` (retried as "${relaxed}" after zero results)`;
+						if (candidates.length) retryNote = `retried as "${relaxed}"`;
 					}
 				}
 			}
 			const visible = candidates.slice(0, params.max_results ?? DEFAULT_MAX_RESULTS);
 			const enginesUsed = [...new Set(fused.flatMap((r) => r.engines))];
-			const snip = 240;
-
-			let text = `${visible.length} result${visible.length === 1 ? '' : 's'} for "${params.query}": ${enginesUsed.length} engine${enginesUsed.length === 1 ? '' : 's'} contributed (${enginesUsed.join(', ')})${retryNote}\n\n`;
-			if (!visible.length) {
-				const fatal = outcomes.find((o) => o.engine === 'none' && o.error);
-				text = fatal
-					? `No results for "${params.query}": ${fatal.error}\n`
-					: `No results for "${params.query}". Broaden the query.\n`;
-			}
 			visible.forEach((r, i) => {
 				lastSearch.urlRank.set(normalizeUrl(r.url), {
 					rank: i + 1,
 					engines: r.engines,
 					query: params.query,
 				});
-				const title = fmtSnippet(stripSiteSuffix(cleanSnippet(r.title), hostOf(r.url)), 100) || r.url;
-				const snippet = fmtSnippet(cleanSnippet(r.snippet), snip);
-				const via = formatEvidence(r);
-				const when = normalizeHitDate(r.date);
-				text += `${i + 1}. [${title}](${r.url})${when ? ` — ${when}` : ''}${via}\n`;
-				if (snippet) text += `   ${snippet}\n`;
-				text += '\n';
 			});
-			if (errored.length) {
-				text += `⚠ engines: ${errored.map((e) => `${e.engine} (${e.error.slice(0, 80)})`).join('; ')}\n`;
-			}
+			const text = buildSearchText(
+				{
+					query: params.query,
+					hits: visible,
+					enginesUsed,
+					errored,
+					fatal: visible.length
+						? undefined
+						: outcomes.find((o): o is EngineOutcome & { error: string } => o.engine === 'none' && !!o.error),
+					retryNote,
+				},
+				{ wrap: terminalWrapWidth() },
+			);
 
 			return {
 				content: [{ type: 'text', text }],
@@ -256,9 +213,7 @@ export function registerWebTools(pi: ExtensionAPI) {
 			};
 		},
 		renderResult(result, options, theme) {
-			const lines = resultText(result).split('\n');
-			if (!options.expanded) return new Text(previewWithHint(lines, theme), 0, 0);
-			return new Text(colorizedResult(resultText(result), theme), 0, 0);
+			return new WrappedSearchText(resultText(result), theme, options.expanded);
 		},
 	});
 
@@ -280,7 +235,7 @@ export function registerWebTools(pi: ExtensionAPI) {
 			const url = typeof args.url === 'string' ? args.url : '…';
 			const s = Array.isArray(args.sections) ? args.sections.join(', ') : '…';
 			const tail = args.sections ? ` sections=[${s}]` : '';
-			return new Text(`${header} ${theme.fg('muted', url + tail)}`, 0, 0);
+			return new Text(`${header} ${theme.fg('accent', url)}${theme.fg('muted', tail)}`, 0, 0);
 		},
 
 		async execute(_id, params, signal) {
@@ -353,11 +308,13 @@ type FetchDetails = {
 	title: string;
 	renderer: string;
 	outline: string[];
+	sections: { index: number; heading: string; text: string }[];
 	section_count: number;
 	chars?: number;
 	sections_requested?: number[];
 	sections_missing?: number[];
 	sections_truncated?: number;
+	sections_clipped?: number[];
 	error?: string;
 };
 
@@ -369,6 +326,7 @@ interface FetchResult {
 }
 
 const PREVIEW_LINES = 10;
+const SEARCH_PREVIEW_LINES = 8;
 
 /** Display-only: the model-facing markers (`indices …`, `- *N*`) render as
  * genuine ANSI italics in the TUI; the content text itself is untouched. */
@@ -378,49 +336,95 @@ function italicizeIndices(text: string, theme: Theme): string {
 		.replace(/^- \*(\d+)\* /gm, (_, n: string) => `- ${theme.italic(n)} `);
 }
 
+function italicizeRanks(text: string, theme: Theme): string {
+	return text.replace(/^(\s*)(\d+)\.(?=\s)/gm, (m, pad: string, n: string) => `${pad}${theme.italic(n)}.`);
+}
+
+const ANSI_SGR = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g');
+
+const DEFAULT_WRAP_COLS = 90;
+const MAX_WRAP_COLS = 240;
+
+function terminalWrapWidth(): number {
+	const cols = process.stdout.columns ?? DEFAULT_WRAP_COLS;
+	return Math.min(MAX_WRAP_COLS, Math.max(60, cols - 4));
+}
+
+class WrappedSearchText implements Component {
+	private raw: string;
+	private theme: Theme;
+	private expanded: boolean;
+	constructor(raw: string, theme: Theme, expanded: boolean) {
+		this.raw = raw;
+		this.theme = theme;
+		this.expanded = expanded;
+	}
+	invalidate(): void {}
+	render(width: number): string[] {
+		const wrapped = wrapAll(this.raw, Math.max(40, width));
+		const lines = wrapped.split('\n');
+		const expanded = this.expanded || lines.length <= SEARCH_PREVIEW_LINES;
+		const shown = expanded ? lines : lines.slice(0, SEARCH_PREVIEW_LINES);
+		const colored = italicizeRanks(shown.join('\n'), this.theme)
+			.split('\n')
+			.map((l) => colorLine(l, this.theme))
+			.join('\n');
+		const out = replaceTabs(colored).split('\n');
+		if (!expanded) {
+			const hint = moreLinesHint(shown.length, lines.length, this.theme);
+			if (hint) out.push(hint);
+		}
+		return out;
+	}
+}
+
+function colorLine(l: string, theme: Theme): string {
+	const plain = l.replace(ANSI_SGR, '');
+	if (
+		plain.includes('…[section truncated') ||
+		plain.includes('…(continues at outline index') ||
+		plain.startsWith('…[clipped by tool]') ||
+		plain.startsWith('…[truncated]')
+	)
+		return theme.fg('warning', l);
+	if (plain.startsWith('## ') && plain.endsWith(READ_FIRST_TAG))
+		return theme.fg('accent', l.slice(0, l.length - READ_FIRST_TAG.length)) + theme.bold(READ_FIRST_TAG);
+	if (/^\s*· recommended first read$/.test(plain)) return theme.bold(l.trimEnd());
+	if (plain.startsWith('## ')) return theme.fg('accent', l);
+	if (plain.startsWith('- ')) return theme.fg('dim', l);
+	if (plain.startsWith('[rendered via')) return theme.fg('muted', l);
+	if (/^\s*\d+\. /.test(plain)) return theme.fg('dim', l);
+	return l;
+}
+
 function colorizedResult(text: string, theme: Theme): string {
 	return text
 		.split('\n')
-		.map((l) => {
-			if (
-				l.includes('…[section truncated') ||
-				l.includes('…(continues at outline index') ||
-				l.startsWith('…[clipped by tool]') ||
-				l.startsWith('…[truncated]')
-			)
-				return theme.fg('warning', l);
-			if (l.startsWith('## [')) return theme.fg('accent', l);
-			if (l.startsWith('- ')) return theme.fg('dim', l);
-			if (l.startsWith('[rendered via')) return theme.fg('muted', l);
-			if (/^\d+\. /.test(l)) return theme.fg('toolOutput', l);
-			return l;
-		})
+		.map((l) => colorLine(l, theme))
 		.join('\n');
 }
 
-/** Per-hit evidence line: "⇐ serper#1, exa#2"; singles are unverified. */
-function formatEvidence(r: FusedHit): string {
-	if (!r.engineRanks?.length) return r.engines.length > 1 ? ` ← ${r.engines.length} engines` : '';
-	if (r.engineRanks.length === 1) return ` ⇐ ${r.engineRanks[0].engine} only (single-engine: unverified)`;
-	const shown = r.engineRanks
-		.slice(0, 6)
-		.map((e) => `${e.engine}#${e.rank}`)
-		.join(', ');
-	const rest = r.engineRanks.length > 6 ? `, +${r.engineRanks.length - 6} more` : '';
-	return ` ⇐ ${shown}${rest}`;
-}
-
 function resultText(result: { content: { type: string; text?: string }[] }): string {
 	return (result.content[0] as { type: string; text: string } | undefined)?.text ?? '';
 }
 
-function previewWithHint(lines: string[], theme: Theme): string {
-	const shown = lines.slice(0, PREVIEW_LINES);
-	const remaining = lines.length - shown.length;
+function moreLinesHint(shown: number, total: number, theme: Theme): string {
+	const remaining = total - shown;
+	if (remaining <= 0) return '';
+	let hint: string;
+	try {
+		hint = keyText('app.tools.expand') ? keyHint('app.tools.expand', 'to expand') : rawKeyHint('ctrl+o', 'to expand');
+	} catch {
+		hint = rawKeyHint('ctrl+o', 'to expand');
+	}
+	return theme.fg('muted', `… (${remaining} more lines,`) + ' ' + hint + theme.fg('muted', ')');
+}
+
+function previewWithHint(lines: string[], theme: Theme, maxLines = PREVIEW_LINES): string {
+	const shown = lines.slice(0, maxLines);
 	const text = shown.map((l) => theme.fg('toolOutput', replaceTabs(l))).join('\n');
-	if (remaining > 0)
-		return text + theme.fg('muted', `\n… (${remaining} more lines, ${keyHint('app.tools.expand', 'to expand')})`);
-	return text;
+	const hint = moreLinesHint(shown.length, lines.length, theme);
+	return hint ? `${text}\n${hint}` : text;
 }
 
 function replaceTabs(text: string): string {
@@ -439,7 +443,6 @@ function fetchErrorResult(details: FetchDetails): FetchResult {
 
 function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: number[]): FetchResult {
 	const want = [...new Set(sections.map((i) => Math.max(0, Math.min(res.sections.length - 1, i))))];
-	const chosen = want.map((i) => res.sections[i]).filter(Boolean);
 	const body = res.outline.length
 		? `indices - 0 - ${res.outline.length - 1} section${res.outline.length === 1 ? '' : 's'}\n${res.outline.map((h, i) => `- *${i}* ${h}`).join('\n')}\n\n`
 		: '';
@@ -471,17 +474,48 @@ function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: num
 	if (res.renderer && res.renderer !== 'local') text += `\n\n[rendered via ${res.renderer}]`;
 	const clipped = text.length > MAX_FETCH_TEXT ? text.slice(0, MAX_FETCH_TEXT) + '\n…[clipped by tool]' : text;
 	const missing = sections.filter((i) => i >= res.sections.length);
+	// continuation cuts (res.continued) vs local budget clips (clippedSections)
 	const truncatedCount = want.filter((idx) => res.continued?.includes(idx)).length;
+
+	// same budget as the text view; headings and the '…' marker count against it
+	let budget = MAX_FETCH_TEXT;
+	const structuredSections: { index: number; heading: string; text: string }[] = [];
+	const clippedSections: number[] = [];
+	for (const idx of want) {
+		if (budget <= 0) {
+			clippedSections.push(idx);
+			continue;
+		}
+		const s = res.sections[idx];
+		if (s === undefined) continue;
+		const heading = res.outline[idx] ?? '';
+		const fullCost = heading.length + s.length;
+		if (fullCost <= budget) {
+			structuredSections.push({ index: idx, heading, text: s });
+			budget -= fullCost;
+			continue;
+		}
+		if (heading.length >= budget) {
+			structuredSections.push({ index: idx, heading: heading.slice(0, Math.max(0, budget - 1)) + '…', text: '' });
+		} else {
+			const textLen = Math.max(0, budget - heading.length - 1);
+			structuredSections.push({ index: idx, heading, text: s.slice(0, textLen) + (textLen < s.length ? '…' : '') });
+		}
+		clippedSections.push(idx);
+		budget = 0;
+	}
 	const details = {
 		url: res.url,
 		title: res.title,
 		renderer: res.renderer,
 		outline: res.outline,
-		section_count: chosen.length,
+		sections: structuredSections,
+		section_count: structuredSections.length,
 		chars: clipped.length,
 		sections_requested: [...new Set(sections)],
 		sections_missing: [...new Set(missing)],
 		sections_truncated: truncatedCount,
+		sections_clipped: clippedSections,
 		...(res.publishedAt ? { publishedAt: res.publishedAt } : {}),
 		...(res.paywalled ? { paywalled: true } : {}),
 		...(res.error ? { error: res.error } : {}),

package/src/transport.ts

@@ -203,7 +203,7 @@ function decodeBody(bytes: Uint8Array, contentType: string): string {
 	return looksMojibake(text) ? repairMojibake(text) : text;
 }
 
-async function readBodyBounded(res: Response, maxBytes: number): Promise<Uint8Array> {
+export async function readBodyBounded(res: Response, maxBytes: number): Promise<Uint8Array> {
 	const declared = Number(res.headers.get('content-length') ?? 0);
 	if (declared > maxBytes) throw new Error('page too large');
 	const reader = res.body?.getReader();
@@ -218,7 +218,10 @@ async function readBodyBounded(res: Response, maxBytes: number): Promise<Uint8Ar
 		const { done, value } = await reader.read();
 		if (done) break;
 		size += value.byteLength;
-		if (size > maxBytes) throw new Error('page too large');
+		if (size > maxBytes) {
+			await reader.cancel();
+			throw new Error('page too large');
+		}
 		chunks.push(value);
 	}
 	const out = new Uint8Array(size);

package/src/view.ts

@@ -0,0 +1,223 @@
+import type { FusedHit } from './fuse';
+import { hostOf } from './urls';
+
+const SNIPPET_MAX = 240;
+const CHAPTER_TITLE_MAX = 100;
+const TOC_TITLE_MAX = 60;
+const ERROR_SNIP = 40;
+const DEFAULT_WRAP = 100;
+const MAX_WRAP = 240;
+
+export const READ_FIRST_TAG = ' · recommended first read';
+
+interface ErroredEngine {
+	engine: string;
+	error: string;
+}
+
+export interface SearchViewOpts {
+	wrap?: number;
+}
+
+export interface SearchView {
+	query: string;
+	hits: FusedHit[];
+	enginesUsed: string[];
+	errored: ErroredEngine[];
+	fatal?: { engine: string; error: string };
+	retryNote?: string;
+}
+
+function fmtSnippet(s: string, max: number): string {
+	const t = s.replace(/\s+/g, ' ').trim();
+	return t.length > max ? t.slice(0, max) + '…' : t;
+}
+
+function cleanSnippet(s: string): string {
+	return s
+		.replace(/\\(.)/g, '$1')
+		.replace(/!\[[^\]]*\]\([^)]*\)/g, '')
+		.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
+		.replace(/\[\[\d+\]\]\([^)]*\)\)?/g, '')
+		.replace(/\[\[\d+\]\]/g, '')
+		.replace(/\(https?:\/\/[^)]*\)/g, ' ')
+		.replace(/#{2,6}\s+/g, '')
+		.replace(/\|/g, ' ')
+		.replace(/(^|\s)-+(?=\s|$)/g, '$1')
+		.replace(/(^|[()?!:;])\s?\d{1,2}:\d{2}\s+/g, '$1 ')
+		.replace(/\(\s*\)/g, '')
+		.replace(/\((?=[0-9+\-–·\s]*[+\-–·])[0-9+\-–·\s]*\)/g, ' ')
+		.replace(/\d{1,4}[KM]?\s*•\s*\d{1,4}(?:mo|d|w|y|h)\s+ago/gi, ' ')
+		.replace(/\s+([?:!;])/g, '$1')
+		.replace(/(^|\s)([*]{1,3}|_{1,2}|`)(\S(?:[^*`_]*\S)?)\2(?=\s|$|[.,;:!?)]|\b)/g, '$1$3')
+		.replace(/\s+/g, ' ')
+		.trim();
+}
+
+function stripSiteSuffix(title: string, host: string): string {
+	const m = /^(.{3,})\s+[|–—-]\s+([^|–—-]{2,40})$/.exec(title.trim());
+	if (!m) return title;
+	const suffix = m[2].trim();
+	const head = m[1].trim();
+	const hostBase = host.replace(/^www\./, '').split('.')[0];
+	if (host && suffix.toLowerCase().includes(hostBase)) return head;
+	if (suffix.length <= 25 && !/\s/.test(suffix)) return head;
+	return title;
+}
+
+function normalizeHitDate(raw: string | undefined): string | undefined {
+	if (!raw) return undefined;
+	const t = Date.parse(raw);
+	if (!Number.isNaN(t)) return new Date(t).toISOString().slice(0, 10);
+	const rel = /^(\d+)\s*(m|min|minute|minutes|h|hour|hours|d|day|days|w|week|weeks|mo|month|months)\b/i.exec(
+		raw.trim(),
+	);
+	if (rel) {
+		const n = parseInt(rel[1], 10);
+		const unit = rel[2].toLowerCase();
+		const ms = unit.startsWith('h')
+			? 3_600_000
+			: unit.startsWith('mo')
+				? 2_592_000_000
+				: unit.startsWith('m')
+					? 60_000
+					: unit.startsWith('d')
+						? 86_400_000
+						: 604_800_000;
+		return new Date(Date.now() - n * ms).toISOString().slice(0, 10);
+	}
+	if (/^yesterday$/i.test(raw.trim())) return new Date(Date.now() - 86_400_000).toISOString().slice(0, 10);
+	return undefined;
+}
+
+function formatEvidence(r: FusedHit): string {
+	if (!r.engineRanks?.length) return r.engines.length > 1 ? ` ← ${r.engines.length} engines` : '';
+	if (r.engineRanks.length === 1) return ` ⇐ ${r.engineRanks[0].engine} only (single-engine: unverified)`;
+	const shown = r.engineRanks
+		.slice(0, 6)
+		.map((e) => `${e.engine}#${e.rank}`)
+		.join(', ');
+	const rest = r.engineRanks.length > 6 ? `, +${r.engineRanks.length - 6} more` : '';
+	return ` ⇐ ${shown}${rest}`;
+}
+
+function cleanTitle(s: string, host: string): string {
+	return stripSiteSuffix(cleanSnippet(s), host).replace(/\s+[—–]\s+/g, ': ');
+}
+
+function tocTitle(r: FusedHit): string {
+	return fmtSnippet(cleanTitle(r.title, hostOf(r.url)), TOC_TITLE_MAX) || fmtSnippet(r.url, TOC_TITLE_MAX);
+}
+
+function chapterTitle(r: FusedHit): string {
+	return fmtSnippet(cleanTitle(r.title, hostOf(r.url)), CHAPTER_TITLE_MAX) || r.url;
+}
+
+function compactWarn(e: ErroredEngine): string {
+	const status = /HTTP\s+(\d{3})/i.exec(e.error)?.[1];
+	return status ? `${status}: ${e.engine}` : `${e.engine}: ${e.error.slice(0, ERROR_SNIP)}`;
+}
+
+export function buildSearchText(v: SearchView, opts: SearchViewOpts = {}): string {
+	const count = v.hits.length;
+	const width = Math.max(40, Math.min(MAX_WRAP, opts.wrap ?? DEFAULT_WRAP));
+	const engineWord = v.enginesUsed.length === 1 ? 'engine' : 'engines';
+	if (!count) {
+		return v.fatal ? `No results: ${v.fatal.error}\n` : `No results. Broaden the query.\n`;
+	}
+	let text = `${v.enginesUsed.length} ${engineWord} [${v.enginesUsed.join(', ')}]`;
+	if (v.retryNote) text += ` · ${v.retryNote}`;
+	const warns = v.errored.map(compactWarn);
+	if (warns.length) text += ` ⚠ ${warns.join(' · ')}`;
+	text += '\n';
+	const rows = v.hits.map((r, i) => {
+		const host = hostOf(r.url);
+		return `${String(i + 1).padStart(2)}. ${tocTitle(r)}${host ? ` · ${host}` : ''}`;
+	});
+	text += `\n${rows.join('\n')}\n`;
+	v.hits.forEach((r, i) => {
+		const title = chapterTitle(r);
+		const snippet = fmtSnippet(cleanSnippet(r.snippet), SNIPPET_MAX);
+		const via = formatEvidence(r).trim();
+		const when = normalizeHitDate(r.date);
+		let meta = r.url;
+		if (when) meta += ` · ${when}`;
+		if (via) meta += ` ${via}`;
+		text += `\n## ${i + 1}. ${title}${i === 0 && count > 1 ? READ_FIRST_TAG : ''}\n`;
+		text += `   ${meta}\n`;
+		if (snippet) text += `   ${snippet}\n`;
+	});
+	return wrapAll(text, width);
+}
+
+export function wrapAll(text: string, width: number): string {
+	return text
+		.split('\n')
+		.map((l) => wrapLine(l, width))
+		.join('\n');
+}
+
+function wrapLine(line: string, width: number): string {
+	if (line.length <= width) return line;
+	const hasTag = line.endsWith(READ_FIRST_TAG);
+	const body = hasTag ? line.slice(0, line.length - READ_FIRST_TAG.length) : line;
+	const m = /^(##\s*\d+\.\s+|\s*\d+\.\s+|\s*)/.exec(body);
+	const head = m?.[1] ?? '';
+	const indent = head.replace(/\S/g, ' ');
+	const words = body.slice(head.length).split(' ');
+	const limit = width - indent.length;
+	const out: string[] = [];
+	let cur = '';
+	for (const word of words) {
+		if (!word) continue;
+		const fits = cur ? cur.length + 1 + word.length <= limit : word.length <= limit;
+		if (fits) {
+			cur = cur ? `${cur} ${word}` : word;
+			continue;
+		}
+		if (cur) {
+			out.push(cur);
+			cur = '';
+		}
+		if (word.length > limit) {
+			for (const chunk of breakToken(word, limit)) {
+				if (cur) {
+					out.push(cur);
+					cur = '';
+				}
+				cur = chunk;
+			}
+		} else {
+			cur = word;
+		}
+	}
+	if (cur) out.push(cur);
+	const lines = out.map((l, i) => (i ? indent + l : head + l));
+	if (hasTag) {
+		const last = lines.length - 1;
+		if (last >= 0 && lines[last].length + READ_FIRST_TAG.length <= width) {
+			lines[last] += READ_FIRST_TAG;
+		} else {
+			lines.push(indent + READ_FIRST_TAG.trimStart());
+		}
+	}
+	return lines.join('\n');
+}
+
+function breakToken(token: string, limit: number): string[] {
+	const out: string[] = [];
+	let from = 0;
+	while (token.length - from > limit) {
+		let cut = from + limit;
+		for (let i = cut - 1; i > from; i--) {
+			if (i < token.length && /[/\-_.?=&]/.test(token[i])) {
+				cut = i + 1;
+				break;
+			}
+		}
+		out.push(token.slice(from, cut));
+		from = cut;
+	}
+	out.push(token.slice(from));
+	return out;
+}