pi-wsearch 0.3.2 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-wsearch",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "Pi extension: web.search + web.fetch",
   "license": "MIT",
   "type": "module",
@@ -45,10 +45,10 @@
     ]
   },
   "peerDependencies": {
-    "@earendil-works/pi-ai": "*",
-    "@earendil-works/pi-agent-core": "*",
-    "@earendil-works/pi-coding-agent": "*",
-    "@earendil-works/pi-tui": "*",
+    "@earendil-works/pi-ai": "^1.3.0",
+    "@earendil-works/pi-agent-core": "^1.3.0",
+    "@earendil-works/pi-coding-agent": "^1.3.0",
+    "@earendil-works/pi-tui": "^1.3.0",
     "typebox": "*"
   },
   "devDependencies": {
@@ -59,7 +59,11 @@
     "oxlint": "^1.86.0",
     "typescript": "^5.9.3"
   },
-  "dependencies": {},
+  "dependencies": {
+    "htmlparser2": "^12.0.0",
+    "parse5": "^8.0.1",
+    "parse5-htmlparser2-tree-adapter": "^8.0.1"
+  },
   "main": "index.ts",
   "files": [
     "index.ts",

package/src/dom.ts

@@ -0,0 +1,328 @@
+import { parse } from 'parse5';
+import { adapter } from 'parse5-htmlparser2-tree-adapter';
+import { parseDocument } from 'htmlparser2';
+
+interface DomNode {
+	type: string;
+	name?: string;
+	attribs?: Record<string, string>;
+	data?: string;
+	children?: DomNode[];
+}
+
+const SKIP = new Set(['script', 'style', 'noscript', 'svg', 'nav', 'footer', 'aside', 'head', 'title']);
+
+const BLOCK = new Set(['p', 'div', 'section', 'article', 'blockquote', 'pre']);
+
+function isKatex(n: DomNode): boolean {
+	return (n.attribs?.['class'] ?? '').split(/\s+/).includes('katex');
+}
+
+function toDom(html: string): DomNode {
+	return parse(html, { treeAdapter: adapter }) as unknown as DomNode;
+}
+
+function inlineOf(n: DomNode): string {
+	const t = n.type;
+	if (t === 'text') return n.data ?? '';
+	if (t === 'comment' || t === 'directive') return '';
+	const name = n.name ?? '';
+	if (!name || SKIP.has(name) || isKatex(n)) return '';
+	if (name === 'math') {
+		const tex = annotationTex(n);
+		return tex ? `$${tex}$` : '';
+	}
+	if (name === 'br' || name === 'hr' || name === 'li') return ' ';
+	return (n.children ?? []).map(inlineOf).join('');
+}
+
+function annotationTex(n: DomNode): string | undefined {
+	const found: DomNode[] = [];
+	allDescendants(n, 'annotation', found);
+	for (const c of found) {
+		const enc = (c.attribs?.['encoding'] ?? '').toLowerCase();
+		if (enc === 'application/x-tex' || enc.includes('x-tex')) return inlineOf(c).trim();
+	}
+	return undefined;
+}
+
+function allDescendants(n: DomNode, name: string, out: DomNode[]): void {
+	for (const c of n.children ?? []) {
+		if (c.type === 'tag') {
+			if (c.name === name) out.push(c);
+			allDescendants(c, name, out);
+		}
+	}
+}
+
+function listText(n: DomNode, out: string[]): void {
+	for (const c of n.children ?? []) {
+		if (c.type === 'tag' && c.name === 'li') {
+			newline(out);
+			out.push('• ');
+			liText(c, out);
+		}
+	}
+}
+
+function liText(n: DomNode, out: string[]): void {
+	for (const c of n.children ?? []) {
+		const t = c.type;
+		if (t === 'text') {
+			out.push((c.data ?? '').replace(/\s+/g, ' '));
+			continue;
+		}
+		if (t === 'comment' || t === 'directive') continue;
+		const name = c.name ?? '';
+		if (SKIP.has(name) || isKatex(c)) continue;
+		if (name === 'li') {
+			newline(out);
+			out.push('• ');
+			liText(c, out);
+			continue;
+		}
+		if (name === 'ul' || name === 'ol') {
+			listText(c, out);
+			continue;
+		}
+		if (name === 'br' || name === 'hr') {
+			out.push('\n');
+			continue;
+		}
+		if (name === 'pre') {
+			out.push('\n');
+			out.push(inlineOf(c).replace(/\n/g, '\uE000'));
+			continue;
+		}
+		if (name === 'math') {
+			const tex = annotationTex(c);
+			if (tex) out.push(`$${tex}$`);
+			continue;
+		}
+		if (BLOCK.has(name) || name === 'table') {
+			newline(out);
+			liText(c, out);
+			continue;
+		}
+		out.push(inlineOf(c));
+	}
+}
+
+function renderTable(n: DomNode, out: string[]): void {
+	const rows: string[][] = [];
+	for (const tr of (() => {
+		const found: DomNode[] = [];
+		allDescendants(n, 'tr', found);
+		return found;
+	})()) {
+		const cells: string[] = [];
+		for (const cell of tr.children ?? []) {
+			if (cell.type === 'tag' && (cell.name === 'td' || cell.name === 'th')) {
+				cells.push(inlineOf(cell).replace(/\s+/g, ' ').trim().replace(/\|/g, '\\|'));
+			}
+		}
+		if (cells.length) rows.push(cells);
+	}
+	newline(out);
+	if (rows.length < 2 || Math.max(...rows.map((r) => r.length)) < 2) {
+		out.push(rows.map((r) => r.join(' ')).join('\n'));
+		out.push('\n');
+		return;
+	}
+	const cols = Math.max(...rows.map((r) => r.length));
+	const line = (r: string[]) => `| ${r.concat(Array(Math.max(0, cols - r.length)).fill(' ')).join(' | ')} |`;
+	out.push(
+		[line(rows[0]), `| ${Array.from({ length: cols }, () => '---').join(' | ')} |`, ...rows.slice(1).map(line)].join(
+			'\n',
+		),
+	);
+	out.push('\n');
+}
+
+function newline(out: string[]): void {
+	if (out.length > 0 && !out[out.length - 1].endsWith('\n')) out.push('\n');
+}
+
+export function treeToMarkdownish(html: string): string {
+	const doc = toDom(html);
+	const out: string[] = [];
+	renderBlocks(doc.children ?? [], out);
+	return out.join('');
+}
+
+function renderBlocks(nodes: DomNode[], out: string[]): void {
+	for (const n of nodes) renderBlock(n, out);
+}
+
+function renderBlock(n: DomNode, out: string[]): void {
+	const t = n.type;
+	if (t === 'text') {
+		out.push(n.data ?? '');
+		return;
+	}
+	if (t === 'comment' || t === 'directive' || t === 'style' || t === 'script') return;
+	const name = n.name ?? '';
+	if (!name || SKIP.has(name) || isKatex(n)) return;
+	switch (name) {
+		case 'h1':
+		case 'h2':
+		case 'h3':
+		case 'h4':
+		case 'h5':
+		case 'h6':
+			newline(out);
+			out.push(`${'#'.repeat(Number(name[1]))} ${inlineOf(n).trim()}\n`);
+			return;
+		case 'li':
+			newline(out);
+			out.push('• ');
+			liText(n, out);
+			return;
+		case 'ul':
+		case 'ol':
+			newline(out);
+			listText(n, out);
+			newline(out);
+			return;
+		case 'pre':
+			newline(out);
+			out.push(inlineOf(n).replace(/\n/g, '\uE000'));
+			newline(out);
+			return;
+		case 'math': {
+			const tex = annotationTex(n);
+			if (tex) out.push(` $${tex}$ `);
+			return;
+		}
+		case 'br':
+		case 'hr':
+			newline(out);
+			return;
+		case 'table':
+			renderTable(n, out);
+			return;
+		case 'p':
+		case 'div':
+		case 'section':
+		case 'article':
+		case 'blockquote':
+			newline(out);
+			renderBlocks(n.children ?? [], out);
+			newline(out);
+			return;
+		default:
+			renderBlocks(n.children ?? [], out);
+	}
+}
+
+const DATE_KEYWORDS = ['datePublished', 'dateCreated', 'uploadDate', 'publishedAt'];
+const DATE_META_NAMES = [
+	'article:published_time',
+	'datePublished',
+	'publish-date',
+	'pubdate',
+	'parsely-pub-date',
+	'date',
+];
+
+function toIso(v: string | undefined): string | undefined {
+	if (!v) return undefined;
+	const ms = Date.parse(v.trim());
+	if (Number.isNaN(ms) || ms < 0) return undefined;
+	return new Date(ms).toISOString().slice(0, 10);
+}
+
+function jsonLdDate(nodes: DomNode[]): string | undefined {
+	const pick = (value: unknown): string | undefined => {
+		if (typeof value === 'string') {
+			const iso = toIso(value);
+			if (iso) return iso;
+		}
+		return undefined;
+	};
+	const walk = (value: unknown): string | undefined => {
+		if (Array.isArray(value)) {
+			for (const v of value) {
+				const hit = walk(v);
+				if (hit) return hit;
+			}
+			return undefined;
+		}
+		if (typeof value === 'object' && value !== null) {
+			for (const key of DATE_KEYWORDS) {
+				const hit = pick((value as Record<string, unknown>)[key]);
+				if (hit) return hit;
+			}
+			for (const v of Object.values(value as Record<string, unknown>)) {
+				const hit = walk(v);
+				if (hit) return hit;
+			}
+		}
+		return undefined;
+	};
+	const visit = (n: DomNode): string | undefined => {
+		if (n.type === 'tag' && n.name === 'script') {
+			const type = (n.attribs?.['type'] ?? '').toLowerCase().trim();
+			if (type === 'application/ld+json' || type === 'application/json' || type === 'ld+json') {
+				try {
+					const hit = walk(JSON.parse(n.data ?? ''));
+					if (hit) return hit;
+				} catch {}
+			}
+		}
+		for (const c of n.children ?? []) {
+			const hit = visit(c);
+			if (hit) return hit;
+		}
+		return undefined;
+	};
+	for (const n of nodes) {
+		const hit = visit(n);
+		if (hit) return hit;
+	}
+	return undefined;
+}
+
+export interface HeadScan {
+	title: string;
+	publishedAt?: string;
+}
+
+export function headScan(html: string): HeadScan {
+	let doc: DomNode;
+	try {
+		doc = parseDocument(html) as unknown as DomNode;
+	} catch {
+		const title = /<title[^>]*>([^<]*)<\/title>/i.exec(html)?.[1]?.trim()?.slice(0, 200) ?? '';
+		return { title };
+	}
+	const nodes = doc.children ?? [];
+	let title = '';
+	let ogTitle = '';
+	let metaDate: string | undefined;
+	let timeDate: string | undefined;
+
+	const visit = (n: DomNode): void => {
+		if (n.type !== 'tag') return;
+		const name = n.name ?? '';
+		const attribs = n.attribs ?? {};
+		if (name === 'title' && !title) {
+			title = (n.children ?? [])
+				.filter((c) => c.type === 'text')
+				.map((c) => c.data ?? '')
+				.join('')
+				.trim()
+				.slice(0, 200);
+		}
+		if (name === 'meta') {
+			const key = (attribs['property'] ?? attribs['name'] ?? '').toLowerCase().trim();
+			if (!ogTitle && key === 'og:title') ogTitle = (attribs['content'] ?? '').trim().slice(0, 200);
+			if (!metaDate && DATE_META_NAMES.includes(key)) metaDate = toIso(attribs['content']);
+		}
+		if (name === 'time' && !timeDate && attribs['datetime']) timeDate = toIso(attribs['datetime']);
+		for (const c of n.children ?? []) visit(c);
+	};
+	for (const n of nodes) visit(n);
+
+	return { title: title || ogTitle, publishedAt: jsonLdDate(nodes) ?? metaDate ?? timeDate };
+}

package/src/scrape.ts

@@ -18,6 +18,7 @@ import {
 	type RemoteDoc,
 	type Renderer,
 } from './transport';
+import { headScan, treeToMarkdownish } from './dom';
 
 export interface ScrapeResult {
 	url: string;
@@ -94,35 +95,6 @@ export function decodeHtmlEntities(s: string): string {
 	});
 }
 
-function stripKatexSpans(h: string): string {
-	let out = '';
-	let i = 0;
-	for (;;) {
-		const start = /<span[^>]*class="[^"]*katex[^"]*"[^>]*>/i.exec(h.slice(i));
-		if (!start || start.index === undefined) {
-			out += h.slice(i);
-			break;
-		}
-		const abs = i + start.index;
-		out += h.slice(i, abs);
-		let depth = 1;
-		let j = abs + start[0].length;
-		while (depth > 0 && j < h.length) {
-			const next = /<span\b[^>]*>|<\/span>/gi;
-			next.lastIndex = j;
-			const tag = next.exec(h);
-			if (!tag) {
-				j = h.length;
-				break;
-			}
-			j = tag.index + tag[0].length;
-			depth += tag[0].startsWith('</') ? -1 : 1;
-		}
-		i = j;
-	}
-	return out;
-}
-
 export function extractHtml(html: string): { title: string; text: string; publishedAt?: string; paywalled: boolean } {
 	const paywalled =
 		/class="[^"]*(?:fc-ab-root|metering-modal|paywall|pw-overlay|subscription-overlay|regwall)[^"]*"/i.test(html) ||
@@ -130,61 +102,8 @@ export function extractHtml(html: string): { title: string; text: string; publis
 		/<meta[^>]+(?:property|name)=["'](?:isAccessibleForFree|tunnel:[^"']*paywall)["'][^>]+content=["'](?:false|yes)["']/i.test(
 			html,
 		);
-	const title =
-		/<title[^>]*>([^<]*)<\/title>/i.exec(html)?.[1]?.trim()?.slice(0, 200) ||
-		/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i.exec(html)?.[1]?.slice(0, 200) ||
-		'';
-	const publishedAt = extractPublishedDate(html);
-	let h = html
-		.replace(/<script[\s\S]*?<\/script>/gi, ' ')
-		.replace(/<style[\s\S]*?<\/style>/gi, ' ')
-		.replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
-		.replace(/<svg[\s\S]*?<\/svg>/gi, ' ')
-		.replace(/<!--[\s\S]*?-->/g, ' ')
-		.replace(/<nav[\s\S]*?<\/nav>/gi, ' ')
-		.replace(/<footer[\s\S]*?<\/footer>/gi, ' ')
-		.replace(/<aside[\s\S]*?<\/aside>/gi, ' ');
-	h = h.replace(/<pre[\s>][\s\S]*?<\/pre>/gi, (m) => m.replace(/\n/g, '\uE000'));
-	h = h.replace(/<table[\s>][\s\S]*?<\/table>/gi, (table) => {
-		const rows = [...table.matchAll(/<tr[\s>][\s\S]*?<\/tr>/gi)].map((r) =>
-			[...r[0].matchAll(/<t([hd])[\s>][\s\S]*?<\/t\1>/gi)].map((c) =>
-				c[0]
-					.replace(/<[^>]+>/g, ' ')
-					.replace(/\s+/g, ' ')
-					.replace(/\|/g, '\\|')
-					.trim(),
-			),
-		);
-		const grid = rows.filter((r) => r.length);
-		if (grid.length < 2) return table;
-		const cols = Math.max(...grid.map((g) => g.length));
-		if (cols < 2) return table;
-		const line = (g: string[]) => {
-			const cells = g.slice(0, cols);
-			while (cells.length < cols) cells.push(' ');
-			return `| ${cells.join(' | ')} |`;
-		};
-		const md = [line(grid[0]), `| ${Array.from({ length: cols }, () => '---').join(' | ')} |`];
-		for (const g of grid.slice(1)) md.push(line(g));
-		return `\n${md.join('\n')}\n`;
-	});
-	h = stripKatexSpans(h);
-	h = h.replace(/<math[\s>][\s\S]*?<\/math>/gi, (m) => {
-		const tex = /<annotation[^>]*encoding=["']application\/x-tex["'][^>]*>([\s\S]*?)<\/annotation>/i.exec(m)?.[1];
-		return tex?.trim() ? ` $${tex.trim()}$ ` : ' ';
-	});
-	h = h.replace(
-		/<h([1-6])[^>]*>(.*?)<\/h\1>/gis,
-		(_, n, inner) => `\n${'#'.repeat(Number(n))} ${inner.replace(/<[^>]+>/g, ' ').trim()}\n`,
-	);
-	h = h
-		.replace(/<li[^>]*>/gi, '\n• ')
-		.replace(/<\/(p|div|li|tr|section|article|blockquote|table|ul|ol|pre)>/gi, '\n')
-		.replace(/<(br|hr)[^>]*>/gi, '\n');
-	// strip tags first, THEN decode entities: escaped markup shown as literal
-	// text (`&lt;limit...&gt;`) must survive as text, not be stripped as a tag
-	h = h.replace(/<[^>]+>/g, ' ');
-	h = h.replace(/\uE000/g, '\n');
+	const { title, publishedAt } = headScan(html);
+	let h = treeToMarkdownish(html).replace(/\uE000/g, '\n');
 	h = stripToAscii(decodeHtmlEntities(h));
 	const lines = h
 		.split('\n')
@@ -230,21 +149,6 @@ export function extractHtml(html: string): { title: string; text: string; publis
 	return { title, publishedAt, paywalled, text: out.join('\n').replace(/\n{3,}/g, '\n\n') };
 }
 
-function extractPublishedDate(html: string): string | undefined {
-	const iso = (v: string | undefined) => {
-		if (!v) return undefined;
-		const t = Date.parse(v.trim());
-		return Number.isNaN(t) || t < 0 ? undefined : new Date(t).toISOString().slice(0, 10);
-	};
-	const ld = /"datePublished"\s*:\s*"([^"]+)"/.exec(html)?.[1];
-	const meta =
-		/<meta[^>]+(?:property|name)=["'](?:article:published_time|datePublished|publish-date|pubdate|parsely-pub-date|date)["'][^>]+content=["']([^"']+)["']/i.exec(
-			html,
-		)?.[1];
-	const timeEl = /<time[^>]+datetime=["']([^"']+)["']/i.exec(html)?.[1];
-	return iso(ld) ?? iso(meta) ?? iso(timeEl);
-}
-
 const NAV_LABEL_RE =
 	/^(home|menu|about( us)?|docs?|documentation|downloads?|license|licence|support|purchase|buy|pricing|search|login|log ?in|sign ?(in|up|out)|register|account|blog|news|contact|careers?|jobs|store|shop|forum|community|tutorials?|api|guide|guides|reference|examples?|faq|help|terms|privacy|cookies?|sitemap|rss|changelog|releases?|twitter|facebook|github|discord|youtube|linkedin|mastodon|x)$/i;
 

package/src/tools.ts

@@ -4,7 +4,7 @@
  */
 import { Type } from 'typebox';
 import type { ExtensionAPI, ExtensionCommandContext, Theme } from '@earendil-works/pi-coding-agent';
-import { keyHint } from '@earendil-works/pi-coding-agent';
+import { keyHint, keyText, rawKeyHint } from '@earendil-works/pi-coding-agent';
 import { Text, type Component } from '@earendil-works/pi-tui';
 import { appendFileSync, mkdirSync, rmSync, writeFileSync, existsSync } from 'node:fs';
 import { dirname, join } from 'node:path';
@@ -363,17 +363,18 @@ class WrappedSearchText implements Component {
 	render(width: number): string[] {
 		const wrapped = wrapAll(this.raw, Math.max(40, width));
 		const lines = wrapped.split('\n');
-		let shown = lines;
-		let hint = '';
-		if (!this.expanded && lines.length > SEARCH_PREVIEW_LINES) {
-			shown = lines.slice(0, SEARCH_PREVIEW_LINES);
-			hint = moreLinesHint(shown.length, lines.length, this.theme);
-		}
+		const expanded = this.expanded || lines.length <= SEARCH_PREVIEW_LINES;
+		const shown = expanded ? lines : lines.slice(0, SEARCH_PREVIEW_LINES);
 		const colored = italicizeRanks(shown.join('\n'), this.theme)
 			.split('\n')
 			.map((l) => colorLine(l, this.theme))
 			.join('\n');
-		return replaceTabs(colored + hint).split('\n');
+		const out = replaceTabs(colored).split('\n');
+		if (!expanded) {
+			const hint = moreLinesHint(shown.length, lines.length, this.theme);
+			if (hint) out.push(hint);
+		}
+		return out;
 	}
 }
 
@@ -410,13 +411,20 @@ function resultText(result: { content: { type: string; text?: string }[] }): str
 function moreLinesHint(shown: number, total: number, theme: Theme): string {
 	const remaining = total - shown;
 	if (remaining <= 0) return '';
-	return theme.fg('muted', `\n… (${remaining} more lines, ${keyHint('app.tools.expand', 'to expand')})`);
+	let hint: string;
+	try {
+		hint = keyText('app.tools.expand') ? keyHint('app.tools.expand', 'to expand') : rawKeyHint('ctrl+o', 'to expand');
+	} catch {
+		hint = rawKeyHint('ctrl+o', 'to expand');
+	}
+	return theme.fg('muted', `… (${remaining} more lines,`) + ' ' + hint + theme.fg('muted', ')');
 }
 
 function previewWithHint(lines: string[], theme: Theme, maxLines = PREVIEW_LINES): string {
 	const shown = lines.slice(0, maxLines);
 	const text = shown.map((l) => theme.fg('toolOutput', replaceTabs(l))).join('\n');
-	return text + moreLinesHint(shown.length, lines.length, theme);
+	const hint = moreLinesHint(shown.length, lines.length, theme);
+	return hint ? `${text}\n${hint}` : text;
 }
 
 function replaceTabs(text: string): string {

package/src/transport.ts

@@ -203,7 +203,7 @@ function decodeBody(bytes: Uint8Array, contentType: string): string {
 	return looksMojibake(text) ? repairMojibake(text) : text;
 }
 
-async function readBodyBounded(res: Response, maxBytes: number): Promise<Uint8Array> {
+export async function readBodyBounded(res: Response, maxBytes: number): Promise<Uint8Array> {
 	const declared = Number(res.headers.get('content-length') ?? 0);
 	if (declared > maxBytes) throw new Error('page too large');
 	const reader = res.body?.getReader();
@@ -218,7 +218,10 @@ async function readBodyBounded(res: Response, maxBytes: number): Promise<Uint8Ar
 		const { done, value } = await reader.read();
 		if (done) break;
 		size += value.byteLength;
-		if (size > maxBytes) throw new Error('page too large');
+		if (size > maxBytes) {
+			await reader.cancel();
+			throw new Error('page too large');
+		}
 		chunks.push(value);
 	}
 	const out = new Uint8Array(size);