pi-webfind 0.5.2 → 0.6.0

package/lib/net.ts

@@ -0,0 +1,93 @@
+// Host-level state shared by engines and the fetcher: per-host cooldowns
+// (HTTP 429), the process-wide offline flag (ENOTFOUND/ENETUNREACH), and the
+// single rate-limit gap for the r.jina.ai relay.
+
+const cooldowns = new Map<string, number>();
+
+/** Put `host` on cooldown for `ms` milliseconds (e.g. after an HTTP 429). */
+export function setHostCooldown(host: string, ms: number): void {
+	cooldowns.set(host, Date.now() + ms);
+}
+
+/** Epoch ms until which `host` is cooling down (0 = not cooling). */
+export function hostCooldownUntil(host: string): number {
+	return cooldowns.get(host) ?? 0;
+}
+
+// ------------------------------------------------------------- offline flag
+
+const OFFLINE_MS = 10_000;
+let offlineUntil = 0;
+let offlineCode = "";
+
+/**
+ * Mark the network as offline for 10 s. Called on ENOTFOUND/ENETUNREACH from
+ * the search hosts (a typo'd target host must not black out the network —
+ * `markOfflineIfSearchHost` guards the single-host case).
+ */
+export function markOffline(code: string): void {
+	offlineUntil = Date.now() + OFFLINE_MS;
+	offlineCode = code;
+}
+
+/** Any successful/answered request clears the flag. */
+export function markOnline(): void {
+	offlineUntil = 0;
+}
+
+/** Throws while the offline flag is set (called at the top of every network fn). */
+export function assertOnline(): void {
+	if (Date.now() < offlineUntil) throw new Error(`network unavailable (${offlineCode})`);
+}
+
+/** True while the offline flag is set (tools surface "Network appears offline"). */
+export function isOffline(): boolean {
+	return Date.now() < offlineUntil;
+}
+
+/**
+ * ENOTFOUND from a single non-search host is usually a typo'd URL, not an
+ * outage — only search/archive infrastructure trips the process-wide flag on
+ * its own (or any two distinct hosts failing within 2 s).
+ */
+const SEARCH_HOSTS = /^(html|lite)\.duckduckgo\.com$|^(www\.)?bing\.com$|^search\.brave\.com$|^archive\.org$|^r\.jina\.ai$/;
+const recentNotFound: Array<{ host: string; at: number }> = [];
+
+export function noteNotFound(host: string): void {
+	if (SEARCH_HOSTS.test(host)) {
+		markOffline("ENOTFOUND");
+		return;
+	}
+	const now = Date.now();
+	recentNotFound.push({ host, at: now });
+	while (recentNotFound.length > 0 && now - recentNotFound[0]!.at > 2_000) recentNotFound.shift();
+	const distinct = new Set(recentNotFound.map((r) => r.host));
+	if (distinct.size >= 2) markOffline("ENOTFOUND");
+}
+
+/** Wait so that consecutive r.jina.ai requests stay ≥ `JINA_RATE_MS` apart. Single shared gap. */
+let lastJinaCall = 0;
+/** With a free JINA_API_KEY the reader relay allows ~200 rpm instead of ~20 — throttle accordingly. */
+export const JINA_RATE_MS = process.env.JINA_API_KEY ? 300 : 3_500;
+export async function jinaGap(signal?: AbortSignal): Promise<void> {
+	const wait = lastJinaCall + JINA_RATE_MS - Date.now();
+	if (wait > 0) await sleep(wait, signal);
+	lastJinaCall = Date.now();
+}
+
+/** Authorization header for r.jina.ai when JINA_API_KEY is set (raises the rate cap). */
+export function jinaAuth(): Record<string, string> {
+	const key = process.env.JINA_API_KEY;
+	return key ? { Authorization: `Bearer ${key}` } : {};
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+	return new Promise((resolve) => {
+		if (signal?.aborted) return resolve();
+		const t = setTimeout(resolve, ms);
+		signal?.addEventListener("abort", () => {
+			clearTimeout(t);
+			resolve();
+		}, { once: true });
+	});
+}

package/lib/rank.ts

@@ -6,7 +6,8 @@
  * Zero dependencies, no model calls.
  */
 
-const WORD_RE = /[a-z0-9_#+.-]+/g;
+const WORD_RE = /[\p{L}\p{N}_#+-]+(?:\.[\p{L}\p{N}_#+-]+)*/gu;
+const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
 
 function stem(t: string): string {
 	if (t.length > 4 && t.endsWith("ing")) return t.slice(0, -3);
@@ -15,8 +16,26 @@ function stem(t: string): string {
 	return t;
 }
 
-function tokenize(s: string): string[] {
-	return (s.toLowerCase().match(WORD_RE) ?? []).map(stem).filter((t) => t.length > 0);
+export function tokenize(s: string): string[] {
+	const out: string[] = [];
+	for (const m of s.toLowerCase().matchAll(WORD_RE)) {
+		const w = m[0];
+		if (CJK_RE.test(w)) {
+			CJK_RE.lastIndex = 0;
+			for (const run of w.match(CJK_RE) ?? []) {
+				if (run.length === 1) out.push(run);
+				for (let i = 0; i + 1 < run.length; i++) out.push(run.slice(i, i + 2));
+			}
+			for (const r of w.replace(CJK_RE, " ").trim().split(/\s+/)) if (r) out.push(stem(r));
+			continue;
+		}
+		out.push(stem(w));
+		// dotted identifiers: also emit the last segment (react.useEffect -> useeffect)
+		const dot = w.lastIndexOf(".");
+		const tail = dot > 0 ? w.slice(dot + 1) : "";
+		if (tail && /\p{L}/u.test(tail)) out.push(stem(tail));
+	}
+	return out;
 }
 
 export interface Passage {
@@ -25,6 +44,8 @@ export interface Passage {
 	heading: string;
 	/** index in document order */
 	pos: number;
+	/** "code" = a whole fenced block (or chunk of one); exempt from MIN_PASSAGE */
+	kind: "prose" | "code";
 }
 
 const MIN_PASSAGE = 24;
@@ -35,7 +56,23 @@ export function splitPassages(text: string): Passage[] {
 	const lines = text.split("\n");
 	let heading = "";
 	let buf: string[] = [];
+	const FENCE_RE = /^\s*(```|~~~)/;
+	let fence: string[] | null = null;
 	for (const line of lines) {
+		if (fence) {
+			fence.push(line);
+			if (FENCE_RE.test(line)) {
+				pushFence(fence);
+				fence = null;
+			}
+			continue;
+		}
+		if (FENCE_RE.test(line)) {
+			push(buf.join("\n"));
+			buf = [];
+			fence = [line];
+			continue;
+		}
 		if (line.trim() === "") {
 			push(buf.join("\n"));
 			buf = [];
@@ -50,28 +87,50 @@ export function splitPassages(text: string): Passage[] {
 		}
 		buf.push(line);
 	}
+	if (fence) pushFence([...fence, fence[0].match(FENCE_RE)![1]!]); // unterminated fence at EOF
 	push(buf.join("\n").trim());
 	return passages;
 
+	/** Whole fenced block as one code passage (or ~900-char line-chunks when huge). */
+	function pushFence(fl: string[]) {
+		const opener = fl[0]!, closer = fl[fl.length - 1]!, body = fl.slice(1, -1);
+		if (body.join("\n").trim().length === 0) return;
+		const emit = (chunk: string[]) =>
+			passages.push({ text: [opener, ...chunk, closer].join("\n"), heading, pos: passages.length, kind: "code" });
+		if (body.join("\n").length <= 1200) return emit(body);
+		let chunk: string[] = [];
+		let size = 0;
+		for (const l of body) {
+			if (size + l.length > 900 && chunk.length) {
+				emit(chunk);
+				chunk = [];
+				size = 0;
+			}
+			chunk.push(l);
+			size += l.length + 1;
+		}
+		if (chunk.length) emit(chunk);
+	}
+
 	/** Long blobs (infoboxes, template junk, minified docs) split on sentence boundaries so scoring can discriminate. */
 	function push(raw: string) {
 		const t = raw.trim();
 		if (t.length < MIN_PASSAGE) return;
 		if (t.length <= 1200) {
-			passages.push({ text: t, heading, pos: passages.length });
+			passages.push({ text: t, heading, pos: passages.length, kind: "prose" });
 			return;
 		}
 		const sentences = t.match(/[^.!?\n]+[.!?]?\s*/g) ?? [t];
 		let cur = "";
 		for (const sen of sentences) {
 			if (cur.length + sen.length > 900 && cur.length >= MIN_PASSAGE) {
-				passages.push({ text: cur.trim(), heading, pos: passages.length });
+				passages.push({ text: cur.trim(), heading, pos: passages.length, kind: "prose" });
 				cur = sen;
 			} else {
 				cur += sen;
 			}
 		}
-		if (cur.trim().length >= MIN_PASSAGE) passages.push({ text: cur.trim(), heading, pos: passages.length });
+		if (cur.trim().length >= MIN_PASSAGE) passages.push({ text: cur.trim(), heading, pos: passages.length, kind: "prose" });
 	}
 }
 
@@ -89,28 +148,33 @@ export function scorePassages(passages: Passage[], query: string): Array<{ p: Pa
 	const qTokens = [...new Set(tokenize(query))];
 	if (qTokens.length === 0) return passages.map((p) => ({ p, score: 0 }));
 
-	const tokenSets: Array<Set<string>> = passages.map((p) => new Set(tokenize(p.text)));
+	const tfs: Array<Map<string, number>> = passages.map((p) => termFreq(tokenize(p.text)));
 	const N = passages.length;
 	const avgLen = passages.reduce((a, p) => a + p.text.length, 0) / Math.max(N, 1);
 	const k1 = 1.5;
 	const b = 0.75;
 	const wantCode = queryCodeish(query);
+	// document frequency per query token, computed once
+	const df = new Map<string, number>(qTokens.map((q) => [q, tfs.reduce((n, m) => n + (m.has(q) ? 1 : 0), 0)]));
+	// phrase bonus: multi-token query occurring verbatim in the passage
+	const phrase = query.toLowerCase().replace(/\s+/g, " ").trim();
 
 	return passages.map((p, i) => {
-		const tokens = tokenSets[i];
+		const tf = tfs[i];
 		const len = p.text.length;
 		// junk penalty: template/URL soup — real prose has few of these per 100 chars
 		const junk = (p.text.match(/\{\{|\}\}|\[\[|\]\]|https?:\/\//g) ?? []).length;
 		const junkDensity = junk / Math.max(len / 100, 1);
 		let score = 0;
 		for (const q of qTokens) {
-			if (!tokens.has(q)) continue;
-			let dfq = 0;
-			for (const ts of tokenSets) if (ts.has(q)) dfq++;
+			const n = tf.get(q) ?? 0;
+			if (!n) continue;
+			const dfq = df.get(q)!;
 			const idf = Math.log(1 + (N - dfq + 0.5) / (dfq + 0.5));
-			score += (idf * 1 * (k1 + 1)) / (1 + k1 * (1 - b + b * (len / avgLen)));
+			score += (idf * n * (k1 + 1)) / (n + k1 * (1 - b + b * (len / avgLen)));
 		}
 		if (p.heading && qTokens.some((q) => p.heading.toLowerCase().includes(q))) score *= 1.5;
+		if (qTokens.length >= 2 && phrase.length >= 6 && p.text.toLowerCase().replace(/\s+/g, " ").includes(phrase)) score *= 1.5;
 		if (wantCode && /```|\t|=>|function |const |def |class /.test(p.text)) score *= 1.2;
 		if (junkDensity > 1) score *= 0.3;
 		else if (junkDensity > 0.4) score *= 0.6;
@@ -118,6 +182,12 @@ export function scorePassages(passages: Passage[], query: string): Array<{ p: Pa
 	});
 }
 
+export function termFreq(tokens: string[]): Map<string, number> {
+	const m = new Map<string, number>();
+	for (const t of tokens) m.set(t, (m.get(t) ?? 0) + 1);
+	return m;
+}
+
 export interface PickedPassage {
 	heading: string;
 	text: string;
@@ -128,17 +198,20 @@ export interface PickedPassage {
  * Pick the passages worth showing for a query within a char budget:
  * document intro first, then top-scoring passages in original order.
  * `total` = passages in the full document (for the "N of M shown" footer).
+ * `passages` = the picked set incl. scores — lets callers re-rank (e.g. deep
+ * mode picks the single best-scoring passage instead of the intro).
  */
 export function topPassages(
 	text: string,
 	query: string,
 	budgetChars = 6000,
 	introChars = 600,
-): { picked: PickedPassage[]; total: number } {
+): { picked: PickedPassage[]; total: number; passages: PickedPassage[] } {
 	const all = splitPassages(text);
-	if (all.length === 0) return { picked: [], total: 0 };
+	if (all.length === 0) return { picked: [], total: 0, passages: [] };
 	if (!query.trim()) {
-		return { picked: [{ heading: all[0].heading, text: text.slice(0, budgetChars), score: 0 }], total: all.length };
+		const introOnly = { heading: all[0].heading, text: text.slice(0, budgetChars), score: 0 };
+		return { picked: [introOnly], total: all.length, passages: [introOnly] };
 	}
 
 	const scored = scorePassages(all, query);
@@ -165,5 +238,5 @@ export function topPassages(
 		.sort((a, b) => a[0] - b[0])
 		.map(([, v]) => v);
 	picked.unshift({ heading: intro.heading, text: intro.text.slice(0, introChars), score: 0 });
-	return { picked, total: all.length };
+	return { picked, total: all.length, passages: picked };
 }

package/lib/safe.ts

@@ -0,0 +1,94 @@
+/**
+ * SSRF-safe URL validation: protocol/credential/name checks plus DNS
+ * resolution — a public-looking name that resolves to a private address is
+ * rejected (nip.io / lvh.me style bypasses), and redirect hops are re-checked
+ * by the fetcher before each new connection.
+ *
+ * Residual risk (documented, deferred to WP-10): the address is validated at
+ * lookup time, not pinned into the TCP connection, so a DNS-rebinding race
+ * between resolveSafe() and connect() remains theoretically open. Node's
+ * global fetch exposes no supported way to inject a custom dns.lookup.
+ */
+import { BlockList, isIP } from "node:net";
+import { lookup as dnsLookup } from "node:dns/promises";
+
+export type Lookup = (host: string) => Promise<Array<{ address: string; family: number }>>;
+
+/** Test seam: tests replace `lookup` with a table; production leaves it alone. */
+export const safeConfig: { lookup: Lookup } = { lookup: (h) => dnsLookup(h, { all: true, verbatim: true }) };
+
+const PRIVATE = new BlockList();
+for (const [net, bits] of [
+	["127.0.0.0", 8],
+	["10.0.0.0", 8],
+	["172.16.0.0", 12],
+	["192.168.0.0", 16],
+	["169.254.0.0", 16],
+	["100.64.0.0", 10],
+	["0.0.0.0", 8],
+] as const)
+	PRIVATE.addSubnet(net, bits, "ipv4");
+for (const [net, bits] of [
+	["::1", 128],
+	["::", 128],
+	["fc00::", 7],
+	["fe80::", 10],
+	["64:ff9b::", 96],
+] as const)
+	PRIVATE.addSubnet(net, bits, "ipv6");
+// NOTE: deliberately NOT adding ::ffff:0:0/96 — Node compares v4-mapped IPv6
+// against the v4 rules above (8.8.8.8 stays allowed, ::ffff:127.0.0.1 blocked).
+
+const BLOCKED_NAME = /^(localhost|localhost\.localdomain)$|\.(local|internal|localhost|home\.arpa)$/i;
+
+export function isPrivateAddress(addr: string): boolean {
+	const fam = isIP(addr);
+	if (fam === 0) return true; // unparseable addresses fail closed
+	return PRIVATE.check(addr, fam === 6 ? "ipv6" : "ipv4");
+}
+
+/** Protocol, credentials, name and *resolved address* check. Throws `Blocked …`; returns the parsed URL. */
+export async function resolveSafe(input: string | URL, lookup: Lookup = safeConfig.lookup): Promise<URL> {
+	let url: URL;
+	try {
+		url = typeof input === "string" ? new URL(input) : input;
+	} catch {
+		throw new Error(`Invalid URL: ${String(input)}`);
+	}
+	if (!/^https?:$/.test(url.protocol)) throw new Error(`Blocked protocol: ${url.protocol} (use http/https)`);
+	if (url.username || url.password) throw new Error("Blocked URL: credentials in URL");
+	const host = url.hostname.replace(/\.$/, "").toLowerCase();
+	const literal = host.startsWith("[") ? host.slice(1, -1) : host;
+	if (BLOCKED_NAME.test(host)) throw new Error(`Blocked host (SSRF protection): ${host}`);
+	if (isIP(literal)) {
+		if (isPrivateAddress(literal)) throw new Error(`Blocked host (SSRF protection): ${host}`);
+		return url;
+	}
+	let addrs: Array<{ address: string; family: number }>;
+	try {
+		addrs = await lookup(host);
+	} catch (e) {
+		throw new Error(`DNS lookup failed for ${host} (${(e as NodeJS.ErrnoException)?.code ?? "unknown"})`);
+	}
+	if (addrs.length === 0) throw new Error(`DNS lookup failed for ${host} (no addresses)`);
+	const bad = addrs.find((a) => isPrivateAddress(a.address));
+	if (bad) throw new Error(`Blocked host (SSRF protection): ${host} resolves to ${bad.address}`);
+	return url;
+}
+
+/** Synchronous subset (protocol, credentials, blocked names, literal IPs) — used for cache keys and adapter routing. */
+export function assertSafeUrl(rawUrl: string): URL {
+	let url: URL;
+	try {
+		url = new URL(rawUrl);
+	} catch {
+		throw new Error(`Invalid URL: ${rawUrl}`);
+	}
+	if (!/^https?:$/.test(url.protocol)) throw new Error(`Blocked protocol: ${url.protocol} (use http/https)`);
+	if (url.username || url.password) throw new Error("Blocked URL: credentials in URL");
+	const host = url.hostname.replace(/\.$/, "").toLowerCase();
+	const literal = host.startsWith("[") ? host.slice(1, -1) : host;
+	if (BLOCKED_NAME.test(host)) throw new Error(`Blocked host (SSRF protection): ${host}`);
+	if (isIP(literal) && isPrivateAddress(literal)) throw new Error(`Blocked host (SSRF protection): ${host}`);
+	return url;
+}

package/lib/version.ts

@@ -1,4 +1,4 @@
 // Single source of truth for the tool's version, used in UA strings.
 // Kept in sync with package.json manually on release (release checklist item).
-export const VERSION = "0.5.2";
+export const VERSION = "0.6.0";
 export const TOOL_UA = `pi-webfind/${VERSION} (free web research toolkit for pi coding agent; +https://github.com/jawwadzafar/pi-webfind)`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-webfind",
-  "version": "0.5.2",
+  "version": "0.6.0",
   "description": "Claude Code-style web research for the pi coding agent \u2014 WebSearch, query-aware fetch (PDFs, Wayback, site adapters, markdown extraction), Stack Overflow, GitHub, HN, Wikipedia, npm. 100% free, no API keys.",
   "type": "module",
   "keywords": [
@@ -20,37 +20,41 @@
     "type": "git",
     "url": "git+https://github.com/jawwadzafar/pi-webfind.git"
   },
-  "pi": {
-    "extensions": [
-      "./extensions"
-    ],
-    "image": "https://jawwadzafar.github.io/pi-webfind/screenshot.png",
-    "video": "https://jawwadzafar.github.io/pi-webfind/demo.mp4"
+  "scripts": {
+    "test": "node --test tests/*.test.ts",
+    "typecheck": "tsc --noEmit",
+    "docs:dev": "vitepress dev docs",
+    "docs:build": "vitepress build docs",
+    "docs:preview": "vitepress preview docs"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",
     "typebox": "*"
   },
+  "devDependencies": {
+    "@earendil-works/pi-coding-agent": "0.85.0",
+    "@earendil-works/pi-tui": "0.85.0",
+    "typebox": "1.3.7",
+    "vitepress": "^1.6.3"
+  },
   "files": [
     "extensions/",
     "lib/",
     "README.md",
-    "LICENSE"
+    "LICENSE",
+    "themes/"
   ],
   "engines": {
     "node": ">=20"
   },
   "author": "Jawwad Zafar <zafarjawwad@gmail.com>",
   "homepage": "https://jawwadzafar.github.io/pi-webfind",
-  "scripts": {
-    "docs:dev": "vitepress dev docs",
-    "docs:build": "vitepress build docs",
-    "docs:preview": "vitepress preview docs"
-  },
-  "devDependencies": {
-    "vitepress": "^1.6.3"
-  },
   "bugs": {
     "url": "https://github.com/jawwadzafar/pi-webfind/issues"
+  },
+  "pi": {
+    "themes": [
+      "./themes"
+    ]
   }
 }

package/themes/claude-dark.json

@@ -0,0 +1,80 @@
+{
+	"$schema": "https://raw.githubusercontent.com/earendil-works/pi/main/packages/coding-agent/src/modes/interactive/theme/theme-schema.json",
+	"name": "Claude Dark",
+	"vars": {
+		"cyan": "#d7c197",
+		"blue": "#7aa2f7",
+		"green": "#6ec876",
+		"red": "#ff6b80",
+		"yellow": "#d9a05b",
+		"text": "#e8e4dd",
+		"gray": "#8a857c",
+		"dimGray": "#5e594f",
+		"darkGray": "#3e3a34",
+		"accent": "#d7c197",
+		"selectedBg": "#3a3630",
+		"userMsgBg": "#2d2a26",
+		"toolPendingBg": "#262320",
+		"toolSuccessBg": "#232a23",
+		"toolErrorBg": "#33241f",
+		"customMsgBg": "#2a2724"
+	},
+	"colors": {
+		"accent": "accent",
+		"border": "blue",
+		"borderAccent": "cyan",
+		"borderMuted": "darkGray",
+		"success": "green",
+		"error": "red",
+		"warning": "yellow",
+		"muted": "gray",
+		"dim": "dimGray",
+		"text": "text",
+		"thinkingText": "gray",
+		"selectedBg": "selectedBg",
+		"scrollbarTrack": "darkGray",
+		"scrollbarThumb": "text",
+		"searchMatchBg": "selectedBg",
+		"searchMatchText": "text",
+		"userMessageBg": "userMsgBg",
+		"userMessageText": "text",
+		"customMessageBg": "customMsgBg",
+		"customMessageText": "text",
+		"customMessageLabel": "#d7c197",
+		"toolPendingBg": "toolPendingBg",
+		"toolSuccessBg": "toolSuccessBg",
+		"toolErrorBg": "toolErrorBg",
+		"toolTitle": "#e8e4dd",
+		"toolOutput": "gray",
+		"mdHeading": "#e8e4dd",
+		"mdLink": "#d7c197",
+		"mdLinkUrl": "#8a857c",
+		"mdCode": "accent",
+		"mdCodeBlock": "green",
+		"mdCodeBlockBorder": "gray",
+		"mdQuote": "gray",
+		"mdQuoteBorder": "gray",
+		"mdHr": "gray",
+		"mdListBullet": "accent",
+		"toolDiffAdded": "green",
+		"toolDiffRemoved": "red",
+		"toolDiffContext": "gray",
+		"syntaxComment": "#6A9955",
+		"syntaxKeyword": "#569CD6",
+		"syntaxFunction": "#DCDCAA",
+		"syntaxVariable": "#9CDCFE",
+		"syntaxString": "#CE9178",
+		"syntaxNumber": "#B5CEA8",
+		"syntaxType": "#4EC9B0",
+		"syntaxOperator": "#D4D4D4",
+		"syntaxPunctuation": "#D4D4D4",
+		"thinkingOff": "darkGray",
+		"thinkingMinimal": "#6e6e6e",
+		"thinkingLow": "#5f87af",
+		"thinkingMedium": "#81a2be",
+		"thinkingHigh": "#b294bb",
+		"thinkingXhigh": "#d183e8",
+		"thinkingMax": "#ff5fff",
+		"bashMode": "green"
+	}
+}