pi-webfind 0.5.1 → 0.6.0

package/README.md

@@ -5,7 +5,7 @@
 <h1 align="center">pi-webfind</h1>
 
 <p align="center">
-  <a href="https://www.npmjs.com/package/pi-webfind"><img alt="npm" src="https://img.shields.io/npm/v/pi-webfind?style=flat-square" /></a>
+  <a href="https://www.npmjs.com/package/pi-webfind"><img alt="npm version" src="https://img.shields.io/npm/v/pi-webfind?style=flat-square&color=cb3837" /></a>
   <a href="https://github.com/jawwadzafar/pi-webfind/actions/workflows/deploy-docs.yml"><img alt="docs" src="https://img.shields.io/website?url=https%3A%2F%2Fjawwadzafar.github.io%2Fpi-webfind%2F&style=flat-square&label=docs" /></a>
   <a href="https://github.com/jawwadzafar/pi-webfind/blob/main/LICENSE"><img alt="license" src="https://img.shields.io/npm/l/pi-webfind?style=flat-square" /></a>
   <a href="https://github.com/jawwadzafar/pi-webfind/stargazers"><img alt="stars" src="https://img.shields.io/github/stars/jawwadzafar/pi-webfind?style=flat-square&color=b5bd68" /></a>
@@ -106,8 +106,20 @@ with it:
 - **Never executes or writes fetched content**; it reads URLs and returns text
 - Politeness throttle per host; fake-browser UA only where required, honest
   `pi-webfind/x.y` UA everywhere it matters
-- The only optional credential is `GITHUB_TOKEN` (lifts GitHub's 10 req/min
-  anonymous limit). Everything else is keyless by design.
+- The only optional credentials are listed below. Everything else is keyless by design.
+
+## Optional accelerators (still fully free without them)
+
+| Env var | Effect when set | Without it |
+| --- | --- | --- |
+| `GITHUB_TOKEN` | Lifts GitHub API from 10 to ~30 req/min in site adapters | 10 req/min anonymous |
+| `JINA_API_KEY` | Reader relay throttle 3.5 s → 300 ms; enables `s.jina.ai` search leg in `engine:"multi"` | 3.5 s gap, no jina search leg |
+| `BRAVE_API_KEY` | Adds Brave's official JSON API as the first `engine:"multi"` attempt | scraped Brave only |
+| `TAVILY_API_KEY` | Adds Tavily as a `engine:"multi"` attempt | no Tavily |
+
+Every var is optional; with none set, behavior is identical to the baseline:
+keyless scraping with the fallback ladder. Keys are never required and never
+sent anywhere except their own API host.
 
 ## Limits (be honest about free)
 

package/extensions/web-search.ts

@@ -20,7 +20,10 @@ import {
 	cacheSet,
 	ddgSearch,
 	multiSearch,
+	relevanceGate,
+	searchRace,
 	type Recency,
+	type SearchOutcome,
 	type SearchResult,
 } from "../lib/engine.ts";
 import {
@@ -31,11 +34,13 @@ import {
 	searchWikipedia,
 } from "../lib/apis.ts";
 import { smartFetch, type FetchOptions } from "../lib/fetcher.ts";
+import { isOffline } from "../lib/net.ts";
 
 const MAX = (n?: number, dflt = 8, cap = 20) => Math.min(Math.max(n ?? dflt, 1), cap);
 const clip = (s: string, n: number) => (s.length > n ? s.slice(0, n - 1) + "…" : s);
 const secs = (ms: number) => `${(ms / 1000).toFixed(1)}s`;
 
+/** Display row — structurally accepts SearchResult (engine rows) and ApiResult (API rows). */
 interface Row {
 	title: string;
 	url: string;
@@ -43,6 +48,10 @@ interface Row {
 	meta?: string;
 	/** publication date when the engine provides one */
 	date?: string;
+	/** "indexed" = crawl stamp (e.g. Bing pubDate), not a publication date */
+	dateKind?: "published" | "indexed";
+	/** every engine that returned this URL (set by multiSearch fusion) */
+	engines?: string[];
 	/** deep mode: query-relevant excerpt fetched from the page itself */
 	excerpt?: string;
 }
@@ -51,7 +60,10 @@ function fmtResults(results: Row[]): string {
 	if (results.length === 0) return "No results found.";
 	return results
 		.map((r, i) => {
-			const lines = [`${i + 1}. ${r.title}`, `   ${r.url}${r.date ? `  (${r.date})` : ""}`];
+			// (indexed 2026-09-04) marks a crawl stamp, not a publication date
+			const date = r.date ? (r.dateKind === "indexed" ? `  (indexed ${r.date})` : `  (${r.date})`) : "";
+			const via = (r.engines ?? []).length > 0 ? `  · ${(r.engines ?? []).join("+")}` : "";
+			const lines = [`${i + 1}. ${r.title}`, `   ${r.url}${date}${via}`];
 			if (r.meta) lines.push(`   ${r.meta}`);
 			if (r.excerpt) lines.push(`   excerpt: ${clip(r.excerpt, 400)}`);
 			else if (r.snippet) lines.push(`   ${clip(r.snippet, 250)}`);
@@ -76,6 +88,22 @@ interface Theme {
  *     ⎿ via ddg · (ctrl+o to expand)
  * Green dot marks state, live ticking elapsed while running.
  */
+// Status-aware dot (Claude Code parity): dim while running, red on error,
+// green on success. Truecolor only when the terminal advertises it —
+// theme.fg() throws on raw hex, so non-truecolor uses theme names.
+const TRUECOLOR =
+	process.env.COLORTERM === "truecolor" || /256color|truecolor/.test(process.env.TERM ?? "");
+const dot = (t: Theme, phase: string | undefined) =>
+	phase === "error"
+		? TRUECOLOR
+			? "\x1b[38;2;255;107;128m⏺ \x1b[39m"
+			: t.fg("error", "⏺ ")
+		: phase === "ok"
+			? TRUECOLOR
+				? "\x1b[38;2;78;186;101m⏺ \x1b[39m"
+				: t.fg("success", "⏺ ")
+			: t.fg("dim", "⏺ ");
+
 function makeRenderers(
 	toolName: string,
 	argDetail: (args: any) => string,
@@ -91,9 +119,9 @@ function makeRenderers(
 			}
 			const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
 			text.setText(
-				t.fg("success", "⏺ ") +
+				dot(t, context.state.phase) +
 					t.fg("toolTitle", t.bold(toolName)) +
-					t.fg("dim", `(${JSON.stringify(clip(String(argDetail(args ?? {})), 70))})`),
+					t.fg("text", `(${JSON.stringify(clip(String(argDetail(args ?? {})), 70))})`),
 			);
 			return text;
 		},
@@ -107,9 +135,19 @@ function makeRenderers(
 			const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
 			const state = context.state ?? {};
 
-			// live elapsed ticking while partial (1s interval, like pi's bash renderer)
+			// live elapsed ticking while partial (1s interval, like pi's bash renderer);
+			// unref'd so it can't hold the process, and self-cleaning when the row is
+			// dropped by /new or compaction (lastComponent falsy)
 			if (state.startedAt !== undefined && options.isPartial && !state.interval) {
-				state.interval = setInterval(() => context.invalidate?.(), 1000);
+				state.interval = setInterval(() => {
+					if (!context.lastComponent) {
+						clearInterval(state.interval);
+						state.interval = undefined;
+						return;
+					}
+					context.invalidate?.();
+				}, 1000);
+				state.interval.unref?.();
 			}
 			if (!options.isPartial && state.interval) {
 				clearInterval(state.interval);
@@ -117,13 +155,21 @@ function makeRenderers(
 			}
 
 			if (options.isPartial) {
-				const step = result?.details?.status ?? "Searching…";
+				if (state.phase !== "running") {
+					state.phase = "running";
+					context.invalidate?.();
+				}
+				const step = result?.details?.step ?? "Searching…";
 				const elapsed = state.startedAt !== undefined ? secs(Date.now() - state.startedAt) : "";
 				text.setText(t.fg("warning", `  ⎿ ${step}${elapsed ? ` · ${elapsed}` : ""}`));
 				return text;
 			}
 
 			const isError = result?.isError || result?.details?.error;
+			if (state.phase !== (isError ? "error" : "ok")) {
+				state.phase = isError ? "error" : "ok";
+				context.invalidate?.();
+			}
 			if (isError) {
 				const msg = result?.details?.error ?? result?.content?.[0]?.text ?? "failed";
 				text.setText(t.fg("error", `  ⎿ ${clip(String(msg), 160)}`));
@@ -187,7 +233,7 @@ function registerSearchTool(
 			const started = Date.now();
 			onUpdate?.({
 				content: [{ type: "text", text: `${label}…` }],
-				details: { status: `Searching ${JSON.stringify(clip(params.query, 50))}…` },
+				details: { step: `Searching ${JSON.stringify(clip(params.query, 50))}…` },
 			});
 			try {
 				const results = await run(params.query, MAX(params.max), signal);
@@ -217,24 +263,28 @@ export default function (pi: ExtensionAPI) {
 		description:
 			"General web search — free, no API key (DuckDuckGo + Brave). Returns titles, URLs, snippets. " +
 			"Use for news, articles, broad topics. Use fetch_page to read a result in full. " +
-			"engine='multi' merges both engines in parallel for best coverage.",
+			"engine='multi' merges both engines in parallel for best coverage. " +
+			"For research questions, run 2-3 queries with varied phrasings (add a year, quote the exact error, add 'docs' or 'github'); " +
+			"set deep:true when you need facts rather than links — often removes the need for fetch_page; " +
+			"prefer recent results for fast-moving topics and cite the date you relied on.",
 		promptSnippet: "Free multi-engine web search (DDG + Brave) with recency filter",
 		promptGuidelines: [
-			"For anything factual or current, search before answering. Run 2-3 searches with different phrasings (add a year, quote the exact error, add 'docs' or 'github') — one query is rarely enough.",
-			"Set deep:true on web_search when you need facts rather than links; use fetch_page with query when you need one page in depth. Cite URLs inline.",
-			"Read at least two independent sources before stating a conclusion. Prefer primary sources (official docs, repos, papers) over aggregators, and say when sources disagree or are thin.",
-			"Results include dates when available — prefer recent ones for fast-moving topics and mention which date you relied on.",
+			"For multi-facet questions (comparisons, 'X vs Y', ecosystem scans), run 2-3 differently-phrased web_search calls in one turn instead of one; dedupe URLs, then fetch_page only the 1-2 results that actually answer the question.",
+			"Prefer deep:true over separate fetch_page calls when you need facts from several results; it returns excerpts inline in one round-trip.",
+			"Never fetch URLs the user must authenticate to (their email, admin panels, private dashboards) — even private-looking hostnames are rejected by SSRF protection; ask the user to paste the content instead.",
+			"Use fetch_page for a specific URL; use web_search first when you'd have to guess the URL.",
 		],
 		parameters: Type.Object({
 			query: Type.String({ description: "Search query" }),
 			max_results: Type.Optional(Type.Number({ description: "Max results, 1-20 (default 8)" })),
 			recency: Type.Optional(Type.String({ description: "d=day, w=week, m=month, y=year (optional)" })),
-			engine: Type.Optional(Type.String({ description: "ddg (default) | brave | bing | multi (parallel merge)" })),
+			lang: Type.Optional(Type.String({ description: "BCP-47 language-region, e.g. 'es-ES', 'ja-JP'. Biases results to that locale." })),
+			engine: Type.Optional(Type.String({ description: "auto (default: ddg+bing race) | ddg | brave | bing | multi (all in parallel)" })),
 			refresh: Type.Optional(Type.Boolean({ description: "Skip the 10-minute cache" })),
 			deep: Type.Optional(
 				Type.Union([Type.Boolean(), Type.Number()], {
 					description:
-						"Read the top results and attach a query-relevant excerpt to each. true = 4 results; or a number 1-8. Use when you need facts, not just links — often removes the need for fetch_page.",
+						"Read the top results and attach a query-relevant excerpt to each. true = 4 results; or a number 1-8. Use when you need facts, not just links.",
 				}),
 			),
 		}),
@@ -244,30 +294,37 @@ export default function (pi: ExtensionAPI) {
 			const recency = (["d", "w", "m", "y"] as const).includes(params.recency as any)
 				? (params.recency as Recency)
 				: undefined;
-			const engine = params.engine ?? "ddg";
-			const cacheKey = `s:${engine}:${recency ?? ""}:${maxResults}:${params.query}`;
-			const run = async () => {
-				if (engine === "multi") {
-					onUpdate?.({ content: [{ type: "text", text: "…" }], details: { status: "querying ddg + brave in parallel…" } });
-					const r = await multiSearch(params.query, maxResults, recency, signal);
-					return { results: r.results as Row[], engines: r.engines, errors: r.errors };
+			const engine = params.engine ?? "auto";
+			const deepN = params.deep === true ? 4 : typeof params.deep === "number" ? Math.min(Math.max(Math.round(params.deep), 1), 8) : 0;
+			const lang = typeof params.lang === "string" && /^[a-z]{2}(-[A-Za-z]{2,4})?$/.test(params.lang.trim()) ? params.lang.trim() : undefined;
+			const cacheKey = `s:${engine}:${recency ?? ""}:${lang ?? ""}:${maxResults}:${params.query}:deep${deepN}`;
+			const run = async (): Promise<SearchOutcome> => {
+				if (engine === "multi" || engine === "auto") {
+					onUpdate?.({
+						content: [{ type: "text", text: "…" }],
+						details: { step: engine === "multi" ? "querying ddg + brave + bing in parallel…" : "querying ddg + bing…" },
+					});
+					const r = engine === "multi"
+						? await multiSearch(params.query, maxResults, recency, signal, lang)
+						: await searchRace(params.query, maxResults, recency, signal, lang);
+					return r;
 				}
 				if (engine === "brave") {
-					onUpdate?.({ content: [{ type: "text", text: "…" }], details: { status: "querying brave…" } });
-					return { results: (await braveSearch(params.query, maxResults, recency, signal)) as Row[], engines: ["brave"], errors: [] as string[] };
+					onUpdate?.({ content: [{ type: "text", text: "…" }], details: { step: "querying brave…" } });
+					const rows = await braveSearch(params.query, maxResults, recency, signal, lang);
+					const kept = relevanceGate(params.query, rows) as Row[];
+					return { results: kept as unknown as SearchResult[], engines: ["brave"], errors: [], stats: { brave: { got: rows.length, kept: kept.length } } };
 				}
 				if (engine === "bing") {
-					onUpdate?.({ content: [{ type: "text", text: "…" }], details: { status: "querying bing rss…" } });
-					return { results: (await bingRssSearch(params.query, maxResults, recency, signal)) as Row[], engines: ["bing"], errors: [] as string[] };
-				}
-				onUpdate?.({ content: [{ type: "text", text: "…" }], details: { status: "querying duckduckgo…" } });
-				try {
-					return { results: (await ddgSearch(params.query, maxResults, recency, signal)) as Row[], engines: ["ddg"], errors: [] as string[] };
-				} catch (err) {
-					// ddg fully failed — structured bing rss before giving up
-					onUpdate?.({ content: [{ type: "text", text: "…" }], details: { status: "ddg failed — trying bing rss…" } });
-					return { results: (await bingRssSearch(params.query, maxResults, recency, signal)) as Row[], engines: ["bing"], errors: [String((err as Error)?.message ?? err)] };
+					onUpdate?.({ content: [{ type: "text", text: "…" }], details: { step: "querying bing rss…" } });
+					const rows = await bingRssSearch(params.query, maxResults, recency, signal, lang);
+					const kept = relevanceGate(params.query, rows) as Row[];
+					return { results: kept as unknown as SearchResult[], engines: ["bing"], errors: [], stats: { bing: { got: rows.length, kept: kept.length } } };
 				}
+				onUpdate?.({ content: [{ type: "text", text: "…" }], details: { step: "querying duckduckgo…" } });
+				const rows = await ddgSearch(params.query, maxResults, recency, signal, lang);
+				const kept = relevanceGate(params.query, rows) as Row[];
+				return { results: kept as unknown as SearchResult[], engines: ["ddg"], errors: [], stats: { ddg: { got: rows.length, kept: kept.length } } };
 			};
 			try {
 				let cachedHit = false;
@@ -275,19 +332,26 @@ export default function (pi: ExtensionAPI) {
 					const hit = cacheGet(cacheKey);
 					if (hit) {
 						cachedHit = true;
+						// rows carry their engine set post-fusion; derive engines from them so the
+						// renderer shows "via cache · ddg + bing", not "via cache · cache"
+						const hitEngines = hit.engines.length > 0
+							? hit.engines
+							: [...new Set(hit.results.flatMap((r) => r.engines ?? [r.engine]))];
 						return {
-							content: [{ type: "text", text: `[cached]\n${fmtResults(hit)}` }],
-							details: { cached: true, results: hit, count: hit.length, durationMs: Date.now() - started },
+							content: [{ type: "text", text: `[cached]\n${fmtResults(hit.results)}` }],
+							details: { cached: true, results: hit.results, engines: hitEngines, count: hit.results.length, durationMs: Date.now() - started },
 						};
 					}
 				}
-				const { results, engines, errors } = await run();
+				const { results, engines, errors, stats } = await run();
+				if (results.length === 0) {
+					throw new Error(`all engines failed — ${errors.join("; ") || "no results from any engine"}`);
+				}
 				// deep mode: read top results in parallel, attach query-relevant excerpts
-				const deepN = params.deep === true ? 4 : typeof params.deep === "number" ? Math.min(Math.max(Math.round(params.deep), 1), 8) : 0;
 				if (deepN > 0 && results.length > 0 && !cachedHit) {
 					onUpdate?.({
 						content: [{ type: "text", text: "…" }],
-						details: { status: `reading top ${Math.min(deepN, results.length)} results for excerpts…` },
+						details: { step: `reading top ${Math.min(deepN, results.length)} results for excerpts…` },
 					});
 					const top = results.slice(0, deepN);
 					const deadline = Date.now() + 25_000;
@@ -303,52 +367,51 @@ export default function (pi: ExtensionAPI) {
 									signal,
 								} satisfies FetchOptions);
 								const body = page.text.replace(/\n\n\[\d+ of \d+ passages shown[^\n]*\n?\n?$/, "");
-								const first = body.split("\n\n").find((p) => !/^(via |\[|\d+\.)/.test(p.trim()));
+								// pick the highest-scoring passage (its heading prefix aids the model),
+								// falling back to the intro only when nothing scored above zero
+								const best = (page.passages ?? [])
+									.filter((p) => p.score > 0)
+									.sort((a, b) => b.score - a.score)[0];
+								const first = best
+									? best.heading
+										? `${best.heading}\n${best.text}`
+										: best.text
+									: body
+											.split("\n\n")
+											.find((p) => !/^(via |\[|\d+\.)/.test(p.trim()));
 								if (first && first.trim().length > 80) {
 									(r as Row).excerpt = clip(first.trim().replace(/\n+/g, " "), 500);
 								}
+								if (page.date) (r as Row).date = page.date;
 							} catch {
 								/* ship without excerpt */
 							}
 						}),
 					);
 				}
-				if (results.length > 0) cacheSet(cacheKey, results);
+				if (results.length > 0) cacheSet(cacheKey, { results: results as SearchResult[], engines });
+				// surface gated-out rows in the header so the model knows what was filtered
+				const gated = Object.entries(stats)
+					.filter(([, s]) => s.kept < s.got)
+					.map(([name, s]) => `${name} ${s.got}→${s.kept} kept`);
 				return {
 					content: [
 						{
 							type: "text",
-							text: `[via ${engines.join(" + ")}]${errors.length ? ` (failed: ${errors.join("; ")})` : ""}\n\n${fmtResults(results)}`,
+							text: `[via ${engines.join(" + ")}${gated.length ? ` · ${gated.join(", ")}` : ""}]${errors.length ? ` (failed: ${errors.join("; ")})` : ""}\n\n${fmtResults(results)}`,
 						},
 					],
-					details: { results, count: results.length, engines, errors, durationMs: Date.now() - started },
+					details: { results, count: results.length, engines, errors, stats, durationMs: Date.now() - started },
 				};
 			} catch (err: any) {
-				if (engine !== "multi") {
-					onUpdate?.({ content: [{ type: "text", text: "…" }], details: { status: "primary engine failed — trying multi…" } });
-					try {
-						const r = await multiSearch(params.query, maxResults, recency, signal);
-						if (r.results.length > 0) {
-							cacheSet(cacheKey, r.results);
-							return {
-								content: [
-									{
-										type: "text",
-										text: `${fmtResults(r.results)}\n\n[primary engine '${engine}' failed: ${err?.message ?? err}]`,
-									},
-								],
-								details: { results: r.results, count: r.results.length, engines: r.engines, durationMs: Date.now() - started },
-							};
-						}
-					} catch {
-						/* fall through */
-					}
-				}
+				const advice = isOffline()
+					? "Network appears offline."
+					: "Engines may be rate-limited — wait a minute or retry with refresh=true.";
 				return {
 					content: [
 						{
 							type: "text",
-							text: `Search error: ${err?.message ?? err}. Engines may be rate-limited — wait a minute or retry with engine='multi', refresh=true.`,
+							text: `Search error: ${err?.message ?? err}. ${advice}`,
 						},
 					],
 					details: { error: err?.message ?? String(err), durationMs: Date.now() - started },
@@ -360,7 +423,7 @@ export default function (pi: ExtensionAPI) {
 			"Web Search",
 			(args) => String(args.query ?? ""),
 			(d) => {
-				const eng = (d.engines ?? []).join(" + ") || (d.cached ? "cache" : "ddg");
+				const eng = (d.engines ?? []).join(" + ") || (d.cached ? "cache" : "");
 				const line1 = `Found ${d.count ?? 0} results in ${secs(d.durationMs ?? 0)}`;
 				const line2 = d.cached
 					? `via cache · ${eng}`
@@ -380,7 +443,9 @@ export default function (pi: ExtensionAPI) {
 			"Fetch a URL and return readable content. Handles HTML (article extraction, nav/ads stripped), " +
 			"JSON (pretty-printed), plain text, and PDFs (text extraction). On 401/403/429/503 automatically " +
 			"retries via the Wayback Machine; thin/SPA pages re-rendered via a reader proxy. SSRF-protected. " +
-				"Pass query to get the most relevant passages of a long page instead of its head. Cached 1h.",
+			"Pass query to get the most relevant passages of a long page instead of its head. Cached 1h. " +
+			"no_jina=true opts out of the reader proxy (raw HTML only, e.g. for JSON/text or when the " +
+			"reader proxy would leak custom headers).",
 		promptSnippet: "Fetch a URL → readable text; handles PDFs, JSON, bot-walls (Wayback fallback)",
 		parameters: Type.Object({
 			url: Type.String({ description: "URL to fetch (http/https only)" }),
@@ -391,6 +456,7 @@ export default function (pi: ExtensionAPI) {
 				}),
 			),
 			max_chars: Type.Optional(Type.Number({ description: "Max text chars (default 8000, max 50000)" })),
+			offset: Type.Optional(Type.Number({ description: "Char offset into the document for paging (from the truncation footer)" })),
 			raw: Type.Optional(Type.Boolean({ description: "Return raw HTML instead of extracted text" })),
 			timeout: Type.Optional(Type.Number({ description: "Timeout ms (1000-60000, default 15000)" })),
 			headers: Type.Optional(
@@ -400,6 +466,7 @@ export default function (pi: ExtensionAPI) {
 			),
 			no_cache: Type.Optional(Type.Boolean({ description: "Skip the 1-hour cache" })),
 			no_wayback: Type.Optional(Type.Boolean({ description: "Disable Wayback Machine fallback" })),
+			no_jina: Type.Optional(Type.Boolean({ description: "Disable the reader-proxy fallback" })),
 			allow_http_errors: Type.Optional(
 				Type.Boolean({
 					description:
@@ -411,20 +478,29 @@ export default function (pi: ExtensionAPI) {
 			const started = Date.now();
 			try {
 				const u = new URL(params.url);
-				onUpdate?.({ content: [{ type: "text", text: "…" }], details: { status: `fetching ${u.host}…` } });
+				onUpdate?.({ content: [{ type: "text", text: "…" }], details: { step: `fetching ${u.host}…` } });
 				const r = await smartFetch(params.url, {
 					query: (params.query as string | undefined)?.trim() || undefined,
 					maxChars: MAX(params.max_chars, 8000, 50_000),
+					offset: typeof params.offset === "number" ? Math.max(0, Math.floor(params.offset)) : undefined,
 					raw: params.raw,
 					timeoutMs: params.timeout ? Math.min(Math.max(params.timeout, 1000), 60_000) : undefined,
 					headers: params.headers as Record<string, string> | undefined,
 					waybackEnabled: !params.no_wayback,
 					allowHttpErrors: params.allow_http_errors,
+					jinaEnabled: params.no_jina === true ? false : undefined,
+					jinaQuery: (params.query as string | undefined)?.trim() || undefined,
 					signal,
 				} satisfies FetchOptions);
 				const tags = [
 					`HTTP ${r.status}`,
-					r.source === "wayback" ? `Wayback ${r.waybackDate}` : null,
+					r.source !== "direct" ? `via ${r.source}` : null,
+					r.source === "wayback" && r.waybackDate ? r.waybackDate : null,
+					r.date ?? null,
+					typeof r.offset === "number" && r.totalChars !== undefined
+						? `${r.offset}\u2013${r.offset + r.text.length} of ${r.totalChars}`
+						: null,
+					...(r.notes ?? []),
 					r.fromCache ? "cached" : null,
 				].filter(Boolean).join(" · ");
 				return {
@@ -434,7 +510,11 @@ export default function (pi: ExtensionAPI) {
 						source: r.source,
 						fromCache: r.fromCache,
 						chars: r.text.length,
+						totalChars: r.totalChars,
+						offset: r.offset,
+						date: r.date,
 						truncated: r.truncated,
+						notes: r.notes,
 						host: u.host,
 						preview: r.text.slice(0, 300),
 						durationMs: Date.now() - started,
@@ -454,10 +534,12 @@ export default function (pi: ExtensionAPI) {
 			(d) => {
 				const line1 = `Read ${d.chars ?? 0} chars in ${secs(d.durationMs ?? 0)}`;
 				const bits = [`HTTP ${d.status ?? "?"}`];
-				if (d.source === "wayback") bits.push("via Wayback");
-				if (d.source === "jina") bits.push("via r.jina.ai");
+				if (d.source && d.source !== "direct") bits.push(`via ${d.source}`);
+				if (d.date) bits.push(d.date);
+				if (typeof d.offset === "number" && d.totalChars) bits.push(`${d.offset}\u2013${(d.offset ?? 0) + (d.chars ?? 0)} of ${d.totalChars}`);
 				if (d.fromCache) bits.push("cached");
 				if (d.truncated) bits.push("truncated");
+				if ((d.notes ?? []).length > 0) bits.push(...d.notes);
 				return { ok: !d.error, line1, line2: bits.join(" · "), rows: undefined, preview: d.preview };
 			},
 		),
@@ -524,7 +606,7 @@ export default function (pi: ExtensionAPI) {
 		}),
 		async execute(_id, params, signal, onUpdate) {
 			const started = Date.now();
-			onUpdate?.({ content: [{ type: "text", text: "…" }], details: { status: "searching github…" } });
+			onUpdate?.({ content: [{ type: "text", text: "…" }], details: { step: "searching github…" } });
 			try {
 				const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
 				const results = await searchGithubRepos(params.query, MAX(params.max), signal, token);
@@ -553,7 +635,10 @@ export default function (pi: ExtensionAPI) {
 		handler: async (args, ctx) => {
 			const topic = (args ?? "").trim();
 			if (!topic) {
-				ctx.ui.notify("Usage: /research <topic>", "warning");
+				ctx.ui.notify(
+					"Usage: /research <topic> — the model is prompted to answer your actual question, not just summarize the topic",
+					"warning",
+				);
 				return;
 			}
 			const setStatus = (s: string) => {
@@ -619,7 +704,7 @@ export default function (pi: ExtensionAPI) {
 			const pages: Array<{ title: string; url: string; text: string }> = [];
 			for (const p of picked) {
 				try {
-					const r = await smartFetch(p.url, { maxChars: 4000, timeoutMs: 15_000 });
+					const r = await smartFetch(p.url, { maxChars: 12_000, timeoutMs: 30_000 });
 					pages.push({ title: p.title, url: p.url, text: r.text });
 					steps.push(`✓ fetched: ${clip(new URL(p.url).host, 40)}`);
 				} catch (e: any) {
@@ -640,11 +725,14 @@ export default function (pi: ExtensionAPI) {
 					material += `### ${p.title}\n${p.url}\n\n${p.text}\n\n`;
 				}
 			}
-			material += `---\nSynthesize the above into a research briefing:
-- Group findings by sub-topic (like "three-way comparisons", "benchmarks", etc.)
-- For each group, list the documents worth reading: [source name](url) — one line on what it covers
+			material += `---\nThe user asked: "${topic.replace(/`/g, "'")}"
+
+Synthesize the above into a research briefing that ANSWERS THIS QUESTION:
+- Lead with the best current answer supported by the material, with citations
+- Then group findings by sub-topic (like "three-way comparisons", "benchmarks", etc.)
+- For each group, list documents worth reading: [source name](url) — one line on what it covers
 - End with "Recurring conclusions": 3-6 bullets of the consensus/tensions across sources
-- Cite only what appears above; if sources are thin, say so.`;
+- Cite only what appears above; if sources are thin or don't address the question, say so explicitly.`;
 
 			pi.sendUserMessage(material);
 			setWidget([...steps, "handed to model for synthesis"]);