pi-unsloth-webtools 0.7.6 → 0.9.0

package/index.ts

@@ -1,13 +1,16 @@
 import { defineTool, type ExtensionAPI, type ExtensionContext, type Theme } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
 import { truncateToWidth } from "@earendil-works/pi-tui";
-import { collapseWhitespace } from "./html-to-md.ts";
+import { collapseWhitespace, visibleChars } from "./html-to-md.ts";
 import { SEARCH_TIMEOUT_MS, webSearch as defaultWebSearch } from "./web-search.ts";
-import { DEFAULT_FETCH_TIMEOUT_MS, fetchPageText as defaultFetchPageText } from "./web-fetch.ts";
-import { renderPageText as defaultRenderPageText } from "./web-render.ts";
-import { loadDefaultFetchSettings, loadDefaultFetchTimeoutMs, loadJinaApiKey } from "./settings.ts";
-import { readConfig, readConfigWithStatus, toggleWebRender } from "./config.ts";
-import { WebToolsConfigOverlay } from "./config-ui.ts";
+import {
+  DEFAULT_FETCH_TIMEOUT_MS,
+  fetchPageOutcome as defaultFetchPageOutcome,
+  fetchPageText as defaultFetchPageText,
+  type FetchPageOutcome,
+} from "./web-fetch.ts";
+import { renderPageWithLightpanda as defaultRenderLocalPageText } from "./lightpanda.ts";
+import { loadDefaultFetchSettings, loadDefaultFetchTimeoutMs, loadLightpandaSettings } from "./settings.ts";
 
 function toolCallLine(theme: Theme, name: string, detail: string) {
   const line = theme.fg("toolTitle", theme.bold(name)) + (detail ? ` ${theme.fg("accent", detail)}` : "");
@@ -23,6 +26,7 @@ function positiveNumber(value: unknown): number | undefined {
   const n = Math.floor(value);
   return n > 0 ? n : undefined;
 }
+
 async function fetchDefaults(cwd: string | undefined, params: { timeoutMs?: unknown; maxChars?: unknown }) {
   const timeoutParam = positiveNumber(params.timeoutMs);
   const maxCharsParam = positiveNumber(params.maxChars);
@@ -32,6 +36,7 @@ async function fetchDefaults(cwd: string | undefined, params: { timeoutMs?: unkn
     maxChars: maxCharsParam ?? defaults.maxChars,
     allowPrivateAddresses: defaults.allowPrivateAddresses,
     allowLocalFiles: defaults.allowLocalFiles,
+    transport: defaults.transport,
   };
 }
 
@@ -39,24 +44,53 @@ function fetchWasForbidden(text: string): boolean {
   return /^Failed to fetch URL: HTTP 403\b/m.test(text);
 }
 
-const FORBIDDEN_FALLBACK_NOTE = "Direct fetch failed with HTTP 403; rendered via the Jina Reader instead.";
+function renderFailed(rendered: string): boolean {
+  return (
+    rendered.startsWith("Failed to render URL:") ||
+    rendered.startsWith("Blocked:") ||
+    rendered.startsWith("(page returned no readable text)")
+  );
+}
+
+function renderImprovesPage(rendered: string, fetched: string): boolean {
+  return visibleChars(rendered) > visibleChars(fetched) * 1.5;
+}
+
+const LOCAL_FORBIDDEN_NOTE = "Direct fetch failed with HTTP 403; rendered locally with Lightpanda instead.";
+const LOCAL_THIN_NOTE = "Direct fetch returned little content; rendered locally with Lightpanda instead.";
+const INCOMPLETE_NOTE = "\n\n*(JavaScript-rendered page; content may be incomplete)*";
+const RENDER_HEADER_LINE_RE = /^(?:URL: |Rendered via |Title: |Author: |Date: |Site: )/;
+
+function renderedProseChars(text: string): number {
+  const prose = text
+    .split("\n")
+    .filter((line) => !RENDER_HEADER_LINE_RE.test(line))
+    .join("\n");
+  return visibleChars(prose);
+}
+
+const INTERSTITIAL_MAX_PROSE_CHARS = 4000;
+const INTERSTITIAL_MARKERS = [
+  "just a moment",
+  "attention required",
+  "you have been blocked",
+  "enable javascript and cookies to continue",
+  "checking if the site connection is secure",
+  "access denied",
+  "please enable cookies",
+  "cf-error-details",
+];
+
+function looksLikeInterstitial(text: string): boolean {
+  if (renderedProseChars(text) > INTERSTITIAL_MAX_PROSE_CHARS) return false;
+  const lowered = text.toLowerCase();
+  return INTERSTITIAL_MARKERS.some((marker) => lowered.includes(marker));
+}
 
 const WebSearchParams = Type.Object({
   query: Type.Optional(
     Type.String({ description: "The search query" }),
   ),
-  url: Type.Optional(
-    Type.String({
-      description:
-        "A URL to fetch full page content from (instead of searching). Use this to read a page found in search results.",
-    }),
-  ),
-  maxChars: Type.Optional(
-    Type.Number({
-      description:
-        "Truncate the fetched page to this many characters (only used with the url parameter)",
-    }),
-  ),
   maxResults: Type.Optional(
     Type.Number({
       minimum: 1,
@@ -87,55 +121,43 @@ const WebFetchParams = Type.Object({
   ),
 });
 
-const WebRenderParams = Type.Object({
-  url: Type.String({ description: "Public URL of the page to render" }),
-  maxChars: Type.Optional(
-    Type.Number({
-      description: "Truncate the returned content to this many characters (default: no limit)",
-    }),
-  ),
-  timeoutMs: Type.Optional(
-    Type.Number({
-      minimum: 1000,
-      description: "Overall timeout in milliseconds (default: 60000)",
-    }),
-  ),
-});
-
 export interface WebToolsDeps {
   fetchPageText?: typeof defaultFetchPageText;
+  fetchPageOutcome?: typeof defaultFetchPageOutcome;
   webSearch?: typeof defaultWebSearch;
-  renderPageText?: typeof defaultRenderPageText;
-  webRenderEnabled?: () => Promise<boolean>;
+  renderLocalPageText?: typeof defaultRenderLocalPageText;
 }
 
 export function createWebTools(deps: WebToolsDeps = {}) {
-  const fetchPageText = deps.fetchPageText ?? defaultFetchPageText;
   const webSearch = deps.webSearch ?? defaultWebSearch;
-  const renderPageText = deps.renderPageText ?? defaultRenderPageText;
-  const webRenderEnabled =
-    deps.webRenderEnabled ?? (async () => (await readConfig()).webRenderEnabled !== false);
+  const renderLocalPageText = deps.renderLocalPageText ?? defaultRenderLocalPageText;
+  const legacyFetchPageText = deps.fetchPageText;
+  const fetchOutcome: typeof defaultFetchPageOutcome =
+    deps.fetchPageOutcome ??
+    (legacyFetchPageText
+      ? async (url, options) => ({ text: await legacyFetchPageText(url, options), hint: null })
+      : defaultFetchPageOutcome);
 
-  const renderOnForbidden = async (
+  const renderFallback = async (
     url: string,
-    failedText: string,
+    outcome: FetchPageOutcome,
     options: { timeoutMs: number; maxChars?: number; signal?: AbortSignal; cwd?: string },
   ): Promise<string | null> => {
-    if (!fetchWasForbidden(failedText)) return null;
-    try {
-      if (!(await webRenderEnabled())) return null;
-      const apiKey = await loadJinaApiKey(options.cwd);
-      const rendered = await renderPageText(url, {
-        timeoutMs: options.timeoutMs,
-        maxChars: options.maxChars,
-        signal: options.signal,
-        apiKey,
-      });
-      if (rendered.startsWith("Failed to render URL:") || rendered.startsWith("Blocked:")) return null;
-      return FORBIDDEN_FALLBACK_NOTE + "\n\n" + rendered;
-    } catch {
-      return null;
-    }
+    const forbidden = fetchWasForbidden(outcome.text);
+    const thin = outcome.hint !== null && !forbidden;
+    if (!forbidden && !thin) return null;
+    const incomplete = outcome.text + INCOMPLETE_NOTE;
+    const settings = await loadLightpandaSettings(options.cwd).catch(() => null);
+    const rendered = await renderLocalPageText(url, {
+      timeoutMs: options.timeoutMs,
+      maxChars: options.maxChars,
+      signal: options.signal,
+      settings,
+    }).catch(() => null);
+    if (rendered === null) return thin ? incomplete : null;
+    if (renderFailed(rendered) || looksLikeInterstitial(rendered)) return thin ? incomplete : null;
+    if (thin && !renderImprovesPage(rendered, outcome.text)) return incomplete;
+    return (forbidden ? LOCAL_FORBIDDEN_NOTE : LOCAL_THIN_NOTE) + "\n\n" + rendered;
   };
 
   return {
@@ -143,38 +165,17 @@ export function createWebTools(deps: WebToolsDeps = {}) {
       name: "web_search",
       label: "Web Search",
       description:
-        "Search the web and return snippets for the top results. Pass url instead of query to read a page found in the same search; use web_fetch for known URLs.",
+        "Search the web and return snippets for the top results. Use web_fetch to read a page found in the results.",
       promptSnippet: "Search the web and return snippets",
       promptGuidelines: [
-        "Web tool order: web_search to discover, web_fetch for a known URL, web_render only when web_fetch cannot read the page.",
+        "Web tool order: web_search to discover, then web_fetch to read a page.",
       ],
       parameters: WebSearchParams,
       renderCall(args, theme) {
-        const url = collapsedArg(args.url);
         const query = collapsedArg(args.query);
-        return toolCallLine(theme, "web_search", url || (query ? `"${query}"` : ""));
+        return toolCallLine(theme, "web_search", query ? `"${query}"` : "");
       },
       async execute(_toolCallId, params, signal, onUpdate, _ctx) {
-        if (params.url?.trim()) {
-          const url = params.url.trim();
-          onUpdate?.({ content: [{ type: "text", text: `Fetching ${url}...` }], details: {} });
-          const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
-          const { timeoutMs, maxChars, allowPrivateAddresses, allowLocalFiles } = await fetchDefaults(cwd, params);
-          const text = await fetchPageText(url, {
-            timeoutMs,
-            signal: signal ?? undefined,
-            maxChars,
-            allowPrivateAddresses,
-            allowLocalFiles,
-          });
-          const rendered = await renderOnForbidden(url, text, {
-            timeoutMs,
-            maxChars,
-            signal: signal ?? undefined,
-            cwd,
-          });
-          return { content: [{ type: "text", text: rendered ?? text }], details: {} };
-        }
         onUpdate?.({ content: [{ type: "text", text: "Searching the web..." }], details: {} });
         const timeoutParam = positiveNumber(params.timeoutMs);
         const searchCwd = (_ctx as ExtensionContext | undefined)?.cwd;
@@ -194,13 +195,16 @@ export function createWebTools(deps: WebToolsDeps = {}) {
       description:
         "Fetch a URL and return its readable text. HTML pages are converted to Markdown using a " +
         "main-content heuristic: article/main scoping plus hidden-element and boilerplate " +
-        "stripping. Non-HTML text is returned as-is. GitHub repo root pages are rewritten to the " +
+        "stripping. HTTP 403 responses are retried over a browser-fingerprint transport, and pages " +
+        "that look JavaScript-rendered (or still refused) are rendered locally with Lightpanda when " +
+        "the binary is installed; no third-party rendering service is involved. " +
+        "Non-HTML text is returned as-is. GitHub repo root pages are rewritten to the " +
         "README API, so the README is returned instead of the repo page's UI chrome. " +
         "Private/loopback/link-local targets and local files (file:// URLs, absolute, ~/ or ./ paths, including " +
         "PDFs) are supported by default; opt out with webFetch.allowPrivateAddresses: false or " +
         "webFetch.allowLocalFiles: false in settings. The download size is capped.",
       promptGuidelines: [
-        "web_fetch retries an HTTP 403 through web_render automatically; never call web_render again for the same URL after a 403.",
+        "web_fetch renders JavaScript-rendered pages locally with Lightpanda when its binary is installed; it never calls a third-party rendering service.",
       ],
       promptSnippet: "Fetch a web page and return readable text content",
       parameters: WebFetchParams,
@@ -210,105 +214,31 @@ export function createWebTools(deps: WebToolsDeps = {}) {
       async execute(_toolCallId, params, signal, onUpdate, _ctx) {
         onUpdate?.({ content: [{ type: "text", text: `Fetching ${params.url}...` }], details: {} });
         const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
-        const { timeoutMs, maxChars, allowPrivateAddresses, allowLocalFiles } = await fetchDefaults(cwd, params);
-        const text = await fetchPageText(params.url, {
+        const { timeoutMs, maxChars, allowPrivateAddresses, allowLocalFiles, transport } = await fetchDefaults(cwd, params);
+        const deadlineMs = Date.now() + timeoutMs;
+        const outcome = await fetchOutcome(params.url, {
           timeoutMs,
+          deadlineMs,
           signal: signal ?? undefined,
           maxChars,
           allowPrivateAddresses,
           allowLocalFiles,
+          transport,
         });
-        const rendered = await renderOnForbidden(params.url, text, {
-          timeoutMs,
+        const rendered = await renderFallback(params.url, outcome, {
+          timeoutMs: Math.max(1, deadlineMs - Date.now()),
           maxChars,
           signal: signal ?? undefined,
           cwd,
         });
-        return { content: [{ type: "text", text: rendered ?? text }], details: {} };
-      },
-    }),
-    webRenderTool: defineTool({
-      name: "web_render",
-      label: "Web Render",
-      description:
-        "Render a public web page to Markdown through the third-party Jina Reader (r.jina.ai). " +
-        "Use it when web_fetch cannot read the page: JavaScript-rendered pages, or pages where web_fetch returned \"(page returned no readable text)\". " +
-        "The target URL is sent to Jina; local files and non-public addresses are refused. " +
-        "An optional JINA_API_KEY or unslothWebTools.jinaApiKey raises the Reader's rate limits.",
-      promptSnippet: "Render a JavaScript-rendered page to Markdown via the Jina Reader",
-      parameters: WebRenderParams,
-      renderCall(args, theme) {
-        return toolCallLine(theme, "web_render", collapsedArg(args.url));
-      },
-      async execute(_toolCallId, params, signal, onUpdate, _ctx) {
-        onUpdate?.({ content: [{ type: "text", text: `Rendering ${params.url}...` }], details: {} });
-        const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
-        const { timeoutMs, maxChars } = await fetchDefaults(cwd, params);
-        const apiKey = await loadJinaApiKey(cwd);
-        const text = await renderPageText(params.url, {
-          timeoutMs,
-          maxChars,
-          signal: signal ?? undefined,
-          apiKey,
-        });
-        return { content: [{ type: "text", text }], details: {} };
+        return { content: [{ type: "text", text: rendered ?? outcome.text }], details: {} };
       },
     }),
   };
 }
 
 export default function (pi: ExtensionAPI) {
-  const { webSearchTool, webFetchTool, webRenderTool } = createWebTools();
+  const { webSearchTool, webFetchTool } = createWebTools();
   pi.registerTool(webSearchTool);
   pi.registerTool(webFetchTool);
-  pi.registerTool(webRenderTool);
-
-  pi.on("session_start", async (_event, ctx) => {
-    try {
-      const { config, corrupted } = await readConfigWithStatus();
-      if (corrupted && ctx.hasUI) {
-        ctx.ui.notify("Web tools config was corrupt and was reset to defaults", "warning");
-      }
-      if (!config.webRenderEnabled) {
-        pi.setActiveTools(pi.getActiveTools().filter((name) => name !== "web_render"));
-      }
-    } catch (error) {
-      console.error("Failed to load web tools config:", error);
-    }
-  });
-
-  pi.registerCommand("webtools-config", {
-    description: "Open the web tools settings window (web_render on/off)",
-    handler: async (_args, ctx) => {
-      if (!ctx.hasUI) {
-        ctx.ui.notify("/webtools-config requires interactive mode", "error");
-        return;
-      }
-      await ctx.ui.custom<void>(
-        async (tui, theme, _keybindings, done) => {
-          const overlay = new WebToolsConfigOverlay({
-            tui,
-            theme,
-            done,
-            onToggle: async (key) => {
-              if (key !== "webRenderEnabled") return;
-              const enabled = await toggleWebRender();
-              const active = pi.getActiveTools();
-              pi.setActiveTools(
-                enabled
-                  ? [...new Set([...active, "web_render"])]
-                  : active.filter((name) => name !== "web_render"),
-              );
-            },
-          });
-          await overlay.load();
-          return overlay;
-        },
-        {
-          overlay: true,
-          overlayOptions: { anchor: "center", width: "90%", minWidth: 60, maxHeight: "90%" },
-        },
-      );
-    },
-  });
 }

package/lightpanda.ts

@@ -0,0 +1,276 @@
+import { spawn as spawnProcess } from "node:child_process";
+import { visibleChars } from "./html-to-md.ts";
+import {
+  checkUrlAccess,
+  MAX_SIGNAL_TIMEOUT_MS,
+  normalizeUrlScheme,
+  type WebsitePolicy,
+} from "./web-access.ts";
+import {
+  pagePrefixedMarkdown,
+  parseLocalPath,
+  resolveAndValidateHost,
+  TRUNCATED_BODY_NOTICE,
+  truncatePageText,
+  type ResolvedHost,
+} from "./web-fetch.ts";
+
+const DEFAULT_BINARY = "lightpanda";
+const DEFAULT_WAIT_UNTIL = "networkidle";
+const DEFAULT_TIMEOUT_MS = 60_000;
+const MAX_DUMP_BYTES = 4 * 1024 * 1024;
+const PROBE_TIMEOUT_MS = 5_000;
+const METADATA_LINE_RE = /^(?:Title|Author|Date|Site|URL|Rendered via): /;
+const PROVENANCE = "Rendered via Lightpanda (local browser; nothing sent to a third party).";
+const CANCELLED_MESSAGE = "Failed to render URL: cancelled.";
+const TIMED_OUT_MESSAGE = "Failed to render URL: timed out.";
+const LOCAL_FILE_MESSAGE = "Blocked: the local renderer cannot fetch local files.";
+const NO_TEXT_MESSAGE = "(page returned no readable text)";
+
+export type LightpandaDump = "html" | "markdown";
+export type LightpandaWaitUntil = "networkidle" | "load" | "domcontentloaded" | "done";
+
+export interface LightpandaSettings {
+  enabled: boolean;
+  binaryPath: string | null;
+  command: string[] | null;
+}
+
+interface LightpandaProcess {
+  stdout: AsyncIterable<Buffer> | null;
+  stderr: AsyncIterable<Buffer> | null;
+  on(event: "error", listener: (error: Error) => void): unknown;
+  on(event: "close", listener: (code: number | null) => void): unknown;
+  kill(signal?: string): unknown;
+}
+
+export type LightpandaSpawn = (
+  binary: string,
+  args: string[],
+  options: { stdio: ["ignore", "pipe", "pipe"] },
+) => LightpandaProcess;
+
+export interface LightpandaRenderOptions {
+  timeoutMs?: number;
+  maxChars?: number;
+  signal?: AbortSignal;
+  websitePolicy?: WebsitePolicy | null;
+  settings?: LightpandaSettings | null;
+  binaryPath?: string | null;
+  command?: string[] | null;
+  dump?: LightpandaDump;
+  waitUntil?: LightpandaWaitUntil;
+  spawn?: LightpandaSpawn;
+  resolve?: (hostname: string, signal?: AbortSignal, allowPrivateAddresses?: boolean) => Promise<ResolvedHost>;
+}
+
+interface RunResult {
+  code: number | null;
+  stdout: string;
+  stderr: string;
+  timedOut: boolean;
+  cancelled: boolean;
+  truncated: boolean;
+  missing: boolean;
+}
+
+const availability = new Map<string, Promise<boolean>>();
+
+function binaryAvailable(launch: string[], spawn: LightpandaSpawn): Promise<boolean> {
+  const key = launch.join(" ");
+  const cached = availability.get(key);
+  if (cached) return cached;
+  const probe = new Promise<boolean>((resolve) => {
+    let settled = false;
+    const finish = (value: boolean) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(value);
+    };
+    const timer = setTimeout(() => {
+      try {
+        child.kill("SIGKILL");
+      } catch {}
+      finish(false);
+    }, PROBE_TIMEOUT_MS);
+    let child: LightpandaProcess;
+    try {
+      child = spawn(launch[0], [...launch.slice(1), "version"], { stdio: ["ignore", "pipe", "pipe"] });
+    } catch {
+      finish(false);
+      return;
+    }
+    child.on("error", () => finish(false));
+    child.on("close", (code) => finish(code === 0));
+  });
+  availability.set(key, probe);
+  return probe;
+}
+
+function runLightpanda(
+  launch: string[],
+  args: string[],
+  options: { timeoutMs: number; signal?: AbortSignal; spawn: LightpandaSpawn },
+): Promise<RunResult> {
+  return new Promise<RunResult>((resolve) => {
+    const state = {
+      code: null as number | null,
+      stdout: [] as Buffer[],
+      stderr: [] as string[],
+      timedOut: false,
+      cancelled: false,
+      truncated: false,
+      missing: false,
+      bytes: 0,
+    };
+    let settled = false;
+    const finish = (error: Error | null) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      options.signal?.removeEventListener("abort", onAbort);
+      if (error && (error as NodeJS.ErrnoException).code === "ENOENT") state.missing = true;
+      const stdout = Buffer.concat(state.stdout).toString("utf-8");
+      resolve({ ...state, stdout, stderr: state.stderr.join("") });
+    };
+    const kill = () => {
+      try {
+        child.kill("SIGKILL");
+      } catch {}
+    };
+    const onAbort = () => {
+      state.cancelled = true;
+      kill();
+    };
+    const timer = setTimeout(() => {
+      state.timedOut = true;
+      kill();
+    }, options.timeoutMs);
+    let child: LightpandaProcess;
+    try {
+      child = options.spawn(launch[0], [...launch.slice(1), ...args], { stdio: ["ignore", "pipe", "pipe"] });
+    } catch (error) {
+      finish(error instanceof Error ? error : new Error(String(error)));
+      return;
+    }
+    const collect = async (stream: AsyncIterable<Buffer> | null, push: (chunk: Buffer) => void): Promise<void> => {
+      if (!stream) return;
+      try {
+        for await (const chunk of stream) push(chunk);
+      } catch {}
+    };
+    const stdoutDone = collect(child.stdout, (chunk) => {
+      if (state.bytes >= MAX_DUMP_BYTES) {
+        state.truncated = true;
+        return;
+      }
+      state.bytes += chunk.length;
+      if (state.bytes > MAX_DUMP_BYTES) {
+        state.truncated = true;
+        state.stdout.push(chunk.subarray(0, chunk.length - (state.bytes - MAX_DUMP_BYTES)));
+        return;
+      }
+      state.stdout.push(chunk);
+    });
+    const stderrDone = collect(child.stderr, (chunk) => state.stderr.push(chunk.toString("utf-8")));
+    child.on("error", (error) => finish(error));
+    child.on("close", (code) => {
+      state.code = code;
+      void Promise.all([stdoutDone, stderrDone]).then(() => finish(null));
+    });
+    options.signal?.addEventListener("abort", onAbort, { once: true });
+    if (options.signal?.aborted) onAbort();
+  });
+}
+
+function bodyChars(text: string): number {
+  const body = text
+    .split("\n")
+    .filter((line) => !METADATA_LINE_RE.test(line))
+    .join("\n");
+  return visibleChars(body);
+}
+
+function withProvenance(prefixed: string, url: string): string {
+  const lines = prefixed.split("\n");
+  let index = 0;
+  while (index < lines.length && METADATA_LINE_RE.test(lines[index])) index++;
+  const header = [`URL: ${url}`, PROVENANCE];
+  if (index === 0) return `${header.join("\n")}\n\n${prefixed}`;
+  return [...lines.slice(0, index), ...header, ...lines.slice(index)].join("\n");
+}
+
+export function lightpandaBinary(options: LightpandaRenderOptions = {}): string {
+  return (
+    options.binaryPath ??
+    options.settings?.binaryPath ??
+    process.env.PI_LIGHTPANDA_BIN?.trim() ??
+    DEFAULT_BINARY
+  );
+}
+
+export function lightpandaLaunch(options: LightpandaRenderOptions = {}): string[] {
+  const command = options.command ?? options.settings?.command ?? null;
+  if (command && command.length) return command;
+  return [lightpandaBinary(options)];
+}
+
+export async function renderPageWithLightpanda(
+  url: string,
+  options: LightpandaRenderOptions = {},
+): Promise<string | null> {
+  if (options.settings && options.settings.enabled === false) return null;
+  const target = typeof url === "string" ? url.trim() : "";
+  const policy = options.websitePolicy ?? null;
+  if (!target) return checkUrlAccess("", policy)[1];
+  if (parseLocalPath(target) !== null) return LOCAL_FILE_MESSAGE;
+  const normalized = normalizeUrlScheme(target);
+  const [allowed, reason, hostname] = checkUrlAccess(normalized, policy);
+  if (!allowed) return reason;
+  const launch = lightpandaLaunch(options);
+  const spawn = options.spawn ?? (spawnProcess as unknown as LightpandaSpawn);
+  if (!(await binaryAvailable(launch, spawn))) return null;
+  const timeoutMs = Math.min(
+    MAX_SIGNAL_TIMEOUT_MS,
+    Math.max(1, Math.floor(options.timeoutMs ?? DEFAULT_TIMEOUT_MS)),
+  );
+  const timeoutSignal = AbortSignal.timeout(timeoutMs);
+  const signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
+  const resolve = options.resolve ?? resolveAndValidateHost;
+  const resolved = await resolve(hostname, signal, false);
+  if (options.signal?.aborted) return CANCELLED_MESSAGE;
+  if (timeoutSignal.aborted) return TIMED_OUT_MESSAGE;
+  if (!resolved.ok) {
+    return resolved.reason.startsWith("Blocked:") ? resolved.reason : `Failed to render URL: ${resolved.reason}`;
+  }
+  const dump = options.dump ?? "html";
+  const result = await runLightpanda(
+    launch,
+    [
+      "fetch",
+      "--dump",
+      dump,
+      "--wait-until",
+      options.waitUntil ?? DEFAULT_WAIT_UNTIL,
+      "--block-private-networks",
+      "--http-timeout",
+      String(timeoutMs),
+      normalized,
+    ],
+    { timeoutMs, signal: options.signal, spawn },
+  );
+  if (result.missing) return null;
+  if (result.cancelled) return CANCELLED_MESSAGE;
+  if (result.timedOut) return TIMED_OUT_MESSAGE;
+  if (result.code !== 0) {
+    return `Failed to render URL: Lightpanda exited with code ${result.code ?? "unknown"}.`;
+  }
+  const raw = result.stdout;
+  const rendered =
+    dump === "html" ? pagePrefixedMarkdown(result.truncated ? raw + TRUNCATED_BODY_NOTICE : raw) : raw.trim();
+  if (bodyChars(rendered) === 0) return NO_TEXT_MESSAGE;
+  const withHeader =
+    dump === "html" || !result.truncated ? withProvenance(rendered, normalized) : withProvenance(rendered + TRUNCATED_BODY_NOTICE, normalized);
+  return truncatePageText(withHeader, options.maxChars);
+}