pi-unsloth-webtools 0.8.0 → 0.9.0

package/lightpanda.ts

@@ -0,0 +1,276 @@
+import { spawn as spawnProcess } from "node:child_process";
+import { visibleChars } from "./html-to-md.ts";
+import {
+  checkUrlAccess,
+  MAX_SIGNAL_TIMEOUT_MS,
+  normalizeUrlScheme,
+  type WebsitePolicy,
+} from "./web-access.ts";
+import {
+  pagePrefixedMarkdown,
+  parseLocalPath,
+  resolveAndValidateHost,
+  TRUNCATED_BODY_NOTICE,
+  truncatePageText,
+  type ResolvedHost,
+} from "./web-fetch.ts";
+
+const DEFAULT_BINARY = "lightpanda";
+const DEFAULT_WAIT_UNTIL = "networkidle";
+const DEFAULT_TIMEOUT_MS = 60_000;
+const MAX_DUMP_BYTES = 4 * 1024 * 1024;
+const PROBE_TIMEOUT_MS = 5_000;
+const METADATA_LINE_RE = /^(?:Title|Author|Date|Site|URL|Rendered via): /;
+const PROVENANCE = "Rendered via Lightpanda (local browser; nothing sent to a third party).";
+const CANCELLED_MESSAGE = "Failed to render URL: cancelled.";
+const TIMED_OUT_MESSAGE = "Failed to render URL: timed out.";
+const LOCAL_FILE_MESSAGE = "Blocked: the local renderer cannot fetch local files.";
+const NO_TEXT_MESSAGE = "(page returned no readable text)";
+
+export type LightpandaDump = "html" | "markdown";
+export type LightpandaWaitUntil = "networkidle" | "load" | "domcontentloaded" | "done";
+
+export interface LightpandaSettings {
+  enabled: boolean;
+  binaryPath: string | null;
+  command: string[] | null;
+}
+
+interface LightpandaProcess {
+  stdout: AsyncIterable<Buffer> | null;
+  stderr: AsyncIterable<Buffer> | null;
+  on(event: "error", listener: (error: Error) => void): unknown;
+  on(event: "close", listener: (code: number | null) => void): unknown;
+  kill(signal?: string): unknown;
+}
+
+export type LightpandaSpawn = (
+  binary: string,
+  args: string[],
+  options: { stdio: ["ignore", "pipe", "pipe"] },
+) => LightpandaProcess;
+
+export interface LightpandaRenderOptions {
+  timeoutMs?: number;
+  maxChars?: number;
+  signal?: AbortSignal;
+  websitePolicy?: WebsitePolicy | null;
+  settings?: LightpandaSettings | null;
+  binaryPath?: string | null;
+  command?: string[] | null;
+  dump?: LightpandaDump;
+  waitUntil?: LightpandaWaitUntil;
+  spawn?: LightpandaSpawn;
+  resolve?: (hostname: string, signal?: AbortSignal, allowPrivateAddresses?: boolean) => Promise<ResolvedHost>;
+}
+
+interface RunResult {
+  code: number | null;
+  stdout: string;
+  stderr: string;
+  timedOut: boolean;
+  cancelled: boolean;
+  truncated: boolean;
+  missing: boolean;
+}
+
+const availability = new Map<string, Promise<boolean>>();
+
+function binaryAvailable(launch: string[], spawn: LightpandaSpawn): Promise<boolean> {
+  const key = launch.join(" ");
+  const cached = availability.get(key);
+  if (cached) return cached;
+  const probe = new Promise<boolean>((resolve) => {
+    let settled = false;
+    const finish = (value: boolean) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(value);
+    };
+    const timer = setTimeout(() => {
+      try {
+        child.kill("SIGKILL");
+      } catch {}
+      finish(false);
+    }, PROBE_TIMEOUT_MS);
+    let child: LightpandaProcess;
+    try {
+      child = spawn(launch[0], [...launch.slice(1), "version"], { stdio: ["ignore", "pipe", "pipe"] });
+    } catch {
+      finish(false);
+      return;
+    }
+    child.on("error", () => finish(false));
+    child.on("close", (code) => finish(code === 0));
+  });
+  availability.set(key, probe);
+  return probe;
+}
+
+function runLightpanda(
+  launch: string[],
+  args: string[],
+  options: { timeoutMs: number; signal?: AbortSignal; spawn: LightpandaSpawn },
+): Promise<RunResult> {
+  return new Promise<RunResult>((resolve) => {
+    const state = {
+      code: null as number | null,
+      stdout: [] as Buffer[],
+      stderr: [] as string[],
+      timedOut: false,
+      cancelled: false,
+      truncated: false,
+      missing: false,
+      bytes: 0,
+    };
+    let settled = false;
+    const finish = (error: Error | null) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      options.signal?.removeEventListener("abort", onAbort);
+      if (error && (error as NodeJS.ErrnoException).code === "ENOENT") state.missing = true;
+      const stdout = Buffer.concat(state.stdout).toString("utf-8");
+      resolve({ ...state, stdout, stderr: state.stderr.join("") });
+    };
+    const kill = () => {
+      try {
+        child.kill("SIGKILL");
+      } catch {}
+    };
+    const onAbort = () => {
+      state.cancelled = true;
+      kill();
+    };
+    const timer = setTimeout(() => {
+      state.timedOut = true;
+      kill();
+    }, options.timeoutMs);
+    let child: LightpandaProcess;
+    try {
+      child = options.spawn(launch[0], [...launch.slice(1), ...args], { stdio: ["ignore", "pipe", "pipe"] });
+    } catch (error) {
+      finish(error instanceof Error ? error : new Error(String(error)));
+      return;
+    }
+    const collect = async (stream: AsyncIterable<Buffer> | null, push: (chunk: Buffer) => void): Promise<void> => {
+      if (!stream) return;
+      try {
+        for await (const chunk of stream) push(chunk);
+      } catch {}
+    };
+    const stdoutDone = collect(child.stdout, (chunk) => {
+      if (state.bytes >= MAX_DUMP_BYTES) {
+        state.truncated = true;
+        return;
+      }
+      state.bytes += chunk.length;
+      if (state.bytes > MAX_DUMP_BYTES) {
+        state.truncated = true;
+        state.stdout.push(chunk.subarray(0, chunk.length - (state.bytes - MAX_DUMP_BYTES)));
+        return;
+      }
+      state.stdout.push(chunk);
+    });
+    const stderrDone = collect(child.stderr, (chunk) => state.stderr.push(chunk.toString("utf-8")));
+    child.on("error", (error) => finish(error));
+    child.on("close", (code) => {
+      state.code = code;
+      void Promise.all([stdoutDone, stderrDone]).then(() => finish(null));
+    });
+    options.signal?.addEventListener("abort", onAbort, { once: true });
+    if (options.signal?.aborted) onAbort();
+  });
+}
+
+function bodyChars(text: string): number {
+  const body = text
+    .split("\n")
+    .filter((line) => !METADATA_LINE_RE.test(line))
+    .join("\n");
+  return visibleChars(body);
+}
+
+function withProvenance(prefixed: string, url: string): string {
+  const lines = prefixed.split("\n");
+  let index = 0;
+  while (index < lines.length && METADATA_LINE_RE.test(lines[index])) index++;
+  const header = [`URL: ${url}`, PROVENANCE];
+  if (index === 0) return `${header.join("\n")}\n\n${prefixed}`;
+  return [...lines.slice(0, index), ...header, ...lines.slice(index)].join("\n");
+}
+
+export function lightpandaBinary(options: LightpandaRenderOptions = {}): string {
+  return (
+    options.binaryPath ??
+    options.settings?.binaryPath ??
+    process.env.PI_LIGHTPANDA_BIN?.trim() ??
+    DEFAULT_BINARY
+  );
+}
+
+export function lightpandaLaunch(options: LightpandaRenderOptions = {}): string[] {
+  const command = options.command ?? options.settings?.command ?? null;
+  if (command && command.length) return command;
+  return [lightpandaBinary(options)];
+}
+
+export async function renderPageWithLightpanda(
+  url: string,
+  options: LightpandaRenderOptions = {},
+): Promise<string | null> {
+  if (options.settings && options.settings.enabled === false) return null;
+  const target = typeof url === "string" ? url.trim() : "";
+  const policy = options.websitePolicy ?? null;
+  if (!target) return checkUrlAccess("", policy)[1];
+  if (parseLocalPath(target) !== null) return LOCAL_FILE_MESSAGE;
+  const normalized = normalizeUrlScheme(target);
+  const [allowed, reason, hostname] = checkUrlAccess(normalized, policy);
+  if (!allowed) return reason;
+  const launch = lightpandaLaunch(options);
+  const spawn = options.spawn ?? (spawnProcess as unknown as LightpandaSpawn);
+  if (!(await binaryAvailable(launch, spawn))) return null;
+  const timeoutMs = Math.min(
+    MAX_SIGNAL_TIMEOUT_MS,
+    Math.max(1, Math.floor(options.timeoutMs ?? DEFAULT_TIMEOUT_MS)),
+  );
+  const timeoutSignal = AbortSignal.timeout(timeoutMs);
+  const signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
+  const resolve = options.resolve ?? resolveAndValidateHost;
+  const resolved = await resolve(hostname, signal, false);
+  if (options.signal?.aborted) return CANCELLED_MESSAGE;
+  if (timeoutSignal.aborted) return TIMED_OUT_MESSAGE;
+  if (!resolved.ok) {
+    return resolved.reason.startsWith("Blocked:") ? resolved.reason : `Failed to render URL: ${resolved.reason}`;
+  }
+  const dump = options.dump ?? "html";
+  const result = await runLightpanda(
+    launch,
+    [
+      "fetch",
+      "--dump",
+      dump,
+      "--wait-until",
+      options.waitUntil ?? DEFAULT_WAIT_UNTIL,
+      "--block-private-networks",
+      "--http-timeout",
+      String(timeoutMs),
+      normalized,
+    ],
+    { timeoutMs, signal: options.signal, spawn },
+  );
+  if (result.missing) return null;
+  if (result.cancelled) return CANCELLED_MESSAGE;
+  if (result.timedOut) return TIMED_OUT_MESSAGE;
+  if (result.code !== 0) {
+    return `Failed to render URL: Lightpanda exited with code ${result.code ?? "unknown"}.`;
+  }
+  const raw = result.stdout;
+  const rendered =
+    dump === "html" ? pagePrefixedMarkdown(result.truncated ? raw + TRUNCATED_BODY_NOTICE : raw) : raw.trim();
+  if (bodyChars(rendered) === 0) return NO_TEXT_MESSAGE;
+  const withHeader =
+    dump === "html" || !result.truncated ? withProvenance(rendered, normalized) : withProvenance(rendered + TRUNCATED_BODY_NOTICE, normalized);
+  return truncatePageText(withHeader, options.maxChars);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools that began as a port of the Unsloth Studio codebase and now diverge from it (multi-engine search, opt-in SSRF guard, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -23,11 +23,10 @@
   "license": "AGPL-3.0",
   "files": [
     "index.ts",
-    "config.ts",
-    "config-ui.ts",
     "web-search.ts",
     "web-fetch.ts",
-    "web-render.ts",
+    "tls-fetch.ts",
+    "lightpanda.ts",
     "web-access.ts",
     "html-to-md.ts",
     "engines.ts",
@@ -39,7 +38,8 @@
     "settings.ts",
     "agent-dir.ts",
     "README.md",
-    "LICENSE"
+    "LICENSE",
+    "scripts/install-lightpanda.sh"
   ],
   "pi": {
     "extensions": [
@@ -58,6 +58,11 @@
     "test": "vitest run",
     "test:unit": "vitest run --exclude **/smoke.test.ts",
     "test:smoke": "vitest run test/smoke.test.ts",
+    "test:lightpanda": "vitest run test/lightpanda-live.test.ts",
+    "compare:fetch": "node scripts/fetch-compare.ts",
+    "compare:browsers": "node scripts/compare-browsers.ts",
+    "camoufox:warmup": "node scripts/camoufox-warmup.ts",
+    "stealth:matrix": "node scripts/stealth-matrix.ts",
     "typecheck": "tsc --noEmit",
     "check:package": "node -e \"const fs=require('fs');const pkg=require('./package.json');const listed=new Set(pkg.files);const bad=[];for(const f of pkg.files){if(!fs.existsSync(f))bad.push('missing file: '+f)}for(const f of fs.readdirSync('.').filter(f=>f.endsWith('.ts'))){if(!listed.has(f))bad.push('unlisted source: '+f)}if(bad.length){console.error(bad.join('\\n'));process.exit(1)}\"",
     "prepublishOnly": "npm run typecheck && npm run lint && npm run test:unit && npm run check:package && npm run check:publint",
@@ -65,6 +70,9 @@
     "check:publint": "publint",
     "check:attw": "attw --pack"
   },
+  "dependencies": {
+    "wreq-js": "^3.2.0"
+  },
   "devDependencies": {
     "@earendil-works/pi-coding-agent": "^0.87.0",
     "@earendil-works/pi-tui": "^0.87.0",

package/scripts/install-lightpanda.sh

@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="lightpanda-io/browser"
+DEFAULT_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/pi-unsloth-webtools/lightpanda"
+PINNED_FALLBACK_VERSION="1.0.0"
+
+DIR="$DEFAULT_DIR"
+VERSION=""
+FORCE=0
+ALLOW_SHIM=1
+PRINT_PATH=0
+
+usage() {
+  cat <<'EOF'
+Install the Lightpanda browser used by this package's local rendering tier.
+
+Usage: bash scripts/install-lightpanda.sh [options]
+
+  --version=X    Release tag to install (default: the newest stable release, e.g. 1.0.0;
+                 pass --version=nightly for the nightly build)
+  --dir=PATH     Install directory (default: $XDG_DATA_HOME/pi-unsloth-webtools/lightpanda)
+  --force        Reinstall even when a working binary is already present
+  --no-shim      Fail instead of bundling a newer libc when the system glibc is too old
+  --print-path   Print only the launcher path (for scripts and CI)
+  -h, --help     Show this message
+
+On Linux, if the release needs a newer glibc than the system provides, the script
+downloads Debian's libc6 for the current stable suite, extracts it into the install
+directory, and writes a launcher shim that runs the binary with that libc. Point
+`webRender.lightpandaPath` (settings.json) or `PI_LIGHTPANDA_BIN` at the printed path.
+EOF
+}
+
+for arg in "$@"; do
+  case "$arg" in
+    --version=*) VERSION="${arg#*=}" ;;
+    --dir=*) DIR="${arg#*=}" ;;
+    --force) FORCE=1 ;;
+    --no-shim) ALLOW_SHIM=0 ;;
+    --print-path) PRINT_PATH=1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown option: $arg" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+need() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    echo "missing required command: $1" >&2
+    exit 1
+  fi
+}
+
+need curl
+need awk
+need sed
+
+os="$(uname -s)"
+arch="$(uname -m)"
+deb_arch=""
+case "$os/$arch" in
+  Linux/x86_64) asset="lightpanda-x86_64-linux"; deb_arch="amd64" ;;
+  Linux/aarch64|Linux/arm64) asset="lightpanda-aarch64-linux"; deb_arch="arm64" ;;
+  Darwin/x86_64) asset="lightpanda-x86_64-macos" ;;
+  Darwin/arm64) asset="lightpanda-aarch64-macos" ;;
+  MINGW*|MSYS*|CYGWIN*)
+    cat >&2 <<'EOF'
+Lightpanda ships no native Windows build. Run it inside WSL2 instead:
+
+  1. in WSL:   bash scripts/install-lightpanda.sh
+               (it prints a path such as /home/<you>/.local/share/pi-unsloth-webtools/lightpanda/lightpanda-run)
+
+  2. in Windows settings.json, drive that binary through wsl.exe:
+
+     {
+       "webRender": {
+         "lightpandaCommand": ["wsl.exe", "-e", "/home/<you>/.local/share/pi-unsloth-webtools/lightpanda/lightpanda-run"]
+       }
+     }
+
+Everything else in this package (search, the browser-fingerprint transport) runs natively on Windows.
+EOF
+    exit 1 ;;
+  *) echo "unsupported platform: $os/$arch" >&2; exit 1 ;;
+esac
+
+if [ -z "$VERSION" ]; then
+  VERSION="$(curl -fsSL "https://api.github.com/repos/$REPO/releases?per_page=30" 2>/dev/null \
+    | grep -o '"tag_name":[[:space:]]*"[^"]*"' \
+    | sed -n 's/.*"\([^"]*\)"$/\1/p' \
+    | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \
+    | head -1 || true)"
+  if [ -z "$VERSION" ]; then
+    echo "could not resolve the newest release; falling back to $PINNED_FALLBACK_VERSION" >&2
+    VERSION="$PINNED_FALLBACK_VERSION"
+  fi
+fi
+
+mkdir -p "$DIR"
+binary="$DIR/lightpanda"
+shim="$DIR/lightpanda-run"
+
+report_path() {
+  if [ "$PRINT_PATH" = 1 ]; then
+    printf '%s\n' "$1"
+  else
+    echo "already installed and working: $1" >&2
+  fi
+  exit 0
+}
+
+if [ "$FORCE" = 0 ] && [ -x "$shim" ] && "$shim" version >/dev/null 2>&1; then
+  report_path "$shim"
+fi
+if [ "$FORCE" = 0 ] && [ -x "$binary" ] && "$binary" version >/dev/null 2>&1; then
+  report_path "$binary"
+fi
+
+url="https://github.com/$REPO/releases/download/$VERSION/$asset"
+if [ "$PRINT_PATH" = 1 ]; then
+  echo "downloading $VERSION ($asset)" >&2
+  curl -fL --silent --show-error "$url" -o "$binary.part"
+else
+  echo "downloading $VERSION ($asset)"
+  curl -fL --progress-bar "$url" -o "$binary.part"
+fi
+chmod +x "$binary.part"
+mv -f "$binary.part" "$binary"
+rm -f "$shim"
+
+run_path="$binary"
+if ! "$binary" version >/dev/null 2>&1; then
+  run_path=""
+  if [ "$ALLOW_SHIM" = 0 ] || [ "$os" != "Linux" ]; then
+    echo "the $VERSION binary cannot run on this system:" >&2
+    "$binary" version 2>&1 | head -3 >&2
+    exit 1
+  fi
+  required="$({ "$binary" version 2>&1 || true; } | sed -n "s/.*GLIBC_\([0-9][0-9.]*\).*/\1/p" | sort -V | tail -1)"
+  if [ -z "$required" ]; then
+    echo "the $VERSION binary failed to start and the cause is not a glibc version:" >&2
+    "$binary" version 2>&1 | head -3 >&2
+    exit 1
+  fi
+  system_glibc="$(ldd --version 2>/dev/null | head -1 | sed -n 's/.* \([0-9][0-9.]*\)$/\1/p')"
+  echo "binary needs glibc $required, system has ${system_glibc:-unknown}: bundling a newer libc"
+  need dpkg-deb
+  glibc_dir="$DIR/glibc"
+  rm -rf "$glibc_dir"
+  for suite in trixie stable sid; do
+    index="https://deb.debian.org/debian/dists/$suite/main/binary-$deb_arch/Packages.gz"
+    deb_path="$(curl -fsSL "$index" 2>/dev/null | gzip -dc 2>/dev/null \
+      | awk '/^Package: libc6$/{found=1} found && /^Filename:/{print $2; exit}' || true)"
+    [ -n "$deb_path" ] || continue
+    echo "fetching libc6 from debian/$suite"
+    curl -fsSL "https://deb.debian.org/debian/$deb_path" -o "$DIR/libc6.deb" || continue
+    mkdir -p "$glibc_dir"
+    dpkg-deb -x "$DIR/libc6.deb" "$glibc_dir"
+    rm -f "$DIR/libc6.deb"
+    loader="$(find "$glibc_dir" -path "*-linux-gnu/ld-linux-*.so.1" | head -1)"
+    [ -n "$loader" ] || continue
+    bundled_glibc="$("$loader" --version 2>/dev/null | head -1 | sed -n 's/.* \([0-9][0-9.]*\)$/\1/p')"
+    if [ -z "$bundled_glibc" ]; then
+      rm -rf "$glibc_dir"
+      continue
+    fi
+    newest="$(printf '%s\n%s\n' "$required" "$bundled_glibc" | sort -V | tail -1)"
+    if [ "$newest" != "$bundled_glibc" ]; then
+      echo "bundled glibc $bundled_glibc is older than required $required, trying the next suite" >&2
+      rm -rf "$glibc_dir"
+      continue
+    fi
+    libdir="$(dirname "$loader")"
+    {
+      echo "#!/bin/sh"
+      echo "exec \"$loader\" --library-path \"$libdir\" \"$binary\" \"\$@\""
+    } > "$shim"
+    chmod +x "$shim"
+    if "$shim" version >/dev/null 2>&1; then
+      run_path="$shim"
+      echo "bundled glibc $bundled_glibc at $glibc_dir"
+      break
+    fi
+    rm -rf "$glibc_dir" "$shim"
+  done
+  if [ -z "$run_path" ]; then
+    echo "could not make $VERSION runnable on this system." >&2
+    echo "Options: upgrade the distribution, run Lightpanda in a container, or install an older" >&2
+    echo "release that targets this glibc (for example 0.2.9)." >&2
+    exit 1
+  fi
+fi
+
+installed_version="$("$run_path" version 2>/dev/null | head -1)"
+if [ "$PRINT_PATH" = 1 ]; then
+  printf '%s\n' "$run_path"
+  exit 0
+fi
+
+echo
+echo "Lightpanda $installed_version installed"
+echo "  run this:          $run_path"
+echo "  configure one of:"
+echo "    export PI_LIGHTPANDA_BIN=$run_path"
+echo "    settings.json:   { \"webRender\": { \"lightpandaPath\": \"$run_path\" } }"
+echo "  verify:            $run_path fetch --dump markdown --wait-until networkidle https://example.com/"

package/settings.ts

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { agentDir } from "./agent-dir.ts";
+import type { FetchTransport } from "./web-fetch.ts";
 
 async function readJson(file: string): Promise<Record<string, unknown> | undefined> {
   try {
@@ -49,6 +50,23 @@ function pickBoolean(data: Record<string, unknown>, paths: string[][]): boolean
   return undefined;
 }
 
+function pickStringArray(data: Record<string, unknown>, paths: string[][]): string[] | undefined {
+  for (const path of paths) {
+    let cur: unknown = data;
+    for (const key of path) {
+      if (cur && typeof cur === "object" && !Array.isArray(cur)) cur = (cur as Record<string, unknown>)[key];
+      else {
+        cur = undefined;
+        break;
+      }
+    }
+    if (!Array.isArray(cur)) continue;
+    const values = cur.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0);
+    if (values.length) return values;
+  }
+  return undefined;
+}
+
 function pickString(data: Record<string, unknown>, paths: string[][]): string | undefined {
   for (const path of paths) {
     let cur: unknown = data;
@@ -94,11 +112,27 @@ const ALLOW_LOCAL_FILES_PATHS: string[][] = [
   ["webFetch", "allowLocalFiles"],
 ];
 
-const JINA_API_KEY_PATHS: string[][] = [
-  ["unslothWebTools", "jinaApiKey"],
-  ["webRender", "jinaApiKey"],
+const TRANSPORT_PATHS: string[][] = [
+  ["unslothWebTools", "transport"],
+  ["webFetch", "transport"],
+];
+
+const TRANSPORT_VALUES: ReadonlySet<string> = new Set(["tls-first", "direct-first", "off"]);
+
+const LIGHTPANDA_ENABLED_PATHS: string[][] = [
+  ["unslothWebTools", "lightpandaEnabled"],
+  ["webRender", "lightpandaEnabled"],
+];
+
+const LIGHTPANDA_PATH_PATHS: string[][] = [
+  ["unslothWebTools", "lightpandaPath"],
+  ["webRender", "lightpandaPath"],
 ];
 
+const LIGHTPANDA_COMMAND_PATHS: string[][] = [
+  ["unslothWebTools", "lightpandaCommand"],
+  ["webRender", "lightpandaCommand"],
+];
 function clampMaxResults(value: number): number {
   return Math.min(20, Math.max(1, value));
 }
@@ -151,11 +185,13 @@ export async function loadDefaultFetchSettings(cwd?: string): Promise<{
   timeoutMs?: number;
   allowPrivateAddresses: boolean;
   allowLocalFiles: boolean;
+  transport: FetchTransport;
 }> {
   let maxChars: number | undefined;
   let timeoutMs: number | undefined;
   let allowPrivateAddresses = true;
   let allowLocalFiles = true;
+  let transport: FetchTransport = "tls-first";
   for (const data of await settingsEntries(cwd)) {
     const c = pickNumber(data, FETCH_MAX_CHARS_PATHS);
     if (c !== undefined && c > 0) maxChars = c;
@@ -165,17 +201,27 @@ export async function loadDefaultFetchSettings(cwd?: string): Promise<{
     if (p !== undefined) allowPrivateAddresses = p;
     const l = pickBoolean(data, ALLOW_LOCAL_FILES_PATHS);
     if (l !== undefined) allowLocalFiles = l;
+    const mode = pickString(data, TRANSPORT_PATHS);
+    if (mode !== undefined && TRANSPORT_VALUES.has(mode)) transport = mode as FetchTransport;
   }
-  return { maxChars, timeoutMs, allowPrivateAddresses, allowLocalFiles };
+  return { maxChars, timeoutMs, allowPrivateAddresses, allowLocalFiles, transport };
 }
 
-export async function loadJinaApiKey(cwd?: string): Promise<string | undefined> {
-  let result: string | undefined;
+export async function loadLightpandaSettings(cwd?: string): Promise<{
+  enabled: boolean;
+  binaryPath: string | null;
+  command: string[] | null;
+}> {
+  let enabled = true;
+  let binaryPath: string | null = null;
+  let command: string[] | null = null;
   for (const data of await settingsEntries(cwd)) {
-    const candidate = pickString(data, JINA_API_KEY_PATHS);
-    if (candidate !== undefined) result = candidate;
+    const e = pickBoolean(data, LIGHTPANDA_ENABLED_PATHS);
+    if (e !== undefined) enabled = e;
+    const p = pickString(data, LIGHTPANDA_PATH_PATHS);
+    if (p !== undefined) binaryPath = p;
+    const c = pickStringArray(data, LIGHTPANDA_COMMAND_PATHS);
+    if (c !== undefined) command = c;
   }
-  if (result !== undefined) return result;
-  const env = process.env.JINA_API_KEY?.trim();
-  return env ? env : undefined;
+  return { enabled, binaryPath, command };
 }