pi-unsloth-webtools 0.9.0 → 0.10.0

package/index.ts

@@ -10,7 +10,13 @@ import {
   type FetchPageOutcome,
 } from "./web-fetch.ts";
 import { renderPageWithLightpanda as defaultRenderLocalPageText } from "./lightpanda.ts";
-import { loadDefaultFetchSettings, loadDefaultFetchTimeoutMs, loadLightpandaSettings } from "./settings.ts";
+import {
+  loadDefaultEngines,
+  loadDefaultEngineWeights,
+  loadDefaultFetchSettings,
+  loadDefaultMaxPerHost,
+  loadLightpandaSettings,
+} from "./settings.ts";
 
 function toolCallLine(theme: Theme, name: string, detail: string) {
   const line = theme.fg("toolTitle", theme.bold(name)) + (detail ? ` ${theme.fg("accent", detail)}` : "");
@@ -179,12 +185,20 @@ export function createWebTools(deps: WebToolsDeps = {}) {
         onUpdate?.({ content: [{ type: "text", text: "Searching the web..." }], details: {} });
         const timeoutParam = positiveNumber(params.timeoutMs);
         const searchCwd = (_ctx as ExtensionContext | undefined)?.cwd;
-        const searchTimeoutMs = timeoutParam ?? (await loadDefaultFetchTimeoutMs(searchCwd)) ?? SEARCH_TIMEOUT_MS;
+        const searchSettings = await loadDefaultFetchSettings(searchCwd);
+        const searchEngines = await loadDefaultEngines(searchCwd);
+        const searchEngineWeights = await loadDefaultEngineWeights(searchCwd);
+        const searchMaxPerHost = await loadDefaultMaxPerHost(searchCwd);
+        const searchTimeoutMs = timeoutParam ?? searchSettings.timeoutMs ?? SEARCH_TIMEOUT_MS;
         const text = await webSearch(params.query, {
           signal: signal ?? undefined,
           timeoutMs: searchTimeoutMs,
           maxResults: positiveNumber(params.maxResults),
           cwd: searchCwd,
+          transport: searchSettings.transport,
+          engines: searchEngines,
+          engineWeights: searchEngineWeights,
+          maxPerHost: searchMaxPerHost,
         });
         return { content: [{ type: "text", text }], details: {} };
       },

package/lightpanda.ts

@@ -1,4 +1,7 @@
 import { spawn as spawnProcess } from "node:child_process";
+import { existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
 import { visibleChars } from "./html-to-md.ts";
 import {
   checkUrlAccess,
@@ -201,11 +204,31 @@ function withProvenance(prefixed: string, url: string): string {
   return [...lines.slice(0, index), ...header, ...lines.slice(index)].join("\n");
 }
 
+function installedLauncher(): string | null {
+  let dataHome = process.env.XDG_DATA_HOME?.trim();
+  if (!dataHome) {
+    try {
+      const home = homedir();
+      if (!home) return null;
+      dataHome = join(home, ".local", "share");
+    } catch {
+      return null;
+    }
+  }
+  const dir = join(dataHome, "pi-unsloth-webtools", "lightpanda");
+  for (const name of ["lightpanda-run", "lightpanda"]) {
+    const candidate = join(dir, name);
+    if (existsSync(candidate)) return candidate;
+  }
+  return null;
+}
+
 export function lightpandaBinary(options: LightpandaRenderOptions = {}): string {
   return (
     options.binaryPath ??
     options.settings?.binaryPath ??
     process.env.PI_LIGHTPANDA_BIN?.trim() ??
+    installedLauncher() ??
     DEFAULT_BINARY
   );
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools that began as a port of the Unsloth Studio codebase and now diverge from it (multi-engine search, opt-in SSRF guard, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -59,10 +59,8 @@
     "test:unit": "vitest run --exclude **/smoke.test.ts",
     "test:smoke": "vitest run test/smoke.test.ts",
     "test:lightpanda": "vitest run test/lightpanda-live.test.ts",
-    "compare:fetch": "node scripts/fetch-compare.ts",
-    "compare:browsers": "node scripts/compare-browsers.ts",
     "camoufox:warmup": "node scripts/camoufox-warmup.ts",
-    "stealth:matrix": "node scripts/stealth-matrix.ts",
+    "engine:eval": "node scripts/engine-eval.ts",
     "typecheck": "tsc --noEmit",
     "check:package": "node -e \"const fs=require('fs');const pkg=require('./package.json');const listed=new Set(pkg.files);const bad=[];for(const f of pkg.files){if(!fs.existsSync(f))bad.push('missing file: '+f)}for(const f of fs.readdirSync('.').filter(f=>f.endsWith('.ts'))){if(!listed.has(f))bad.push('unlisted source: '+f)}if(bad.length){console.error(bad.join('\\n'));process.exit(1)}\"",
     "prepublishOnly": "npm run typecheck && npm run lint && npm run test:unit && npm run check:package && npm run check:publint",

package/scripts/install-lightpanda.sh

@@ -10,6 +10,7 @@ VERSION=""
 FORCE=0
 ALLOW_SHIM=1
 PRINT_PATH=0
+NO_CONFIGURE=0
 
 usage() {
   cat <<'EOF'
@@ -23,12 +24,15 @@ Usage: bash scripts/install-lightpanda.sh [options]
   --force        Reinstall even when a working binary is already present
   --no-shim      Fail instead of bundling a newer libc when the system glibc is too old
   --print-path   Print only the launcher path (for scripts and CI)
+  --no-configure Skip writing webRender.lightpandaPath into the global settings
   -h, --help     Show this message
 
 On Linux, if the release needs a newer glibc than the system provides, the script
 downloads Debian's libc6 for the current stable suite, extracts it into the install
-directory, and writes a launcher shim that runs the binary with that libc. Point
-`webRender.lightpandaPath` (settings.json) or `PI_LIGHTPANDA_BIN` at the printed path.
+directory, and writes a launcher shim that runs the binary with that libc.
+The script also writes `webRender.lightpandaPath` into the global settings so the
+extension picks the binary up with no further setup; `--no-configure` skips that,
+and `--print-path` prints only the launcher path for scripts and CI.
 EOF
 }
 
@@ -39,6 +43,7 @@ for arg in "$@"; do
     --force) FORCE=1 ;;
     --no-shim) ALLOW_SHIM=0 ;;
     --print-path) PRINT_PATH=1 ;;
+    --no-configure) NO_CONFIGURE=1 ;;
     -h|--help) usage; exit 0 ;;
     *) echo "unknown option: $arg" >&2; usage >&2; exit 2 ;;
   esac
@@ -100,11 +105,96 @@ mkdir -p "$DIR"
 binary="$DIR/lightpanda"
 shim="$DIR/lightpanda-run"
 
+settings_file_for_agent() {
+  local agent_dir="${PI_CODING_AGENT_DIR:-${PI_AGENT_DIR:-$HOME/.pi/agent}}"
+  case "$agent_dir" in
+    "~") agent_dir="$HOME" ;;
+    "~/"*) agent_dir="$HOME/${agent_dir#\~/}" ;;
+  esac
+  printf '%s\n' "$agent_dir/settings.json"
+}
+
+configure_renderer() {
+  local binary_path="$1"
+  local settings_file
+  local status
+  settings_file="$(settings_file_for_agent)"
+  if [ "$NO_CONFIGURE" = 1 ]; then
+    echo "settings:   skipped (--no-configure); set PI_LIGHTPANDA_BIN or webRender.lightpandaPath to $binary_path"
+    return
+  fi
+  if ! command -v node >/dev/null 2>&1; then
+    echo "settings:   node not found; set PI_LIGHTPANDA_BIN=$binary_path or webRender.lightpandaPath yourself" >&2
+    return
+  fi
+  status="$(SETTINGS_FILE="$settings_file" LIGHTPANDA_PATH="$binary_path" node --input-type=module <<'NODE'
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+
+const settingsFile = process.env.SETTINGS_FILE;
+const binaryPath = process.env.LIGHTPANDA_PATH;
+function fail(message) {
+  console.error(message);
+  process.exit(3);
+}
+
+let settings = {};
+let mode;
+if (existsSync(settingsFile)) {
+  try {
+    settings = JSON.parse(readFileSync(settingsFile, "utf8"));
+  } catch {
+    fail(`settings:   keeping ${settingsFile}: not valid JSON`);
+  }
+  if (!settings || typeof settings !== "object" || Array.isArray(settings)) {
+    fail(`settings:   keeping ${settingsFile}: not a JSON object`);
+  }
+  mode = statSync(settingsFile).mode & 0o777;
+}
+
+const sections = [settings.unslothWebTools, settings.webRender].filter(
+  (section) => section && typeof section === "object" && !Array.isArray(section),
+);
+const alreadyConfigured = sections.some(
+  (section) =>
+    (typeof section.lightpandaPath === "string" && section.lightpandaPath.length > 0) ||
+    (Array.isArray(section.lightpandaCommand) && section.lightpandaCommand.length > 0),
+);
+if (alreadyConfigured) {
+  console.log("kept");
+  process.exit(0);
+}
+
+if (settings.webRender !== undefined && (!settings.webRender || typeof settings.webRender !== "object" || Array.isArray(settings.webRender))) {
+  fail(`settings:   keeping ${settingsFile}: webRender is not a JSON object`);
+}
+
+settings.webRender = { ...(settings.webRender ?? {}), lightpandaPath: binaryPath };
+const temporary = `${settingsFile}.tmp.${process.pid}`;
+try {
+  mkdirSync(dirname(settingsFile), { recursive: true });
+  writeFileSync(temporary, JSON.stringify(settings, null, 2) + "\n", mode === undefined ? {} : { mode });
+  renameSync(temporary, settingsFile);
+} catch (error) {
+  rmSync(temporary, { force: true });
+  fail(`settings:   keeping ${settingsFile}: ${error instanceof Error ? error.message : String(error)}`);
+}
+console.log("configured");
+NODE
+)" || status=""
+  case "$status" in
+    configured) echo "settings:   ${settings_file} now sets webRender.lightpandaPath" ;;
+    kept) echo "settings:   ${settings_file} already configures a lightpanda path or command; left unchanged" ;;
+    *) echo "settings:   could not update ${settings_file} automatically" >&2 ;;
+  esac
+}
+
 report_path() {
   if [ "$PRINT_PATH" = 1 ]; then
     printf '%s\n' "$1"
   else
     echo "already installed and working: $1" >&2
+    configure_renderer "$1"
   fi
   exit 0
 }
@@ -200,7 +290,5 @@ fi
 echo
 echo "Lightpanda $installed_version installed"
 echo "  run this:          $run_path"
-echo "  configure one of:"
-echo "    export PI_LIGHTPANDA_BIN=$run_path"
-echo "    settings.json:   { \"webRender\": { \"lightpandaPath\": \"$run_path\" } }"
 echo "  verify:            $run_path fetch --dump markdown --wait-until networkidle https://example.com/"
+configure_renderer "$run_path"

package/settings.ts

@@ -50,6 +50,26 @@ function pickBoolean(data: Record<string, unknown>, paths: string[][]): boolean
   return undefined;
 }
 
+function pickNumberRecord(data: Record<string, unknown>, paths: string[][]): Record<string, number> | undefined {
+  for (const path of paths) {
+    let cur: unknown = data;
+    for (const key of path) {
+      if (cur && typeof cur === "object" && !Array.isArray(cur)) cur = (cur as Record<string, unknown>)[key];
+      else {
+        cur = undefined;
+        break;
+      }
+    }
+    if (!cur || typeof cur !== "object" || Array.isArray(cur)) continue;
+    const record: Record<string, number> = {};
+    for (const [key, value] of Object.entries(cur as Record<string, unknown>)) {
+      if (typeof value === "number" && Number.isFinite(value) && value > 0) record[key] = value;
+    }
+    if (Object.keys(record).length) return record;
+  }
+  return undefined;
+}
+
 function pickStringArray(data: Record<string, unknown>, paths: string[][]): string[] | undefined {
   for (const path of paths) {
     let cur: unknown = data;
@@ -133,6 +153,21 @@ const LIGHTPANDA_COMMAND_PATHS: string[][] = [
   ["unslothWebTools", "lightpandaCommand"],
   ["webRender", "lightpandaCommand"],
 ];
+
+const ENGINE_WEIGHTS_PATHS: string[][] = [
+  ["unslothWebTools", "engineWeights"],
+  ["webSearch", "engineWeights"],
+];
+
+const MAX_PER_HOST_PATHS: string[][] = [
+  ["unslothWebTools", "maxPerHost"],
+  ["webSearch", "maxPerHost"],
+];
+
+const ENGINE_PATHS: string[][] = [
+  ["unslothWebTools", "engines"],
+  ["webSearch", "engines"],
+];
 function clampMaxResults(value: number): number {
   return Math.min(20, Math.max(1, value));
 }
@@ -163,6 +198,28 @@ export async function loadDefaultMaxResults(cwd?: string): Promise<number> {
   return result;
 }
 
+export async function loadDefaultEngines(cwd?: string): Promise<string[] | undefined> {
+  let result: string[] | undefined;
+  for (const data of await settingsEntries(cwd)) {
+    const candidate = pickStringArray(data, ENGINE_PATHS);
+    if (candidate !== undefined) result = candidate;
+  }
+  return result;
+}
+
+export async function loadDefaultEngineWeights(cwd?: string): Promise<Record<string, number> | undefined> {
+  let result: Record<string, number> | undefined;
+  for (const data of await settingsEntries(cwd)) {
+    const candidate = pickNumberRecord(data, ENGINE_WEIGHTS_PATHS);
+    if (candidate !== undefined) result = candidate;
+  }
+  return result;
+}
+
+export async function loadDefaultMaxPerHost(cwd?: string): Promise<number | undefined> {
+  return loadFetchSetting(cwd, MAX_PER_HOST_PATHS, (n) => n >= 0);
+}
+
 async function loadFetchSetting(cwd: string | undefined, paths: string[][], valid: (n: number) => boolean): Promise<number | undefined> {
   let result: number | undefined;
   for (const data of await settingsEntries(cwd)) {

package/tls-fetch.ts

@@ -49,15 +49,19 @@ interface ImpersonationModule {
 
 export type ImpersonationLoader = () => Promise<ImpersonationModule | null>;
 
+export type FetchTransport = "tls-first" | "direct-first" | "off";
+
 export interface TlsHopOptions {
   url: URL;
-  pinnedIp: string;
-  family: number;
+  pinnedIp?: string;
+  family?: number;
   timeoutMs: number;
   signal?: AbortSignal;
   maxBytes: number;
   maxPdfBytes?: number;
   profile?: string;
+  method?: string;
+  body?: string;
   extraHeaders?: Record<string, string>;
   loader?: ImpersonationLoader;
 }
@@ -85,7 +89,7 @@ async function loadImpersonationModule(): Promise<ImpersonationModule | null> {
 const transports = new Map<string, Promise<ImpersonationTransport>>();
 
 function transportKey(options: TlsHopOptions): string {
-  return `${options.profile ?? DEFAULT_PROFILE}|${options.url.hostname}|${options.pinnedIp}|${options.family}`;
+  return `${options.profile ?? DEFAULT_PROFILE}|${options.url.hostname}|${options.pinnedIp ?? ""}|${options.family ?? 0}`;
 }
 
 function evictTransports(): void {
@@ -107,12 +111,13 @@ async function transportFor(
   const key = transportKey(options);
   const existing = transports.get(key);
   if (existing) return existing;
+  const transportOptions: Record<string, unknown> = {
+    browser: options.profile ?? DEFAULT_PROFILE,
+    os: DEFAULT_OS,
+  };
+  if (options.pinnedIp) transportOptions.resolve = { [options.url.hostname]: options.pinnedIp };
   const created = module
-    .createTransport({
-      resolve: { [options.url.hostname]: options.pinnedIp },
-      browser: options.profile ?? DEFAULT_PROFILE,
-      os: DEFAULT_OS,
-    })
+    .createTransport(transportOptions)
     .catch((error: unknown) => {
       transports.delete(key);
       throw error;
@@ -179,12 +184,15 @@ export async function impersonatedRequest(options: TlsHopOptions): Promise<TlsHo
   const signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
   let response: ImpersonationResponse;
   try {
-    response = await module.fetch(options.url.toString(), {
+    const init: Record<string, unknown> = {
       transport,
+      method: options.method ?? "GET",
       redirect: "manual",
       signal,
       headers: hopHeaders(options.extraHeaders),
-    });
+    };
+    if (options.body !== undefined) init.body = options.body;
+    response = await module.fetch(options.url.toString(), init);
   } catch (error) {
     throw new Error(failureMessage(error, options.signal, signal));
   }

package/web-fetch.ts

@@ -23,7 +23,7 @@ import {
   stripIpv6Brackets,
   type WebsitePolicy,
 } from "./web-access.ts";
-import { impersonatedRequest, type TlsHopOptions, type TlsHopResponse } from "./tls-fetch.ts";
+import { impersonatedRequest, type FetchTransport, type TlsHopOptions, type TlsHopResponse } from "./tls-fetch.ts";
 import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown, visibleChars } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
 import { INVALID_CHARREFS } from "./entities.ts";
@@ -210,7 +210,7 @@ export interface ResolvedHost {
   alternates?: { ip: string; family: number }[];
 }
 
-export type FetchTransport = "tls-first" | "direct-first" | "off";
+export type { FetchTransport };
 
 type TransportKind = "direct" | "tls";
 export interface FetchSeams {

package/web-search.ts

@@ -6,8 +6,10 @@ import {
   EmptySweepError,
   SearchCancelled,
   SearchTimeoutError,
+  type SearchEngineOptions,
   type SearchResult,
 } from "./engines.ts";
+import type { FetchTransport } from "./tls-fetch.ts";
 
 export { EmptySweepError, SearchCancelled, SearchTimeoutError } from "./engines.ts";
 
@@ -24,15 +26,20 @@ export type SearchClient = (
   maxResults: number,
   signal?: AbortSignal,
   timeoutMs?: number,
+  options?: SearchEngineOptions,
 ) => Promise<SearchResult[]>;
 export async function ddgSearch(
   query: string,
   maxResults = MAX_RESULTS,
   signal?: AbortSignal,
   timeoutMs = SEARCH_TIMEOUT_MS,
+  options: SearchEngineOptions = {},
 ): Promise<SearchResult[]> {
   if (signal?.aborted) throw new SearchCancelled();
-  return autoTextSearch(query, maxResults, timeoutMs, signal);
+  return autoTextSearch(query, maxResults, timeoutMs, signal, {
+    ...options,
+    transport: options.transport ?? "tls-first",
+  });
 }
 
 const POLICY_OVERFETCH = 4;
@@ -44,6 +51,10 @@ export interface WebSearchOptions {
   websitePolicy?: WebsitePolicy | null;
   client?: SearchClient;
   cwd?: string;
+  transport?: FetchTransport;
+  engines?: string[];
+  engineWeights?: Record<string, number>;
+  maxPerHost?: number;
 }
 
 export { loadDefaultMaxResults };
@@ -71,7 +82,13 @@ export async function webSearch(
         (policy?.blockedDomains?.length ?? 0) > 0,
     );
     const wanted = restricted ? maxResults * POLICY_OVERFETCH : maxResults;
-    const results = await client(effectiveQuery, wanted, signal, timeoutMs);
+    const results = await client(effectiveQuery, wanted, signal, timeoutMs, {
+      transport: options.transport,
+      policy,
+      engines: options.engines,
+      engineWeights: options.engineWeights,
+      maxPerHost: options.maxPerHost,
+    });
     if (signal?.aborted) return "Search cancelled.";
     if (!results.length) return EMPTY_SEARCH_RESULTS[0];
     const allowed: SearchResult[] = [];