pi-web-toolkit 0.3.2 → 0.3.3

package/extensions/utils/config.ts

@@ -0,0 +1,170 @@
+/**
+ * pi-web-toolkit runtime configuration
+ *
+ * Reads user-level toolkit configuration without requiring users to modify
+ * shell profiles. Environment variables remain the highest-priority override.
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+
+export const DEFAULT_SEARXNG_URL = "http://localhost:8080";
+
+export interface ToolkitCommandsConfig {
+  scrapling?: string;
+  agentBrowser?: string;
+  firecrawl?: string;
+}
+
+export type FirecrawlRunner = "installed" | "npx" | "bunx";
+
+export interface ToolkitConfig {
+  searxngUrl?: string;
+  firecrawlFallback?: boolean;
+  firecrawlRunner?: FirecrawlRunner;
+  commands?: ToolkitCommandsConfig;
+}
+
+export type ToolkitCommandName = "scrapling" | "agentBrowser" | "firecrawl";
+
+const COMMAND_DEFAULTS: Record<ToolkitCommandName, string> = {
+  scrapling: "scrapling",
+  agentBrowser: "agent-browser",
+  firecrawl: "firecrawl",
+};
+
+const COMMAND_ENV_VARS: Record<ToolkitCommandName, string> = {
+  scrapling: "SCRAPLING_BIN",
+  agentBrowser: "AGENT_BROWSER_BIN",
+  firecrawl: "FIRECRAWL_BIN",
+};
+
+const FIRECRAWL_RUNNERS = ["installed", "npx", "bunx"] as const;
+
+function isFirecrawlRunner(value: string): value is FirecrawlRunner {
+  return (FIRECRAWL_RUNNERS as readonly string[]).includes(value);
+}
+
+export function getDefaultToolkitConfigPath(): string {
+  const configHome = process.env.XDG_CONFIG_HOME?.trim() || path.join(os.homedir(), ".config");
+  return path.join(configHome, "pi-web-toolkit", "config.json");
+}
+
+export function getToolkitConfigPath(): string {
+  const configured = process.env.PI_WEB_TOOLKIT_CONFIG?.trim();
+  return configured || getDefaultToolkitConfigPath();
+}
+
+function parseConfigFile(filePath: string, required: boolean): ToolkitConfig {
+  if (!existsSync(filePath)) {
+    if (required) {
+      throw new Error(`Toolkit config file not found: ${filePath}`);
+    }
+    return {};
+  }
+
+  let raw: string;
+  try {
+    raw = readFileSync(filePath, "utf8");
+  } catch (err: any) {
+    throw new Error(`Unable to read toolkit config at ${filePath}: ${err.message ?? String(err)}`);
+  }
+
+  try {
+    const parsed = JSON.parse(raw) as unknown;
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+      throw new Error("expected a JSON object");
+    }
+    validateToolkitConfig(parsed as Record<string, unknown>);
+    return parsed as ToolkitConfig;
+  } catch (err: any) {
+    throw new Error(`Invalid toolkit config at ${filePath}: ${err.message ?? String(err)}`);
+  }
+}
+
+function validateOptionalString(value: unknown, key: string): void {
+  if (value !== undefined && typeof value !== "string") {
+    throw new Error(`${key} must be a string`);
+  }
+}
+
+function validateToolkitConfig(value: Record<string, unknown>): void {
+  validateOptionalString(value.searxngUrl, "searxngUrl");
+
+  if (value.firecrawlFallback !== undefined && typeof value.firecrawlFallback !== "boolean") {
+    throw new Error("firecrawlFallback must be a boolean");
+  }
+
+  if (value.firecrawlRunner !== undefined) {
+    if (typeof value.firecrawlRunner !== "string" || !isFirecrawlRunner(value.firecrawlRunner)) {
+      throw new Error("firecrawlRunner must be one of: installed, npx, bunx");
+    }
+  }
+
+  if (value.commands !== undefined) {
+    if (typeof value.commands !== "object" || value.commands === null || Array.isArray(value.commands)) {
+      throw new Error("commands must be an object");
+    }
+    const commands = value.commands as Record<string, unknown>;
+    validateOptionalString(commands.scrapling, "commands.scrapling");
+    validateOptionalString(commands.agentBrowser, "commands.agentBrowser");
+    validateOptionalString(commands.firecrawl, "commands.firecrawl");
+  }
+}
+
+export function readToolkitConfig(): ToolkitConfig {
+  const filePath = getToolkitConfigPath();
+  const required = Boolean(process.env.PI_WEB_TOOLKIT_CONFIG?.trim());
+  return parseConfigFile(filePath, required);
+}
+
+function normalizeUrl(url: string): string {
+  return url.replace(/\/+$/, "");
+}
+
+export function getSearxngUrl(): string {
+  const envUrl = process.env.SEARXNG_URL?.trim();
+  if (envUrl) return normalizeUrl(envUrl);
+
+  const cfgUrl = readToolkitConfig().searxngUrl?.trim();
+  if (cfgUrl) return normalizeUrl(cfgUrl);
+
+  return DEFAULT_SEARXNG_URL;
+}
+
+export function getToolkitCommand(name: ToolkitCommandName): string {
+  const envVar = COMMAND_ENV_VARS[name];
+  const envCommand = process.env[envVar]?.trim();
+  if (envCommand) return envCommand;
+
+  const cfgCommand = readToolkitConfig().commands?.[name]?.trim();
+  if (cfgCommand) return cfgCommand;
+
+  return COMMAND_DEFAULTS[name];
+}
+
+export function isFirecrawlFallbackEnabled(): boolean {
+  const envValue = process.env.PI_WEB_FIRECRAWL_FALLBACK;
+  if (envValue !== undefined) {
+    const v = envValue.trim().toLowerCase();
+    return !(v === "0" || v === "false" || v === "no" || v === "off");
+  }
+
+  const cfgValue = readToolkitConfig().firecrawlFallback;
+  if (cfgValue !== undefined) return cfgValue;
+
+  return true;
+}
+
+export function getFirecrawlRunner(): FirecrawlRunner {
+  const envValue = process.env.PI_WEB_FIRECRAWL_RUNNER?.trim().toLowerCase();
+  if (envValue) {
+    if (!isFirecrawlRunner(envValue)) {
+      throw new Error("PI_WEB_FIRECRAWL_RUNNER must be one of: installed, npx, bunx");
+    }
+    return envValue;
+  }
+
+  return readToolkitConfig().firecrawlRunner ?? "installed";
+}

package/extensions/utils/firecrawl.ts

@@ -19,6 +19,7 @@ import { mkdtemp, rm } from "node:fs/promises";
 import * as os from "node:os";
 import * as path from "node:path";
 import { runCLI } from "./cli-runner";
+import { getFirecrawlRunner, getToolkitCommand, isFirecrawlFallbackEnabled, type FirecrawlRunner } from "./config";
 
 // ---------------------------------------------------------------------------
 // Shared types
@@ -33,8 +34,7 @@ export type FirecrawlFailureKind = "graceful-skip" | "hard-error";
  * the single opt-out for a strict local-only / no-cloud-egress policy.
  */
 export function isFirecrawlEnabled(): boolean {
-  const v = (process.env.PI_WEB_FIRECRAWL_FALLBACK ?? "").trim().toLowerCase();
-  return !(v === "0" || v === "false" || v === "no" || v === "off");
+  return isFirecrawlFallbackEnabled();
 }
 
 export interface FirecrawlFailure {
@@ -159,6 +159,29 @@ export interface FirecrawlCliResult {
   exitCode: number;
 }
 
+export interface FirecrawlCliInvocation {
+  command: string;
+  args: string[];
+}
+
+/**
+ * Build the command used to invoke the official Firecrawl CLI. `npx` and
+ * `bunx` are opt-in runners because they may run or download packages at
+ * fallback time.
+ */
+export function buildFirecrawlCliInvocation(
+  args: string[],
+  runner: FirecrawlRunner = getFirecrawlRunner(),
+): FirecrawlCliInvocation {
+  if (runner === "npx") {
+    return { command: "npx", args: ["-y", "firecrawl-cli", ...args] };
+  }
+  if (runner === "bunx") {
+    return { command: "bunx", args: ["firecrawl-cli", ...args] };
+  }
+  return { command: getToolkitCommand("firecrawl"), args };
+}
+
 /**
  * Run the firecrawl CLI under an isolated temporary HOME with no key env, so
  * it can only ever operate in keyless mode (no stored credentials, no
@@ -178,7 +201,8 @@ export async function runFirecrawlCli(
     delete env.FIRECRAWL_OAUTH_TOKEN;
     env.HOME = home;
     env.XDG_CONFIG_HOME = path.join(home, ".config");
-    return await runCLI({ command: "firecrawl", args, env, signal, timeout });
+    const invocation = buildFirecrawlCliInvocation(args);
+    return await runCLI({ command: invocation.command, args: invocation.args, env, signal, timeout });
   } finally {
     await rm(home, { recursive: true, force: true }).catch(() => { /* best-effort */ });
   }

package/extensions/utils/scrapling.ts

@@ -1,4 +1,5 @@
 import { runCLI } from "./cli-runner";
+import { getToolkitCommand } from "./config";
 
 /**
  * Run a scrapling CLI command with optional abort signal.
@@ -7,7 +8,7 @@ export function runScrapling(
   args: string[],
   signal?: AbortSignal,
 ): Promise<{ stdout: string; stderr: string; exitCode: number }> {
-  return runCLI({ command: "scrapling", args, signal });
+  return runCLI({ command: getToolkitCommand("scrapling"), args, signal });
 }
 
 /**

package/extensions/utils/web-search-core.ts

@@ -0,0 +1,146 @@
+/**
+ * web_search execution core
+ *
+ * Keeps SearXNG-first search behavior behind a testable boundary. Firecrawl
+ * remains fallback-only and missing fallback runners never replace the primary
+ * SearXNG failure/no-result UX.
+ */
+
+import type { FirecrawlSearchOutput } from "./firecrawl";
+import { shouldFallbackSearch } from "./firecrawl";
+
+export interface WebSearchCoreInput {
+  query: string;
+  language?: string;
+  results?: number;
+}
+
+export interface WebSearchResultItem {
+  title: string;
+  url: string;
+  content?: string;
+  engine?: string;
+  score?: number;
+}
+
+interface SearxResponse {
+  query: string;
+  results: WebSearchResultItem[];
+  suggestions?: string[];
+}
+
+export interface WebSearchCoreResult {
+  query: string;
+  totalResults: number;
+  results: WebSearchResultItem[];
+  suggestions?: string[];
+  viaFirecrawl: boolean;
+  creditsUsed?: number;
+}
+
+export interface WebSearchCoreDeps {
+  searxngUrl: string;
+  fetchImpl: typeof fetch;
+  firecrawlSearch: (query: string, options: { limit: number }, signal?: AbortSignal) => Promise<FirecrawlSearchOutput>;
+  signal?: AbortSignal;
+}
+
+function normalizeSearxngUrl(url: string): string {
+  return url.replace(/\/+$/, "");
+}
+
+export async function runWebSearchCore(
+  input: WebSearchCoreInput,
+  deps: WebSearchCoreDeps,
+): Promise<WebSearchCoreResult> {
+  const searxngUrl = normalizeSearxngUrl(deps.searxngUrl);
+  const maxResults = Math.floor(Math.min(60, Math.max(1, input.results ?? 20)));
+  const language = input.language ?? "";
+
+  const allResults: WebSearchResultItem[] = [];
+  const seenUrls = new Set<string>();
+  let suggestions: string[] | undefined;
+  let finalQuery = input.query;
+  const MAX_PAGES = 3;
+
+  let localOk = true;
+  let localError: string | undefined;
+
+  try {
+    for (let page = 1; page <= MAX_PAGES; page++) {
+      const searchParams = new URLSearchParams({
+        q: input.query,
+        format: "json",
+        pageno: String(page),
+      });
+      if (language) searchParams.set("language", language);
+
+      const response = await deps.fetchImpl(`${searxngUrl}/search?${searchParams.toString()}`, {
+        method: "GET",
+        headers: { Accept: "application/json" },
+        signal: deps.signal,
+      });
+
+      if (!response.ok) {
+        const body = await response.text().catch(() => "");
+        throw new Error(`SearXNG error: ${response.status} ${response.statusText}\n${body}`);
+      }
+
+      const data = (await response.json()) as SearxResponse;
+      finalQuery = data.query;
+
+      if (data.suggestions && data.suggestions.length > 0 && !suggestions) {
+        suggestions = data.suggestions;
+      }
+
+      if (!data.results || data.results.length === 0) {
+        break;
+      }
+
+      for (const r of data.results) {
+        if (!seenUrls.has(r.url)) {
+          seenUrls.add(r.url);
+          allResults.push(r);
+        }
+      }
+
+      if (allResults.length >= maxResults) {
+        break;
+      }
+    }
+  } catch (err: any) {
+    localOk = false;
+    localError = err.message ?? String(err);
+  }
+
+  if (shouldFallbackSearch(localOk, allResults.length)) {
+    const fb = await deps.firecrawlSearch(input.query, { limit: Math.min(maxResults, 10) }, deps.signal);
+    if (fb.ok && fb.results.length > 0) {
+      const fbResults: WebSearchResultItem[] = fb.results.slice(0, maxResults).map((r) => ({
+        title: r.title ?? "(untitled)",
+        url: r.url,
+        content: r.description,
+        engine: "firecrawl",
+      }));
+      return {
+        query: input.query,
+        totalResults: fbResults.length,
+        results: fbResults,
+        viaFirecrawl: true,
+        creditsUsed: fb.creditsUsed,
+      };
+    }
+  }
+
+  if (!localOk) {
+    throw new Error(`Failed to query SearXNG at ${searxngUrl}: ${localError}`);
+  }
+
+  return {
+    query: finalQuery,
+    totalResults: allResults.length,
+    results: allResults.slice(0, maxResults),
+    suggestions,
+    viaFirecrawl: false,
+  };
+}

package/extensions/web_search.ts

@@ -19,12 +19,13 @@ import {
 } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
 import { Type, type Static } from "typebox";
+import { getSearxngUrl } from "./utils/config";
 import { writeWithFallback } from "./utils/output-sink";
-import { searchKeyless, shouldFallbackSearch } from "./utils/firecrawl";
+import { searchKeyless } from "./utils/firecrawl";
+import { runWebSearchCore } from "./utils/web-search-core";
 import { abbreviateUrl, getDomain, getErrorText, normalizeWhitespace } from "./utils/render-helpers";
 
 
-
 interface SearxResult {
   title: string;
   url: string;
@@ -33,12 +34,6 @@ interface SearxResult {
   score?: number;
 }
 
-interface SearxResponse {
-  query: string;
-  results: SearxResult[];
-  suggestions?: string[];
-}
-
 export const WebSearchParamsSchema = Type.Object({
   query: Type.String({ description: "Search query" }),
   language: Type.Optional(Type.String({ description: "Language code (e.g. en, en-US, de). Omit to use SearXNG default.", default: "" })),
@@ -67,122 +62,53 @@ const webSearchTool = defineTool({
   parameters: WebSearchParamsSchema,
 
   async execute(_toolCallId, params, signal) {
-    const searxngUrl = (process.env.SEARXNG_URL || "http://localhost:8080").replace(/\/$/, "");
-    const maxResults = Math.floor(Math.min(60, Math.max(1, params.results ?? 20)));
-    const language = params.language ?? "";
-
-    const allResults: SearxResult[] = [];
-    const seenUrls = new Set<string>();
-    let suggestions: string[] | undefined;
-    let finalQuery = params.query;
-    let fullOutputPath: string | undefined;
-    const MAX_PAGES = 3;
-
-    let localOk = true;
-    let localError: string | undefined;
-
-    try {
-      for (let page = 1; page <= MAX_PAGES; page++) {
-        const searchParams = new URLSearchParams({
-          q: params.query,
-          format: "json",
-          pageno: String(page),
-        });
-        if (language) searchParams.set("language", language);
-
-        const response = await fetch(`${searxngUrl}/search?${searchParams.toString()}`, {
-          method: "GET",
-          headers: { Accept: "application/json" },
-          signal,
-        });
-
-        if (!response.ok) {
-          const body = await response.text().catch(() => "");
-          throw new Error(`SearXNG error: ${response.status} ${response.statusText}\n${body}`);
-        }
-
-        const data = (await response.json()) as SearxResponse;
-        finalQuery = data.query;
-
-        if (data.suggestions && data.suggestions.length > 0 && !suggestions) {
-          suggestions = data.suggestions;
-        }
-
-        if (!data.results || data.results.length === 0) {
-          break;
-        }
-
-        for (const r of data.results) {
-          if (!seenUrls.has(r.url)) {
-            seenUrls.add(r.url);
-            allResults.push(r);
-          }
-        }
-
-        if (allResults.length >= maxResults) {
-          break;
-        }
-      }
-    } catch (err: any) {
-      localOk = false;
-      localError = err.message ?? String(err);
-    }
+    const result = await runWebSearchCore(params, {
+      searxngUrl: getSearxngUrl(),
+      fetchImpl: fetch,
+      firecrawlSearch: searchKeyless,
+      signal,
+    });
 
-    // Firecrawl keyless fallback: when SearXNG errored OR returned nothing.
-    if (shouldFallbackSearch(localOk, allResults.length)) {
-      const fb = await searchKeyless(params.query, { limit: Math.min(maxResults, 10) }, signal);
-      if (fb.ok && fb.results.length > 0) {
-        const fbResults: SearxResult[] = fb.results.slice(0, maxResults).map((r) => ({
-          title: r.title ?? "(untitled)",
-          url: r.url,
-          content: r.description,
-          engine: "firecrawl",
-        }));
-        const creditTag = fb.creditsUsed !== undefined ? `, ${fb.creditsUsed} credits` : "";
-        const lines: string[] = [`Results for "${params.query}" (via Firecrawl keyless${creditTag}):`, ""];
-        for (let i = 0; i < fbResults.length; i++) {
-          const r = fbResults[i];
-          lines.push(`${i + 1}. ${r.title}`);
-          lines.push(`   URL: ${r.url}`);
-          if (r.content) lines.push(`   ${r.content.replace(/\s+/g, " ").trim()}`);
-          if (r.engine) lines.push(`   [engine: ${r.engine}]`);
-          lines.push("");
-        }
-        const rawText = lines.join("\n");
-        const sink = await writeWithFallback(rawText, {
-          tmpPrefix: "pi-web-search-firecrawl-",
-          alwaysWriteFile: true,
-        });
-        return {
-          content: [{ type: "text", text: sink.text }],
-          details: { query: params.query, totalResults: fbResults.length, results: fbResults, fullOutputPath: sink.fullOutputPath, viaFirecrawl: true, creditsUsed: fb.creditsUsed },
-        };
+    if (result.viaFirecrawl) {
+      const creditTag = result.creditsUsed !== undefined ? `, ${result.creditsUsed} credits` : "";
+      const lines: string[] = [`Results for "${params.query}" (via Firecrawl keyless${creditTag}):`, ""];
+      for (let i = 0; i < result.results.length; i++) {
+        const r = result.results[i];
+        lines.push(`${i + 1}. ${r.title}`);
+        lines.push(`   URL: ${r.url}`);
+        if (r.content) lines.push(`   ${r.content.replace(/\s+/g, " ").trim()}`);
+        if (r.engine) lines.push(`   [engine: ${r.engine}]`);
+        lines.push("");
       }
-      // Graceful skip or empty Firecrawl: fall through to local handling.
-    }
-
-    if (!localOk) {
-      throw new Error(`Failed to query SearXNG at ${searxngUrl}: ${localError}`);
+      const rawText = lines.join("\n");
+      const sink = await writeWithFallback(rawText, {
+        tmpPrefix: "pi-web-search-firecrawl-",
+        alwaysWriteFile: true,
+      });
+      return {
+        content: [{ type: "text", text: sink.text }],
+        details: { query: params.query, totalResults: result.totalResults, results: result.results, fullOutputPath: sink.fullOutputPath, viaFirecrawl: true, creditsUsed: result.creditsUsed },
+      };
     }
 
-    if (allResults.length === 0) {
-      let text = `No results found for "${finalQuery}".`;
-      if (suggestions && suggestions.length > 0) {
-        text += `\n\nSuggestions:\n${suggestions.map((s) => `- ${s}`).join("\n")}`;
+    if (result.results.length === 0) {
+      let text = `No results found for "${result.query}".`;
+      if (result.suggestions && result.suggestions.length > 0) {
+        text += `\n\nSuggestions:\n${result.suggestions.map((s) => `- ${s}`).join("\n")}`;
       }
       return {
         content: [{ type: "text", text }],
-        details: { query: finalQuery, totalResults: 0, results: [] as SearxResult[], fullOutputPath: undefined as string | undefined, viaFirecrawl: false, creditsUsed: undefined },
+        details: { query: result.query, totalResults: 0, results: [] as SearxResult[], fullOutputPath: undefined as string | undefined, viaFirecrawl: false, creditsUsed: undefined },
       };
     }
 
     const lines: string[] = [
-      `Results for "${finalQuery}":`,
+      `Results for "${result.query}":`,
       "",
     ];
 
-    for (let i = 0; i < Math.min(maxResults, allResults.length); i++) {
-      const r = allResults[i];
+    for (let i = 0; i < result.results.length; i++) {
+      const r = result.results[i];
       lines.push(`${i + 1}. ${r.title}`);
       lines.push(`   URL: ${r.url}`);
       if (r.content) {
@@ -200,11 +126,10 @@ const webSearchTool = defineTool({
       tmpPrefix: "pi-web-search-",
       alwaysWriteFile: true,
     });
-    fullOutputPath = sink.fullOutputPath;
 
     return {
       content: [{ type: "text", text: sink.text }],
-      details: { query: finalQuery, totalResults: allResults.length, results: allResults.slice(0, maxResults), fullOutputPath, viaFirecrawl: false, creditsUsed: undefined },
+      details: { query: result.query, totalResults: result.totalResults, results: result.results, fullOutputPath: sink.fullOutputPath, viaFirecrawl: false, creditsUsed: undefined },
     };
   },