pi-web-toolkit 0.3.1 → 0.3.3

package/extensions/web_search.ts

@@ -19,12 +19,13 @@ import {
 } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
 import { Type, type Static } from "typebox";
+import { getSearxngUrl } from "./utils/config";
 import { writeWithFallback } from "./utils/output-sink";
-import { searchKeyless, shouldFallbackSearch } from "./utils/firecrawl";
+import { searchKeyless } from "./utils/firecrawl";
+import { runWebSearchCore } from "./utils/web-search-core";
 import { abbreviateUrl, getDomain, getErrorText, normalizeWhitespace } from "./utils/render-helpers";
 
 
-
 interface SearxResult {
   title: string;
   url: string;
@@ -33,12 +34,6 @@ interface SearxResult {
   score?: number;
 }
 
-interface SearxResponse {
-  query: string;
-  results: SearxResult[];
-  suggestions?: string[];
-}
-
 export const WebSearchParamsSchema = Type.Object({
   query: Type.String({ description: "Search query" }),
   language: Type.Optional(Type.String({ description: "Language code (e.g. en, en-US, de). Omit to use SearXNG default.", default: "" })),
@@ -51,139 +46,69 @@ const webSearchTool = defineTool({
   name: "web_search",
   label: "Web Search",
   description: [
-    "Search the web using a SearXNG instance.",
+    "Primary local-first tool for web discovery via a SearXNG instance.",
     "Returns a list of results with title, URL, and snippet.",
     "Automatically aggregates up to 3 pages of SearXNG results when more than ~20 are needed.",
+    "Use web_search as the first attempt for web search; it automatically tries Firecrawl keyless only if SearXNG fails or returns nothing.",
     "Use web_search when the user asks about current events, facts, or anything",
     "that requires up-to-date information beyond the model's training data.",
     `Output is truncated to ${DEFAULT_MAX_LINES} lines or ${formatSize(DEFAULT_MAX_BYTES)}; if truncated, full output is saved to a temp file.`,
   ].join(" "),
-  promptSnippet: "Search the web for current information",
+  promptSnippet: "Local web search via SearXNG",
   promptGuidelines: [
-    "Use web_search when the user asks about recent events, current data, or external facts.",
-    "Use web_search to verify claims, find documentation, or discover resources online.",
-    "If web_search returns no results but includes suggestions, consider using a suggested query to refine your search.",
-    "If web_search returns multiple (2–5) relevant results that all need to be read, prefer web_batch_fetch to fetch them in parallel instead of calling web_fetch repeatedly.",
+    "Use web_search for current/external facts, verification, docs, and discovery.",
+    "If 2–5 results need reading, use web_batch_fetch; retry suggested queries when results are empty.",
   ],
   parameters: WebSearchParamsSchema,
 
   async execute(_toolCallId, params, signal) {
-    const searxngUrl = (process.env.SEARXNG_URL || "http://localhost:8080").replace(/\/$/, "");
-    const maxResults = Math.floor(Math.min(60, Math.max(1, params.results ?? 20)));
-    const language = params.language ?? "";
-
-    const allResults: SearxResult[] = [];
-    const seenUrls = new Set<string>();
-    let suggestions: string[] | undefined;
-    let finalQuery = params.query;
-    let fullOutputPath: string | undefined;
-    const MAX_PAGES = 3;
-
-    let localOk = true;
-    let localError: string | undefined;
-
-    try {
-      for (let page = 1; page <= MAX_PAGES; page++) {
-        const searchParams = new URLSearchParams({
-          q: params.query,
-          format: "json",
-          pageno: String(page),
-        });
-        if (language) searchParams.set("language", language);
-
-        const response = await fetch(`${searxngUrl}/search?${searchParams.toString()}`, {
-          method: "GET",
-          headers: { Accept: "application/json" },
-          signal,
-        });
-
-        if (!response.ok) {
-          const body = await response.text().catch(() => "");
-          throw new Error(`SearXNG error: ${response.status} ${response.statusText}\n${body}`);
-        }
-
-        const data = (await response.json()) as SearxResponse;
-        finalQuery = data.query;
-
-        if (data.suggestions && data.suggestions.length > 0 && !suggestions) {
-          suggestions = data.suggestions;
-        }
-
-        if (!data.results || data.results.length === 0) {
-          break;
-        }
-
-        for (const r of data.results) {
-          if (!seenUrls.has(r.url)) {
-            seenUrls.add(r.url);
-            allResults.push(r);
-          }
-        }
-
-        if (allResults.length >= maxResults) {
-          break;
-        }
-      }
-    } catch (err: any) {
-      localOk = false;
-      localError = err.message ?? String(err);
-    }
+    const result = await runWebSearchCore(params, {
+      searxngUrl: getSearxngUrl(),
+      fetchImpl: fetch,
+      firecrawlSearch: searchKeyless,
+      signal,
+    });
 
-    // Firecrawl keyless fallback: when SearXNG errored OR returned nothing.
-    if (shouldFallbackSearch(localOk, allResults.length)) {
-      const fb = await searchKeyless(params.query, { limit: Math.min(maxResults, 10) }, signal);
-      if (fb.ok && fb.results.length > 0) {
-        const fbResults: SearxResult[] = fb.results.slice(0, maxResults).map((r) => ({
-          title: r.title ?? "(untitled)",
-          url: r.url,
-          content: r.description,
-          engine: "firecrawl",
-        }));
-        const creditTag = fb.creditsUsed !== undefined ? `, ${fb.creditsUsed} credits` : "";
-        const lines: string[] = [`Results for "${params.query}" (via Firecrawl keyless${creditTag}):`, ""];
-        for (let i = 0; i < fbResults.length; i++) {
-          const r = fbResults[i];
-          lines.push(`${i + 1}. ${r.title}`);
-          lines.push(`   URL: ${r.url}`);
-          if (r.content) lines.push(`   ${r.content.replace(/\s+/g, " ").trim()}`);
-          if (r.engine) lines.push(`   [engine: ${r.engine}]`);
-          lines.push("");
-        }
-        const rawText = lines.join("\n");
-        const sink = await writeWithFallback(rawText, {
-          tmpPrefix: "pi-web-search-firecrawl-",
-          alwaysWriteFile: true,
-        });
-        return {
-          content: [{ type: "text", text: sink.text }],
-          details: { query: params.query, totalResults: fbResults.length, results: fbResults, fullOutputPath: sink.fullOutputPath, viaFirecrawl: true, creditsUsed: fb.creditsUsed },
-        };
+    if (result.viaFirecrawl) {
+      const creditTag = result.creditsUsed !== undefined ? `, ${result.creditsUsed} credits` : "";
+      const lines: string[] = [`Results for "${params.query}" (via Firecrawl keyless${creditTag}):`, ""];
+      for (let i = 0; i < result.results.length; i++) {
+        const r = result.results[i];
+        lines.push(`${i + 1}. ${r.title}`);
+        lines.push(`   URL: ${r.url}`);
+        if (r.content) lines.push(`   ${r.content.replace(/\s+/g, " ").trim()}`);
+        if (r.engine) lines.push(`   [engine: ${r.engine}]`);
+        lines.push("");
       }
-      // Graceful skip or empty Firecrawl: fall through to local handling.
-    }
-
-    if (!localOk) {
-      throw new Error(`Failed to query SearXNG at ${searxngUrl}: ${localError}`);
+      const rawText = lines.join("\n");
+      const sink = await writeWithFallback(rawText, {
+        tmpPrefix: "pi-web-search-firecrawl-",
+        alwaysWriteFile: true,
+      });
+      return {
+        content: [{ type: "text", text: sink.text }],
+        details: { query: params.query, totalResults: result.totalResults, results: result.results, fullOutputPath: sink.fullOutputPath, viaFirecrawl: true, creditsUsed: result.creditsUsed },
+      };
     }
 
-    if (allResults.length === 0) {
-      let text = `No results found for "${finalQuery}".`;
-      if (suggestions && suggestions.length > 0) {
-        text += `\n\nSuggestions:\n${suggestions.map((s) => `- ${s}`).join("\n")}`;
+    if (result.results.length === 0) {
+      let text = `No results found for "${result.query}".`;
+      if (result.suggestions && result.suggestions.length > 0) {
+        text += `\n\nSuggestions:\n${result.suggestions.map((s) => `- ${s}`).join("\n")}`;
       }
       return {
         content: [{ type: "text", text }],
-        details: { query: finalQuery, totalResults: 0, results: [] as SearxResult[], fullOutputPath: undefined as string | undefined, viaFirecrawl: false, creditsUsed: undefined },
+        details: { query: result.query, totalResults: 0, results: [] as SearxResult[], fullOutputPath: undefined as string | undefined, viaFirecrawl: false, creditsUsed: undefined },
       };
     }
 
     const lines: string[] = [
-      `Results for "${finalQuery}":`,
+      `Results for "${result.query}":`,
       "",
     ];
 
-    for (let i = 0; i < Math.min(maxResults, allResults.length); i++) {
-      const r = allResults[i];
+    for (let i = 0; i < result.results.length; i++) {
+      const r = result.results[i];
       lines.push(`${i + 1}. ${r.title}`);
       lines.push(`   URL: ${r.url}`);
       if (r.content) {
@@ -201,11 +126,10 @@ const webSearchTool = defineTool({
       tmpPrefix: "pi-web-search-",
       alwaysWriteFile: true,
     });
-    fullOutputPath = sink.fullOutputPath;
 
     return {
       content: [{ type: "text", text: sink.text }],
-      details: { query: finalQuery, totalResults: allResults.length, results: allResults.slice(0, maxResults), fullOutputPath, viaFirecrawl: false, creditsUsed: undefined },
+      details: { query: result.query, totalResults: result.totalResults, results: result.results, fullOutputPath: sink.fullOutputPath, viaFirecrawl: false, creditsUsed: undefined },
     };
   },