pi-web-toolkit 0.1.2 → 0.2.0

package/README.md

@@ -13,9 +13,9 @@ Web research toolkit for [pi](https://pi.dev) agents. Search via SearXNG, fetch
 
 | Tool | Backend | Purpose | Current Limit |
 |------|---------|---------|---------------|
-| **`web_search`** | [SearXNG](https://github.com/searxng/searxng) | Search the web with scored, ranked results from multiple engines — always the first step in web research | 10 results (max 50) |
+| **`web_search`** | [SearXNG](https://github.com/searxng/searxng) | Search the web with scored, ranked results from multiple engines — always the first step in web research | 20 results (max 60, auto-pages up to 3 pages) |
 | **`web_fetch`** | [scrapling](https://github.com/D4Vinci/Scrapling) | Fetch a single static page as clean markdown | — |
-| **`web_batch_fetch`** | [scrapling](https://github.com/D4Vinci/Scrapling) | Fetch 2–10 pages in parallel for research synthesis | 3 concurrent (max 5) |
+| **`web_batch_fetch`** | [scrapling](https://github.com/D4Vinci/Scrapling) | Fetch 2–15 pages in parallel for research synthesis | 3 concurrent (max 5) |
 | **`web_browse`** | [agent-browser](https://github.com/vercel-labs/agent-browser) | Interact with a page (click, scroll, fill) then extract content | 25 actions |
 
 ## Quick Start

package/docs/guide.md

@@ -32,7 +32,7 @@ User asks about something external / current
 
 | | `web_fetch` | `web_browse` | `web_batch_fetch` |
 |--|-------------|--------------|-------------------|
-| **Pages** | 1 | 1 | 2–10 |
+| **Pages** | 1 | 1 | 2–15 |
 | **Browser** | Yes (scrapling) | Yes (agent-browser) | Yes (scrapling) |
 | **Interaction** | ❌ No | ✅ Click, fill, scroll, wait | ❌ No |
 | **Selector** | ✅ Per-URL | ✅ Final state | ✅ Applied to all |

package/docs/tools.md

@@ -2,18 +2,22 @@
 
 ## `web_search`
 
-Search the web via SearXNG. Returns ranked results with title, URL, and snippet.
+Search the web via SearXNG. Returns ranked results with title, URL, and snippet. Automatically aggregates up to 3 pages of SearXNG results when more than ~20 are needed.
 
 ```typescript
 {
   query: string,           // Search query
   language?: string,       // Language code (en, de, fr...). Default: "auto"
-  results?: number,        // Max results (1–50). Default: 10
+  results?: number,        // Max results (1–60). Default: 20. Automatically pages through SearXNG (up to 3 pages) if needed.
 }
 ```
 
 **When to use:** The user asks about current events, facts, or anything requiring up-to-date information. This is always the **first step** of web research.
 
+**Empty results behavior:** When no results are found, `web_search` returns a list of **suggestions** — alternative queries that SearXNG believes may yield better results. The agent can use these suggestions to automatically refine and retry the search.
+
+**Pagination:** `web_search` automatically fetches up to 3 pages from SearXNG and deduplicates by URL. You do not need to call it multiple times for deeper results.
+
 ---
 
 ## `web_fetch`

package/extensions/utils/agent-browser.ts

@@ -5,7 +5,7 @@
  * command building, process spawning, JSON parsing, and session cleanup.
  */
 
-import { spawn } from "node:child_process";
+import { runCLI } from "./cli-runner";
 
 export interface BrowseAction {
   type: "click" | "fill" | "type" | "press" | "wait" | "wait_selector" | "scroll";
@@ -26,7 +26,7 @@ export interface AgentBrowserBatchItem {
 }
 
 function requireString(action: BrowseAction, field: "selector" | "value" | "key"): string {
-  const value = action[field];
+  const value = action[field] as string | undefined;
   if (typeof value !== "string" || value.length === 0) {
     throw new Error(`Action "${action.type}" requires non-empty ${field}`);
   }
@@ -34,11 +34,11 @@ function requireString(action: BrowseAction, field: "selector" | "value" | "key"
 }
 
 function requireInteger(action: BrowseAction, field: "ms" | "amount"): number {
-  const value = action[field];
-  if (!Number.isInteger(value) || value < 0) {
+  const value = action[field] as number | undefined;
+  if (!Number.isInteger(value) || (value as number) < 0) {
     throw new Error(`Action "${action.type}" requires non-negative integer ${field}`);
   }
-  return value;
+  return value as number;
 }
 
 function waitForSelectorScript(selector: string, state: "attached" | "visible" | "hidden"): string {
@@ -128,7 +128,7 @@ export function buildBatchCommands(
   return commands;
 }
 
-export function runAgentBrowserBatch(
+export async function runAgentBrowserBatch(
   commands: string[][],
   options: { session: string; headless: boolean; signal?: AbortSignal; timeout?: number },
 ): Promise<AgentBrowserBatchItem[]> {
@@ -136,99 +136,44 @@ export function runAgentBrowserBatch(
   if (!options.headless) args.push("--headed");
   args.push("batch", "--bail", "--json");
 
-  return new Promise((resolve, reject) => {
-    const proc = spawn("agent-browser", args, {
-      shell: false,
-      stdio: ["pipe", "pipe", "pipe"],
+  try {
+    const result = await runCLI({
+      command: "agent-browser",
+      args,
+      stdin: JSON.stringify(commands),
+      timeout: options.timeout,
+      signal: options.signal,
     });
 
-    let stdout = "";
-    let stderr = "";
-    let timeoutId: NodeJS.Timeout | undefined;
-    let settled = false;
-
-    const cleanup = () => {
-      if (timeoutId) clearTimeout(timeoutId);
-      if (options.signal) options.signal.removeEventListener("abort", kill);
-    };
-
-    const settleReject = (err: Error) => {
-      if (settled) return;
-      settled = true;
-      cleanup();
-      reject(err);
-    };
-
-    const kill = () => proc.kill("SIGTERM");
-
-    proc.stdout.on("data", (data: Buffer) => {
-      stdout += data.toString();
-    });
-
-    proc.stderr.on("data", (data: Buffer) => {
-      stderr += data.toString();
-    });
-
-    if (options.timeout) {
-      timeoutId = setTimeout(() => {
-        proc.kill("SIGTERM");
-        settleReject(new Error(`agent-browser timed out after ${options.timeout}ms`));
-      }, options.timeout);
+    if (result.exitCode !== 0 && !result.stdout.trim()) {
+      throw new Error(`agent-browser failed (exit ${result.exitCode}):\n${result.stderr || "unknown error"}`);
     }
 
-    proc.on("close", (code) => {
-      if (settled) return;
-      settled = true;
-      cleanup();
-
-      if (code !== 0 && !stdout.trim()) {
-        reject(new Error(`agent-browser failed (exit ${code}):\n${stderr || "unknown error"}`));
-        return;
-      }
-
-      try {
-        const results = JSON.parse(stdout) as AgentBrowserBatchItem[];
-        resolve(results);
-      } catch (err: any) {
-        reject(new Error(
-          `Failed to parse agent-browser output: ${err.message}\nstdout: ${stdout}\nstderr: ${stderr}`
-        ));
-      }
-    });
-
-    proc.on("error", (err: any) => {
-      if (err.code === "ENOENT") {
-        settleReject(new Error(
-          "agent-browser is not installed.\n\nInstall it with:\n  npm i -g agent-browser && agent-browser install\n\nThen run: agent-browser doctor"
-        ));
-      } else {
-        settleReject(err);
-      }
-    });
-
-    if (options.signal) {
-      if (options.signal.aborted) kill();
-      else options.signal.addEventListener("abort", kill, { once: true });
+    try {
+      return JSON.parse(result.stdout) as AgentBrowserBatchItem[];
+    } catch (err: any) {
+      throw new Error(
+        `Failed to parse agent-browser output: ${err.message}\nstdout: ${result.stdout}\nstderr: ${result.stderr}`
+      );
     }
-
-    proc.stdin.write(JSON.stringify(commands));
-    proc.stdin.end();
-  });
+  } catch (err: any) {
+    if (err.message === "agent-browser is not installed") {
+      throw new Error(
+        "agent-browser is not installed.\n\nInstall it with:\n  npm i -g agent-browser && agent-browser install\n\nThen run: agent-browser doctor"
+      );
+    }
+    throw err;
+  }
 }
 
-export function closeAgentBrowserSession(session: string, signal?: AbortSignal): Promise<void> {
-  return new Promise((resolve) => {
-    const proc = spawn("agent-browser", ["--session", session, "close"], {
-      shell: false,
-      stdio: ["ignore", "ignore", "ignore"],
+export async function closeAgentBrowserSession(session: string, signal?: AbortSignal): Promise<void> {
+  try {
+    await runCLI({
+      command: "agent-browser",
+      args: ["--session", session, "close"],
+      signal,
     });
-    const done = () => resolve();
-    proc.on("close", done);
-    proc.on("error", done);
-    if (signal) {
-      const kill = () => proc.kill("SIGTERM");
-      if (signal.aborted) kill();
-      else signal.addEventListener("abort", kill, { once: true });
-    }
-  });
+  } catch {
+    // Best-effort cleanup — ignore errors
+  }
 }

package/extensions/utils/cli-runner.ts

@@ -0,0 +1,108 @@
+/**
+ * CLI runner — abstracted process spawning
+ *
+ * Provides a single interface for running external CLI commands
+ * with consistent signal handling, timeout support, and stdout/stderr
+ * collection. Enables testability by allowing the runner to be swapped.
+ */
+
+import { spawn, type ChildProcess } from "node:child_process";
+
+export interface CLIRunOptions {
+  command: string;
+  args: string[];
+  /** Data to write to stdin. If omitted, stdin is ignored. */
+  stdin?: string;
+  /** Timeout in milliseconds. If exceeded, the process is killed. */
+  timeout?: number;
+  /** AbortSignal for cancellation. */
+  signal?: AbortSignal;
+}
+
+export interface CLIRunResult {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+}
+
+/**
+ * Run an external CLI command and capture its output.
+ *
+ * Handles:
+ * - stdout/stderr collection
+ * - optional stdin feeding
+ * - optional timeout (SIGTERM)
+ * - AbortSignal cancellation (SIGTERM)
+ * - process spawn errors (e.g. ENOENT)
+ */
+export function runCLI(options: CLIRunOptions): Promise<CLIRunResult> {
+  return new Promise((resolve, reject) => {
+    const stdio = options.stdin
+      ? ["pipe", "pipe", "pipe"]
+      : ["ignore", "pipe", "pipe"];
+
+    const proc = spawn(options.command, options.args, {
+      shell: false,
+      stdio: stdio as any,
+    }) as ChildProcess;
+
+    let stdout = "";
+    let stderr = "";
+    let timeoutId: NodeJS.Timeout | undefined;
+    let settled = false;
+
+    const cleanup = () => {
+      if (timeoutId) clearTimeout(timeoutId);
+      if (options.signal) options.signal.removeEventListener("abort", kill);
+    };
+
+    const settleReject = (err: Error) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      reject(err);
+    };
+
+    const kill = () => proc.kill("SIGTERM");
+
+    proc.stdout?.on("data", (data: Buffer) => {
+      stdout += data.toString();
+    });
+
+    proc.stderr?.on("data", (data: Buffer) => {
+      stderr += data.toString();
+    });
+
+    if (options.timeout) {
+      timeoutId = setTimeout(() => {
+        proc.kill("SIGTERM");
+        settleReject(new Error(`${options.command} timed out after ${options.timeout}ms`));
+      }, options.timeout);
+    }
+
+    proc.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      resolve({ stdout, stderr, exitCode: code ?? 1 });
+    });
+
+    proc.on("error", (err: any) => {
+      if (err.code === "ENOENT") {
+        settleReject(new Error(`${options.command} is not installed`));
+      } else {
+        settleReject(err);
+      }
+    });
+
+    if (options.signal) {
+      if (options.signal.aborted) kill();
+      else options.signal.addEventListener("abort", kill, { once: true });
+    }
+
+    if (options.stdin && proc.stdin) {
+      proc.stdin.write(options.stdin);
+      proc.stdin.end();
+    }
+  });
+}