pi-web-toolkit 0.1.1 → 0.1.2

package/README.md

@@ -1,79 +1,125 @@
 # pi-web-toolkit
 
-Web research toolkit for [pi](https://pi.dev) agents. Search, fetch, browse, and batch-read the web.
+[![npm version](https://badge.fury.io/js/pi-web-toolkit.svg)](https://www.npmjs.com/package/pi-web-toolkit)
+[![CI](https://github.com/Wade11s/pi-web-toolkit/actions/workflows/ci.yml/badge.svg)](https://github.com/Wade11s/pi-web-toolkit/actions)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+![Node.js](https://img.shields.io/badge/node-%3E%3D22-339933)
+
+**100% open-source. Zero API keys. Zero fees.**
+
+Web research toolkit for [pi](https://pi.dev) agents. Search via SearXNG, fetch static pages with scrapling, browse interactively via agent-browser, and batch-read sources in parallel. All self-hosted, all local, all free — with built-in truncation safety and LLM-optimized prompt guidelines.
 
 ## Features
 
-| Tool | Purpose |
-|------|---------|
-| **`web_search`** | Search the web via SearXNG |
-| **`web_fetch`** | Fetch a single static page as clean markdown |
-| **`web_browse`** | Interact with a page (click, scroll, fill) then extract content |
-| **`web_batch_fetch`** | Fetch 2–10 pages in parallel for research synthesis |
+| Tool | Backend | Purpose | Current Limit |
+|------|---------|---------|---------------|
+| **`web_search`** | [SearXNG](https://github.com/searxng/searxng) | Search the web with scored, ranked results from multiple engines — always the first step in web research | 10 results (max 50) |
+| **`web_fetch`** | [scrapling](https://github.com/D4Vinci/Scrapling) | Fetch a single static page as clean markdown | — |
+| **`web_batch_fetch`** | [scrapling](https://github.com/D4Vinci/Scrapling) | Fetch 2–10 pages in parallel for research synthesis | 3 concurrent (max 5) |
+| **`web_browse`** | [agent-browser](https://github.com/vercel-labs/agent-browser) | Interact with a page (click, scroll, fill) then extract content | 25 actions |
 
-## Installation
+## Quick Start
 
-### Option 1: From npm (recommended)
+### 1. Install external dependencies
 
 ```bash
-pi install pi-web-toolkit
+# SearXNG (for search)
+docker run -d --name searxng -p 8080:8080 -v searxng:/etc/searxng searxng/searxng
+export SEARXNG_URL="http://localhost:8080"
+
+# scrapling (for fetch & batch fetch)
+uv tool install "scrapling[all]"
+scrapling install
+
+# agent-browser (for browse)
+npm i -g agent-browser && agent-browser install
 ```
 
-### Option 2: From GitHub
+**Verify dependencies:**
+```bash
+# SearXNG
+curl -s "$SEARXNG_URL" | head
+
+# scrapling
+scrapling --help
 
+# agent-browser
+agent-browser doctor
+```
+
+### 2. Install the extension
+#### From npm
+```bash
+pi install npm:pi-web-toolkit
+```
+#### From GitHub
 ```bash
 pi install git:github.com/Wade11s/pi-web-toolkit
 ```
 
-## Requirements
-
-- **Node.js ≥ 20** — for running pi extensions
-- **SearXNG** — for `web_search`
-  ```bash
-  # Set your SearXNG instance URL (default: http://localhost:8080)
-  export SEARXNG_URL="http://localhost:8080"
-
-  # Self-host with Docker
-  docker run -d -p 8080:8080 -v searxng:/etc/searxng searxng/searxng
-  ```
-- **scrapling** — for `web_fetch` and `web_batch_fetch`
-  ```bash
-  # recommended: install scrapling via uv
-  uv tool install "scrapling[all]"
-  scrapling install
-  ```
-- **agent-browser** — for `web_browse`
-  ```bash
-  npm i -g agent-browser && agent-browser install
-  ```
-  Verify installation:
-  ```bash
-  agent-browser doctor
-  ```
+## Configuration
+
+All tools are configured via **environment variables** at runtime — no rebuild or restart required.
+
+| Variable | Default | Used By | Description |
+|----------|---------|---------|-------------|
+| `SEARXNG_URL` | `http://localhost:8080` | `web_search` | Your SearXNG instance endpoint |
+
+Set before starting pi:
+
+```bash
+export SEARXNG_URL="https://searxng.example.com"
+```
 
 ## Project Structure
 
 ```
 pi-web-toolkit/
 ├── extensions/
+│   ├── index.ts              # Unified entry point — registers all 4 tools
 │   ├── utils/
-│   │   └── scrapling.ts    # scrapling CLI wrapper
-│   ├── web_search.ts       # web_search
-│   ├── web_fetch.ts        # web_fetch
-│   ├── web_browse.ts       # web_browse (agent-browser)
-│   └── web_batch_fetch.ts  # web_batch_fetch
+│   │   ├── scrapling.ts      # Reusable scrapling CLI wrapper (shared by fetch + batch)
+│   │   └── agent-browser.ts  # agent-browser CLI wrapper (shared by web_browse)
+│   ├── web_search.ts         # SearXNG search tool
+│   ├── web_fetch.ts          # Single-page scrapling fetcher
+│   ├── web_batch_fetch.ts    # Parallel scrapling fetcher
+│   └── web_browse.ts         # Interactive browser automation (agent-browser)
 ├── docs/
-│   ├── tools.md
-│   └── guide.md
+│   ├── tools.md              # Full parameter specs
+│   └── guide.md              # Decision tree & tool comparison
+├── CHANGELOG.md
 ├── package.json
 ├── README.md
 └── LICENSE
 ```
 
+**Design principles:**
+- **Unified registration** — `index.ts` is the single source of truth for what pi loads.
+- **Shared utilities** — `utils/scrapling.ts` and `utils/agent-browser.ts` encapsulate the CLI wrappers and fallback logic; tool files import only from `utils/`, never from each other.
+- **Per-tool isolation** — each tool owns its own schema, execute logic, and TUI renderer; no cross-imports except via `utils/`.
+- **Runtime config** — environment variables are read at execute time, not build time.
+
 ## Reference
 
 - [Tool Reference](docs/tools.md) — Full parameter specs and usage examples for each tool.
 - [Usage Guide](docs/guide.md) — Decision tree and tool comparison.
+- [Changelog](CHANGELOG.md) — Release history and migration notes.
+
+## Contributing
+
+```bash
+# Local development
+pi install ./
+
+# Type-check (no build step; pi loads TypeScript directly)
+npx tsc --noEmit
+
+# Verify external CLI dependencies
+scrapling --help
+agent-browser doctor
+```
+
+Pull requests welcome. Please keep changes scoped to a single tool or concern and follow [Conventional Commits](https://www.conventionalcommits.org/).
 
 ## License
 

package/docs/agents/domain.md

@@ -0,0 +1,51 @@
+# Domain Docs
+
+How the engineering skills should consume this repo's domain documentation when exploring the codebase.
+
+## Before exploring, read these
+
+- **`CONTEXT.md`** at the repo root, or
+- **`CONTEXT-MAP.md`** at the repo root if it exists — it points at one `CONTEXT.md` per context. Read each one relevant to the topic.
+- **`docs/adr/`** — read ADRs that touch the area you're about to work in. In multi-context repos, also check `src/<context>/docs/adr/` for context-scoped decisions.
+
+If any of these files don't exist, **proceed silently**. Don't flag their absence; don't suggest creating them upfront. The producer skill (`/grill-with-docs`) creates them lazily when terms or decisions actually get resolved.
+
+## File structure
+
+Single-context repo (most repos):
+
+```
+/
+├── CONTEXT.md
+├── docs/adr/
+│   ├── 0001-event-sourced-orders.md
+│   └── 0002-postgres-for-write-model.md
+└── src/
+```
+
+Multi-context repo (presence of `CONTEXT-MAP.md` at the root):
+
+```
+/
+├── CONTEXT-MAP.md
+├── docs/adr/                          ← system-wide decisions
+└── src/
+    ├── ordering/
+    │   ├── CONTEXT.md
+    │   └── docs/adr/                  ← context-specific decisions
+    └── billing/
+        ├── CONTEXT.md
+        └── docs/adr/
+```
+
+## Use the glossary's vocabulary
+
+When your output names a domain concept (in an issue title, a refactor proposal, a hypothesis, a test name), use the term as defined in `CONTEXT.md`. Don't drift to synonyms the glossary explicitly avoids.
+
+If the concept you need isn't in the glossary yet, that's a signal — either you're inventing language the project doesn't use (reconsider) or there's a real gap (note it for `/grill-with-docs`).
+
+## Flag ADR conflicts
+
+If your output contradicts an existing ADR, surface it explicitly rather than silently overriding:
+
+> _Contradicts ADR-0007 (event-sourced orders) — but worth reopening because…_

package/docs/agents/issue-tracker.md

@@ -0,0 +1,22 @@
+# Issue tracker: GitHub
+
+Issues and PRDs for this repo live as GitHub issues. Use the `gh` CLI for all operations.
+
+## Conventions
+
+- **Create an issue**: `gh issue create --title "..." --body "..."`. Use a heredoc for multi-line bodies.
+- **Read an issue**: `gh issue view <number> --comments`, filtering comments by `jq` and also fetching labels.
+- **List issues**: `gh issue list --state open --json number,title,body,labels,comments --jq '[.[] | {number, title, body, labels: [.labels[].name], comments: [.comments[].body]}]'` with appropriate `--label` and `--state` filters.
+- **Comment on an issue**: `gh issue comment <number> --body "..."`
+- **Apply / remove labels**: `gh issue edit <number> --add-label "..."` / `--remove-label "..."`
+- **Close**: `gh issue close <number> --comment "..."`
+
+Infer the repo from `git remote -v` — `gh` does this automatically when run inside a clone.
+
+## When a skill says "publish to the issue tracker"
+
+Create a GitHub issue.
+
+## When a skill says "fetch the relevant ticket"
+
+Run `gh issue view <number> --comments`.

package/docs/agents/triage-labels.md

@@ -0,0 +1,15 @@
+# Triage Labels
+
+The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker.
+
+| Label in mattpocock/skills | Label in our tracker | Meaning                                  |
+| -------------------------- | -------------------- | ---------------------------------------- |
+| `needs-triage`             | `needs-triage`       | Maintainer needs to evaluate this issue  |
+| `needs-info`               | `needs-info`         | Waiting on reporter for more information |
+| `ready-for-agent`          | `ready-for-agent`    | Fully specified, ready for an AFK agent  |
+| `ready-for-human`          | `ready-for-human`    | Requires human implementation            |
+| `wontfix`                  | `wontfix`            | Will not be actioned                     |
+
+When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the corresponding label string from this table.
+
+Edit the right-hand column to match whatever vocabulary you actually use.

package/extensions/utils/agent-browser.ts

@@ -0,0 +1,234 @@
+/**
+ * agent-browser CLI wrapper
+ *
+ * Encapsulates all low-level interaction with the agent-browser command:
+ * command building, process spawning, JSON parsing, and session cleanup.
+ */
+
+import { spawn } from "node:child_process";
+
+export interface BrowseAction {
+  type: "click" | "fill" | "type" | "press" | "wait" | "wait_selector" | "scroll";
+  selector?: string;
+  value?: string;
+  key?: string;
+  ms?: number;
+  direction?: "down" | "up" | "bottom" | "top";
+  amount?: number;
+  state?: "attached" | "visible" | "hidden";
+}
+
+export interface AgentBrowserBatchItem {
+  success: boolean;
+  command: string[];
+  result?: any;
+  error?: string | null;
+}
+
+function requireString(action: BrowseAction, field: "selector" | "value" | "key"): string {
+  const value = action[field];
+  if (typeof value !== "string" || value.length === 0) {
+    throw new Error(`Action "${action.type}" requires non-empty ${field}`);
+  }
+  return value;
+}
+
+function requireInteger(action: BrowseAction, field: "ms" | "amount"): number {
+  const value = action[field];
+  if (!Number.isInteger(value) || value < 0) {
+    throw new Error(`Action "${action.type}" requires non-negative integer ${field}`);
+  }
+  return value;
+}
+
+function waitForSelectorScript(selector: string, state: "attached" | "visible" | "hidden"): string {
+  const selectorLiteral = JSON.stringify(selector);
+  const stateLiteral = JSON.stringify(state);
+  return `await new Promise((resolve, reject) => {
+    const selector = ${selectorLiteral};
+    const state = ${stateLiteral};
+    const deadline = Date.now() + 30000;
+    const isVisible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+    const check = () => {
+      const el = document.querySelector(selector);
+      const ok = state === "attached" ? !!el : state === "hidden" ? !isVisible(el) : isVisible(el);
+      if (ok) return resolve(true);
+      if (Date.now() > deadline) return reject(new Error(\`Timed out waiting for ${state} selector: ${selector}\`));
+      setTimeout(check, 100);
+    };
+    check();
+  })`;
+}
+
+export function buildBatchCommands(
+  url: string,
+  actions: BrowseAction[],
+  selector?: string,
+): string[][] {
+  const commands: string[][] = [["open", url]];
+
+  for (const action of actions) {
+    switch (action.type) {
+      case "click":
+        commands.push(["click", requireString(action, "selector")]);
+        break;
+      case "fill":
+        commands.push(["fill", requireString(action, "selector"), requireString(action, "value")]);
+        break;
+      case "type":
+        commands.push(["type", requireString(action, "selector"), requireString(action, "value")]);
+        break;
+      case "press": {
+        if (action.selector) {
+          commands.push(["focus", action.selector]);
+        }
+        commands.push(["press", requireString(action, "key")]);
+        break;
+      }
+      case "wait":
+        commands.push(["wait", String(requireInteger(action, "ms"))]);
+        break;
+      case "wait_selector": {
+        const state = action.state ?? "visible";
+        const waitSelector = requireString(action, "selector");
+        if (state === "visible") {
+          commands.push(["wait", waitSelector]);
+        } else {
+          commands.push(["eval", waitForSelectorScript(waitSelector, state)]);
+        }
+        break;
+      }
+      case "scroll": {
+        const dir = action.direction ?? "down";
+        if (dir === "top") {
+          commands.push(["eval", "window.scrollTo(0, 0)"]);
+        } else if (dir === "bottom") {
+          commands.push(["eval", "window.scrollTo(0, document.body.scrollHeight)"]);
+        } else {
+          commands.push(["scroll", dir, String(action.amount ?? 500)]);
+        }
+        break;
+      }
+      default:
+        throw new Error(`Unsupported browser action: ${(action as BrowseAction).type}`);
+    }
+  }
+
+  // Extract content
+  if (selector) {
+    commands.push(["get", "text", selector, "--json"]);
+  } else {
+    commands.push(["snapshot", "-i", "--json"]);
+  }
+
+  // Metadata
+  commands.push(["get", "title", "--json"]);
+  commands.push(["get", "url", "--json"]);
+
+  return commands;
+}
+
+export function runAgentBrowserBatch(
+  commands: string[][],
+  options: { session: string; headless: boolean; signal?: AbortSignal; timeout?: number },
+): Promise<AgentBrowserBatchItem[]> {
+  const args = ["--session", options.session];
+  if (!options.headless) args.push("--headed");
+  args.push("batch", "--bail", "--json");
+
+  return new Promise((resolve, reject) => {
+    const proc = spawn("agent-browser", args, {
+      shell: false,
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+
+    let stdout = "";
+    let stderr = "";
+    let timeoutId: NodeJS.Timeout | undefined;
+    let settled = false;
+
+    const cleanup = () => {
+      if (timeoutId) clearTimeout(timeoutId);
+      if (options.signal) options.signal.removeEventListener("abort", kill);
+    };
+
+    const settleReject = (err: Error) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      reject(err);
+    };
+
+    const kill = () => proc.kill("SIGTERM");
+
+    proc.stdout.on("data", (data: Buffer) => {
+      stdout += data.toString();
+    });
+
+    proc.stderr.on("data", (data: Buffer) => {
+      stderr += data.toString();
+    });
+
+    if (options.timeout) {
+      timeoutId = setTimeout(() => {
+        proc.kill("SIGTERM");
+        settleReject(new Error(`agent-browser timed out after ${options.timeout}ms`));
+      }, options.timeout);
+    }
+
+    proc.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+
+      if (code !== 0 && !stdout.trim()) {
+        reject(new Error(`agent-browser failed (exit ${code}):\n${stderr || "unknown error"}`));
+        return;
+      }
+
+      try {
+        const results = JSON.parse(stdout) as AgentBrowserBatchItem[];
+        resolve(results);
+      } catch (err: any) {
+        reject(new Error(
+          `Failed to parse agent-browser output: ${err.message}\nstdout: ${stdout}\nstderr: ${stderr}`
+        ));
+      }
+    });
+
+    proc.on("error", (err: any) => {
+      if (err.code === "ENOENT") {
+        settleReject(new Error(
+          "agent-browser is not installed.\n\nInstall it with:\n  npm i -g agent-browser && agent-browser install\n\nThen run: agent-browser doctor"
+        ));
+      } else {
+        settleReject(err);
+      }
+    });
+
+    if (options.signal) {
+      if (options.signal.aborted) kill();
+      else options.signal.addEventListener("abort", kill, { once: true });
+    }
+
+    proc.stdin.write(JSON.stringify(commands));
+    proc.stdin.end();
+  });
+}
+
+export function closeAgentBrowserSession(session: string, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    const proc = spawn("agent-browser", ["--session", session, "close"], {
+      shell: false,
+      stdio: ["ignore", "ignore", "ignore"],
+    });
+    const done = () => resolve();
+    proc.on("close", done);
+    proc.on("error", done);
+    if (signal) {
+      const kill = () => proc.kill("SIGTERM");
+      if (signal.aborted) kill();
+      else signal.addEventListener("abort", kill, { once: true });
+    }
+  });
+}

package/extensions/utils/scrapling.ts

@@ -34,3 +34,43 @@ export function runScrapling(
     }
   });
 }
+
+/**
+ * Run scrapling fetch with automatic fallback to HTTP GET on failure.
+ *
+ * @param url       Target URL
+ * @param tmpFile   Output markdown file path
+ * @param options   { selector?: string; stealthy?: boolean; noGetFallback?: boolean }
+ * @param signal    Optional AbortSignal
+ * @returns         { ok: true } or { ok: false, stderr: string }
+ */
+export async function runScraplingWithFallback(
+  url: string,
+  tmpFile: string,
+  options: { selector?: string; stealthy?: boolean; noGetFallback?: boolean },
+  signal?: AbortSignal,
+): Promise<{ ok: boolean; stderr?: string }> {
+  const cmd = options.stealthy ? "stealthy-fetch" : "fetch";
+  const args = ["extract", cmd, url, tmpFile, "--ai-targeted"];
+  if (options.selector) {
+    args.push("--css-selector", options.selector);
+  }
+
+  const result = await runScrapling(args, signal);
+  if (result.exitCode === 0) {
+    return { ok: true };
+  }
+
+  if (!options.noGetFallback) {
+    const fallback = await runScrapling(
+      ["extract", "get", url, tmpFile, "--ai-targeted"],
+      signal,
+    );
+    if (fallback.exitCode === 0) {
+      return { ok: true };
+    }
+    return { ok: false, stderr: result.stderr || fallback.stderr };
+  }
+
+  return { ok: false, stderr: result.stderr };
+}

package/extensions/web_batch_fetch.ts

@@ -24,7 +24,7 @@ import { Type, type Static } from "typebox";
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
-import { runScrapling } from "./utils/scrapling";
+import { runScraplingWithFallback } from "./utils/scrapling";
 
 interface FetchTask {
   url: string;
@@ -37,18 +37,15 @@ async function fetchOne(
   stealthy: boolean,
   signal?: AbortSignal,
 ): Promise<{ url: string; content: string; size: number; ok: boolean; error?: string }> {
-  const cmd = stealthy ? "stealthy-fetch" : "fetch";
-  const args = ["extract", cmd, task.url, task.tmpFile, "--ai-targeted"];
-  if (selector) args.push("--css-selector", selector);
-
-  const { stderr, exitCode } = await runScrapling(args, signal);
-
-  if (exitCode !== 0) {
-    // Fallback to GET
-    const fallback = await runScrapling(["extract", "get", task.url, task.tmpFile, "--ai-targeted"], signal);
-    if (fallback.exitCode !== 0) {
-      return { url: task.url, content: "", size: 0, ok: false, error: stderr || fallback.stderr };
-    }
+  const { ok: fetchOk, stderr } = await runScraplingWithFallback(
+    task.url,
+    task.tmpFile,
+    { selector, stealthy },
+    signal,
+  );
+
+  if (!fetchOk) {
+    return { url: task.url, content: "", size: 0, ok: false, error: stderr };
   }
 
   try {

package/extensions/web_browse.ts

@@ -22,236 +22,15 @@ import {
 import { StringEnum } from "@earendil-works/pi-ai";
 import { Text } from "@earendil-works/pi-tui";
 import { Type, type Static } from "typebox";
-import { spawn } from "node:child_process";
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
-
-interface BrowseAction {
-  type: "click" | "fill" | "type" | "press" | "wait" | "wait_selector" | "scroll";
-  selector?: string;
-  value?: string;
-  key?: string;
-  ms?: number;
-  direction?: "down" | "up" | "bottom" | "top";
-  amount?: number;
-  state?: "attached" | "visible" | "hidden";
-}
-
-interface AgentBrowserBatchItem {
-  success: boolean;
-  command: string[];
-  result?: any;
-  error?: string | null;
-}
-
-function requireString(action: BrowseAction, field: "selector" | "value" | "key"): string {
-  const value = action[field];
-  if (typeof value !== "string" || value.length === 0) {
-    throw new Error(`Action "${action.type}" requires non-empty ${field}`);
-  }
-  return value;
-}
-
-function requireInteger(action: BrowseAction, field: "ms" | "amount"): number {
-  const value = action[field];
-  if (!Number.isInteger(value) || value < 0) {
-    throw new Error(`Action "${action.type}" requires non-negative integer ${field}`);
-  }
-  return value;
-}
-
-function waitForSelectorScript(selector: string, state: "attached" | "visible" | "hidden"): string {
-  const selectorLiteral = JSON.stringify(selector);
-  const stateLiteral = JSON.stringify(state);
-  return `await new Promise((resolve, reject) => {
-    const selector = ${selectorLiteral};
-    const state = ${stateLiteral};
-    const deadline = Date.now() + 30000;
-    const isVisible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
-    const check = () => {
-      const el = document.querySelector(selector);
-      const ok = state === "attached" ? !!el : state === "hidden" ? !isVisible(el) : isVisible(el);
-      if (ok) return resolve(true);
-      if (Date.now() > deadline) return reject(new Error(\`Timed out waiting for ${state} selector: ${selector}\`));
-      setTimeout(check, 100);
-    };
-    check();
-  })`;
-}
-
-function buildBatchCommands(
-  url: string,
-  actions: BrowseAction[],
-  selector?: string,
-): string[][] {
-  const commands: string[][] = [["open", url]];
-
-  for (const action of actions) {
-    switch (action.type) {
-      case "click":
-        commands.push(["click", requireString(action, "selector")]);
-        break;
-      case "fill":
-        commands.push(["fill", requireString(action, "selector"), requireString(action, "value")]);
-        break;
-      case "type":
-        commands.push(["type", requireString(action, "selector"), requireString(action, "value")]);
-        break;
-      case "press": {
-        if (action.selector) {
-          commands.push(["focus", action.selector]);
-        }
-        commands.push(["press", requireString(action, "key")]);
-        break;
-      }
-      case "wait":
-        commands.push(["wait", String(requireInteger(action, "ms"))]);
-        break;
-      case "wait_selector": {
-        const state = action.state ?? "visible";
-        const waitSelector = requireString(action, "selector");
-        if (state === "visible") {
-          commands.push(["wait", waitSelector]);
-        } else {
-          commands.push(["eval", waitForSelectorScript(waitSelector, state)]);
-        }
-        break;
-      }
-      case "scroll": {
-        const dir = action.direction ?? "down";
-        if (dir === "top") {
-          commands.push(["eval", "window.scrollTo(0, 0)"]);
-        } else if (dir === "bottom") {
-          commands.push(["eval", "window.scrollTo(0, document.body.scrollHeight)"]);
-        } else {
-          commands.push(["scroll", dir, String(action.amount ?? 500)]);
-        }
-        break;
-      }
-      default:
-        throw new Error(`Unsupported browser action: ${(action as BrowseAction).type}`);
-    }
-  }
-
-  // Extract content
-  if (selector) {
-    commands.push(["get", "text", selector, "--json"]);
-  } else {
-    commands.push(["snapshot", "-i", "--json"]);
-  }
-
-  // Metadata
-  commands.push(["get", "title", "--json"]);
-  commands.push(["get", "url", "--json"]);
-
-  return commands;
-}
-
-function runAgentBrowserBatch(
-  commands: string[][],
-  options: { session: string; headless: boolean; signal?: AbortSignal; timeout?: number },
-): Promise<AgentBrowserBatchItem[]> {
-  const args = ["--session", options.session];
-  if (!options.headless) args.push("--headed");
-  args.push("batch", "--bail", "--json");
-
-  return new Promise((resolve, reject) => {
-    const proc = spawn("agent-browser", args, {
-      shell: false,
-      stdio: ["pipe", "pipe", "pipe"],
-    });
-
-    let stdout = "";
-    let stderr = "";
-    let timeoutId: NodeJS.Timeout | undefined;
-    let settled = false;
-
-    const cleanup = () => {
-      if (timeoutId) clearTimeout(timeoutId);
-      if (options.signal) options.signal.removeEventListener("abort", kill);
-    };
-
-    const settleReject = (err: Error) => {
-      if (settled) return;
-      settled = true;
-      cleanup();
-      reject(err);
-    };
-
-    const kill = () => proc.kill("SIGTERM");
-
-    proc.stdout.on("data", (data: Buffer) => {
-      stdout += data.toString();
-    });
-
-    proc.stderr.on("data", (data: Buffer) => {
-      stderr += data.toString();
-    });
-
-    if (options.timeout) {
-      timeoutId = setTimeout(() => {
-        proc.kill("SIGTERM");
-        settleReject(new Error(`agent-browser timed out after ${options.timeout}ms`));
-      }, options.timeout);
-    }
-
-    proc.on("close", (code) => {
-      if (settled) return;
-      settled = true;
-      cleanup();
-
-      if (code !== 0 && !stdout.trim()) {
-        reject(new Error(`agent-browser failed (exit ${code}):\n${stderr || "unknown error"}`));
-        return;
-      }
-
-      try {
-        const results = JSON.parse(stdout) as AgentBrowserBatchItem[];
-        resolve(results);
-      } catch (err: any) {
-        reject(new Error(
-          `Failed to parse agent-browser output: ${err.message}\nstdout: ${stdout}\nstderr: ${stderr}`
-        ));
-      }
-    });
-
-    proc.on("error", (err: any) => {
-      if (err.code === "ENOENT") {
-        settleReject(new Error(
-          "agent-browser is not installed.\n\nInstall it with:\n  npm i -g agent-browser && agent-browser install\n\nThen run: agent-browser doctor"
-        ));
-      } else {
-        settleReject(err);
-      }
-    });
-
-    if (options.signal) {
-      if (options.signal.aborted) kill();
-      else options.signal.addEventListener("abort", kill, { once: true });
-    }
-
-    proc.stdin.write(JSON.stringify(commands));
-    proc.stdin.end();
-  });
-}
-
-function closeAgentBrowserSession(session: string, signal?: AbortSignal): Promise<void> {
-  return new Promise((resolve) => {
-    const proc = spawn("agent-browser", ["--session", session, "close"], {
-      shell: false,
-      stdio: ["ignore", "ignore", "ignore"],
-    });
-    const done = () => resolve();
-    proc.on("close", done);
-    proc.on("error", done);
-    if (signal) {
-      const kill = () => proc.kill("SIGTERM");
-      if (signal.aborted) kill();
-      else signal.addEventListener("abort", kill, { once: true });
-    }
-  });
-}
+import {
+  type BrowseAction,
+  buildBatchCommands,
+  runAgentBrowserBatch,
+  closeAgentBrowserSession,
+} from "./utils/agent-browser";
 
 export const WebBrowseActionSchema = Type.Object({
   type: StringEnum(["click", "fill", "type", "press", "wait", "wait_selector", "scroll"] as const),

package/extensions/web_fetch.ts

@@ -23,7 +23,7 @@ import { Type, type Static } from "typebox";
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
-import { runScrapling } from "./utils/scrapling";
+import { runScraplingWithFallback } from "./utils/scrapling";
 
 export const WebFetchParamsSchema = Type.Object({
   url: Type.String({ description: "Full URL to fetch (e.g. https://example.com/article)" }),
@@ -57,24 +57,15 @@ const webFetchTool = defineTool({
     let tmpFull: string | undefined;
 
     try {
-      const cmd = params.stealthy ? "stealthy-fetch" : "fetch";
-      const args = ["extract", cmd, params.url, tmpFile, "--ai-targeted"];
-      if (params.selector) {
-        args.push("--css-selector", params.selector);
-      }
-
-      const { stdout, stderr, exitCode } = await runScrapling(args, signal);
+      const { ok, stderr } = await runScraplingWithFallback(
+        params.url,
+        tmpFile,
+        { selector: params.selector, stealthy: params.stealthy, noGetFallback: params.stealthy },
+        signal,
+      );
 
-      if (exitCode !== 0) {
-        // Try fallback to simple HTTP GET if fetch/stealthy-fetch failed
-        if (!params.stealthy) {
-          const fallback = await runScrapling(["extract", "get", params.url, tmpFile, "--ai-targeted"], signal);
-          if (fallback.exitCode !== 0) {
-            throw new Error(`Failed to fetch ${params.url}\n\nscrapling error:\n${stderr || fallback.stderr}`);
-          }
-        } else {
-          throw new Error(`Failed to fetch ${params.url}\n\nscrapling error:\n${stderr}`);
-        }
+      if (!ok) {
+        throw new Error(`Failed to fetch ${params.url}\n\nscrapling error:\n${stderr}`);
       }
 
       const content = await fs.promises.readFile(tmpFile, "utf-8");

package/extensions/web_search.ts

@@ -24,7 +24,7 @@ import { mkdtemp, writeFile } from "node:fs/promises";
 import * as os from "node:os";
 import * as path from "node:path";
 
-const SEARXNG_URL = (process.env.SEARXNG_URL || "http://localhost:8080").replace(/\/$/, "");
+
 
 interface SearxResult {
   title: string;
@@ -66,6 +66,7 @@ const webSearchTool = defineTool({
   parameters: WebSearchParamsSchema,
 
   async execute(_toolCallId, params, signal) {
+    const searxngUrl = (process.env.SEARXNG_URL || "http://localhost:8080").replace(/\/$/, "");
     const maxResults = Math.floor(Math.min(50, Math.max(1, params.results ?? 10)));
     const searchParams = new URLSearchParams({
       q: params.query,
@@ -73,7 +74,7 @@ const webSearchTool = defineTool({
       language: params.language ?? "auto",
     });
 
-    const url = `${SEARXNG_URL}/search?${searchParams.toString()}`;
+    const url = `${searxngUrl}/search?${searchParams.toString()}`;
 
     let fullOutputPath: string | undefined;
 
@@ -140,7 +141,7 @@ const webSearchTool = defineTool({
         details: { query: data.query, totalResults: data.results.length, results: data.results.slice(0, maxResults), fullOutputPath },
       };
     } catch (err: any) {
-      throw new Error(`Failed to query SearXNG at ${SEARXNG_URL}: ${err.message ?? err}`);
+      throw new Error(`Failed to query SearXNG at ${searxngUrl}: ${err.message ?? err}`);
     }
   },
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-web-toolkit",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Web research toolkit for the pi coding agent. Search via SearXNG, fetch static pages with scrapling, browse interactively via agent-browser, and batch-read sources in parallel.",
   "author": "Wade Huang <fastwade11@gmail.com>",
   "license": "MIT",