pi-web-toolkit 0.1.1 → 0.2.0

package/README.md

@@ -1,79 +1,125 @@
 # pi-web-toolkit
 
-Web research toolkit for [pi](https://pi.dev) agents. Search, fetch, browse, and batch-read the web.
+[![npm version](https://badge.fury.io/js/pi-web-toolkit.svg)](https://www.npmjs.com/package/pi-web-toolkit)
+[![CI](https://github.com/Wade11s/pi-web-toolkit/actions/workflows/ci.yml/badge.svg)](https://github.com/Wade11s/pi-web-toolkit/actions)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+![Node.js](https://img.shields.io/badge/node-%3E%3D22-339933)
+
+**100% open-source. Zero API keys. Zero fees.**
+
+Web research toolkit for [pi](https://pi.dev) agents. Search via SearXNG, fetch static pages with scrapling, browse interactively via agent-browser, and batch-read sources in parallel. All self-hosted, all local, all free — with built-in truncation safety and LLM-optimized prompt guidelines.
 
 ## Features
 
-| Tool | Purpose |
-|------|---------|
-| **`web_search`** | Search the web via SearXNG |
-| **`web_fetch`** | Fetch a single static page as clean markdown |
-| **`web_browse`** | Interact with a page (click, scroll, fill) then extract content |
-| **`web_batch_fetch`** | Fetch 2–10 pages in parallel for research synthesis |
+| Tool | Backend | Purpose | Current Limit |
+|------|---------|---------|---------------|
+| **`web_search`** | [SearXNG](https://github.com/searxng/searxng) | Search the web with scored, ranked results from multiple engines — always the first step in web research | 20 results (max 60, auto-pages up to 3 pages) |
+| **`web_fetch`** | [scrapling](https://github.com/D4Vinci/Scrapling) | Fetch a single static page as clean markdown | — |
+| **`web_batch_fetch`** | [scrapling](https://github.com/D4Vinci/Scrapling) | Fetch 2–15 pages in parallel for research synthesis | 3 concurrent (max 5) |
+| **`web_browse`** | [agent-browser](https://github.com/vercel-labs/agent-browser) | Interact with a page (click, scroll, fill) then extract content | 25 actions |
 
-## Installation
+## Quick Start
 
-### Option 1: From npm (recommended)
+### 1. Install external dependencies
 
 ```bash
-pi install pi-web-toolkit
+# SearXNG (for search)
+docker run -d --name searxng -p 8080:8080 -v searxng:/etc/searxng searxng/searxng
+export SEARXNG_URL="http://localhost:8080"
+
+# scrapling (for fetch & batch fetch)
+uv tool install "scrapling[all]"
+scrapling install
+
+# agent-browser (for browse)
+npm i -g agent-browser && agent-browser install
 ```
 
-### Option 2: From GitHub
+**Verify dependencies:**
+```bash
+# SearXNG
+curl -s "$SEARXNG_URL" | head
+
+# scrapling
+scrapling --help
 
+# agent-browser
+agent-browser doctor
+```
+
+### 2. Install the extension
+#### From npm
+```bash
+pi install npm:pi-web-toolkit
+```
+#### From GitHub
 ```bash
 pi install git:github.com/Wade11s/pi-web-toolkit
 ```
 
-## Requirements
-
-- **Node.js ≥ 20** — for running pi extensions
-- **SearXNG** — for `web_search`
-  ```bash
-  # Set your SearXNG instance URL (default: http://localhost:8080)
-  export SEARXNG_URL="http://localhost:8080"
-
-  # Self-host with Docker
-  docker run -d -p 8080:8080 -v searxng:/etc/searxng searxng/searxng
-  ```
-- **scrapling** — for `web_fetch` and `web_batch_fetch`
-  ```bash
-  # recommended: install scrapling via uv
-  uv tool install "scrapling[all]"
-  scrapling install
-  ```
-- **agent-browser** — for `web_browse`
-  ```bash
-  npm i -g agent-browser && agent-browser install
-  ```
-  Verify installation:
-  ```bash
-  agent-browser doctor
-  ```
+## Configuration
+
+All tools are configured via **environment variables** at runtime — no rebuild or restart required.
+
+| Variable | Default | Used By | Description |
+|----------|---------|---------|-------------|
+| `SEARXNG_URL` | `http://localhost:8080` | `web_search` | Your SearXNG instance endpoint |
+
+Set before starting pi:
+
+```bash
+export SEARXNG_URL="https://searxng.example.com"
+```
 
 ## Project Structure
 
 ```
 pi-web-toolkit/
 ├── extensions/
+│   ├── index.ts              # Unified entry point — registers all 4 tools
 │   ├── utils/
-│   │   └── scrapling.ts    # scrapling CLI wrapper
-│   ├── web_search.ts       # web_search
-│   ├── web_fetch.ts        # web_fetch
-│   ├── web_browse.ts       # web_browse (agent-browser)
-│   └── web_batch_fetch.ts  # web_batch_fetch
+│   │   ├── scrapling.ts      # Reusable scrapling CLI wrapper (shared by fetch + batch)
+│   │   └── agent-browser.ts  # agent-browser CLI wrapper (shared by web_browse)
+│   ├── web_search.ts         # SearXNG search tool
+│   ├── web_fetch.ts          # Single-page scrapling fetcher
+│   ├── web_batch_fetch.ts    # Parallel scrapling fetcher
+│   └── web_browse.ts         # Interactive browser automation (agent-browser)
 ├── docs/
-│   ├── tools.md
-│   └── guide.md
+│   ├── tools.md              # Full parameter specs
+│   └── guide.md              # Decision tree & tool comparison
+├── CHANGELOG.md
 ├── package.json
 ├── README.md
 └── LICENSE
 ```
 
+**Design principles:**
+- **Unified registration** — `index.ts` is the single source of truth for what pi loads.
+- **Shared utilities** — `utils/scrapling.ts` and `utils/agent-browser.ts` encapsulate the CLI wrappers and fallback logic; tool files import only from `utils/`, never from each other.
+- **Per-tool isolation** — each tool owns its own schema, execute logic, and TUI renderer; no cross-imports except via `utils/`.
+- **Runtime config** — environment variables are read at execute time, not build time.
+
 ## Reference
 
 - [Tool Reference](docs/tools.md) — Full parameter specs and usage examples for each tool.
 - [Usage Guide](docs/guide.md) — Decision tree and tool comparison.
+- [Changelog](CHANGELOG.md) — Release history and migration notes.
+
+## Contributing
+
+```bash
+# Local development
+pi install ./
+
+# Type-check (no build step; pi loads TypeScript directly)
+npx tsc --noEmit
+
+# Verify external CLI dependencies
+scrapling --help
+agent-browser doctor
+```
+
+Pull requests welcome. Please keep changes scoped to a single tool or concern and follow [Conventional Commits](https://www.conventionalcommits.org/).
 
 ## License
 

package/docs/agents/domain.md

@@ -0,0 +1,51 @@
+# Domain Docs
+
+How the engineering skills should consume this repo's domain documentation when exploring the codebase.
+
+## Before exploring, read these
+
+- **`CONTEXT.md`** at the repo root, or
+- **`CONTEXT-MAP.md`** at the repo root if it exists — it points at one `CONTEXT.md` per context. Read each one relevant to the topic.
+- **`docs/adr/`** — read ADRs that touch the area you're about to work in. In multi-context repos, also check `src/<context>/docs/adr/` for context-scoped decisions.
+
+If any of these files don't exist, **proceed silently**. Don't flag their absence; don't suggest creating them upfront. The producer skill (`/grill-with-docs`) creates them lazily when terms or decisions actually get resolved.
+
+## File structure
+
+Single-context repo (most repos):
+
+```
+/
+├── CONTEXT.md
+├── docs/adr/
+│   ├── 0001-event-sourced-orders.md
+│   └── 0002-postgres-for-write-model.md
+└── src/
+```
+
+Multi-context repo (presence of `CONTEXT-MAP.md` at the root):
+
+```
+/
+├── CONTEXT-MAP.md
+├── docs/adr/                          ← system-wide decisions
+└── src/
+    ├── ordering/
+    │   ├── CONTEXT.md
+    │   └── docs/adr/                  ← context-specific decisions
+    └── billing/
+        ├── CONTEXT.md
+        └── docs/adr/
+```
+
+## Use the glossary's vocabulary
+
+When your output names a domain concept (in an issue title, a refactor proposal, a hypothesis, a test name), use the term as defined in `CONTEXT.md`. Don't drift to synonyms the glossary explicitly avoids.
+
+If the concept you need isn't in the glossary yet, that's a signal — either you're inventing language the project doesn't use (reconsider) or there's a real gap (note it for `/grill-with-docs`).
+
+## Flag ADR conflicts
+
+If your output contradicts an existing ADR, surface it explicitly rather than silently overriding:
+
+> _Contradicts ADR-0007 (event-sourced orders) — but worth reopening because…_

package/docs/agents/issue-tracker.md

@@ -0,0 +1,22 @@
+# Issue tracker: GitHub
+
+Issues and PRDs for this repo live as GitHub issues. Use the `gh` CLI for all operations.
+
+## Conventions
+
+- **Create an issue**: `gh issue create --title "..." --body "..."`. Use a heredoc for multi-line bodies.
+- **Read an issue**: `gh issue view <number> --comments`, filtering comments by `jq` and also fetching labels.
+- **List issues**: `gh issue list --state open --json number,title,body,labels,comments --jq '[.[] | {number, title, body, labels: [.labels[].name], comments: [.comments[].body]}]'` with appropriate `--label` and `--state` filters.
+- **Comment on an issue**: `gh issue comment <number> --body "..."`
+- **Apply / remove labels**: `gh issue edit <number> --add-label "..."` / `--remove-label "..."`
+- **Close**: `gh issue close <number> --comment "..."`
+
+Infer the repo from `git remote -v` — `gh` does this automatically when run inside a clone.
+
+## When a skill says "publish to the issue tracker"
+
+Create a GitHub issue.
+
+## When a skill says "fetch the relevant ticket"
+
+Run `gh issue view <number> --comments`.

package/docs/agents/triage-labels.md

@@ -0,0 +1,15 @@
+# Triage Labels
+
+The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker.
+
+| Label in mattpocock/skills | Label in our tracker | Meaning                                  |
+| -------------------------- | -------------------- | ---------------------------------------- |
+| `needs-triage`             | `needs-triage`       | Maintainer needs to evaluate this issue  |
+| `needs-info`               | `needs-info`         | Waiting on reporter for more information |
+| `ready-for-agent`          | `ready-for-agent`    | Fully specified, ready for an AFK agent  |
+| `ready-for-human`          | `ready-for-human`    | Requires human implementation            |
+| `wontfix`                  | `wontfix`            | Will not be actioned                     |
+
+When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the corresponding label string from this table.
+
+Edit the right-hand column to match whatever vocabulary you actually use.

package/docs/guide.md

@@ -32,7 +32,7 @@ User asks about something external / current
 
 | | `web_fetch` | `web_browse` | `web_batch_fetch` |
 |--|-------------|--------------|-------------------|
-| **Pages** | 1 | 1 | 2–10 |
+| **Pages** | 1 | 1 | 2–15 |
 | **Browser** | Yes (scrapling) | Yes (agent-browser) | Yes (scrapling) |
 | **Interaction** | ❌ No | ✅ Click, fill, scroll, wait | ❌ No |
 | **Selector** | ✅ Per-URL | ✅ Final state | ✅ Applied to all |

package/docs/tools.md

@@ -2,18 +2,22 @@
 
 ## `web_search`
 
-Search the web via SearXNG. Returns ranked results with title, URL, and snippet.
+Search the web via SearXNG. Returns ranked results with title, URL, and snippet. Automatically aggregates up to 3 pages of SearXNG results when more than ~20 are needed.
 
 ```typescript
 {
   query: string,           // Search query
   language?: string,       // Language code (en, de, fr...). Default: "auto"
-  results?: number,        // Max results (1–50). Default: 10
+  results?: number,        // Max results (1–60). Default: 20. Automatically pages through SearXNG (up to 3 pages) if needed.
 }
 ```
 
 **When to use:** The user asks about current events, facts, or anything requiring up-to-date information. This is always the **first step** of web research.
 
+**Empty results behavior:** When no results are found, `web_search` returns a list of **suggestions** — alternative queries that SearXNG believes may yield better results. The agent can use these suggestions to automatically refine and retry the search.
+
+**Pagination:** `web_search` automatically fetches up to 3 pages from SearXNG and deduplicates by URL. You do not need to call it multiple times for deeper results.
+
 ---
 
 ## `web_fetch`

package/extensions/utils/agent-browser.ts

@@ -0,0 +1,179 @@
+/**
+ * agent-browser CLI wrapper
+ *
+ * Encapsulates all low-level interaction with the agent-browser command:
+ * command building, process spawning, JSON parsing, and session cleanup.
+ */
+
+import { runCLI } from "./cli-runner";
+
+export interface BrowseAction {
+  type: "click" | "fill" | "type" | "press" | "wait" | "wait_selector" | "scroll";
+  selector?: string;
+  value?: string;
+  key?: string;
+  ms?: number;
+  direction?: "down" | "up" | "bottom" | "top";
+  amount?: number;
+  state?: "attached" | "visible" | "hidden";
+}
+
+export interface AgentBrowserBatchItem {
+  success: boolean;
+  command: string[];
+  result?: any;
+  error?: string | null;
+}
+
+function requireString(action: BrowseAction, field: "selector" | "value" | "key"): string {
+  const value = action[field] as string | undefined;
+  if (typeof value !== "string" || value.length === 0) {
+    throw new Error(`Action "${action.type}" requires non-empty ${field}`);
+  }
+  return value;
+}
+
+function requireInteger(action: BrowseAction, field: "ms" | "amount"): number {
+  const value = action[field] as number | undefined;
+  if (!Number.isInteger(value) || (value as number) < 0) {
+    throw new Error(`Action "${action.type}" requires non-negative integer ${field}`);
+  }
+  return value as number;
+}
+
+function waitForSelectorScript(selector: string, state: "attached" | "visible" | "hidden"): string {
+  const selectorLiteral = JSON.stringify(selector);
+  const stateLiteral = JSON.stringify(state);
+  return `await new Promise((resolve, reject) => {
+    const selector = ${selectorLiteral};
+    const state = ${stateLiteral};
+    const deadline = Date.now() + 30000;
+    const isVisible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+    const check = () => {
+      const el = document.querySelector(selector);
+      const ok = state === "attached" ? !!el : state === "hidden" ? !isVisible(el) : isVisible(el);
+      if (ok) return resolve(true);
+      if (Date.now() > deadline) return reject(new Error(\`Timed out waiting for ${state} selector: ${selector}\`));
+      setTimeout(check, 100);
+    };
+    check();
+  })`;
+}
+
+export function buildBatchCommands(
+  url: string,
+  actions: BrowseAction[],
+  selector?: string,
+): string[][] {
+  const commands: string[][] = [["open", url]];
+
+  for (const action of actions) {
+    switch (action.type) {
+      case "click":
+        commands.push(["click", requireString(action, "selector")]);
+        break;
+      case "fill":
+        commands.push(["fill", requireString(action, "selector"), requireString(action, "value")]);
+        break;
+      case "type":
+        commands.push(["type", requireString(action, "selector"), requireString(action, "value")]);
+        break;
+      case "press": {
+        if (action.selector) {
+          commands.push(["focus", action.selector]);
+        }
+        commands.push(["press", requireString(action, "key")]);
+        break;
+      }
+      case "wait":
+        commands.push(["wait", String(requireInteger(action, "ms"))]);
+        break;
+      case "wait_selector": {
+        const state = action.state ?? "visible";
+        const waitSelector = requireString(action, "selector");
+        if (state === "visible") {
+          commands.push(["wait", waitSelector]);
+        } else {
+          commands.push(["eval", waitForSelectorScript(waitSelector, state)]);
+        }
+        break;
+      }
+      case "scroll": {
+        const dir = action.direction ?? "down";
+        if (dir === "top") {
+          commands.push(["eval", "window.scrollTo(0, 0)"]);
+        } else if (dir === "bottom") {
+          commands.push(["eval", "window.scrollTo(0, document.body.scrollHeight)"]);
+        } else {
+          commands.push(["scroll", dir, String(action.amount ?? 500)]);
+        }
+        break;
+      }
+      default:
+        throw new Error(`Unsupported browser action: ${(action as BrowseAction).type}`);
+    }
+  }
+
+  // Extract content
+  if (selector) {
+    commands.push(["get", "text", selector, "--json"]);
+  } else {
+    commands.push(["snapshot", "-i", "--json"]);
+  }
+
+  // Metadata
+  commands.push(["get", "title", "--json"]);
+  commands.push(["get", "url", "--json"]);
+
+  return commands;
+}
+
+export async function runAgentBrowserBatch(
+  commands: string[][],
+  options: { session: string; headless: boolean; signal?: AbortSignal; timeout?: number },
+): Promise<AgentBrowserBatchItem[]> {
+  const args = ["--session", options.session];
+  if (!options.headless) args.push("--headed");
+  args.push("batch", "--bail", "--json");
+
+  try {
+    const result = await runCLI({
+      command: "agent-browser",
+      args,
+      stdin: JSON.stringify(commands),
+      timeout: options.timeout,
+      signal: options.signal,
+    });
+
+    if (result.exitCode !== 0 && !result.stdout.trim()) {
+      throw new Error(`agent-browser failed (exit ${result.exitCode}):\n${result.stderr || "unknown error"}`);
+    }
+
+    try {
+      return JSON.parse(result.stdout) as AgentBrowserBatchItem[];
+    } catch (err: any) {
+      throw new Error(
+        `Failed to parse agent-browser output: ${err.message}\nstdout: ${result.stdout}\nstderr: ${result.stderr}`
+      );
+    }
+  } catch (err: any) {
+    if (err.message === "agent-browser is not installed") {
+      throw new Error(
+        "agent-browser is not installed.\n\nInstall it with:\n  npm i -g agent-browser && agent-browser install\n\nThen run: agent-browser doctor"
+      );
+    }
+    throw err;
+  }
+}
+
+export async function closeAgentBrowserSession(session: string, signal?: AbortSignal): Promise<void> {
+  try {
+    await runCLI({
+      command: "agent-browser",
+      args: ["--session", session, "close"],
+      signal,
+    });
+  } catch {
+    // Best-effort cleanup — ignore errors
+  }
+}

package/extensions/utils/cli-runner.ts

@@ -0,0 +1,108 @@
+/**
+ * CLI runner — abstracted process spawning
+ *
+ * Provides a single interface for running external CLI commands
+ * with consistent signal handling, timeout support, and stdout/stderr
+ * collection. Enables testability by allowing the runner to be swapped.
+ */
+
+import { spawn, type ChildProcess } from "node:child_process";
+
+export interface CLIRunOptions {
+  command: string;
+  args: string[];
+  /** Data to write to stdin. If omitted, stdin is ignored. */
+  stdin?: string;
+  /** Timeout in milliseconds. If exceeded, the process is killed. */
+  timeout?: number;
+  /** AbortSignal for cancellation. */
+  signal?: AbortSignal;
+}
+
+export interface CLIRunResult {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+}
+
+/**
+ * Run an external CLI command and capture its output.
+ *
+ * Handles:
+ * - stdout/stderr collection
+ * - optional stdin feeding
+ * - optional timeout (SIGTERM)
+ * - AbortSignal cancellation (SIGTERM)
+ * - process spawn errors (e.g. ENOENT)
+ */
+export function runCLI(options: CLIRunOptions): Promise<CLIRunResult> {
+  return new Promise((resolve, reject) => {
+    const stdio = options.stdin
+      ? ["pipe", "pipe", "pipe"]
+      : ["ignore", "pipe", "pipe"];
+
+    const proc = spawn(options.command, options.args, {
+      shell: false,
+      stdio: stdio as any,
+    }) as ChildProcess;
+
+    let stdout = "";
+    let stderr = "";
+    let timeoutId: NodeJS.Timeout | undefined;
+    let settled = false;
+
+    const cleanup = () => {
+      if (timeoutId) clearTimeout(timeoutId);
+      if (options.signal) options.signal.removeEventListener("abort", kill);
+    };
+
+    const settleReject = (err: Error) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      reject(err);
+    };
+
+    const kill = () => proc.kill("SIGTERM");
+
+    proc.stdout?.on("data", (data: Buffer) => {
+      stdout += data.toString();
+    });
+
+    proc.stderr?.on("data", (data: Buffer) => {
+      stderr += data.toString();
+    });
+
+    if (options.timeout) {
+      timeoutId = setTimeout(() => {
+        proc.kill("SIGTERM");
+        settleReject(new Error(`${options.command} timed out after ${options.timeout}ms`));
+      }, options.timeout);
+    }
+
+    proc.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      resolve({ stdout, stderr, exitCode: code ?? 1 });
+    });
+
+    proc.on("error", (err: any) => {
+      if (err.code === "ENOENT") {
+        settleReject(new Error(`${options.command} is not installed`));
+      } else {
+        settleReject(err);
+      }
+    });
+
+    if (options.signal) {
+      if (options.signal.aborted) kill();
+      else options.signal.addEventListener("abort", kill, { once: true });
+    }
+
+    if (options.stdin && proc.stdin) {
+      proc.stdin.write(options.stdin);
+      proc.stdin.end();
+    }
+  });
+}