pi-web-toolkit 0.2.0 → 0.2.1

package/README.md

@@ -18,6 +18,28 @@ Web research toolkit for [pi](https://pi.dev) agents. Search via SearXNG, fetch
 | **`web_batch_fetch`** | [scrapling](https://github.com/D4Vinci/Scrapling) | Fetch 2–15 pages in parallel for research synthesis | 3 concurrent (max 5) |
 | **`web_browse`** | [agent-browser](https://github.com/vercel-labs/agent-browser) | Interact with a page (click, scroll, fill) then extract content | 25 actions |
 
+## Tools Preview
+
+A quick look at how pi renders toolkit calls while an agent searches, fetches, batches, and browses the web.
+
+<table>
+  <tr>
+    <td width="50%"><strong>Multi-tool research flow</strong><br><img src="docs/assets/screenshots/tools-workflow-preview.png" alt="pi-web-toolkit multi-tool research preview"></td>
+    <td width="50%"><strong><code>web_search</code> expanded results</strong><br><img src="docs/assets/screenshots/web-search-results-expanded.png" alt="web_search expanded results"></td>
+  </tr>
+  <tr>
+    <td width="50%"><strong><code>web_batch_fetch</code> progress</strong><br><img src="docs/assets/screenshots/web-batch-fetch-progress.png" alt="web_batch_fetch progress"></td>
+    <td width="50%"><strong><code>web_batch_fetch</code> results</strong><br><img src="docs/assets/screenshots/web-batch-fetch-results.png" alt="web_batch_fetch results"></td>
+  </tr>
+  <tr>
+    <td width="50%"><strong><code>web_fetch</code> result preview</strong><br><img src="docs/assets/screenshots/web-fetch-summary.png" alt="web_fetch result preview"></td>
+    <td width="50%"><strong><code>web_browse</code> headless browser flow</strong><br><img src="docs/assets/screenshots/web-browse-headless.png" alt="web_browse headless browser flow"></td>
+  </tr>
+  <tr>
+    <td colspan="2"><strong>End-to-end research summary</strong><br><img src="docs/assets/screenshots/web-research-workflow.png" alt="end-to-end web research workflow"></td>
+  </tr>
+</table>
+
 ## Quick Start
 
 ### 1. Install external dependencies
@@ -78,12 +100,19 @@ pi-web-toolkit/
 ├── extensions/
 │   ├── index.ts              # Unified entry point — registers all 4 tools
 │   ├── utils/
+│   │   ├── cli-runner.ts     # Unified CLI process spawning with timeout/AbortSignal
+│   │   ├── content-preview.ts # Intelligent content extraction from scraped pages
+│   │   ├── output-sink.ts    # Truncation + temp-file fallback
+│   │   ├── render-helpers.ts # URL abbreviations, text normalization, error formatting for TUI
 │   │   ├── scrapling.ts      # Reusable scrapling CLI wrapper (shared by fetch + batch)
+│   │   ├── tool-factory.ts   # Common tool registration patterns
 │   │   └── agent-browser.ts  # agent-browser CLI wrapper (shared by web_browse)
 │   ├── web_search.ts         # SearXNG search tool
 │   ├── web_fetch.ts          # Single-page scrapling fetcher
 │   ├── web_batch_fetch.ts    # Parallel scrapling fetcher
 │   └── web_browse.ts         # Interactive browser automation (agent-browser)
+├── test/
+│   └── content-preview/      # Automated test suite with fixtures & snapshots
 ├── docs/
 │   ├── tools.md              # Full parameter specs
 │   └── guide.md              # Decision tree & tool comparison
@@ -95,7 +124,7 @@ pi-web-toolkit/
 
 **Design principles:**
 - **Unified registration** — `index.ts` is the single source of truth for what pi loads.
-- **Shared utilities** — `utils/scrapling.ts` and `utils/agent-browser.ts` encapsulate the CLI wrappers and fallback logic; tool files import only from `utils/`, never from each other.
+- **Shared utilities** — `utils/` modules encapsulate CLI spawning, content extraction, output truncation, TUI formatting, and common registration patterns; tool files import only from `utils/`, never from each other.
 - **Per-tool isolation** — each tool owns its own schema, execute logic, and TUI renderer; no cross-imports except via `utils/`.
 - **Runtime config** — environment variables are read at execute time, not build time.
 
@@ -112,7 +141,10 @@ pi-web-toolkit/
 pi install ./
 
 # Type-check (no build step; pi loads TypeScript directly)
-npx tsc --noEmit
+npm run typecheck
+
+# Run tests
+npm run test
 
 # Verify external CLI dependencies
 scrapling --help

package/extensions/utils/agent-browser.ts

@@ -25,6 +25,48 @@ export interface AgentBrowserBatchItem {
   error?: string | null;
 }
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isBatchItem(value: unknown): value is AgentBrowserBatchItem {
+  return isRecord(value)
+    && typeof value.success === "boolean"
+    && Array.isArray(value.command)
+    && value.command.every((part) => typeof part === "string");
+}
+
+function describeBatchOutput(value: unknown): string {
+  if (Array.isArray(value)) return `array with ${value.length} item(s)`;
+  if (isRecord(value)) return `object with keys: ${Object.keys(value).join(", ") || "(none)"}`;
+  return typeof value;
+}
+
+export function parseAgentBrowserBatchOutput(stdout: string): AgentBrowserBatchItem[] {
+  const parsed = JSON.parse(stdout) as unknown;
+
+  if (Array.isArray(parsed)) {
+    if (parsed.every(isBatchItem)) return parsed;
+    throw new Error(`Expected every batch result item to contain { success, command }; got ${describeBatchOutput(parsed)}`);
+  }
+
+  if (isBatchItem(parsed)) {
+    return [parsed];
+  }
+
+  if (isRecord(parsed)) {
+    for (const key of ["results", "items", "data", "commands"]) {
+      const candidate = parsed[key];
+      if (Array.isArray(candidate)) {
+        if (candidate.every(isBatchItem)) return candidate;
+        throw new Error(`Expected ${key} to contain batch result items; got ${describeBatchOutput(candidate)}`);
+      }
+    }
+  }
+
+  throw new Error(`Expected JSON array of batch results; got ${describeBatchOutput(parsed)}`);
+}
+
 function requireString(action: BrowseAction, field: "selector" | "value" | "key"): string {
   const value = action[field] as string | undefined;
   if (typeof value !== "string" || value.length === 0) {
@@ -150,7 +192,7 @@ export async function runAgentBrowserBatch(
     }
 
     try {
-      return JSON.parse(result.stdout) as AgentBrowserBatchItem[];
+      return parseAgentBrowserBatchOutput(result.stdout);
     } catch (err: any) {
       throw new Error(
         `Failed to parse agent-browser output: ${err.message}\nstdout: ${result.stdout}\nstderr: ${result.stderr}`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-web-toolkit",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Web research toolkit for the pi coding agent. Search via SearXNG, fetch static pages with scrapling, browse interactively via agent-browser, and batch-read sources in parallel.",
   "author": "Wade Huang <fastwade11@gmail.com>",
   "license": "MIT",
@@ -19,7 +19,8 @@
   },
   "scripts": {
     "typecheck": "tsc --noEmit",
-    "test": "npx tsx test/content-preview/test.ts",
+    "test": "npx tsx test/content-preview/test.ts && npx tsx test/agent-browser/test.ts",
+    "test:agent-browser": "npx tsx test/agent-browser/test.ts",
     "test:approve": "npx tsx test/content-preview/test.ts --approve"
   },
   "devDependencies": {