pi-web-providers 1.1.0 → 2.0.0

package/example-config.json

@@ -1,87 +1,5 @@
 {
   "tools": {
-    "search": "codex",
-    "contents": null,
-    "answer": null,
-    "research": null
-  },
-  "genericSettings": {
-    "requestTimeoutMs": 30000,
-    "retryCount": 3,
-    "retryDelayMs": 2000,
-    "researchPollIntervalMs": 3000,
-    "researchTimeoutMs": 21600000,
-    "researchMaxConsecutivePollErrors": 3
-  },
-  "providers": {
-    "claude": {
-      "enabled": false
-    },
-    "codex": {
-      "enabled": true,
-      "native": {
-        "networkAccessEnabled": true,
-        "webSearchEnabled": true,
-        "webSearchMode": "live"
-      }
-    },
-    "custom-cli": {
-      "enabled": false
-    },
-    "exa": {
-      "enabled": false,
-      "apiKey": "EXA_API_KEY",
-      "native": {
-        "type": "auto",
-        "contents": {
-          "text": true
-        }
-      }
-    },
-    "gemini": {
-      "enabled": false,
-      "apiKey": "GOOGLE_API_KEY",
-      "native": {
-        "searchModel": "gemini-2.5-flash",
-        "answerModel": "gemini-2.5-flash",
-        "researchAgent": "deep-research-pro-preview-12-2025"
-      },
-      "policy": {
-        "researchMaxConsecutivePollErrors": 10
-      }
-    },
-    "perplexity": {
-      "enabled": false,
-      "apiKey": "PERPLEXITY_API_KEY",
-      "native": {
-        "answer": {
-          "model": "sonar"
-        },
-        "research": {
-          "model": "sonar-deep-research"
-        }
-      }
-    },
-    "parallel": {
-      "enabled": false,
-      "apiKey": "PARALLEL_API_KEY",
-      "native": {
-        "search": {
-          "mode": "agentic"
-        },
-        "extract": {
-          "excerpts": true,
-          "full_content": false
-        }
-      }
-    },
-    "valyu": {
-      "enabled": false,
-      "apiKey": "VALYU_API_KEY",
-      "native": {
-        "searchType": "all",
-        "responseLength": "short"
-      }
-    }
+    "search": "codex"
   }
 }

package/examples/{custom-cli → custom}/README.md

@@ -1,4 +1,4 @@
-# Custom CLI wrapper examples
+# Custom wrapper examples
 
 These examples keep the wrapper logic small. They are bash scripts that use
 `jq` for JSON handling. Each wrapper uses a different backend pattern:
@@ -30,27 +30,27 @@ You need:
 
 ```bash
 mkdir -p ./wrappers
-cp examples/custom-cli/wrappers/codex-search.sh ./wrappers/
-cp examples/custom-cli/wrappers/gemini-contents.sh ./wrappers/
-cp examples/custom-cli/wrappers/claude-answer.sh ./wrappers/
-cp examples/custom-cli/wrappers/perplexity-research.sh ./wrappers/
+cp examples/custom/wrappers/codex-search.sh ./wrappers/
+cp examples/custom/wrappers/gemini-contents.sh ./wrappers/
+cp examples/custom/wrappers/claude-answer.sh ./wrappers/
+cp examples/custom/wrappers/perplexity-research.sh ./wrappers/
 chmod +x ./wrappers/*.sh
 ```
 
-Then configure `custom-cli` like this:
+Then configure `custom` like this:
 
 ```json
 {
   "tools": {
-    "search": "custom-cli",
-    "contents": "custom-cli",
-    "answer": "custom-cli",
-    "research": "custom-cli"
+    "search": "custom",
+    "contents": "custom",
+    "answer": "custom",
+    "research": "custom"
   },
   "providers": {
-    "custom-cli": {
+    "custom": {
       "enabled": true,
-      "native": {
+      "options": {
         "search": {
           "argv": ["bash", "./wrappers/codex-search.sh"]
         },
@@ -69,8 +69,9 @@ Then configure `custom-cli` like this:
 }
 ```
 
-`web_research` runs as a foreground wrapper command, so polling controls and
-`resumeId` do not apply to `custom-cli`.
+`web_research` uses the same async workflow as every other research provider:
+pi starts the wrapper in the background, tracks the job locally, and writes the
+final report to a file when it finishes.
 
 ## Core command shapes
 
@@ -132,7 +133,7 @@ printf '%s' '{
   "maxResults": 5,
   "options": {},
   "cwd": "'"$PWD"'"
-}' | bash examples/custom-cli/wrappers/codex-search.sh
+}' | bash examples/custom/wrappers/codex-search.sh
 ```
 
 ### Contents
@@ -143,7 +144,7 @@ printf '%s' '{
   "urls": ["https://example.com"],
   "options": {},
   "cwd": "'"$PWD"'"
-}' | bash examples/custom-cli/wrappers/gemini-contents.sh
+}' | bash examples/custom/wrappers/gemini-contents.sh
 ```
 
 ### Answer
@@ -154,7 +155,7 @@ printf '%s' '{
   "query": "What changed in the latest Claude Code release?",
   "options": {},
   "cwd": "'"$PWD"'"
-}' | bash examples/custom-cli/wrappers/claude-answer.sh
+}' | bash examples/custom/wrappers/claude-answer.sh
 ```
 
 ### Research
@@ -165,7 +166,7 @@ printf '%s' '{
   "input": "Compare current local agent CLIs for web-grounded tasks.",
   "options": {},
   "cwd": "'"$PWD"'"
-}' | bash examples/custom-cli/wrappers/perplexity-research.sh
+}' | bash examples/custom/wrappers/perplexity-research.sh
 ```
 
 ## Request and response contract
@@ -196,7 +197,22 @@ printf '%s' '{
 }
 ```
 
-### Contents, answer, and research response
+### Contents response
+
+```json
+{
+  "answers": [
+    {
+      "url": "https://example.com",
+      "content": "# Example\n\nMain page content",
+      "summary": "Optional short summary",
+      "metadata": {}
+    }
+  ]
+}
+```
+
+### Answer and research response
 
 ```json
 {

package/examples/{custom-cli → custom}/wrappers/gemini-contents.sh

@@ -13,11 +13,12 @@ prompt="$(
 Extract the main textual content from these URLs:
 $urls
 
-Return JSON only with these fields:
-- text: the extracted content
-- summary: a short summary
-- itemCount: the number of processed URLs ($url_count)
-- metadata: include the input URLs under metadata.urls
+Return JSON only with this shape:
+- answers: an array with exactly $url_count items
+- each answer must include the input url, and either:
+  - content: the extracted text as a string
+  - or error: a short error string when extraction fails
+- optionally include summary and metadata
 EOF
 )"
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pi-web-providers",
-  "version": "1.1.0",
-  "description": "Configurable web access extension for pi with per-tool provider routing for search, contents, answers, and research across Claude, Codex, Custom CLI, Exa, Gemini, Perplexity, Parallel, and Valyu.",
+  "version": "2.0.0",
+  "description": "Configurable web access extension for pi with per-tool provider routing for search, contents, answers, and research.",
   "type": "module",
   "files": [
     "dist",
@@ -16,12 +16,15 @@
     "coding-agent",
     "web-search",
     "claude",
+    "cloudflare",
     "codex",
-    "custom-cli",
+    "custom",
     "exa",
+    "firecrawl",
     "gemini",
     "perplexity",
     "parallel",
+    "tavily",
     "valyu"
   ],
   "author": "mavam",
@@ -42,20 +45,24 @@
     ]
   },
   "scripts": {
-    "build": "rm -rf dist && esbuild src/index.ts --bundle --format=esm --platform=node --outfile=dist/index.js --external:@mariozechner/pi-coding-agent --external:@mariozechner/pi-ai --external:@mariozechner/pi-tui --external:@sinclair/typebox --external:@anthropic-ai/claude-agent-sdk --external:@google/genai --external:@openai/codex-sdk --external:@perplexity-ai/perplexity_ai --external:exa-js --external:parallel-web --external:valyu-js",
+    "build": "rm -rf dist && esbuild src/index.ts --bundle --format=esm --platform=node --outfile=dist/index.js --external:@mariozechner/pi-coding-agent --external:@mariozechner/pi-ai --external:@mariozechner/pi-tui --external:@sinclair/typebox --external:@anthropic-ai/claude-agent-sdk --external:@google/genai --external:@mendable/firecrawl-js --external:@openai/codex-sdk --external:@perplexity-ai/perplexity_ai --external:@tavily/core --external:cloudflare --external:exa-js --external:parallel-web --external:valyu-js",
     "prepare": "npm run build",
     "prepack": "npm run build",
     "check": "tsc --noEmit",
     "format": "biome format --write .",
     "format:check": "biome format .",
+    "smoke:live": "npm run build && node scripts/live-smoke.mjs",
     "test": "vitest run",
     "test:watch": "vitest"
   },
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.2.71",
     "@google/genai": "^1.44.0",
+    "@mendable/firecrawl-js": "^4.18.1",
     "@openai/codex-sdk": "^0.111.0",
     "@perplexity-ai/perplexity_ai": "^0.26.1",
+    "@tavily/core": "^0.7.2",
+    "cloudflare": "^5.2.0",
     "exa-js": "^2.7.0",
     "parallel-web": "^0.3.1",
     "valyu-js": "^2.5.9",