webpeel 0.21.88 → 0.22.0

package/dist/cli/commands/fetch.js

@@ -747,7 +747,61 @@ export async function runFetch(url, options) {
                 }
             }
             if (!options.silent && !options.json && result.tokens && result.tokens < 50 && !options.render) {
-                console.error(`\x1b[33m💡 Tip: Page returned very little content. Try --render for JavaScript-heavy sites or --stealth if blocked.\x1b[0m`);
+                console.error(`\x1b[33m💡 Tip: Very little content extracted. This may be a JavaScript-rendered page.\x1b[0m`);
+                console.error(`\x1b[33m   Try: webpeel "${url}" --render\x1b[0m`);
+                console.error(`\x1b[33m   For infinite scroll/SPAs: --action 'scroll:bottom' --action 'wait:2000'\x1b[0m`);
+                console.error(`\x1b[33m   Or use --stealth if the site blocks bots.\x1b[0m`);
+            }
+            // Auth wall detection hint
+            if (!options.json && result.authRequired) {
+                let authHost = url;
+                try {
+                    authHost = new URL(url).hostname.replace('www.', '');
+                }
+                catch { /* ignore */ }
+                console.error('');
+                console.error('\x1b[33m🔐 This page requires authentication.\x1b[0m');
+                console.error(`\x1b[36m   1. Create a login profile:  webpeel profile create ${authHost}\x1b[0m`);
+                console.error('\x1b[36m   2. Log in to the site in the browser that opens\x1b[0m');
+                console.error('\x1b[36m   3. Press Ctrl+C when done\x1b[0m');
+                console.error(`\x1b[36m   4. Re-run with:  webpeel "${url}" --profile ${authHost}\x1b[0m`);
+                console.error('');
+            }
+        }
+        // Trust & safety warnings — shown prominently in non-JSON mode
+        if (!options.silent && !options.json) {
+            const trustData = result.trust;
+            const sbData = result.safeBrowsing;
+            // Unsafe: safe browsing threats detected
+            const allThreats = [
+                ...(sbData?.threats ?? []),
+                ...(trustData?.threatFeeds?.threats ?? []),
+            ].filter((t, i, a) => a.indexOf(t) === i);
+            if (sbData && !sbData.safe && allThreats.length > 0) {
+                console.error(`\x1b[31m🚨 UNSAFE — Threats detected: ${allThreats.join(', ')}\x1b[0m`);
+            }
+            else if (trustData?.threatFeeds && !trustData.threatFeeds.safe && trustData.threatFeeds.threats.length > 0) {
+                console.error(`\x1b[31m🚨 UNSAFE — Threat feeds flagged: ${trustData.threatFeeds.threats.join(', ')}\x1b[0m`);
+                if (trustData.threatFeeds.details) {
+                    console.error(`\x1b[31m   ${trustData.threatFeeds.details}\x1b[0m`);
+                }
+            }
+            else if (trustData && trustData.score < 0.5) {
+                // Low trust score
+                const tier = trustData.source?.tier ?? 'unknown';
+                const label = trustData.source?.label ?? '';
+                const reason = tier === 'suspicious'
+                    ? 'Domain shows suspicious signals'
+                    : tier === 'new'
+                        ? 'Domain has limited verifiable presence'
+                        : label || 'Low credibility domain';
+                console.error(`\x1b[33m⚠️  Low trust score (${trustData.score.toFixed(2)}) — ${reason}\x1b[0m`);
+            }
+            // Show any trust warnings
+            if (trustData?.warnings && trustData.warnings.length > 0) {
+                for (const warn of trustData.warnings) {
+                    console.error(`\x1b[33m⚠️  ${warn}\x1b[0m`);
+                }
             }
         }
         // Show metadata header

package/dist/cli/commands/guide.d.ts

@@ -0,0 +1,2 @@
+import type { Command } from 'commander';
+export declare function registerGuideCommand(program: Command): void;

package/dist/cli/commands/guide.js

@@ -0,0 +1,183 @@
+const GUIDE = `
+# WebPeel — AI Usage Guide
+
+WebPeel is a fast web fetcher built for AI agents. It handles JS rendering, Cloudflare protection,
+and 55+ domain-specific extractors automatically. Run any webpeel command with --help for options.
+
+## Quick Decision Tree
+
+- Static page                        → webpeel <url>
+- JavaScript SPA (React/Vue/Angular) → webpeel <url> --render
+- Bot-protected site (Cloudflare)    → webpeel <url> --stealth
+- Infinite scroll / lazy content     → webpeel <url> --render --action 'scroll:bottom' --action 'wait:2000'
+- Need to interact (click, type)     → webpeel <url> --render --action 'click:.button' --action 'wait:1000'
+- Screenshot                         → webpeel screenshot <url>
+- Search the web                     → webpeel search "query"
+- YouTube transcript                 → webpeel <youtube-url>
+- PDF content                        → webpeel <pdf-url>
+- Structured data                    → webpeel <url> --schema product --json
+- Monitor for changes                → webpeel watch <url>
+
+## When to Use --render
+
+Use for ANY site that:
+- Shows blank/minimal content without JavaScript
+- Is a Single Page Application (React, Vue, Angular, Svelte, Next.js)
+- Returns less than 50 tokens of content
+- Has dynamic/interactive elements
+
+Known SPA sites (auto-detected): Google, Airbnb, Booking.com, Expedia, Indeed, Zillow, Polymarket, and more.
+For sites NOT in the auto-list, add --render manually.
+
+Tip: If content looks sparse or empty, ALWAYS retry with --render before concluding the page has no content.
+
+## Browser Actions (--action flag)
+
+Actions require --render (auto-enabled when you pass --action). Chain multiple actions with repeated --action flags.
+
+Available Actions:
+  scroll:bottom         — scroll to page bottom (for infinite scroll / lazy-loaded content)
+  scroll:top            — scroll to top
+  scroll:down:500       — scroll down 500px
+  scroll:0,1500         — scroll to exact coordinates (x,y)
+  wait:2000             — wait 2000ms (useful after navigation or clicks)
+  click:.selector       — click a CSS element
+  type:#input:hello     — type text into an input field
+  waitFor:.selector     — wait for a CSS element to appear in the DOM
+  hover:.element        — hover over an element (for dropdown menus, tooltips)
+
+Common Patterns:
+
+  # Load all lazy content / infinite scroll
+  webpeel <url> --render --action 'scroll:bottom' --action 'wait:2000'
+
+  # Click "Load More" button then extract
+  webpeel <url> --render --action 'click:.load-more' --action 'wait:1000'
+
+  # Fill a search form and submit
+  webpeel <url> --render --action 'type:#search:query' --action 'click:.submit' --action 'wait:2000'
+
+  # Wait for dynamic content to appear
+  webpeel <url> --render --action 'waitFor:.results-list' --action 'wait:500'
+
+## Stealth Mode (--stealth)
+
+Use when:
+- Site returns a Cloudflare challenge page
+- Site blocks bots with fingerprinting or rate limiting
+- Normal --render fails with access denied / 403
+
+  webpeel <url> --stealth
+
+Stealth mode auto-enables --render.
+
+## Authentication (Login-Protected Pages)
+
+Some pages require you to be logged in (e.g. dashboards, profiles, activity feeds).
+WebPeel detects auth walls automatically and tells you what to do.
+
+To access login-protected content:
+
+  1. Create a browser profile:
+     webpeel profile create polymarket
+
+  2. A browser opens — log in to the site normally
+
+  3. Press Ctrl+C when done (cookies are saved)
+
+  4. Fetch with your profile:
+     webpeel "https://polymarket.com/@user" --profile polymarket
+
+Profiles are saved in ~/.webpeel/profiles/ and can be reused.
+
+  webpeel profile list              — see all saved profiles
+  webpeel profile delete <name>     — remove a profile
+
+## 55+ Domain Extractors (automatic)
+
+These sites get instant structured data via dedicated API — no browser needed:
+Amazon, Reddit, YouTube, GitHub, Wikipedia, ESPN, Polymarket, Kalshi, TradingView,
+Hacker News, NPM, PyPI, Stack Overflow, and 40+ more.
+
+If the URL matches a supported domain, WebPeel uses the extractor automatically.
+You never need to configure this.
+
+## Output Options
+
+  Default            → clean markdown (LLM-optimized, 65-98% token reduction)
+  --json             → full JSON with metadata, token count, method used
+  --raw              → full page HTML/text, no smart extraction
+  --budget N         → distill content to N tokens (smart summarization)
+  --schema NAME      → extract structured data (product, article, recipe, job, event, contact, review)
+  --silent           → suppress progress spinner (for piping output)
+  --question "..."   → answer a specific question about the page (BM25, no LLM needed)
+
+## MCP Server
+
+For Claude Desktop, Cursor, VS Code — add to your MCP config:
+
+  {
+    "mcpServers": {
+      "webpeel": {
+        "command": "npx",
+        "args": ["-y", "webpeel", "mcp"]
+      }
+    }
+  }
+
+Available MCP tools: webpeel (smart), webpeel_read, webpeel_see, webpeel_find,
+                     webpeel_extract, webpeel_monitor, webpeel_act
+
+## Troubleshooting
+
+  Very little content?              → Add --render
+  Still blocked?                    → Add --stealth (implies --render)
+  SPA not loading data?             → --render --action 'wait:3000'
+  Screenshot fails?                 → Run: npx playwright install chromium
+  Wrong content for /profile pages? → Content is client-side routed. Use --render.
+  Need to extract specific fields?  → Use --schema or --json with jq
+
+## Examples
+
+  # Fetch a static page
+  webpeel https://example.com
+
+  # Fetch a React SPA (Polymarket, Airbnb, etc.)
+  webpeel https://polymarket.com --render
+
+  # Scroll and load all predictions on Polymarket
+  webpeel https://polymarket.com --render --action 'scroll:bottom' --action 'wait:2000'
+
+  # Get Cloudflare-protected site
+  webpeel https://someprotectedsite.com --stealth
+
+  # Extract product data from Amazon
+  webpeel https://amazon.com/dp/B09X3PRGT7 --schema product --json
+
+  # Get YouTube transcript
+  webpeel https://youtube.com/watch?v=dQw4w9WgXcQ
+
+  # Screenshot on mobile viewport
+  webpeel screenshot https://stripe.com/pricing --viewport mobile
+
+  # Search and get top results
+  webpeel search "best TypeScript ORM 2024"
+
+  # Watch a page for price changes
+  webpeel watch https://store.example.com/product
+
+  # Limit output to 500 tokens
+  webpeel https://longpage.com --budget 500
+
+  # Silent JSON output (pipe-friendly)
+  webpeel https://example.com --json --silent | jq .tokens
+`.trimStart();
+export function registerGuideCommand(program) {
+    program
+        .command('guide')
+        .description('Print AI-optimized usage guide to stdout')
+        .action(() => {
+        process.stdout.write(GUIDE);
+        process.exit(0);
+    });
+}

package/dist/cli/commands/screenshot.js

@@ -119,7 +119,16 @@ export function registerScreenshotCommands(program) {
                 spinner.fail('Screenshot failed');
             }
             if (error instanceof Error) {
-                console.error(`\nError: ${error.message}`);
+                const msg = error.message;
+                // Detect missing browser binary and give an actionable error
+                if (msg.includes("Executable doesn't exist") || msg.includes('browserType.launch') || msg.includes('Chromium is not installed')) {
+                    console.error('\n\x1b[31m❌  Browser not installed.\x1b[0m');
+                    console.error('\x1b[36m   Run: npx playwright install chromium\x1b[0m');
+                    console.error('\x1b[36m   Then retry your screenshot command.\x1b[0m');
+                }
+                else {
+                    console.error(`\nError: ${msg}`);
+                }
             }
             else {
                 console.error('\nError: Unknown error occurred');

package/dist/cli/commands/search.js

@@ -49,6 +49,10 @@ export function registerSearchCommands(program) {
         .option('-s, --silent', 'Silent mode')
         .option('--proxy <url>', 'Proxy URL for requests (http://host:port, socks5://user:pass@host:port)')
         .option('--fetch', 'Also fetch and include content from each result URL')
+        .option('--local', 'Local business search via Google Places / Yelp (requires API key)')
+        .option('--location <location>', 'Location for local search (e.g. "Shibuya, Tokyo", "35.6595,139.7004")')
+        .option('--language <lang>', 'Language code for local search results (e.g. "ja", "fr")')
+        .option('--country <code>', 'ISO 3166-1 alpha-2 country code for local search (e.g. "JP", "FR")')
         .option('--agent', 'Agent mode: sets --json, --silent, and --budget 4000 (override with --budget N)')
         .action(async (query, options) => {
         // --agent sets sensible defaults for AI agents; explicit flags override
@@ -164,6 +168,54 @@ export function registerSearchCommands(program) {
                 process.exit(1);
             }
         }
+        // ── --local: local business search via Google Places / Yelp ─────────
+        if (options.local) {
+            const spinner = isSilent ? null : ora('Searching local businesses...').start();
+            try {
+                const { localSearch } = await import('../../core/local-search.js');
+                const localResults = await localSearch({
+                    query,
+                    location: options.location,
+                    language: options.language,
+                    country: options.country,
+                    limit: count,
+                });
+                if (spinner)
+                    spinner.succeed(`Found ${localResults.results.length} results (${localResults.source})`);
+                if (isJson) {
+                    await writeStdout(JSON.stringify(localResults, null, 2) + '\n');
+                }
+                else {
+                    if (localResults.results.length === 0) {
+                        await writeStdout('No local results found.\n');
+                    }
+                    else {
+                        await writeStdout(`\n📍 Local results for "${query}"${localResults.location ? ` near ${localResults.location}` : ''}\n`);
+                        await writeStdout(`Source: ${localResults.source}\n\n`);
+                        for (const [i, r] of localResults.results.entries()) {
+                            const rating = r.rating ? `⭐${r.rating}` : '';
+                            const reviews = r.reviewCount ? `(${r.reviewCount.toLocaleString()})` : '';
+                            const price = r.priceLevel !== undefined ? ` · ${'$'.repeat(Math.max(1, r.priceLevel))}` : '';
+                            const open = r.isOpen === true ? ' · 🟢 Open' : r.isOpen === false ? ' · 🔴 Closed' : '';
+                            await writeStdout(`${i + 1}. ${r.name} ${rating} ${reviews}${price}${open}\n`);
+                            if (r.address)
+                                await writeStdout(`   ${r.address}\n`);
+                            if (r.googleMapsUrl)
+                                await writeStdout(`   ${r.googleMapsUrl}\n`);
+                            await writeStdout('\n');
+                        }
+                    }
+                }
+                process.exit(0);
+            }
+            catch (err) {
+                if (spinner)
+                    spinner.fail('Local search failed');
+                console.error(`Error: ${err instanceof Error ? err.message : 'Unknown error'}`);
+                console.error('Hint: Set GOOGLE_PLACES_API_KEY or YELP_API_KEY environment variable for local search.');
+                process.exit(1);
+            }
+        }
         const spinner = isSilent ? null : ora('Searching...').start();
         try {
             // Route search through the WebPeel API when a key is configured

package/dist/cli/utils.js

@@ -296,7 +296,9 @@ export async function fetchViaApi(url, options, apiKey, apiUrl) {
         }
     }
     // Map API response to PeelResult shape that the CLI already handles
+    // Spread all API fields first, then override with normalized names
     return {
+        ...data,
         url: data.url || url,
         title: data.metadata?.title || data.title || '',
         content: data.content || '',
@@ -660,6 +662,10 @@ export async function outputResult(result, options, extra = {}) {
             output.focusReduction = result.focusReduction;
         if (result.extracted)
             output.extracted = result.extracted;
+        if (result.trust)
+            output.trust = result.trust;
+        if (result.safeBrowsing)
+            output.safeBrowsing = result.safeBrowsing;
         if (extra.cached)
             output.cached = true;
         if (extra.truncated)

package/dist/cli.js

@@ -37,6 +37,7 @@ import { registerAuthCommands } from './cli/commands/auth.js';
 import { registerScreenshotCommands } from './cli/commands/screenshot.js';
 import { registerJobsCommands } from './cli/commands/jobs.js';
 import { registerMonitorCommands } from './cli/commands/monitor.js';
+import { registerGuideCommand } from './cli/commands/guide.js';
 // ── Early silent/log-level detection (must happen before any async module code) ──
 // Set WEBPEEL_LOG_LEVEL early so logger checks see it when async IIFEs fire.
 if (!process.env.WEBPEEL_LOG_LEVEL && process.argv.includes('--silent')) {
@@ -83,5 +84,6 @@ registerAuthCommands(program);
 registerScreenshotCommands(program);
 registerJobsCommands(program);
 registerMonitorCommands(program);
+registerGuideCommand(program);
 // ── Parse ─────────────────────────────────────────────────────────────────────
 program.parse();

package/dist/core/actions.d.ts

@@ -28,20 +28,38 @@ export interface AutoScrollResult {
     finalHeight: number;
     /** Whether the page content grew during scrolling */
     contentGrew: boolean;
+    /** Whether a virtual/inner scrollable container was found and used */
+    scrollContainerFound?: boolean;
+    /** Total number of DOM mutations detected during scrolling */
+    mutationsDetected?: number;
 }
 /**
  * Normalize a raw actions array to WebPeel's internal PageAction shape.
  * Accepts Firecrawl-style fields (milliseconds, text, direction/amount).
  */
 export declare function normalizeActions(input?: unknown): PageAction[] | undefined;
+/**
+ * Detect the most likely scrollable container on the page.
+ * Returns a CSS selector string for the container, or null if only window scrolling is needed.
+ *
+ * Looks for elements with overflow-y: auto|scroll whose scrollHeight > clientHeight,
+ * preferring the largest such element. Used by autoScroll and scrollThrough.
+ */
+export declare function detectScrollContainer(page: Page): Promise<string | null>;
 /**
  * Intelligently scroll the page to load all lazy/infinite-scroll content.
  *
- * Scrolls to the bottom repeatedly, detecting height changes to determine
- * when new content has loaded. Stops when:
- * - Page height is stable for 2 consecutive checks
+ * Improvements over the basic version:
+ * 1. Detects virtual/inner scroll containers (Polymarket, React virtualized lists)
+ * 2. Uses MutationObserver to detect DOM additions (not just height changes)
+ * 3. Gracefully handles execution context destruction (SPA navigation)
+ * 4. Stability requires BOTH no height change AND no DOM mutations
+ *
+ * Stops when:
+ * - Height is stable AND no DOM mutations for 2 consecutive checks
  * - maxScrolls limit is reached
  * - Total timeout is exceeded
+ * - Execution context is destroyed (SPA navigation)
  */
 export declare function autoScroll(page: Page, options?: AutoScrollOptions): Promise<AutoScrollResult>;
 export declare function executeActions(page: Page, actions: PageAction[], screenshotOptions?: {