webpeel 0.21.78 → 0.21.79

package/dist/cli/commands/search.js

@@ -631,6 +631,122 @@ export function registerSearchCommands(program) {
             process.exit(1);
         }
     });
+    // ── rental command ────────────────────────────────────────────────────────
+    program
+        .command('rental <query>')
+        .alias('car-rental')
+        .description('Search for car rentals via Kayak — e.g. "Punta Gorda FL Apr 1-3"')
+        .option('--json', 'Output as JSON')
+        .option('-s, --silent', 'Silent mode')
+        .action(async (query, options) => {
+        // Parse location: strip date portion from query
+        const location = query.replace(/\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*\s+\d+.*/i, '').trim();
+        const encodedLocation = encodeURIComponent(location.replace(/\s+/g, '-'));
+        // Parse dates: try "Apr 1-3" or "Apr 1 to Apr 3" patterns
+        const year = new Date().getFullYear();
+        let pickupDate = `${year}-04-01`;
+        let returnDate = `${year}-04-03`;
+        const rangeMatch = query.match(/\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*\s+(\d+)\s*[-–to]+\s*(?:(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*\s+)?(\d+)/i);
+        if (rangeMatch) {
+            const months = {
+                jan: '01', feb: '02', mar: '03', apr: '04', may: '05', jun: '06',
+                jul: '07', aug: '08', sep: '09', oct: '10', nov: '11', dec: '12',
+            };
+            const startMonth = months[rangeMatch[1].toLowerCase().slice(0, 3)];
+            const startDay = rangeMatch[2].padStart(2, '0');
+            const endMonth = rangeMatch[3] ? months[rangeMatch[3].toLowerCase().slice(0, 3)] : startMonth;
+            const endDay = rangeMatch[4].padStart(2, '0');
+            pickupDate = `${year}-${startMonth}-${startDay}`;
+            returnDate = `${year}-${endMonth}-${endDay}`;
+        }
+        const searchUrl = `https://www.kayak.com/cars/${encodedLocation}/${pickupDate}/${returnDate}?sort=price_a`;
+        const spinner = options.silent ? null : (await import('ora')).default(`Searching car rentals: ${query}...`).start();
+        try {
+            const result = await peel(searchUrl, { render: true, timeout: 40000 });
+            if (spinner)
+                spinner.succeed('Car rentals loaded');
+            if (options.json) {
+                console.log(JSON.stringify({
+                    query,
+                    location,
+                    pickupDate,
+                    returnDate,
+                    url: searchUrl,
+                    content: result.content,
+                    tokens: result.tokens,
+                }, null, 2));
+            }
+            else {
+                console.log(result.content);
+            }
+            await cleanup();
+            process.exit(0);
+        }
+        catch (error) {
+            if (spinner)
+                spinner.fail('Car rental search failed');
+            console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            await cleanup();
+            process.exit(1);
+        }
+    });
+    // ── cars command ──────────────────────────────────────────────────────────
+    program
+        .command('cars <query>')
+        .description('Search for cars to buy via Cars.com — e.g. "Honda Civic"')
+        .option('--zip <zip>', 'ZIP code for local search', '10001')
+        .option('--distance <miles>', 'Max distance in miles', '30')
+        .option('--max-price <price>', 'Maximum listing price')
+        .option('--min-price <price>', 'Minimum listing price')
+        .option('--json', 'Output as JSON')
+        .option('-s, --silent', 'Silent mode')
+        .action(async (query, options) => {
+        const zip = options.zip || '10001';
+        const distance = options.distance || '30';
+        const maxPrice = options.maxPrice || '';
+        const minPrice = options.minPrice || '';
+        const params = new URLSearchParams({
+            keyword: query,
+            sort: 'list_price',
+            stock_type: 'all',
+            zip,
+            maximum_distance: distance,
+        });
+        if (maxPrice)
+            params.set('list_price_max', maxPrice);
+        if (minPrice)
+            params.set('list_price_min', minPrice);
+        const url = `https://www.cars.com/shopping/results/?${params.toString()}`;
+        const spinner = options.silent ? null : (await import('ora')).default(`Searching cars: ${query}...`).start();
+        try {
+            const result = await peel(url, { timeout: 25000 });
+            if (spinner)
+                spinner.succeed('Cars loaded');
+            if (options.json) {
+                console.log(JSON.stringify({
+                    query,
+                    zip,
+                    distance,
+                    maxPrice,
+                    url,
+                    content: result.content,
+                    tokens: result.tokens,
+                }, null, 2));
+            }
+            else {
+                console.log(result.content);
+            }
+            await cleanup();
+            process.exit(0);
+        }
+        catch (error) {
+            if (spinner)
+                spinner.fail('Car search failed');
+            console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            await cleanup();
+            process.exit(1);
+        }
+    });
     // ── extractors command ────────────────────────────────────────────────────
     program
         .command('extractors')

package/dist/core/browser-fetch.d.ts

@@ -56,6 +56,12 @@ export declare function browserFetch(url: string, options?: {
     waitSelector?: string;
     /** Block resource types for faster loading: 'image', 'stylesheet', 'font', 'media', 'script' */
     blockResources?: string[];
+    /**
+     * Whether the target is a Single-Page Application (Kayak, Google Flights, Expedia, etc).
+     * When true, the DOM stability check uses a longer timeout (12s) to wait for async data loads.
+     * When false (default), a shorter 3s stability window is used.
+     */
+    isSPA?: boolean;
 }): Promise<FetchResult>;
 /**
  * Capture a screenshot of a URL using headless Chromium via Playwright.

package/dist/core/browser-fetch.js

@@ -30,7 +30,7 @@ let activePagesCount = 0;
 export async function browserFetch(url, options = {}) {
     // SECURITY: Validate URL to prevent SSRF
     validateUrl(url);
-    const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, stealth = false, actions, keepPageOpen = false, signal, profileDir, headed = false, storageState, proxy, device = 'desktop', viewportWidth: optViewportWidth, viewportHeight: optViewportHeight, waitUntil: optWaitUntil, waitSelector, blockResources, } = options;
+    const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, stealth = false, actions, keepPageOpen = false, signal, profileDir, headed = false, storageState, proxy, device = 'desktop', viewportWidth: optViewportWidth, viewportHeight: optViewportHeight, waitUntil: optWaitUntil, waitSelector, blockResources, isSPA = false, } = options;
     // Device emulation profiles
     const deviceProfiles = {
         desktop: { width: 1920, height: 1080, userAgent: undefined },
@@ -317,27 +317,37 @@ export async function browserFetch(url, options = {}) {
                 throwIfAborted();
             }
             // DOM stability check: wait for SPA hydration to settle.
-            // Polls innerText length every 500ms — if still growing, keep waiting (max 3s extra).
+            // Polls innerText length every 500ms — if still growing, keep waiting.
+            // SPAs (Kayak, Google Flights, Expedia) get a longer timeout to allow async data loads.
             {
                 const stabilityStart = Date.now();
-                const MAX_STABILITY_WAIT_MS = 3000;
+                // SPA sites (Kayak, Google Flights, Expedia) need up to 12s for results to load.
+                // Normal rendered pages need just 3s extra.
+                const MAX_STABILITY_WAIT_MS = isSPA ? 12000 : 3000;
+                // SPA: must be stable for 2s (4 consecutive 500ms checks). Normal: 1s (2 checks).
+                const STABLE_CHECKS_REQUIRED = isSPA ? 4 : 2;
                 const POLL_INTERVAL_MS = 500;
+                const MIN_CONTENT_LENGTH = 200; // Don't consider near-empty pages stable
                 let prevLength = await page.evaluate('document.body?.innerText?.length || 0').catch(() => 0);
                 let stableCount = 0;
                 while (Date.now() - stabilityStart < MAX_STABILITY_WAIT_MS) {
                     throwIfAborted();
                     await page.waitForTimeout(POLL_INTERVAL_MS);
                     const curLength = await page.evaluate('document.body?.innerText?.length || 0').catch(() => 0);
-                    if (curLength === prevLength) {
-                        stableCount++;
-                        if (stableCount >= 2)
-                            break; // stable for 2 consecutive checks (~1s)
-                    }
-                    else {
+                    if (curLength !== prevLength) {
+                        // Content changed — reset stability counter
                         stableCount = 0;
                     }
+                    else if (curLength >= MIN_CONTENT_LENGTH) {
+                        stableCount++;
+                        if (stableCount >= STABLE_CHECKS_REQUIRED)
+                            break; // stable long enough
+                    }
                     prevLength = curLength;
                 }
+                if (isSPA) {
+                    log.debug(`SPA stability check: ${Date.now() - stabilityStart}ms, length=${prevLength}`);
+                }
             }
             const finalUrl = page.url();
             const contentType = response?.headers()?.['content-type'] || '';

package/dist/core/strategies.js

@@ -232,7 +232,7 @@ function prefetchDns(url) {
     }
 }
 async function fetchWithBrowserStrategy(url, options) {
-    const { userAgent, waitMs, timeoutMs, screenshot, screenshotFullPage, headers, cookies, actions, keepPageOpen, effectiveStealth, signal, profileDir, headed, storageState, proxy, device, viewportWidth, viewportHeight, waitUntil, waitSelector, blockResources, } = options;
+    const { userAgent, waitMs, timeoutMs, screenshot, screenshotFullPage, headers, cookies, actions, keepPageOpen, effectiveStealth, signal, profileDir, headed, storageState, proxy, device, viewportWidth, viewportHeight, waitUntil, waitSelector, blockResources, isSPA, } = options;
     try {
         const result = await browserFetch(url, {
             userAgent,
@@ -256,6 +256,7 @@ async function fetchWithBrowserStrategy(url, options) {
             waitUntil,
             waitSelector,
             blockResources,
+            isSPA,
         });
         return {
             ...result,
@@ -500,6 +501,21 @@ export async function smartFetch(url, options = {}) {
     if (storageState) {
         effectiveForceBrowser = true;
     }
+    // Detect SPA for smarter DOM stability wait
+    const SPA_FETCH_DOMAINS = new Set([
+        'www.google.com', 'flights.google.com', 'www.airbnb.com', 'www.booking.com',
+        'www.expedia.com', 'www.kayak.com', 'www.skyscanner.com', 'www.tripadvisor.com',
+        'www.indeed.com', 'www.glassdoor.com', 'www.zillow.com', 'app.webpeel.dev',
+    ]);
+    const SPA_FETCH_URL_PATTERNS = [
+        /google\.com\/travel/, /google\.com\/maps/, /google\.com\/shopping/,
+    ];
+    let isSPAUrl = false;
+    try {
+        const parsedHostname = new URL(url).hostname;
+        isSPAUrl = SPA_FETCH_DOMAINS.has(parsedHostname) || SPA_FETCH_URL_PATTERNS.some(p => p.test(url));
+    }
+    catch { /* invalid URL — ignore */ }
     const browserOptions = {
         userAgent,
         waitMs,
@@ -521,6 +537,7 @@ export async function smartFetch(url, options = {}) {
         waitUntil,
         waitSelector,
         blockResources,
+        isSPA: isSPAUrl,
     };
     /* ---- Strategy: simple fetch (with optional race) --------------------- */
     if (!shouldUseBrowser) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.78",
+  "version": "0.21.79",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",