webpeel 0.21.78 → 0.21.80

package/dist/cli/commands/search.js

@@ -6,6 +6,31 @@ import { readFileSync } from 'fs';
 import { peel, peelBatch, cleanup } from '../../index.js';
 import { checkUsage, showUsageFooter, loadConfig } from '../../cli-auth.js';
 import { writeStdout, formatListingsCsv } from '../utils.js';
+/**
+ * Parse a date range string like "Mar29-Apr4" into an array of date strings.
+ * Returns ["Mar 29", "Mar 30", ..., "Apr 4"]
+ */
+function parseDateRange(range) {
+    const match = range.match(/(\w{3})\s*(\d{1,2})\s*[-–to]+\s*(\w{3})\s*(\d{1,2})/i);
+    if (!match)
+        return [];
+    const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+    const startMonthIdx = months.findIndex(m => m.toLowerCase() === match[1].toLowerCase().slice(0, 3));
+    const endMonthIdx = months.findIndex(m => m.toLowerCase() === match[3].toLowerCase().slice(0, 3));
+    if (startMonthIdx === -1 || endMonthIdx === -1)
+        return [];
+    const startDay = parseInt(match[2]);
+    const endDay = parseInt(match[4]);
+    const year = new Date().getFullYear();
+    const dates = [];
+    const start = new Date(year, startMonthIdx, startDay);
+    const end = new Date(year, endMonthIdx, endDay);
+    for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) {
+        const mon = months[d.getMonth()];
+        dates.push(`${mon} ${d.getDate()}`);
+    }
+    return dates;
+}
 export function registerSearchCommands(program) {
     // ── search command ────────────────────────────────────────────────────────
     program
@@ -595,9 +620,90 @@ export function registerSearchCommands(program) {
         .option('--one-way', 'One-way flight (default)')
         .option('--round-trip', 'Round-trip flight')
         .option('-n, --count <n>', 'Max flights to show', '10')
+        .option('--dates <range>', 'Compare prices across date range (e.g., "Mar29-Apr4")')
         .option('--json', 'Output as JSON')
         .option('-s, --silent', 'Silent mode')
         .action(async (query, options) => {
+        // ── --dates: compare cheapest flight across a date range ──────────────
+        if (options.dates) {
+            const dates = parseDateRange(options.dates);
+            if (dates.length === 0) {
+                console.error('Could not parse date range. Format: "Mar29-Apr4"');
+                process.exit(1);
+            }
+            const spinner = options.silent ? null : ora(`Comparing flights across ${dates.length} dates...`).start();
+            const tripType = options.roundTrip ? '' : ' one way';
+            const rows = [];
+            for (const date of dates) {
+                if (spinner)
+                    spinner.text = `Fetching flights for ${date}...`;
+                try {
+                    const dateQuery = `Flights from ${query} ${date}${tripType}`;
+                    const encoded = encodeURIComponent(dateQuery);
+                    const url = `https://www.google.com/travel/flights?q=${encoded}`;
+                    const result = await peel(url, { render: true, timeout: 30000 });
+                    // Try to extract cheapest flight from structured data or content
+                    let price = null;
+                    let airline = null;
+                    let time = null;
+                    const flights = result.domainData?.structured?.flights || [];
+                    if (flights.length > 0) {
+                        const cheapest = flights.reduce((a, b) => {
+                            const ap = parseFloat(String(a.price || '').replace(/[^0-9.]/g, '')) || Infinity;
+                            const bp = parseFloat(String(b.price || '').replace(/[^0-9.]/g, '')) || Infinity;
+                            return ap <= bp ? a : b;
+                        });
+                        price = cheapest.priceStr || (cheapest.price ? `$${cheapest.price}` : null);
+                        airline = cheapest.airline || cheapest.carrier || null;
+                        time = cheapest.departTime && cheapest.arriveTime
+                            ? `${cheapest.departTime} → ${cheapest.arriveTime}`
+                            : (cheapest.time || cheapest.departure || null);
+                    }
+                    else {
+                        // Extract from markdown content — look for price patterns
+                        const priceMatch = result.content.match(/\$(\d+)/);
+                        if (priceMatch)
+                            price = `$${priceMatch[1]}`;
+                        const airlineMatch = result.content.match(/\b(American|Delta|United|Southwest|Spirit|JetBlue|Alaska|Frontier|Allegiant|Sun Country)\b/i);
+                        if (airlineMatch)
+                            airline = airlineMatch[1];
+                        const timeMatch = result.content.match(/(\d{1,2}:\d{2}\s*(?:AM|PM))\s*[–—→]\s*(\d{1,2}:\d{2}\s*(?:AM|PM))/i);
+                        if (timeMatch)
+                            time = `${timeMatch[1]} → ${timeMatch[2]}`;
+                    }
+                    const priceNum = price ? parseFloat(price.replace(/[^0-9.]/g, '')) || Infinity : Infinity;
+                    rows.push({ date, price, airline, time, priceNum });
+                }
+                catch {
+                    rows.push({ date, price: null, airline: null, time: null, priceNum: Infinity });
+                }
+            }
+            if (spinner)
+                spinner.succeed(`Compared ${rows.length} dates`);
+            if (options.json) {
+                console.log(JSON.stringify({ query, dateRange: options.dates, rows }, null, 2));
+            }
+            else {
+                // Find best price
+                const best = rows.reduce((a, b) => a.priceNum <= b.priceNum ? a : b);
+                console.log(`\n# ✈️ Flight Price Comparison — ${query}\n`);
+                console.log('| Date | Airline | Time | Price |');
+                console.log('|------|---------|------|-------|');
+                for (const row of rows) {
+                    const star = row.priceNum === best.priceNum ? ' ⭐' : '';
+                    const priceStr = row.price ? `${row.price}${star}` : 'N/A';
+                    const airlineStr = row.airline || 'Unknown';
+                    const timeStr = row.time || '—';
+                    console.log(`| ${row.date} | ${airlineStr} | ${timeStr} | ${priceStr} |`);
+                }
+                if (best.price) {
+                    console.log(`\n⭐ Best price: ${best.date} — ${best.airline || 'Unknown'} ${best.price}`);
+                }
+            }
+            await cleanup();
+            process.exit(0);
+        }
+        // ── Single date (default) ─────────────────────────────────────────────
         const tripType = options.roundTrip ? '' : ' one way';
         const encoded = encodeURIComponent(`Flights from ${query}${tripType}`);
         const url = `https://www.google.com/travel/flights?q=${encoded}`;
@@ -631,6 +737,122 @@ export function registerSearchCommands(program) {
             process.exit(1);
         }
     });
+    // ── rental command ────────────────────────────────────────────────────────
+    program
+        .command('rental <query>')
+        .alias('car-rental')
+        .description('Search for car rentals via Kayak — e.g. "Punta Gorda FL Apr 1-3"')
+        .option('--json', 'Output as JSON')
+        .option('-s, --silent', 'Silent mode')
+        .action(async (query, options) => {
+        // Parse location: strip date portion from query
+        const location = query.replace(/\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*\s+\d+.*/i, '').trim();
+        const encodedLocation = encodeURIComponent(location.replace(/\s+/g, '-'));
+        // Parse dates: try "Apr 1-3" or "Apr 1 to Apr 3" patterns
+        const year = new Date().getFullYear();
+        let pickupDate = `${year}-04-01`;
+        let returnDate = `${year}-04-03`;
+        const rangeMatch = query.match(/\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*\s+(\d+)\s*[-–to]+\s*(?:(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*\s+)?(\d+)/i);
+        if (rangeMatch) {
+            const months = {
+                jan: '01', feb: '02', mar: '03', apr: '04', may: '05', jun: '06',
+                jul: '07', aug: '08', sep: '09', oct: '10', nov: '11', dec: '12',
+            };
+            const startMonth = months[rangeMatch[1].toLowerCase().slice(0, 3)];
+            const startDay = rangeMatch[2].padStart(2, '0');
+            const endMonth = rangeMatch[3] ? months[rangeMatch[3].toLowerCase().slice(0, 3)] : startMonth;
+            const endDay = rangeMatch[4].padStart(2, '0');
+            pickupDate = `${year}-${startMonth}-${startDay}`;
+            returnDate = `${year}-${endMonth}-${endDay}`;
+        }
+        const searchUrl = `https://www.kayak.com/cars/${encodedLocation}/${pickupDate}/${returnDate}?sort=price_a`;
+        const spinner = options.silent ? null : (await import('ora')).default(`Searching car rentals: ${query}...`).start();
+        try {
+            const result = await peel(searchUrl, { render: true, timeout: 40000 });
+            if (spinner)
+                spinner.succeed('Car rentals loaded');
+            if (options.json) {
+                console.log(JSON.stringify({
+                    query,
+                    location,
+                    pickupDate,
+                    returnDate,
+                    url: searchUrl,
+                    content: result.content,
+                    tokens: result.tokens,
+                }, null, 2));
+            }
+            else {
+                console.log(result.content);
+            }
+            await cleanup();
+            process.exit(0);
+        }
+        catch (error) {
+            if (spinner)
+                spinner.fail('Car rental search failed');
+            console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            await cleanup();
+            process.exit(1);
+        }
+    });
+    // ── cars command ──────────────────────────────────────────────────────────
+    program
+        .command('cars <query>')
+        .description('Search for cars to buy via Cars.com — e.g. "Honda Civic"')
+        .option('--zip <zip>', 'ZIP code for local search', '10001')
+        .option('--distance <miles>', 'Max distance in miles', '30')
+        .option('--max-price <price>', 'Maximum listing price')
+        .option('--min-price <price>', 'Minimum listing price')
+        .option('--json', 'Output as JSON')
+        .option('-s, --silent', 'Silent mode')
+        .action(async (query, options) => {
+        const zip = options.zip || '10001';
+        const distance = options.distance || '30';
+        const maxPrice = options.maxPrice || '';
+        const minPrice = options.minPrice || '';
+        const params = new URLSearchParams({
+            keyword: query,
+            sort: 'list_price',
+            stock_type: 'all',
+            zip,
+            maximum_distance: distance,
+        });
+        if (maxPrice)
+            params.set('list_price_max', maxPrice);
+        if (minPrice)
+            params.set('list_price_min', minPrice);
+        const url = `https://www.cars.com/shopping/results/?${params.toString()}`;
+        const spinner = options.silent ? null : (await import('ora')).default(`Searching cars: ${query}...`).start();
+        try {
+            const result = await peel(url, { timeout: 25000 });
+            if (spinner)
+                spinner.succeed('Cars loaded');
+            if (options.json) {
+                console.log(JSON.stringify({
+                    query,
+                    zip,
+                    distance,
+                    maxPrice,
+                    url,
+                    content: result.content,
+                    tokens: result.tokens,
+                }, null, 2));
+            }
+            else {
+                console.log(result.content);
+            }
+            await cleanup();
+            process.exit(0);
+        }
+        catch (error) {
+            if (spinner)
+                spinner.fail('Car search failed');
+            console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            await cleanup();
+            process.exit(1);
+        }
+    });
     // ── extractors command ────────────────────────────────────────────────────
     program
         .command('extractors')

package/dist/core/browser-fetch.d.ts

@@ -56,6 +56,12 @@ export declare function browserFetch(url: string, options?: {
     waitSelector?: string;
     /** Block resource types for faster loading: 'image', 'stylesheet', 'font', 'media', 'script' */
     blockResources?: string[];
+    /**
+     * Whether the target is a Single-Page Application (Kayak, Google Flights, Expedia, etc).
+     * When true, the DOM stability check uses a longer timeout (12s) to wait for async data loads.
+     * When false (default), a shorter 3s stability window is used.
+     */
+    isSPA?: boolean;
 }): Promise<FetchResult>;
 /**
  * Capture a screenshot of a URL using headless Chromium via Playwright.

package/dist/core/browser-fetch.js

@@ -30,7 +30,7 @@ let activePagesCount = 0;
 export async function browserFetch(url, options = {}) {
     // SECURITY: Validate URL to prevent SSRF
     validateUrl(url);
-    const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, stealth = false, actions, keepPageOpen = false, signal, profileDir, headed = false, storageState, proxy, device = 'desktop', viewportWidth: optViewportWidth, viewportHeight: optViewportHeight, waitUntil: optWaitUntil, waitSelector, blockResources, } = options;
+    const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, stealth = false, actions, keepPageOpen = false, signal, profileDir, headed = false, storageState, proxy, device = 'desktop', viewportWidth: optViewportWidth, viewportHeight: optViewportHeight, waitUntil: optWaitUntil, waitSelector, blockResources, isSPA = false, } = options;
     // Device emulation profiles
     const deviceProfiles = {
         desktop: { width: 1920, height: 1080, userAgent: undefined },
@@ -317,27 +317,37 @@ export async function browserFetch(url, options = {}) {
                 throwIfAborted();
             }
             // DOM stability check: wait for SPA hydration to settle.
-            // Polls innerText length every 500ms — if still growing, keep waiting (max 3s extra).
+            // Polls innerText length every 500ms — if still growing, keep waiting.
+            // SPAs (Kayak, Google Flights, Expedia) get a longer timeout to allow async data loads.
             {
                 const stabilityStart = Date.now();
-                const MAX_STABILITY_WAIT_MS = 3000;
+                // SPA sites (Kayak, Google Flights, Expedia) need up to 12s for results to load.
+                // Normal rendered pages need just 3s extra.
+                const MAX_STABILITY_WAIT_MS = isSPA ? 12000 : 3000;
+                // SPA: must be stable for 2s (4 consecutive 500ms checks). Normal: 1s (2 checks).
+                const STABLE_CHECKS_REQUIRED = isSPA ? 4 : 2;
                 const POLL_INTERVAL_MS = 500;
+                const MIN_CONTENT_LENGTH = 200; // Don't consider near-empty pages stable
                 let prevLength = await page.evaluate('document.body?.innerText?.length || 0').catch(() => 0);
                 let stableCount = 0;
                 while (Date.now() - stabilityStart < MAX_STABILITY_WAIT_MS) {
                     throwIfAborted();
                     await page.waitForTimeout(POLL_INTERVAL_MS);
                     const curLength = await page.evaluate('document.body?.innerText?.length || 0').catch(() => 0);
-                    if (curLength === prevLength) {
-                        stableCount++;
-                        if (stableCount >= 2)
-                            break; // stable for 2 consecutive checks (~1s)
-                    }
-                    else {
+                    if (curLength !== prevLength) {
+                        // Content changed — reset stability counter
                         stableCount = 0;
                     }
+                    else if (curLength >= MIN_CONTENT_LENGTH) {
+                        stableCount++;
+                        if (stableCount >= STABLE_CHECKS_REQUIRED)
+                            break; // stable long enough
+                    }
                     prevLength = curLength;
                 }
+                if (isSPA) {
+                    log.debug(`SPA stability check: ${Date.now() - stabilityStart}ms, length=${prevLength}`);
+                }
             }
             const finalUrl = page.url();
             const contentType = response?.headers()?.['content-type'] || '';

package/dist/core/domain-extractors.js

@@ -123,6 +123,7 @@ const REGISTRY = [
     { match: (h) => h === 'redfin.com' || h === 'www.redfin.com', extractor: redfinExtractor },
     // ── Travel ──────────────────────────────────────────────────────────────
     { match: (h, url = '') => (h === 'www.google.com' || h === 'google.com') && url.includes('/travel/flights'), extractor: googleFlightsExtractor },
+    { match: (h, url = '') => (h === 'www.kayak.com' || h === 'kayak.com') && url.includes('/cars/'), extractor: kayakCarRentalExtractor },
 ];
 /**
  * Returns the domain extractor for a URL, or null if none matches.
@@ -6069,3 +6070,273 @@ async function googleFlightsExtractor(_html, url) {
         cleanContent: md.join('\n'),
     };
 }
+// ---------------------------------------------------------------------------
+// Kayak Car Rental extractor
+// ---------------------------------------------------------------------------
+async function kayakCarRentalExtractor(_html, url) {
+    if (!url.includes('/cars/'))
+        return null;
+    // Rental company homepage URLs
+    const rentalCompanyUrls = {
+        'Hertz': 'https://www.hertz.com',
+        'Budget': 'https://www.budget.com',
+        'Avis': 'https://www.avis.com',
+        'Enterprise': 'https://www.enterprise.com',
+        'National': 'https://www.nationalcar.com',
+        'Alamo': 'https://www.alamo.com',
+        'Dollar': 'https://www.dollar.com',
+        'Thrifty': 'https://www.thrifty.com',
+        'Sixt': 'https://www.sixt.com',
+        'Fox': 'https://www.foxrentacar.com',
+        'Payless': 'https://www.paylesscar.com',
+        'Turn': 'https://www.turn.com',
+        'EconomyBookings': 'https://www.economybookings.com',
+        'Priceline': 'https://www.priceline.com',
+        'Expedia': 'https://www.expedia.com',
+        'Turo': 'https://www.turo.com',
+        'KAYAK': 'https://www.kayak.com',
+        'Booking.com': 'https://www.booking.com',
+        'DiscoverCars': 'https://www.discovercars.com',
+        'RentalCars': 'https://www.rentalcars.com',
+        'Car Rental 8': 'https://www.carrental8.com',
+        'Hotwire': 'https://www.hotwire.com',
+    };
+    function getCompanyUrl(company) {
+        return rentalCompanyUrls[company] || `https://www.kayak.com`;
+    }
+    // Parse dates from URL: /cars/Location/YYYY-MM-DD/YYYY-MM-DD
+    let numDays = 1;
+    let pickupDate = '';
+    let dropoffDate = '';
+    let locationName = '';
+    const dateMatch = url.match(/\/cars\/([^/]+)\/(\d{4}-\d{2}-\d{2})\/(\d{4}-\d{2}-\d{2})/);
+    if (dateMatch) {
+        locationName = decodeURIComponent(dateMatch[1]);
+        pickupDate = dateMatch[2];
+        dropoffDate = dateMatch[3];
+        const pickup = new Date(pickupDate);
+        const dropoff = new Date(dropoffDate);
+        numDays = Math.max(1, Math.round((dropoff.getTime() - pickup.getTime()) / (1000 * 60 * 60 * 24)));
+    }
+    // Format date range for display (e.g. "Apr 1–3")
+    function formatDateRange(from, to) {
+        if (!from || !to)
+            return '';
+        const fromDate = new Date(from + 'T12:00:00');
+        const toDate = new Date(to + 'T12:00:00');
+        const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+        const fromMonth = months[fromDate.getUTCMonth()];
+        const toMonth = months[toDate.getUTCMonth()];
+        const fromDay = fromDate.getUTCDate();
+        const toDay = toDate.getUTCDate();
+        if (fromMonth === toMonth)
+            return `${fromMonth} ${fromDay}–${toDay}`;
+        return `${fromMonth} ${fromDay}–${toMonth} ${toDay}`;
+    }
+    // Process content: strip HTML if needed
+    let text = _html;
+    if (text.includes('<!DOCTYPE') || text.includes('<html')) {
+        text = text
+            .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+            .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+            .replace(/<[^>]+>/g, '\n')
+            .replace(/&amp;/g, '&')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&#\d+;/g, '')
+            .replace(/\n{2,}/g, '\n');
+    }
+    const lines = text.split('\n').map(l => l.trim()).filter(Boolean);
+    const KNOWN_COMPANIES = ['Hertz', 'Budget', 'Avis', 'Enterprise', 'National', 'Alamo', 'Dollar', 'Thrifty', 'Sixt', 'Fox', 'Payless', 'Turn', 'EconomyBookings', 'Priceline', 'Expedia', 'Turo', 'KAYAK', 'Booking.com', 'DiscoverCars', 'RentalCars', 'Car Rental 8', 'Hotwire'];
+    const listings = [];
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        // Detect: "or similar {Class}" — this signals a standard car rental listing
+        // The car name is the line BEFORE "or similar"
+        const orSimilarMatch = line.match(/^or similar\s+(.+)$/);
+        if (orSimilarMatch) {
+            const carClass = orSimilarMatch[1].trim();
+            const carName = i > 0 ? lines[i - 1] : '';
+            if (!carName || carName.length > 60)
+                continue;
+            // Look ahead for: pickup location, rating, company, price
+            let location = '';
+            let distanceFromCenter = '';
+            let rating = null;
+            let company = '';
+            let totalPrice = 0;
+            let cancellation = '';
+            for (let j = i + 1; j < Math.min(i + 30, lines.length); j++) {
+                const l = lines[j];
+                // Pickup location
+                if (!location && l.startsWith('Pick-up')) {
+                    const locMatch = l.match(/Pick-up (?:city|airport):\s*(.+)/);
+                    if (locMatch)
+                        location = locMatch[1].trim();
+                    continue;
+                }
+                // Distance from center
+                if (!distanceFromCenter) {
+                    const distM = l.match(/^([\d.]+)\s+mi\s+from\s+city\s+center/);
+                    if (distM) {
+                        distanceFromCenter = `${distM[1]} mi from city center`;
+                        continue;
+                    }
+                }
+                // Rating (number like "9.2", "8.5", "7.2")
+                if (rating === null) {
+                    const ratingM = l.match(/^(\d+\.\d+)$/);
+                    if (ratingM) {
+                        rating = parseFloat(ratingM[1]);
+                        continue;
+                    }
+                }
+                // Company from "X offer from {Company}" or "{Company}" line
+                if (!company) {
+                    const offerMatch = l.match(/offer from (.+)$/);
+                    if (offerMatch) {
+                        company = offerMatch[1].trim();
+                        continue;
+                    }
+                    // Also detect company name standalone
+                    for (const c of KNOWN_COMPANIES) {
+                        if (l === c) {
+                            company = c;
+                            break;
+                        }
+                    }
+                    if (company)
+                        continue;
+                }
+                // Cancellation policy
+                if (!cancellation && (l.includes('Free cancellation') || l.includes('No free cancellation'))) {
+                    cancellation = l;
+                    continue;
+                }
+                // Price — "$NNN" followed by "Total"
+                const priceM = l.match(/^\$(\d[\d,]*)$/);
+                if (priceM) {
+                    const nextLine = lines[j + 1] || '';
+                    if (nextLine === 'Total' || nextLine.includes('Total')) {
+                        totalPrice = parseInt(priceM[1].replace(',', ''));
+                        break;
+                    }
+                }
+                // Also catch price on same line
+                const inlinePriceM = l.match(/\$(\d[\d,]*)\s*Total/);
+                if (inlinePriceM) {
+                    totalPrice = parseInt(inlinePriceM[1].replace(',', ''));
+                    break;
+                }
+                // Stop if we hit another car listing marker
+                if (l.match(/^or similar\s/) || l === 'Show more results')
+                    break;
+            }
+            if (carName && totalPrice > 0) {
+                const normalizedClass = carClass.replace('Full size', 'Full-size');
+                listings.push({
+                    name: carName,
+                    carClass: normalizedClass,
+                    totalPrice,
+                    perDayPrice: Math.round(totalPrice / numDays),
+                    company: company || 'Unknown',
+                    location: location || 'See booking',
+                    distanceFromCenter,
+                    rating,
+                    cancellation,
+                    isTuro: false,
+                });
+            }
+        }
+    }
+    // Deduplicate: first prefer listings with real company info over "Unknown"
+    // Key by name+price; keep the one with best data
+    const byKey = new Map();
+    for (const c of listings) {
+        const key = `${c.name.toLowerCase()}-${c.totalPrice}`;
+        const existing = byKey.get(key);
+        if (!existing) {
+            byKey.set(key, c);
+        }
+        else {
+            // Prefer non-Unknown company, or same company with more info
+            if (existing.company === 'Unknown' && c.company !== 'Unknown') {
+                byKey.set(key, c);
+            }
+        }
+    }
+    const unique = Array.from(byKey.values());
+    if (unique.length === 0)
+        return null;
+    // Filter out Unknown company entries if the total found from page suggests more results exist
+    // Also filter them only if they have no location info (these are likely ad/promo extractions)
+    const knownCompanyListings = unique.filter(c => c.company !== 'Unknown');
+    const finalListings = knownCompanyListings.length > 0 ? knownCompanyListings : unique;
+    // Sort by price
+    finalListings.sort((a, b) => a.totalPrice - b.totalPrice);
+    // Get total count from page if mentioned
+    let totalFound = unique.length;
+    for (const l of lines) {
+        const m = l.match(/^(\d+)\s+results?$/);
+        if (m) {
+            totalFound = parseInt(m[1]);
+            break;
+        }
+        const m2 = l.match(/(\d+)\s+cars?\s+found/);
+        if (m2) {
+            totalFound = parseInt(m2[1]);
+            break;
+        }
+    }
+    // Format location name nicely (e.g. "Punta-Gorda,FL-c34451" → "Punta Gorda, FL")
+    function formatLocation(loc) {
+        return loc
+            .replace(/-c\d+$/, '') // remove trailing "-c12345"
+            .replace(/-/g, ' ') // hyphens to spaces
+            .replace(/,(\S)/g, ', $1'); // ensure space after comma
+    }
+    const dateRange = formatDateRange(pickupDate, dropoffDate);
+    const displayLocation = formatLocation(locationName);
+    const daysLabel = numDays === 1 ? '1 day' : `${numDays} days`;
+    const md = [
+        `# 🚗 Car Rentals — ${displayLocation} · ${dateRange} (${daysLabel})`,
+        '',
+        `*${totalFound} cars found · Source: [Kayak](${url})*`,
+        `*Free cancellation available on most rentals*`,
+        '',
+    ];
+    for (let idx = 0; idx < finalListings.length; idx++) {
+        const c = finalListings[idx];
+        md.push(`## ${idx + 1}. ${c.name} (${c.carClass}) — $${c.totalPrice} total · $${c.perDayPrice}/day`);
+        if (c.distanceFromCenter) {
+            md.push(`📍 ${c.distanceFromCenter}`);
+        }
+        else if (c.location && c.location !== 'See booking') {
+            md.push(`📍 ${c.location}`);
+        }
+        const ratingStr = c.rating !== null ? ` · Rating: ${c.rating}` : '';
+        md.push(`🏪 via ${c.company}${ratingStr}`);
+        if (c.cancellation)
+            md.push(`✅ ${c.cancellation}`);
+        md.push(`🔍 [See price on Kayak](${url})`);
+        md.push(`🛒 [Book on ${c.company}](${getCompanyUrl(c.company)})`);
+        md.push('');
+    }
+    md.push('---');
+    md.push(`📌 *Prices verified via [Kayak](${url}). Click "See price" to confirm current rate, then book with the rental company.*`);
+    return {
+        domain: 'kayak.com/cars',
+        type: 'car-rental',
+        structured: {
+            cars: finalListings,
+            location: displayLocation,
+            pickupDate,
+            dropoffDate,
+            numDays,
+            totalFound,
+            source: 'Kayak',
+            sourceUrl: url,
+        },
+        cleanContent: md.join('\n'),
+    };
+}

package/dist/core/pipeline.js

@@ -426,6 +426,22 @@ export async function fetchContent(ctx) {
                 // @ts-ignore — proprietary module, gitignored
                 const { searchFallback } = await import('./search-fallback.js');
                 const searchResult = await searchFallback(ctx.url);
+                // If DDG/primary returned very little, also try Bing for richer snippets
+                if (!searchResult.cachedContent || searchResult.cachedContent.length < 400) {
+                    try {
+                        const { simpleFetch } = await import('./http-fetch.js');
+                        const bingUrl = `https://www.bing.com/search?q=${encodeURIComponent(ctx.url)}`;
+                        const bingResult = await simpleFetch(bingUrl, ctx.userAgent, 8000);
+                        if (bingResult.html && bingResult.html.length > 500) {
+                            const snippetMatch = bingResult.html.match(/<p[^>]*class="[^"]*snippet[^"]*"[^>]*>(.*?)<\/p>/gi);
+                            if (snippetMatch) {
+                                const bingSnippet = snippetMatch.map(s => s.replace(/<[^>]+>/g, '')).join('\n');
+                                searchResult.cachedContent = (searchResult.cachedContent || '') + '\n\n---\n*Additional context from Bing:*\n' + bingSnippet;
+                            }
+                        }
+                    }
+                    catch { /* Bing fallback is best-effort */ }
+                }
                 if (searchResult.cachedContent && searchResult.cachedContent.length > 50) {
                     ctx.timer.end('fetch');
                     ctx.content = searchResult.cachedContent;

package/dist/core/strategies.js

@@ -232,7 +232,7 @@ function prefetchDns(url) {
     }
 }
 async function fetchWithBrowserStrategy(url, options) {
-    const { userAgent, waitMs, timeoutMs, screenshot, screenshotFullPage, headers, cookies, actions, keepPageOpen, effectiveStealth, signal, profileDir, headed, storageState, proxy, device, viewportWidth, viewportHeight, waitUntil, waitSelector, blockResources, } = options;
+    const { userAgent, waitMs, timeoutMs, screenshot, screenshotFullPage, headers, cookies, actions, keepPageOpen, effectiveStealth, signal, profileDir, headed, storageState, proxy, device, viewportWidth, viewportHeight, waitUntil, waitSelector, blockResources, isSPA, } = options;
     try {
         const result = await browserFetch(url, {
             userAgent,
@@ -256,6 +256,7 @@ async function fetchWithBrowserStrategy(url, options) {
             waitUntil,
             waitSelector,
             blockResources,
+            isSPA,
         });
         return {
             ...result,
@@ -500,6 +501,21 @@ export async function smartFetch(url, options = {}) {
     if (storageState) {
         effectiveForceBrowser = true;
     }
+    // Detect SPA for smarter DOM stability wait
+    const SPA_FETCH_DOMAINS = new Set([
+        'www.google.com', 'flights.google.com', 'www.airbnb.com', 'www.booking.com',
+        'www.expedia.com', 'www.kayak.com', 'www.skyscanner.com', 'www.tripadvisor.com',
+        'www.indeed.com', 'www.glassdoor.com', 'www.zillow.com', 'app.webpeel.dev',
+    ]);
+    const SPA_FETCH_URL_PATTERNS = [
+        /google\.com\/travel/, /google\.com\/maps/, /google\.com\/shopping/,
+    ];
+    let isSPAUrl = false;
+    try {
+        const parsedHostname = new URL(url).hostname;
+        isSPAUrl = SPA_FETCH_DOMAINS.has(parsedHostname) || SPA_FETCH_URL_PATTERNS.some(p => p.test(url));
+    }
+    catch { /* invalid URL — ignore */ }
     const browserOptions = {
         userAgent,
         waitMs,
@@ -521,6 +537,7 @@ export async function smartFetch(url, options = {}) {
         waitUntil,
         waitSelector,
         blockResources,
+        isSPA: isSPAUrl,
     };
     /* ---- Strategy: simple fetch (with optional race) --------------------- */
     if (!shouldUseBrowser) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.78",
+  "version": "0.21.80",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",