webpeel 0.21.79 → 0.21.80

package/dist/cli/commands/search.js

@@ -6,6 +6,31 @@ import { readFileSync } from 'fs';
 import { peel, peelBatch, cleanup } from '../../index.js';
 import { checkUsage, showUsageFooter, loadConfig } from '../../cli-auth.js';
 import { writeStdout, formatListingsCsv } from '../utils.js';
+/**
+ * Parse a date range string like "Mar29-Apr4" into an array of date strings.
+ * Returns ["Mar 29", "Mar 30", ..., "Apr 4"]
+ */
+function parseDateRange(range) {
+    const match = range.match(/(\w{3})\s*(\d{1,2})\s*[-–to]+\s*(\w{3})\s*(\d{1,2})/i);
+    if (!match)
+        return [];
+    const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+    const startMonthIdx = months.findIndex(m => m.toLowerCase() === match[1].toLowerCase().slice(0, 3));
+    const endMonthIdx = months.findIndex(m => m.toLowerCase() === match[3].toLowerCase().slice(0, 3));
+    if (startMonthIdx === -1 || endMonthIdx === -1)
+        return [];
+    const startDay = parseInt(match[2]);
+    const endDay = parseInt(match[4]);
+    const year = new Date().getFullYear();
+    const dates = [];
+    const start = new Date(year, startMonthIdx, startDay);
+    const end = new Date(year, endMonthIdx, endDay);
+    for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) {
+        const mon = months[d.getMonth()];
+        dates.push(`${mon} ${d.getDate()}`);
+    }
+    return dates;
+}
 export function registerSearchCommands(program) {
     // ── search command ────────────────────────────────────────────────────────
     program
@@ -595,9 +620,90 @@ export function registerSearchCommands(program) {
         .option('--one-way', 'One-way flight (default)')
         .option('--round-trip', 'Round-trip flight')
         .option('-n, --count <n>', 'Max flights to show', '10')
+        .option('--dates <range>', 'Compare prices across date range (e.g., "Mar29-Apr4")')
         .option('--json', 'Output as JSON')
         .option('-s, --silent', 'Silent mode')
         .action(async (query, options) => {
+        // ── --dates: compare cheapest flight across a date range ──────────────
+        if (options.dates) {
+            const dates = parseDateRange(options.dates);
+            if (dates.length === 0) {
+                console.error('Could not parse date range. Format: "Mar29-Apr4"');
+                process.exit(1);
+            }
+            const spinner = options.silent ? null : ora(`Comparing flights across ${dates.length} dates...`).start();
+            const tripType = options.roundTrip ? '' : ' one way';
+            const rows = [];
+            for (const date of dates) {
+                if (spinner)
+                    spinner.text = `Fetching flights for ${date}...`;
+                try {
+                    const dateQuery = `Flights from ${query} ${date}${tripType}`;
+                    const encoded = encodeURIComponent(dateQuery);
+                    const url = `https://www.google.com/travel/flights?q=${encoded}`;
+                    const result = await peel(url, { render: true, timeout: 30000 });
+                    // Try to extract cheapest flight from structured data or content
+                    let price = null;
+                    let airline = null;
+                    let time = null;
+                    const flights = result.domainData?.structured?.flights || [];
+                    if (flights.length > 0) {
+                        const cheapest = flights.reduce((a, b) => {
+                            const ap = parseFloat(String(a.price || '').replace(/[^0-9.]/g, '')) || Infinity;
+                            const bp = parseFloat(String(b.price || '').replace(/[^0-9.]/g, '')) || Infinity;
+                            return ap <= bp ? a : b;
+                        });
+                        price = cheapest.priceStr || (cheapest.price ? `$${cheapest.price}` : null);
+                        airline = cheapest.airline || cheapest.carrier || null;
+                        time = cheapest.departTime && cheapest.arriveTime
+                            ? `${cheapest.departTime} → ${cheapest.arriveTime}`
+                            : (cheapest.time || cheapest.departure || null);
+                    }
+                    else {
+                        // Extract from markdown content — look for price patterns
+                        const priceMatch = result.content.match(/\$(\d+)/);
+                        if (priceMatch)
+                            price = `$${priceMatch[1]}`;
+                        const airlineMatch = result.content.match(/\b(American|Delta|United|Southwest|Spirit|JetBlue|Alaska|Frontier|Allegiant|Sun Country)\b/i);
+                        if (airlineMatch)
+                            airline = airlineMatch[1];
+                        const timeMatch = result.content.match(/(\d{1,2}:\d{2}\s*(?:AM|PM))\s*[–—→]\s*(\d{1,2}:\d{2}\s*(?:AM|PM))/i);
+                        if (timeMatch)
+                            time = `${timeMatch[1]} → ${timeMatch[2]}`;
+                    }
+                    const priceNum = price ? parseFloat(price.replace(/[^0-9.]/g, '')) || Infinity : Infinity;
+                    rows.push({ date, price, airline, time, priceNum });
+                }
+                catch {
+                    rows.push({ date, price: null, airline: null, time: null, priceNum: Infinity });
+                }
+            }
+            if (spinner)
+                spinner.succeed(`Compared ${rows.length} dates`);
+            if (options.json) {
+                console.log(JSON.stringify({ query, dateRange: options.dates, rows }, null, 2));
+            }
+            else {
+                // Find best price
+                const best = rows.reduce((a, b) => a.priceNum <= b.priceNum ? a : b);
+                console.log(`\n# ✈️ Flight Price Comparison — ${query}\n`);
+                console.log('| Date | Airline | Time | Price |');
+                console.log('|------|---------|------|-------|');
+                for (const row of rows) {
+                    const star = row.priceNum === best.priceNum ? ' ⭐' : '';
+                    const priceStr = row.price ? `${row.price}${star}` : 'N/A';
+                    const airlineStr = row.airline || 'Unknown';
+                    const timeStr = row.time || '—';
+                    console.log(`| ${row.date} | ${airlineStr} | ${timeStr} | ${priceStr} |`);
+                }
+                if (best.price) {
+                    console.log(`\n⭐ Best price: ${best.date} — ${best.airline || 'Unknown'} ${best.price}`);
+                }
+            }
+            await cleanup();
+            process.exit(0);
+        }
+        // ── Single date (default) ─────────────────────────────────────────────
         const tripType = options.roundTrip ? '' : ' one way';
         const encoded = encodeURIComponent(`Flights from ${query}${tripType}`);
         const url = `https://www.google.com/travel/flights?q=${encoded}`;

package/dist/core/domain-extractors.js

@@ -123,6 +123,7 @@ const REGISTRY = [
     { match: (h) => h === 'redfin.com' || h === 'www.redfin.com', extractor: redfinExtractor },
     // ── Travel ──────────────────────────────────────────────────────────────
     { match: (h, url = '') => (h === 'www.google.com' || h === 'google.com') && url.includes('/travel/flights'), extractor: googleFlightsExtractor },
+    { match: (h, url = '') => (h === 'www.kayak.com' || h === 'kayak.com') && url.includes('/cars/'), extractor: kayakCarRentalExtractor },
 ];
 /**
  * Returns the domain extractor for a URL, or null if none matches.
@@ -6069,3 +6070,273 @@ async function googleFlightsExtractor(_html, url) {
         cleanContent: md.join('\n'),
     };
 }
+// ---------------------------------------------------------------------------
+// Kayak Car Rental extractor
+// ---------------------------------------------------------------------------
+async function kayakCarRentalExtractor(_html, url) {
+    if (!url.includes('/cars/'))
+        return null;
+    // Rental company homepage URLs
+    const rentalCompanyUrls = {
+        'Hertz': 'https://www.hertz.com',
+        'Budget': 'https://www.budget.com',
+        'Avis': 'https://www.avis.com',
+        'Enterprise': 'https://www.enterprise.com',
+        'National': 'https://www.nationalcar.com',
+        'Alamo': 'https://www.alamo.com',
+        'Dollar': 'https://www.dollar.com',
+        'Thrifty': 'https://www.thrifty.com',
+        'Sixt': 'https://www.sixt.com',
+        'Fox': 'https://www.foxrentacar.com',
+        'Payless': 'https://www.paylesscar.com',
+        'Turn': 'https://www.turn.com',
+        'EconomyBookings': 'https://www.economybookings.com',
+        'Priceline': 'https://www.priceline.com',
+        'Expedia': 'https://www.expedia.com',
+        'Turo': 'https://www.turo.com',
+        'KAYAK': 'https://www.kayak.com',
+        'Booking.com': 'https://www.booking.com',
+        'DiscoverCars': 'https://www.discovercars.com',
+        'RentalCars': 'https://www.rentalcars.com',
+        'Car Rental 8': 'https://www.carrental8.com',
+        'Hotwire': 'https://www.hotwire.com',
+    };
+    function getCompanyUrl(company) {
+        return rentalCompanyUrls[company] || `https://www.kayak.com`;
+    }
+    // Parse dates from URL: /cars/Location/YYYY-MM-DD/YYYY-MM-DD
+    let numDays = 1;
+    let pickupDate = '';
+    let dropoffDate = '';
+    let locationName = '';
+    const dateMatch = url.match(/\/cars\/([^/]+)\/(\d{4}-\d{2}-\d{2})\/(\d{4}-\d{2}-\d{2})/);
+    if (dateMatch) {
+        locationName = decodeURIComponent(dateMatch[1]);
+        pickupDate = dateMatch[2];
+        dropoffDate = dateMatch[3];
+        const pickup = new Date(pickupDate);
+        const dropoff = new Date(dropoffDate);
+        numDays = Math.max(1, Math.round((dropoff.getTime() - pickup.getTime()) / (1000 * 60 * 60 * 24)));
+    }
+    // Format date range for display (e.g. "Apr 1–3")
+    function formatDateRange(from, to) {
+        if (!from || !to)
+            return '';
+        const fromDate = new Date(from + 'T12:00:00');
+        const toDate = new Date(to + 'T12:00:00');
+        const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+        const fromMonth = months[fromDate.getUTCMonth()];
+        const toMonth = months[toDate.getUTCMonth()];
+        const fromDay = fromDate.getUTCDate();
+        const toDay = toDate.getUTCDate();
+        if (fromMonth === toMonth)
+            return `${fromMonth} ${fromDay}–${toDay}`;
+        return `${fromMonth} ${fromDay}–${toMonth} ${toDay}`;
+    }
+    // Process content: strip HTML if needed
+    let text = _html;
+    if (text.includes('<!DOCTYPE') || text.includes('<html')) {
+        text = text
+            .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+            .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+            .replace(/<[^>]+>/g, '\n')
+            .replace(/&amp;/g, '&')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&#\d+;/g, '')
+            .replace(/\n{2,}/g, '\n');
+    }
+    const lines = text.split('\n').map(l => l.trim()).filter(Boolean);
+    const KNOWN_COMPANIES = ['Hertz', 'Budget', 'Avis', 'Enterprise', 'National', 'Alamo', 'Dollar', 'Thrifty', 'Sixt', 'Fox', 'Payless', 'Turn', 'EconomyBookings', 'Priceline', 'Expedia', 'Turo', 'KAYAK', 'Booking.com', 'DiscoverCars', 'RentalCars', 'Car Rental 8', 'Hotwire'];
+    const listings = [];
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        // Detect: "or similar {Class}" — this signals a standard car rental listing
+        // The car name is the line BEFORE "or similar"
+        const orSimilarMatch = line.match(/^or similar\s+(.+)$/);
+        if (orSimilarMatch) {
+            const carClass = orSimilarMatch[1].trim();
+            const carName = i > 0 ? lines[i - 1] : '';
+            if (!carName || carName.length > 60)
+                continue;
+            // Look ahead for: pickup location, rating, company, price
+            let location = '';
+            let distanceFromCenter = '';
+            let rating = null;
+            let company = '';
+            let totalPrice = 0;
+            let cancellation = '';
+            for (let j = i + 1; j < Math.min(i + 30, lines.length); j++) {
+                const l = lines[j];
+                // Pickup location
+                if (!location && l.startsWith('Pick-up')) {
+                    const locMatch = l.match(/Pick-up (?:city|airport):\s*(.+)/);
+                    if (locMatch)
+                        location = locMatch[1].trim();
+                    continue;
+                }
+                // Distance from center
+                if (!distanceFromCenter) {
+                    const distM = l.match(/^([\d.]+)\s+mi\s+from\s+city\s+center/);
+                    if (distM) {
+                        distanceFromCenter = `${distM[1]} mi from city center`;
+                        continue;
+                    }
+                }
+                // Rating (number like "9.2", "8.5", "7.2")
+                if (rating === null) {
+                    const ratingM = l.match(/^(\d+\.\d+)$/);
+                    if (ratingM) {
+                        rating = parseFloat(ratingM[1]);
+                        continue;
+                    }
+                }
+                // Company from "X offer from {Company}" or "{Company}" line
+                if (!company) {
+                    const offerMatch = l.match(/offer from (.+)$/);
+                    if (offerMatch) {
+                        company = offerMatch[1].trim();
+                        continue;
+                    }
+                    // Also detect company name standalone
+                    for (const c of KNOWN_COMPANIES) {
+                        if (l === c) {
+                            company = c;
+                            break;
+                        }
+                    }
+                    if (company)
+                        continue;
+                }
+                // Cancellation policy
+                if (!cancellation && (l.includes('Free cancellation') || l.includes('No free cancellation'))) {
+                    cancellation = l;
+                    continue;
+                }
+                // Price — "$NNN" followed by "Total"
+                const priceM = l.match(/^\$(\d[\d,]*)$/);
+                if (priceM) {
+                    const nextLine = lines[j + 1] || '';
+                    if (nextLine === 'Total' || nextLine.includes('Total')) {
+                        totalPrice = parseInt(priceM[1].replace(',', ''));
+                        break;
+                    }
+                }
+                // Also catch price on same line
+                const inlinePriceM = l.match(/\$(\d[\d,]*)\s*Total/);
+                if (inlinePriceM) {
+                    totalPrice = parseInt(inlinePriceM[1].replace(',', ''));
+                    break;
+                }
+                // Stop if we hit another car listing marker
+                if (l.match(/^or similar\s/) || l === 'Show more results')
+                    break;
+            }
+            if (carName && totalPrice > 0) {
+                const normalizedClass = carClass.replace('Full size', 'Full-size');
+                listings.push({
+                    name: carName,
+                    carClass: normalizedClass,
+                    totalPrice,
+                    perDayPrice: Math.round(totalPrice / numDays),
+                    company: company || 'Unknown',
+                    location: location || 'See booking',
+                    distanceFromCenter,
+                    rating,
+                    cancellation,
+                    isTuro: false,
+                });
+            }
+        }
+    }
+    // Deduplicate: first prefer listings with real company info over "Unknown"
+    // Key by name+price; keep the one with best data
+    const byKey = new Map();
+    for (const c of listings) {
+        const key = `${c.name.toLowerCase()}-${c.totalPrice}`;
+        const existing = byKey.get(key);
+        if (!existing) {
+            byKey.set(key, c);
+        }
+        else {
+            // Prefer non-Unknown company, or same company with more info
+            if (existing.company === 'Unknown' && c.company !== 'Unknown') {
+                byKey.set(key, c);
+            }
+        }
+    }
+    const unique = Array.from(byKey.values());
+    if (unique.length === 0)
+        return null;
+    // Filter out Unknown company entries if the total found from page suggests more results exist
+    // Also filter them only if they have no location info (these are likely ad/promo extractions)
+    const knownCompanyListings = unique.filter(c => c.company !== 'Unknown');
+    const finalListings = knownCompanyListings.length > 0 ? knownCompanyListings : unique;
+    // Sort by price
+    finalListings.sort((a, b) => a.totalPrice - b.totalPrice);
+    // Get total count from page if mentioned
+    let totalFound = unique.length;
+    for (const l of lines) {
+        const m = l.match(/^(\d+)\s+results?$/);
+        if (m) {
+            totalFound = parseInt(m[1]);
+            break;
+        }
+        const m2 = l.match(/(\d+)\s+cars?\s+found/);
+        if (m2) {
+            totalFound = parseInt(m2[1]);
+            break;
+        }
+    }
+    // Format location name nicely (e.g. "Punta-Gorda,FL-c34451" → "Punta Gorda, FL")
+    function formatLocation(loc) {
+        return loc
+            .replace(/-c\d+$/, '') // remove trailing "-c12345"
+            .replace(/-/g, ' ') // hyphens to spaces
+            .replace(/,(\S)/g, ', $1'); // ensure space after comma
+    }
+    const dateRange = formatDateRange(pickupDate, dropoffDate);
+    const displayLocation = formatLocation(locationName);
+    const daysLabel = numDays === 1 ? '1 day' : `${numDays} days`;
+    const md = [
+        `# 🚗 Car Rentals — ${displayLocation} · ${dateRange} (${daysLabel})`,
+        '',
+        `*${totalFound} cars found · Source: [Kayak](${url})*`,
+        `*Free cancellation available on most rentals*`,
+        '',
+    ];
+    for (let idx = 0; idx < finalListings.length; idx++) {
+        const c = finalListings[idx];
+        md.push(`## ${idx + 1}. ${c.name} (${c.carClass}) — $${c.totalPrice} total · $${c.perDayPrice}/day`);
+        if (c.distanceFromCenter) {
+            md.push(`📍 ${c.distanceFromCenter}`);
+        }
+        else if (c.location && c.location !== 'See booking') {
+            md.push(`📍 ${c.location}`);
+        }
+        const ratingStr = c.rating !== null ? ` · Rating: ${c.rating}` : '';
+        md.push(`🏪 via ${c.company}${ratingStr}`);
+        if (c.cancellation)
+            md.push(`✅ ${c.cancellation}`);
+        md.push(`🔍 [See price on Kayak](${url})`);
+        md.push(`🛒 [Book on ${c.company}](${getCompanyUrl(c.company)})`);
+        md.push('');
+    }
+    md.push('---');
+    md.push(`📌 *Prices verified via [Kayak](${url}). Click "See price" to confirm current rate, then book with the rental company.*`);
+    return {
+        domain: 'kayak.com/cars',
+        type: 'car-rental',
+        structured: {
+            cars: finalListings,
+            location: displayLocation,
+            pickupDate,
+            dropoffDate,
+            numDays,
+            totalFound,
+            source: 'Kayak',
+            sourceUrl: url,
+        },
+        cleanContent: md.join('\n'),
+    };
+}

package/dist/core/pipeline.js

@@ -426,6 +426,22 @@ export async function fetchContent(ctx) {
                 // @ts-ignore — proprietary module, gitignored
                 const { searchFallback } = await import('./search-fallback.js');
                 const searchResult = await searchFallback(ctx.url);
+                // If DDG/primary returned very little, also try Bing for richer snippets
+                if (!searchResult.cachedContent || searchResult.cachedContent.length < 400) {
+                    try {
+                        const { simpleFetch } = await import('./http-fetch.js');
+                        const bingUrl = `https://www.bing.com/search?q=${encodeURIComponent(ctx.url)}`;
+                        const bingResult = await simpleFetch(bingUrl, ctx.userAgent, 8000);
+                        if (bingResult.html && bingResult.html.length > 500) {
+                            const snippetMatch = bingResult.html.match(/<p[^>]*class="[^"]*snippet[^"]*"[^>]*>(.*?)<\/p>/gi);
+                            if (snippetMatch) {
+                                const bingSnippet = snippetMatch.map(s => s.replace(/<[^>]+>/g, '')).join('\n');
+                                searchResult.cachedContent = (searchResult.cachedContent || '') + '\n\n---\n*Additional context from Bing:*\n' + bingSnippet;
+                            }
+                        }
+                    }
+                    catch { /* Bing fallback is best-effort */ }
+                }
                 if (searchResult.cachedContent && searchResult.cachedContent.length > 50) {
                     ctx.timer.end('fetch');
                     ctx.content = searchResult.cachedContent;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.79",
+  "version": "0.21.80",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",