webpeel 0.21.73 → 0.21.75

package/dist/cli/commands/search.js

@@ -588,6 +588,49 @@ export function registerSearchCommands(program) {
             process.exit(1);
         }
     });
+    // ── flights command ───────────────────────────────────────────────────────
+    program
+        .command('flights <query>')
+        .description('Search for flights (via Google Flights) — e.g. "NYC to Fort Myers Apr 4"')
+        .option('--one-way', 'One-way flight (default)')
+        .option('--round-trip', 'Round-trip flight')
+        .option('-n, --count <n>', 'Max flights to show', '10')
+        .option('--json', 'Output as JSON')
+        .option('-s, --silent', 'Silent mode')
+        .action(async (query, options) => {
+        const tripType = options.roundTrip ? '' : ' one way';
+        const encoded = encodeURIComponent(`Flights from ${query}${tripType}`);
+        const url = `https://www.google.com/travel/flights?q=${encoded}`;
+        const spinner = options.silent ? null : ora(`Searching flights: ${query}...`).start();
+        try {
+            // render is forced automatically by SPA auto-detect, but be explicit here
+            const result = await peel(url, { render: true, timeout: 30000 });
+            if (spinner)
+                spinner.succeed('Flights loaded');
+            if (options.json) {
+                console.log(JSON.stringify({
+                    query,
+                    url,
+                    flights: result.domainData?.structured?.flights || [],
+                    source: 'Google Flights',
+                    content: result.content,
+                    tokens: result.tokens,
+                }, null, 2));
+            }
+            else {
+                console.log(result.content);
+            }
+            await cleanup();
+            process.exit(0);
+        }
+        catch (error) {
+            if (spinner)
+                spinner.fail('Flight search failed');
+            console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            await cleanup();
+            process.exit(1);
+        }
+    });
     // ── extractors command ────────────────────────────────────────────────────
     program
         .command('extractors')

package/dist/core/domain-extractors.js

@@ -121,6 +121,8 @@ const REGISTRY = [
     { match: (h) => h === 'yelp.com' || h === 'www.yelp.com', extractor: yelpExtractor },
     { match: (h) => h === 'zillow.com' || h === 'www.zillow.com', extractor: zillowExtractor },
     { match: (h) => h === 'redfin.com' || h === 'www.redfin.com', extractor: redfinExtractor },
+    // ── Travel ──────────────────────────────────────────────────────────────
+    { match: (h, url = '') => (h === 'www.google.com' || h === 'google.com') && url.includes('/travel/flights'), extractor: googleFlightsExtractor },
 ];
 /**
  * Returns the domain extractor for a URL, or null if none matches.
@@ -5860,3 +5862,177 @@ async function redfinExtractor(_html, url) {
         return null;
     }
 }
+// ---------------------------------------------------------------------------
+// Google Flights extractor
+// ---------------------------------------------------------------------------
+async function googleFlightsExtractor(_html, url) {
+    if (!url.includes('/travel/flights'))
+        return null;
+    // Google Flights is a SPA. The _html parameter is usually readability-processed markdown
+    // (from the pipeline's post-fetch processing), which looks like:
+    //   -   7:15 PM
+    //       7:15 PM on Sat, Apr 4
+    //        – 10:29 PM
+    //       United
+    //       3 hr 14 min
+    //       EWR
+    //       ...
+    //       $188
+    //
+    // This markdown is much easier to parse than raw HTML.
+    let text = _html;
+    // If this is raw HTML (contains <!DOCTYPE or <html), strip HTML tags
+    if (text.includes('<!DOCTYPE') || text.includes('<html')) {
+        text = text
+            .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+            .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+            .replace(/<[^>]+>/g, '\n')
+            .replace(/&amp;/g, '&')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&#\d+;/g, '')
+            .replace(/\n{2,}/g, '\n');
+    }
+    const lines = text.split('\n').map(l => l.trim()).filter(Boolean);
+    const AIRLINES = ['United', 'Delta', 'American', 'JetBlue', 'Spirit', 'Frontier', 'Southwest', 'Breeze', 'Alaska', 'Hawaiian', 'Sun Country', 'Avelo'];
+    const flights = [];
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        // Detect departure time
+        const departMatch = line.match(/^(?:-\s+)?(\d{1,2}:\d{2}\s*[AP]M)$/);
+        if (!departMatch)
+            continue;
+        const departTime = departMatch[1];
+        let departDate = '', arriveTime = '', airline = '', duration = '';
+        let fromAirport = '', toAirport = '', stops = '', bags = '';
+        let price = 0;
+        for (let j = i + 1; j < Math.min(i + 45, lines.length); j++) {
+            const l = lines[j];
+            // Date
+            const dateM = l.match(/on\s+(\w+,\s+\w+\s+\d+)/);
+            if (dateM && !departDate) {
+                departDate = dateM[1];
+                continue;
+            }
+            // Arrival time
+            const arrM = l.match(/^[–\-–—]\s*(\d{1,2}:\d{2}\s*[AP]M)$/) || l.match(/^(\d{1,2}:\d{2}\s*[AP]M)\s+on\s/);
+            if (arrM && !arriveTime && departTime) {
+                arriveTime = arrM[1];
+                continue;
+            }
+            // Arrival time: also check for "10:29 PM on Sat, Apr 4" pattern (second occurrence)
+            if (!arriveTime && l.match(/^\d{1,2}:\d{2}\s*[AP]M\s+on\s/)) {
+                const m = l.match(/^(\d{1,2}:\d{2}\s*[AP]M)/);
+                if (m) {
+                    arriveTime = m[1];
+                    continue;
+                }
+            }
+            // Airline
+            if (!airline) {
+                for (const a of AIRLINES) {
+                    if (l === a || l.startsWith(a + 'Operated') || l.startsWith(a + ' ')) {
+                        airline = a;
+                        break;
+                    }
+                }
+                if (airline)
+                    continue;
+            }
+            // Duration
+            if (!duration && l.match(/^\d+\s+hr\s+\d+\s+min$/)) {
+                duration = l;
+                continue;
+            }
+            // Airport codes
+            if (l.match(/^[A-Z]{3}$/) && !fromAirport) {
+                fromAirport = l;
+                continue;
+            }
+            if (l.match(/^[A-Z]{3}$/) && fromAirport && !toAirport && l !== fromAirport) {
+                toAirport = l;
+                continue;
+            }
+            // Stops
+            if (!stops && (l === 'Nonstop' || l.match(/^\d+\s+stop/))) {
+                stops = l;
+                continue;
+            }
+            // Bags
+            if (l.includes('carry-on bag') && !bags) {
+                bags = l.includes('not included') ? 'Carry-on NOT included (extra fee)' : 'Carry-on included';
+                continue;
+            }
+            // Price — first occurrence only
+            const priceM = l.match(/^\$(\d[\d,]*)$/);
+            if (priceM && !price) {
+                price = parseInt(priceM[1].replace(',', ''));
+                break;
+            }
+        }
+        if (departTime && arriveTime && airline && price) {
+            flights.push({ departTime, arriveTime, departDate, airline, duration, fromAirport, toAirport, stops: stops || 'Unknown', price, priceStr: `$${price}`, bags });
+        }
+    }
+    // Deduplicate
+    const seen = new Set();
+    const unique = flights.filter(f => {
+        const key = `${f.departTime}-${f.airline}-${f.price}`;
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+    if (unique.length === 0)
+        return null;
+    unique.sort((a, b) => a.price - b.price);
+    // Helper: get airline booking URL
+    function getAirlineBookingUrl(airline, from, to) {
+        const fromLower = from.toLowerCase();
+        const toLower = to.toLowerCase();
+        const urlMap = {
+            'United': `https://www.united.com/en-us/flights-from-${fromLower}-to-${toLower}`,
+            'Delta': `https://www.delta.com/flight-search/search`,
+            'JetBlue': `https://www.jetblue.com/booking/flights`,
+            'American': `https://www.aa.com/booking/find-flights`,
+            'Spirit': `https://www.spirit.com/book/flights`,
+            'Frontier': `https://www.flyfrontier.com/booking/`,
+            'Southwest': `https://www.southwest.com/air/booking/`,
+            'Breeze': `https://www.flybreeze.com/home`,
+            'Alaska': `https://www.alaskaair.com/booking/flights`,
+            'Hawaiian': `https://www.hawaiianairlines.com/book-a-trip`,
+            'Sun Country': `https://www.suncountry.com/booking/search`,
+            'Avelo': `https://www.aveloair.com/book`,
+        };
+        return urlMap[airline] || `https://www.google.com/search?q=${encodeURIComponent(`${airline} flights ${from} to ${to}`)}`;
+    }
+    // Parse route from URL
+    const u = new URL(url);
+    const query = (u.searchParams.get('q') || '').replace(/Flights?\s+(from\s+)?/i, '').replace(/\s+one\s+way/i, '').trim();
+    const md = [
+        `# ✈️ Flights — ${query || 'Search Results'}`,
+        '',
+        `*${unique.length} flights found · Source: [Google Flights](${url})*`,
+        `*Prices include taxes + fees for 1 adult. Book directly via airline.*`,
+        '',
+    ];
+    for (let idx = 0; idx < unique.length; idx++) {
+        const f = unique[idx];
+        const bookingUrl = getAirlineBookingUrl(f.airline, f.fromAirport, f.toAirport);
+        md.push(`## ${idx + 1}. ${f.airline} — ${f.priceStr}`);
+        md.push(`🕐 Depart **${f.departTime}** → Arrive **${f.arriveTime}**${f.departDate ? ` · ${f.departDate}` : ''}`);
+        md.push(`🛫 ${f.fromAirport} → ${f.toAirport} · ${f.duration} · ${f.stops}`);
+        if (f.bags)
+            md.push(`🧳 ${f.bags}`);
+        md.push(`🔗 [Book on ${f.airline}](${bookingUrl})`);
+        md.push('');
+    }
+    md.push('---');
+    md.push(`📌 *Prices change frequently. [View live prices on Google Flights](${url})*`);
+    return {
+        domain: 'google.com/travel/flights',
+        type: 'flights',
+        structured: { flights: unique, route: query, source: 'Google Flights', sourceUrl: url },
+        cleanContent: md.join('\n'),
+    };
+}

package/dist/core/pipeline.js

@@ -146,6 +146,43 @@ export function normalizeOptions(ctx) {
     if (autoScrollOpts) {
         ctx.render = true;
     }
+    // Auto-detect SPAs that require browser rendering (no --render flag needed)
+    if (!ctx.render) {
+        const SPA_DOMAINS = new Set([
+            'www.google.com', // Google Flights, Maps, Shopping etc.
+            'flights.google.com',
+            'www.airbnb.com',
+            'www.booking.com',
+            'www.expedia.com',
+            'www.kayak.com',
+            'www.skyscanner.com',
+            'www.tripadvisor.com',
+            'www.indeed.com',
+            'www.glassdoor.com',
+            'www.zillow.com', // already handled but backup
+            'app.webpeel.dev', // our own dashboard is a SPA
+        ]);
+        // More specific: some google.com paths need render, not all
+        const SPA_URL_PATTERNS = [
+            /google\.com\/travel/,
+            /google\.com\/maps/,
+            /google\.com\/shopping/,
+        ];
+        try {
+            const hostname = new URL(ctx.url).hostname;
+            if (SPA_DOMAINS.has(hostname)) {
+                ctx.render = true;
+                log.debug(`Auto-enabling render: SPA domain detected (${hostname})`);
+            }
+            else if (SPA_URL_PATTERNS.some(p => p.test(ctx.url))) {
+                ctx.render = true;
+                log.debug(`Auto-enabling render: SPA URL pattern matched`);
+            }
+        }
+        catch {
+            // Invalid URL — skip SPA detection
+        }
+    }
 }
 // ---------------------------------------------------------------------------
 // Stage 2: handleYouTube
@@ -991,7 +1028,12 @@ export async function postProcess(ctx) {
     if (getDomainExtractor(fetchResult.url) && !ctx.domainApiHandled) {
         try {
             ctx.timer.mark('domainExtract');
-            const ddResult = await extractDomainData(fetchResult.html, fetchResult.url);
+            // Try raw HTML first, then fall back to readability-processed content
+            // (some SPAs like Google Flights have data only after readability processing)
+            let ddResult = await extractDomainData(fetchResult.html, fetchResult.url);
+            if (!ddResult && ctx.content) {
+                ddResult = await extractDomainData(ctx.content, fetchResult.url);
+            }
             ctx.timer.end('domainExtract');
             if (ddResult) {
                 ctx.domainData = ddResult;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.73",
+  "version": "0.21.75",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",