webpeel 0.21.71 → 0.21.73

package/dist/cli/commands/fetch.js

@@ -682,13 +682,21 @@ export async function runFetch(url, options) {
             result = await fetchViaApi(url, peelOptions, fetchApiKey, fetchApiUrl);
         }
         else {
-            // No API key — show helpful message instead of trying local mode
+            // No API key — fall back to local peel() mode (runs locally, no API needed)
             if (spinner)
-                spinner.fail('Authentication required');
-            console.error('No API key configured. Run: webpeel auth <your-key>');
-            console.error('Get a free key at: https://app.webpeel.dev/keys');
-            await cleanup();
-            process.exit(2);
+                spinner.text = 'Fetching locally (no API key)…';
+            const startLocal = Date.now();
+            const { peel } = await import('../../index.js');
+            const localResult = await peel(url, peelOptions);
+            const elapsed = Date.now() - startLocal;
+            // Normalize to the shape fetchViaApi returns
+            result = {
+                ...localResult,
+                elapsed: localResult.elapsed ?? elapsed,
+                method: localResult.method ?? 'local',
+                tokens: localResult.tokens ?? Math.ceil((localResult.content?.length ?? 0) / 4),
+                cached: false,
+            };
         }
         // Update lastUsed timestamp for named profiles
         if (resolvedProfileName) {

package/dist/cli/commands/search.js

@@ -588,4 +588,100 @@ export function registerSearchCommands(program) {
             process.exit(1);
         }
     });
+    // ── extractors command ────────────────────────────────────────────────────
+    program
+        .command('extractors')
+        .alias('list-extractors')
+        .description('List all supported domain extractors')
+        .option('--json', 'Output as JSON')
+        .action((options) => {
+        const extractors = [
+            // Social
+            { domain: 'twitter.com / x.com', category: 'Social', description: 'Tweets, threads, profiles' },
+            { domain: 'reddit.com', category: 'Social', description: 'Subreddits, posts, comments' },
+            { domain: 'instagram.com', category: 'Social', description: 'Photos, reels, profiles' },
+            { domain: 'tiktok.com', category: 'Social', description: 'Video metadata, captions' },
+            { domain: 'pinterest.com', category: 'Social', description: 'Pins, boards' },
+            { domain: 'linkedin.com', category: 'Social', description: 'Profiles, job listings' },
+            { domain: 'facebook.com', category: 'Social', description: 'Marketplace listings' },
+            // Video / Audio
+            { domain: 'youtube.com', category: 'Video', description: 'Transcripts, metadata, comments' },
+            { domain: 'twitch.tv', category: 'Video', description: 'Streams, clips, channel info' },
+            { domain: 'soundcloud.com', category: 'Audio', description: 'Tracks, playlists' },
+            { domain: 'open.spotify.com', category: 'Audio', description: 'Tracks, albums, playlists' },
+            // Tech / Dev
+            { domain: 'github.com', category: 'Dev', description: 'Repos, issues, PRs, code' },
+            { domain: 'stackoverflow.com', category: 'Dev', description: 'Questions, answers' },
+            { domain: 'npmjs.com', category: 'Dev', description: 'Package metadata, readme' },
+            { domain: 'pypi.org', category: 'Dev', description: 'Package metadata, readme' },
+            { domain: 'dev.to', category: 'Dev', description: 'Articles, comments' },
+            // News / Articles
+            { domain: 'news.ycombinator.com', category: 'News', description: 'HN posts, comments, Ask/Show HN' },
+            { domain: 'medium.com', category: 'Articles', description: 'Articles, publications' },
+            { domain: 'substack.com / *.substack.com', category: 'Articles', description: 'Newsletters, posts' },
+            { domain: 'nytimes.com', category: 'News', description: 'Articles, headlines' },
+            { domain: 'bbc.com', category: 'News', description: 'Articles, headlines' },
+            { domain: 'cnn.com', category: 'News', description: 'Articles, headlines' },
+            // Shopping / E-commerce
+            { domain: 'amazon.com', category: 'Shopping', description: 'Products, prices, reviews' },
+            { domain: 'bestbuy.com', category: 'Shopping', description: 'Products, prices, specs' },
+            { domain: 'walmart.com', category: 'Shopping', description: 'Products, prices' },
+            { domain: 'ebay.com', category: 'Shopping', description: 'Listings, prices' },
+            { domain: 'etsy.com', category: 'Shopping', description: 'Handmade listings' },
+            // Local / Real Estate
+            { domain: 'yelp.com', category: 'Local', description: 'Business info, reviews (needs YELP_API_KEY)' },
+            { domain: 'craigslist.org', category: 'Local', description: 'Listings, classifieds' },
+            { domain: 'zillow.com', category: 'Real Estate', description: 'Property listings, estimates' },
+            { domain: 'redfin.com', category: 'Real Estate', description: 'Property listings, prices' },
+            { domain: 'cars.com', category: 'Automotive', description: 'Car listings, prices' },
+            // Knowledge / Academic
+            { domain: 'en.wikipedia.org', category: 'Knowledge', description: 'Articles, structured data' },
+            { domain: 'arxiv.org', category: 'Academic', description: 'Papers, abstracts, metadata' },
+            { domain: 'semanticscholar.org', category: 'Academic', description: 'Papers, citations' },
+            { domain: 'pubmed.ncbi.nlm.nih.gov', category: 'Academic', description: 'Medical papers, abstracts' },
+            { domain: 'imdb.com', category: 'Knowledge', description: 'Movies, TV shows, cast' },
+            { domain: 'allrecipes.com', category: 'Knowledge', description: 'Recipes, ingredients, steps' },
+            // Finance / Markets
+            { domain: 'polymarket.com', category: 'Finance', description: 'Prediction markets' },
+            { domain: 'kalshi.com', category: 'Finance', description: 'Prediction markets' },
+            { domain: 'tradingview.com', category: 'Finance', description: 'Charts, indicators, ideas' },
+            { domain: 'coingecko.com', category: 'Finance', description: 'Crypto prices, market data' },
+            { domain: 'coinmarketcap.com', category: 'Finance', description: 'Crypto prices, market data' },
+            // Sports / Betting
+            { domain: 'espn.com', category: 'Sports', description: 'Scores, stats, news' },
+            { domain: 'draftkings.com', category: 'Betting', description: 'Odds, lines' },
+            { domain: 'fanduel.com', category: 'Betting', description: 'Odds, lines' },
+            { domain: 'betmgm.com', category: 'Betting', description: 'Odds, lines' },
+            // Entertainment
+            { domain: 'producthunt.com', category: 'Tech', description: 'Product launches, upvotes' },
+            // Documents
+            { domain: '*.pdf URLs', category: 'Documents', description: 'PDF text extraction' },
+            // Weather
+            { domain: 'weather.com', category: 'Weather', description: 'Forecasts, conditions' },
+            { domain: 'accuweather.com', category: 'Weather', description: 'Forecasts, conditions' },
+            { domain: 'api.open-meteo.com', category: 'Weather', description: 'Free weather API' },
+        ];
+        if (options.json) {
+            console.log(JSON.stringify(extractors, null, 2));
+            return;
+        }
+        // Group by category
+        const byCategory = new Map();
+        for (const e of extractors) {
+            if (!byCategory.has(e.category))
+                byCategory.set(e.category, []);
+            byCategory.get(e.category).push(e);
+        }
+        console.log(`\n🔌 WebPeel Domain Extractors (${extractors.length} total)\n`);
+        for (const [cat, items] of byCategory) {
+            console.log(`  ${cat}`);
+            for (const item of items) {
+                const pad = 35;
+                const domainPad = item.domain.padEnd(pad);
+                console.log(`    ${domainPad} ${item.description}`);
+            }
+            console.log('');
+        }
+        console.log('  Run `webpeel <url>` to use these automatically based on the URL.');
+    });
 }

package/dist/cli/utils.js

@@ -255,7 +255,37 @@ export async function fetchViaApi(url, options, apiKey, apiUrl) {
         err.statusCode = res.status;
         throw err;
     }
-    const data = await res.json();
+    let data = await res.json();
+    // Handle async job queue mode — API returns { jobId, pollUrl } and we need to poll
+    if (data.jobId && data.pollUrl && !data.content) {
+        const pollEndpoint = `${apiUrl}${data.pollUrl}`;
+        const maxPollMs = 90_000; // 90s max
+        const pollInterval = 1_000; // 1s intervals
+        const start = Date.now();
+        while (Date.now() - start < maxPollMs) {
+            await new Promise(r => setTimeout(r, pollInterval));
+            const pollRes = await fetch(pollEndpoint, {
+                headers: { Authorization: `Bearer ${apiKey}` },
+                signal: AbortSignal.timeout(10_000),
+            });
+            if (!pollRes.ok) {
+                throw new Error(`Job poll failed: HTTP ${pollRes.status}`);
+            }
+            const pollData = await pollRes.json();
+            if (pollData.status === 'completed' || pollData.content) {
+                data = pollData.result || pollData;
+                break;
+            }
+            if (pollData.status === 'failed' || pollData.status === 'error') {
+                throw new Error(pollData.error?.message || pollData.error || 'Job failed on server');
+            }
+            // Still processing — keep polling
+        }
+        // If we exited the loop without data, warn
+        if (!data.content && data.jobId) {
+            throw new Error('Job timed out waiting for server response. Try again or use local mode (unset WEBPEEL_API_KEY).');
+        }
+    }
     // Map API response to PeelResult shape that the CLI already handles
     return {
         url: data.url || url,

package/dist/cli.js

@@ -14,6 +14,20 @@
  *   npx webpeel --help                 - Condensed help
  *   npx webpeel --help-all             - Full option reference
  */
+// ── Auto-load .env from cwd (lightweight, no dotenv dependency) ──────────────
+// Must happen BEFORE any imports that read env vars (e.g., WEBPEEL_API_KEY)
+import { readFileSync, existsSync } from 'fs';
+import { resolve } from 'path';
+{
+    const envPath = resolve(process.cwd(), '.env');
+    if (existsSync(envPath)) {
+        for (const line of readFileSync(envPath, 'utf-8').split('\n')) {
+            const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
+            if (m && !process.env[m[1]])
+                process.env[m[1]] = m[2].replace(/^["']|["']$/g, '');
+        }
+    }
+}
 import { Command } from 'commander';
 import { VERB_ALIASES, cliVersion, checkForUpdates, buildCommanderHelp, buildCondensedHelp, } from './cli/utils.js';
 import { registerFetchCommands } from './cli/commands/fetch.js';

package/dist/core/domain-extractors.js

@@ -5247,100 +5247,212 @@ async function ebayExtractor(html, url) {
 // ---------------------------------------------------------------------------
 // Yelp extractor — parse JSON-LD + meta from stealth-rendered HTML
 // ---------------------------------------------------------------------------
-async function yelpExtractor(html, url) {
+async function yelpExtractor(_html, url) {
+    const YELP_API_KEY = process.env.YELP_API_KEY;
+    // Helper to call Yelp Fusion API
+    async function yelpFetch(path, params) {
+        const base = 'https://api.yelp.com/v3';
+        const qs = params ? '?' + new URLSearchParams(params).toString() : '';
+        const res = await fetch(`${base}${path}${qs}`, {
+            headers: { 'Authorization': `Bearer ${YELP_API_KEY}` },
+        });
+        if (!res.ok) {
+            throw new Error(`Yelp API ${res.status}: ${res.statusText}`);
+        }
+        return res.json();
+    }
     try {
-        const { load } = await import('cheerio');
-        const $ = load(html);
-        // Try JSON-LD structured data first
-        const jsonLdScripts = $('script[type="application/ld+json"]');
-        let businessData = null;
-        jsonLdScripts.each((_, el) => {
-            const raw = $(el).html() || '';
+        const parsed = new URL(url);
+        const pathname = parsed.pathname;
+        const searchParams = parsed.searchParams;
+        // ----------------------------------------------------------------
+        // If no API key, fall back to the legacy HTML-scraping approach
+        // ----------------------------------------------------------------
+        if (!YELP_API_KEY) {
+            // Legacy fallback: minimal result pointing user to Yelp
+            const term = searchParams.get('find_desc') || searchParams.get('cflt') || 'businesses';
+            const loc = searchParams.get('find_loc') || '';
+            const isBiz = pathname.startsWith('/biz/');
+            const cleanContent = isBiz
+                ? `# Yelp Business\n\n*No YELP_API_KEY configured — visit [Yelp](${url}) for details.*`
+                : `# 🔍 Yelp Search: ${term}${loc ? ` in ${loc}` : ''}\n\n*No YELP_API_KEY configured — [View on Yelp](${url})*`;
+            return {
+                domain: 'yelp.com',
+                type: isBiz ? 'business' : 'search',
+                structured: { url },
+                cleanContent,
+            };
+        }
+        // ----------------------------------------------------------------
+        // Business page: /biz/<alias>
+        // ----------------------------------------------------------------
+        if (pathname.startsWith('/biz/')) {
+            const alias = pathname.replace('/biz/', '').split('?')[0].split('#')[0];
+            let biz;
             try {
-                const parsed = JSON.parse(raw);
-                const items = Array.isArray(parsed) ? parsed : [parsed];
-                for (const item of items) {
-                    const type = item['@type'];
-                    if (type === 'Restaurant' || type === 'LocalBusiness' || type === 'FoodEstablishment' ||
-                        type === 'BarOrPub' || type === 'CafeOrCoffeeShop') {
-                        businessData = item;
-                    }
-                }
+                biz = await yelpFetch(`/businesses/${alias}`);
             }
-            catch { /* ignore malformed JSON-LD */ }
-        });
-        // --- Business page ---
-        if (businessData) {
-            const name = businessData.name || '';
-            const rating = businessData.aggregateRating?.ratingValue;
-            const reviewCount = businessData.aggregateRating?.reviewCount;
-            const addr = businessData.address;
+            catch (e) {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel] Yelp biz fetch failed:', e instanceof Error ? e.message : e);
+                return null;
+            }
+            // Fetch reviews (best-effort)
+            let reviews = [];
+            try {
+                const revData = await yelpFetch(`/businesses/${alias}/reviews`, { limit: '3' });
+                reviews = revData.reviews || [];
+            }
+            catch { /* reviews are optional */ }
+            const name = biz.name || alias;
+            const rating = biz.rating != null ? biz.rating.toFixed(1) : '?';
+            const reviewCount = biz.review_count ?? 0;
+            const addr = biz.location;
             const address = addr
-                ? [addr.streetAddress, addr.addressLocality, addr.addressRegion, addr.postalCode].filter(Boolean).join(', ')
+                ? [addr.address1, addr.city, addr.state, addr.zip_code].filter(Boolean).join(', ')
                 : '';
-            const phone = businessData.telephone || '';
-            const cuisine = businessData.servesCuisine || '';
-            const priceRange = businessData.priceRange || '';
-            const description = businessData.description || $('meta[property="og:description"]').attr('content') || '';
-            const hours = businessData.openingHours || '';
+            const phone = biz.display_phone || biz.phone || '';
+            const price = biz.price || '';
+            const categories = (biz.categories || []).map((c) => c.title).join(' | ');
+            const yelpUrl = biz.url || url;
+            // Hours
+            let hoursStr = '';
+            if (biz.hours && biz.hours.length > 0) {
+                const dayNames = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'];
+                const dayMap = {};
+                for (const slot of biz.hours[0].open || []) {
+                    const fmt = (t) => {
+                        const h = parseInt(t.slice(0, 2), 10);
+                        const m = t.slice(2);
+                        const period = h >= 12 ? 'PM' : 'AM';
+                        const h12 = h % 12 || 12;
+                        return `${h12}:${m} ${period}`;
+                    };
+                    const day = slot.day;
+                    if (!dayMap[day])
+                        dayMap[day] = [];
+                    dayMap[day].push(`${fmt(slot.start)}–${fmt(slot.end)}`);
+                }
+                hoursStr = Object.entries(dayMap)
+                    .map(([d, times]) => `${dayNames[parseInt(d, 10)]}: ${times.join(', ')}`)
+                    .join(' | ');
+            }
             const lines = [
-                `# ⭐ Yelp: ${name}`,
-                '',
-                rating && `**Rating:** ${rating}/5 (${reviewCount} reviews)`,
-                cuisine && `**Cuisine:** ${cuisine}`,
-                priceRange && `**Price:** ${priceRange}`,
-                address && `**Address:** ${address}`,
-                phone && `**Phone:** ${phone}`,
-                hours && `**Hours:** ${Array.isArray(hours) ? hours.join(', ') : hours}`,
-                description && `\n${description.substring(0, 500)}`,
+                `# ${name} ⭐ ${rating} (${reviewCount.toLocaleString()} reviews)`,
                 '',
-                `**More info:** [View on Yelp](${url})`,
-                '',
-                '---',
-                '*Source: Yelp*',
-            ].filter(Boolean);
+            ];
+            if (address)
+                lines.push(`📍 ${address}`);
+            if (categories)
+                lines.push(`🏷️ ${categories}${price ? ` | 💰 ${price}` : ''}`);
+            else if (price)
+                lines.push(`💰 ${price}`);
+            if (phone)
+                lines.push(`📞 ${phone}`);
+            if (hoursStr)
+                lines.push(`🕐 ${hoursStr}`);
+            if (biz.is_closed === true)
+                lines.push(`⚠️ *Permanently closed*`);
+            lines.push('');
+            if (reviews.length > 0) {
+                for (const rev of reviews) {
+                    const stars = '⭐'.repeat(Math.round(rev.rating || 0));
+                    const text = (rev.text || '').replace(/\n+/g, ' ').trim().slice(0, 200);
+                    lines.push(`> ${stars} — ${text}${(rev.text || '').length > 200 ? '…' : ''}`);
+                    lines.push('');
+                }
+            }
+            lines.push(`[View on Yelp](${yelpUrl})`);
             return {
                 domain: 'yelp.com',
                 type: 'business',
-                structured: { name, rating, reviewCount, address, phone, cuisine, priceRange, description },
+                structured: { name, rating: parseFloat(rating), reviewCount, address, phone, price, categories, url: yelpUrl },
                 cleanContent: lines.join('\n'),
             };
         }
-        // --- Search page — parse from meta / og tags ---
-        const ogTitle = $('meta[property="og:title"]').attr('content') || '';
-        const ogDescription = $('meta[property="og:description"]').attr('content') || '';
-        // Try to extract listing names from heading tags
-        const listings = [];
-        $('h3, h4').each((_, el) => {
-            const text = $(el).text().trim();
-            if (text && text.length > 2 && text.length < 100) {
-                const anchor = $(el).find('a').first();
-                const href = anchor.attr('href') || '';
-                const fullHref = href.startsWith('/') ? `https://www.yelp.com${href}` : href;
-                listings.push({ name: text, url: fullHref || undefined });
-            }
-        });
-        if (ogTitle || listings.length > 0) {
-            const searchTerm = ogTitle.replace(/\s*-\s*Yelp$/, '').trim();
-            const lines = [
-                `# 🔍 Yelp Search: ${searchTerm || 'Results'}`,
-                ogDescription && `\n${ogDescription}`,
-                listings.length > 0 && `\n**Found ${listings.length} results:**`,
-                ...listings.slice(0, 15).map((l, i) => `${i + 1}. ${l.url ? `[${l.name}](${l.url})` : l.name}`),
-                '',
-                `**Search:** [View on Yelp](${url})`,
-                '',
-                '---',
-                '*Source: Yelp*',
-            ].filter(Boolean);
-            return {
-                domain: 'yelp.com',
-                type: 'search',
-                structured: { query: searchTerm, count: listings.length, listings },
-                cleanContent: lines.join('\n'),
-            };
+        // ----------------------------------------------------------------
+        // Search / Category URL: /search?find_desc=...&find_loc=...
+        //                        /search?cflt=restaurants&find_loc=...
+        // ----------------------------------------------------------------
+        const findDesc = searchParams.get('find_desc') || '';
+        const cflt = searchParams.get('cflt') || '';
+        const findLoc = searchParams.get('find_loc') || '';
+        if (!findLoc && !findDesc && !cflt) {
+            // Not a recognized pattern
+            return null;
         }
-        return null;
+        const apiParams = { limit: '10' };
+        if (findLoc)
+            apiParams.location = findLoc;
+        if (findDesc)
+            apiParams.term = findDesc;
+        if (cflt && !findDesc)
+            apiParams.categories = cflt;
+        let data;
+        try {
+            data = await yelpFetch('/businesses/search', apiParams);
+        }
+        catch (e) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel] Yelp search failed:', e instanceof Error ? e.message : e);
+            return null;
+        }
+        const businesses = data.businesses || [];
+        const total = data.total ?? businesses.length;
+        // Build header
+        const searchLabel = findDesc || cflt || 'Businesses';
+        const locationLabel = findLoc || '';
+        const emoji = cflt === 'restaurants' || findDesc?.toLowerCase().includes('restaurant') ? '🍽️'
+            : findDesc?.toLowerCase().includes('pizza') ? '🍕'
+                : findDesc?.toLowerCase().includes('coffee') || findDesc?.toLowerCase().includes('cafe') ? '☕'
+                    : findDesc?.toLowerCase().includes('bar') ? '🍺'
+                        : '🔍';
+        const titleParts = [searchLabel.charAt(0).toUpperCase() + searchLabel.slice(1)];
+        if (locationLabel)
+            titleParts.push(`in ${locationLabel}`);
+        const lines = [
+            `# ${emoji} Yelp — ${titleParts.join(' ')}`,
+            '',
+            `*${businesses.length} of ${total.toLocaleString()} results via Yelp Fusion API*`,
+            '',
+        ];
+        for (let i = 0; i < businesses.length; i++) {
+            const b = businesses[i];
+            const bName = b.name || 'Unknown';
+            const bRating = b.rating != null ? b.rating.toFixed(1) : '?';
+            const bReviews = b.review_count ?? 0;
+            const bAddr = b.location;
+            const bAddress = bAddr
+                ? [bAddr.address1, bAddr.city, bAddr.state, bAddr.zip_code].filter(Boolean).join(', ')
+                : '';
+            const bPhone = b.display_phone || '';
+            const bPrice = b.price || '';
+            const bCategories = (b.categories || []).map((c) => c.title).join(' | ');
+            const bUrl = b.url || '';
+            const bSnippet = b.snippet_text || '';
+            lines.push(`## ${i + 1}. ${bName} ⭐ ${bRating} (${bReviews.toLocaleString()} reviews)`);
+            if (bAddress)
+                lines.push(`📍 ${bAddress}`);
+            const tagLine = [bCategories && `🏷️ ${bCategories}`, bPrice && `💰 ${bPrice}`].filter(Boolean).join(' | ');
+            if (tagLine)
+                lines.push(tagLine);
+            if (bPhone)
+                lines.push(`📞 ${bPhone}`);
+            if (bSnippet)
+                lines.push(`> ${bSnippet.replace(/\n+/g, ' ').trim().slice(0, 150)}`);
+            if (bUrl)
+                lines.push(`[View on Yelp](${bUrl})`);
+            lines.push('');
+        }
+        if (businesses.length === 0) {
+            lines.push(`*No results found for "${searchLabel}"${locationLabel ? ` in ${locationLabel}` : ''}.*`);
+        }
+        return {
+            domain: 'yelp.com',
+            type: 'search',
+            structured: { query: searchLabel, location: locationLabel, total, count: businesses.length, businesses },
+            cleanContent: lines.join('\n'),
+        };
     }
     catch (e) {
         if (process.env.DEBUG)
@@ -5471,20 +5583,21 @@ async function zillowExtractor(_html, url) {
             const redfinCityUrl = `https://www.redfin.com/${stateCode}/${cityForUrl}`;
             const locationLabel = `${cityName}, ${stateCode}`;
             // Try to fetch live Redfin listings via their API
-            // Map common cities to known Redfin market IDs
-            const marketIdMap = {
-                'NY-New-York': 8, 'NY-Brooklyn': 8, 'NY-Queens': 8, 'NY-Bronx': 8,
-                'NY-Staten-Island': 8, 'NY-Manhattan': 8,
-                'CA-Los-Angeles': 4, 'CA-San-Francisco': 1, 'CA-San-Diego': 5,
-                'TX-Houston': 7, 'TX-Dallas': 24, 'TX-Austin': 22,
-                'FL-Miami': 13, 'FL-Orlando': 15, 'FL-Tampa': 11,
-                'IL-Chicago': 3, 'WA-Seattle': 16, 'MA-Boston': 10,
-                'AZ-Phoenix': 14, 'PA-Philadelphia': 12, 'GA-Atlanta': 9,
-                'CO-Denver': 6, 'MN-Minneapolis': 18, 'OR-Portland': 17,
-                'NV-Las-Vegas': 20, 'NC-Charlotte': 21, 'OH-Columbus': 23,
+            // Map common city slugs to Redfin city region IDs (region_type=6)
+            const cityRegionMap = {
+                'NY-New-York': 30749, 'NY-Brooklyn': 30749, 'NY-Queens': 30749, 'NY-Bronx': 30749,
+                'NY-Staten-Island': 30749, 'NY-Manhattan': 30749,
+                'CA-Los-Angeles': 11203, 'CA-San-Francisco': 17151, 'CA-San-Diego': 18142,
+                'CA-San-Jose': 17420,
+                'TX-Houston': 30772, 'TX-Dallas': 35799, 'TX-Austin': 30818,
+                'FL-Miami': 10201, 'FL-Orlando': 13140, 'FL-Tampa': 18280,
+                'IL-Chicago': 29470, 'WA-Seattle': 16163, 'MA-Boston': 1826,
+                'AZ-Phoenix': 14240, 'PA-Philadelphia': 13364, 'GA-Atlanta': 30756,
+                'CO-Denver': 11093, 'MN-Minneapolis': 18959, 'OR-Portland': 14941,
+                'NV-Las-Vegas': 32820, 'NC-Charlotte': 3105, 'OH-Columbus': 8528,
             };
             const marketKey = `${stateCode}-${cityForUrl}`;
-            const marketId = marketIdMap[marketKey];
+            const marketId = cityRegionMap[marketKey];
             if (marketId) {
                 const payload = await fetchRedfinListings(marketId, 6 /* city */);
                 if (payload?.homes && payload.homes.length > 0) {
@@ -5622,32 +5735,21 @@ async function redfinExtractor(_html, url) {
             const citySlug = stateCity[2];
             const cityName = citySlug.replace(/-/g, ' ');
             const locationLabel = `${cityName}, ${stateCode}`;
-            // No region ID — use a GIS bounding box search via the city name
-            // Try a known NYC region as a broader fallback search
-            // For now, attempt search with region_type=2 (market area)
-            // We'll make a best-effort attempt using a city name search
-            // Since Redfin's autocomplete is blocked, try common market IDs
-            const marketIdMap = {
-                'NY-New-York': 8,
-                'NY-Brooklyn': 8,
-                'NY-Queens': 8,
-                'NY-Bronx': 8,
-                'NY-Staten-Island': 8,
-                'NY-Manhattan': 8,
-                'CA-Los-Angeles': 4,
-                'CA-San-Francisco': 1,
-                'TX-Houston': 7,
-                'TX-Dallas': 24,
-                'FL-Miami': 13,
-                'IL-Chicago': 3,
-                'WA-Seattle': 16,
-                'MA-Boston': 10,
-                'AZ-Phoenix': 14,
-                'PA-Philadelphia': 12,
-                'GA-Atlanta': 9,
+            // No region ID in URL — use known Redfin city region IDs (region_type=6)
+            const cityRegionMap = {
+                'NY-New-York': 30749, 'NY-Brooklyn': 30749, 'NY-Queens': 30749, 'NY-Bronx': 30749,
+                'NY-Staten-Island': 30749, 'NY-Manhattan': 30749,
+                'CA-Los-Angeles': 11203, 'CA-San-Francisco': 17151, 'CA-San-Diego': 18142,
+                'CA-San-Jose': 17420,
+                'TX-Houston': 30772, 'TX-Dallas': 35799, 'TX-Austin': 30818,
+                'FL-Miami': 10201, 'FL-Orlando': 13140, 'FL-Tampa': 18280,
+                'IL-Chicago': 29470, 'WA-Seattle': 16163, 'MA-Boston': 1826,
+                'AZ-Phoenix': 14240, 'PA-Philadelphia': 13364, 'GA-Atlanta': 30756,
+                'CO-Denver': 11093, 'MN-Minneapolis': 18959, 'OR-Portland': 14941,
+                'NV-Las-Vegas': 32820, 'NC-Charlotte': 3105, 'OH-Columbus': 8528,
             };
             const marketKey = `${stateCode}-${citySlug}`;
-            const marketId = marketIdMap[marketKey];
+            const marketId = cityRegionMap[marketKey];
             if (marketId) {
                 const payload = await fetchRedfinListings(marketId, 6 /* city */);
                 if (payload?.homes && payload.homes.length > 0) {

package/dist/core/search-fallback.d.ts

@@ -22,6 +22,7 @@ export interface SearchFallbackResult {
 }
 /**
  * Search for a URL using the best available search provider and extract the snippet.
+ * Richer fallback: tries multiple engines if the first returns < 100 tokens.
  * Returns the title, snippet, and any extracted product data.
  */
 export declare function searchFallback(url: string): Promise<SearchFallbackResult>;

package/dist/core/search-fallback.js

@@ -7,7 +7,7 @@
  *   Google CSE API → Brave API → Google stealth → DDG
  * This avoids direct HTML scraping which is blocked by CAPTCHAs on datacenter IPs.
  */
-import { getBestSearchProvider } from './search-provider.js';
+import { getBestSearchProvider, DuckDuckGoProvider } from './search-provider.js';
 /**
  * Detect if a URL is likely a product page.
  */
@@ -121,8 +121,15 @@ function buildCachedContent(url, title, snippet, productData) {
     lines.push(`*⚠️ Limited content — original page blocked direct access. For full data, configure GOOGLE_SEARCH_KEY or BRAVE_SEARCH_KEY.*`);
     return lines.join('\n');
 }
+/**
+ * Count approximate tokens in a string (1 token ≈ 4 chars).
+ */
+function countTokens(text) {
+    return Math.ceil(text.length / 4);
+}
 /**
  * Search for a URL using the best available search provider and extract the snippet.
+ * Richer fallback: tries multiple engines if the first returns < 100 tokens.
  * Returns the title, snippet, and any extracted product data.
  */
 export async function searchFallback(url) {
@@ -142,16 +149,42 @@ export async function searchFallback(url) {
         }
         const searchQuery = buildSearchQuery(url);
         const { provider, apiKey } = getBestSearchProvider();
-        const results = await provider.searchWeb(searchQuery, {
-            count: 3,
-            apiKey,
-        });
-        if (!results || results.length === 0) {
-            return emptyResult;
+        // Map provider ID to our source type
+        const sourceMap = {
+            duckduckgo: 'duckduckgo',
+            brave: 'google',
+            stealth: 'duckduckgo',
+            google: 'google',
+        };
+        // Try the primary (best) provider first
+        let results = await provider.searchWeb(searchQuery, { count: 5, apiKey }).catch(() => []);
+        // If primary returns sparse results (< 100 tokens), try DDG as secondary
+        const combinedSnippets = [];
+        let title = '';
+        let source = sourceMap[provider.id] ?? 'google';
+        if (results.length > 0) {
+            title = results[0].title?.trim() || '';
+            combinedSnippets.push(...results.map(r => r.snippet?.trim()).filter(Boolean));
+        }
+        const primaryTokens = countTokens(combinedSnippets.join(' '));
+        if (primaryTokens < 100) {
+            // Try DDG as a secondary engine to supplement
+            try {
+                const ddgProvider = new DuckDuckGoProvider();
+                const ddgResults = await ddgProvider.searchWeb(searchQuery, { count: 5 });
+                if (ddgResults.length > 0) {
+                    if (!title)
+                        title = ddgResults[0].title?.trim() || '';
+                    if (source !== 'duckduckgo')
+                        source = 'duckduckgo';
+                    combinedSnippets.push(...ddgResults.map(r => r.snippet?.trim()).filter(Boolean));
+                }
+            }
+            catch { /* ignore secondary failure */ }
         }
-        const topResult = results[0];
-        const title = topResult.title?.trim() || '';
-        const snippet = topResult.snippet?.trim() || '';
+        // Also try Google Cache URL as a last-resort content source
+        const allSnippets = [...new Set(combinedSnippets)]; // deduplicate
+        const snippet = allSnippets.slice(0, 3).join('\n\n');
         if (!title && !snippet) {
             return emptyResult;
         }
@@ -159,14 +192,6 @@ export async function searchFallback(url) {
             ? extractProductData(title, snippet)
             : undefined;
         const cachedContent = buildCachedContent(url, title, snippet, productData);
-        // Map provider ID to our source type
-        const sourceMap = {
-            duckduckgo: 'duckduckgo',
-            brave: 'google',
-            stealth: 'duckduckgo',
-            google: 'google',
-        };
-        const source = sourceMap[provider.id] ?? 'google';
         return {
             title,
             snippet,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.71",
+  "version": "0.21.73",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",
@@ -8,7 +8,7 @@
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "bin": {
-    "webpeel": "dist/cli.bundle.cjs"
+    "webpeel": "dist/cli.js"
   },
   "exports": {
     ".": {