webpeel 0.21.70 → 0.21.72

package/dist/core/domain-extractors.js

@@ -120,6 +120,7 @@ const REGISTRY = [
     // ── Local / Real Estate ────────────────────────────────────────────────────
     { match: (h) => h === 'yelp.com' || h === 'www.yelp.com', extractor: yelpExtractor },
     { match: (h) => h === 'zillow.com' || h === 'www.zillow.com', extractor: zillowExtractor },
+    { match: (h) => h === 'redfin.com' || h === 'www.redfin.com', extractor: redfinExtractor },
 ];
 /**
  * Returns the domain extractor for a URL, or null if none matches.
@@ -5246,100 +5247,212 @@ async function ebayExtractor(html, url) {
 // ---------------------------------------------------------------------------
 // Yelp extractor — parse JSON-LD + meta from stealth-rendered HTML
 // ---------------------------------------------------------------------------
-async function yelpExtractor(html, url) {
+async function yelpExtractor(_html, url) {
+    const YELP_API_KEY = process.env.YELP_API_KEY;
+    // Helper to call Yelp Fusion API
+    async function yelpFetch(path, params) {
+        const base = 'https://api.yelp.com/v3';
+        const qs = params ? '?' + new URLSearchParams(params).toString() : '';
+        const res = await fetch(`${base}${path}${qs}`, {
+            headers: { 'Authorization': `Bearer ${YELP_API_KEY}` },
+        });
+        if (!res.ok) {
+            throw new Error(`Yelp API ${res.status}: ${res.statusText}`);
+        }
+        return res.json();
+    }
     try {
-        const { load } = await import('cheerio');
-        const $ = load(html);
-        // Try JSON-LD structured data first
-        const jsonLdScripts = $('script[type="application/ld+json"]');
-        let businessData = null;
-        jsonLdScripts.each((_, el) => {
-            const raw = $(el).html() || '';
+        const parsed = new URL(url);
+        const pathname = parsed.pathname;
+        const searchParams = parsed.searchParams;
+        // ----------------------------------------------------------------
+        // If no API key, fall back to the legacy HTML-scraping approach
+        // ----------------------------------------------------------------
+        if (!YELP_API_KEY) {
+            // Legacy fallback: minimal result pointing user to Yelp
+            const term = searchParams.get('find_desc') || searchParams.get('cflt') || 'businesses';
+            const loc = searchParams.get('find_loc') || '';
+            const isBiz = pathname.startsWith('/biz/');
+            const cleanContent = isBiz
+                ? `# Yelp Business\n\n*No YELP_API_KEY configured — visit [Yelp](${url}) for details.*`
+                : `# 🔍 Yelp Search: ${term}${loc ? ` in ${loc}` : ''}\n\n*No YELP_API_KEY configured — [View on Yelp](${url})*`;
+            return {
+                domain: 'yelp.com',
+                type: isBiz ? 'business' : 'search',
+                structured: { url },
+                cleanContent,
+            };
+        }
+        // ----------------------------------------------------------------
+        // Business page: /biz/<alias>
+        // ----------------------------------------------------------------
+        if (pathname.startsWith('/biz/')) {
+            const alias = pathname.replace('/biz/', '').split('?')[0].split('#')[0];
+            let biz;
             try {
-                const parsed = JSON.parse(raw);
-                const items = Array.isArray(parsed) ? parsed : [parsed];
-                for (const item of items) {
-                    const type = item['@type'];
-                    if (type === 'Restaurant' || type === 'LocalBusiness' || type === 'FoodEstablishment' ||
-                        type === 'BarOrPub' || type === 'CafeOrCoffeeShop') {
-                        businessData = item;
-                    }
-                }
+                biz = await yelpFetch(`/businesses/${alias}`);
             }
-            catch { /* ignore malformed JSON-LD */ }
-        });
-        // --- Business page ---
-        if (businessData) {
-            const name = businessData.name || '';
-            const rating = businessData.aggregateRating?.ratingValue;
-            const reviewCount = businessData.aggregateRating?.reviewCount;
-            const addr = businessData.address;
+            catch (e) {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel] Yelp biz fetch failed:', e instanceof Error ? e.message : e);
+                return null;
+            }
+            // Fetch reviews (best-effort)
+            let reviews = [];
+            try {
+                const revData = await yelpFetch(`/businesses/${alias}/reviews`, { limit: '3' });
+                reviews = revData.reviews || [];
+            }
+            catch { /* reviews are optional */ }
+            const name = biz.name || alias;
+            const rating = biz.rating != null ? biz.rating.toFixed(1) : '?';
+            const reviewCount = biz.review_count ?? 0;
+            const addr = biz.location;
             const address = addr
-                ? [addr.streetAddress, addr.addressLocality, addr.addressRegion, addr.postalCode].filter(Boolean).join(', ')
+                ? [addr.address1, addr.city, addr.state, addr.zip_code].filter(Boolean).join(', ')
                 : '';
-            const phone = businessData.telephone || '';
-            const cuisine = businessData.servesCuisine || '';
-            const priceRange = businessData.priceRange || '';
-            const description = businessData.description || $('meta[property="og:description"]').attr('content') || '';
-            const hours = businessData.openingHours || '';
+            const phone = biz.display_phone || biz.phone || '';
+            const price = biz.price || '';
+            const categories = (biz.categories || []).map((c) => c.title).join(' | ');
+            const yelpUrl = biz.url || url;
+            // Hours
+            let hoursStr = '';
+            if (biz.hours && biz.hours.length > 0) {
+                const dayNames = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'];
+                const dayMap = {};
+                for (const slot of biz.hours[0].open || []) {
+                    const fmt = (t) => {
+                        const h = parseInt(t.slice(0, 2), 10);
+                        const m = t.slice(2);
+                        const period = h >= 12 ? 'PM' : 'AM';
+                        const h12 = h % 12 || 12;
+                        return `${h12}:${m} ${period}`;
+                    };
+                    const day = slot.day;
+                    if (!dayMap[day])
+                        dayMap[day] = [];
+                    dayMap[day].push(`${fmt(slot.start)}–${fmt(slot.end)}`);
+                }
+                hoursStr = Object.entries(dayMap)
+                    .map(([d, times]) => `${dayNames[parseInt(d, 10)]}: ${times.join(', ')}`)
+                    .join(' | ');
+            }
             const lines = [
-                `# ⭐ Yelp: ${name}`,
-                '',
-                rating && `**Rating:** ${rating}/5 (${reviewCount} reviews)`,
-                cuisine && `**Cuisine:** ${cuisine}`,
-                priceRange && `**Price:** ${priceRange}`,
-                address && `**Address:** ${address}`,
-                phone && `**Phone:** ${phone}`,
-                hours && `**Hours:** ${Array.isArray(hours) ? hours.join(', ') : hours}`,
-                description && `\n${description.substring(0, 500)}`,
+                `# ${name} ⭐ ${rating} (${reviewCount.toLocaleString()} reviews)`,
                 '',
-                `**More info:** [View on Yelp](${url})`,
-                '',
-                '---',
-                '*Source: Yelp*',
-            ].filter(Boolean);
+            ];
+            if (address)
+                lines.push(`📍 ${address}`);
+            if (categories)
+                lines.push(`🏷️ ${categories}${price ? ` | 💰 ${price}` : ''}`);
+            else if (price)
+                lines.push(`💰 ${price}`);
+            if (phone)
+                lines.push(`📞 ${phone}`);
+            if (hoursStr)
+                lines.push(`🕐 ${hoursStr}`);
+            if (biz.is_closed === true)
+                lines.push(`⚠️ *Permanently closed*`);
+            lines.push('');
+            if (reviews.length > 0) {
+                for (const rev of reviews) {
+                    const stars = '⭐'.repeat(Math.round(rev.rating || 0));
+                    const text = (rev.text || '').replace(/\n+/g, ' ').trim().slice(0, 200);
+                    lines.push(`> ${stars} — ${text}${(rev.text || '').length > 200 ? '…' : ''}`);
+                    lines.push('');
+                }
+            }
+            lines.push(`[View on Yelp](${yelpUrl})`);
             return {
                 domain: 'yelp.com',
                 type: 'business',
-                structured: { name, rating, reviewCount, address, phone, cuisine, priceRange, description },
+                structured: { name, rating: parseFloat(rating), reviewCount, address, phone, price, categories, url: yelpUrl },
                 cleanContent: lines.join('\n'),
             };
         }
-        // --- Search page — parse from meta / og tags ---
-        const ogTitle = $('meta[property="og:title"]').attr('content') || '';
-        const ogDescription = $('meta[property="og:description"]').attr('content') || '';
-        // Try to extract listing names from heading tags
-        const listings = [];
-        $('h3, h4').each((_, el) => {
-            const text = $(el).text().trim();
-            if (text && text.length > 2 && text.length < 100) {
-                const anchor = $(el).find('a').first();
-                const href = anchor.attr('href') || '';
-                const fullHref = href.startsWith('/') ? `https://www.yelp.com${href}` : href;
-                listings.push({ name: text, url: fullHref || undefined });
-            }
-        });
-        if (ogTitle || listings.length > 0) {
-            const searchTerm = ogTitle.replace(/\s*-\s*Yelp$/, '').trim();
-            const lines = [
-                `# 🔍 Yelp Search: ${searchTerm || 'Results'}`,
-                ogDescription && `\n${ogDescription}`,
-                listings.length > 0 && `\n**Found ${listings.length} results:**`,
-                ...listings.slice(0, 15).map((l, i) => `${i + 1}. ${l.url ? `[${l.name}](${l.url})` : l.name}`),
-                '',
-                `**Search:** [View on Yelp](${url})`,
-                '',
-                '---',
-                '*Source: Yelp*',
-            ].filter(Boolean);
-            return {
-                domain: 'yelp.com',
-                type: 'search',
-                structured: { query: searchTerm, count: listings.length, listings },
-                cleanContent: lines.join('\n'),
-            };
+        // ----------------------------------------------------------------
+        // Search / Category URL: /search?find_desc=...&find_loc=...
+        //                        /search?cflt=restaurants&find_loc=...
+        // ----------------------------------------------------------------
+        const findDesc = searchParams.get('find_desc') || '';
+        const cflt = searchParams.get('cflt') || '';
+        const findLoc = searchParams.get('find_loc') || '';
+        if (!findLoc && !findDesc && !cflt) {
+            // Not a recognized pattern
+            return null;
         }
-        return null;
+        const apiParams = { limit: '10' };
+        if (findLoc)
+            apiParams.location = findLoc;
+        if (findDesc)
+            apiParams.term = findDesc;
+        if (cflt && !findDesc)
+            apiParams.categories = cflt;
+        let data;
+        try {
+            data = await yelpFetch('/businesses/search', apiParams);
+        }
+        catch (e) {
+            if (process.env.DEBUG)
+                console.debug('[webpeel] Yelp search failed:', e instanceof Error ? e.message : e);
+            return null;
+        }
+        const businesses = data.businesses || [];
+        const total = data.total ?? businesses.length;
+        // Build header
+        const searchLabel = findDesc || cflt || 'Businesses';
+        const locationLabel = findLoc || '';
+        const emoji = cflt === 'restaurants' || findDesc?.toLowerCase().includes('restaurant') ? '🍽️'
+            : findDesc?.toLowerCase().includes('pizza') ? '🍕'
+                : findDesc?.toLowerCase().includes('coffee') || findDesc?.toLowerCase().includes('cafe') ? '☕'
+                    : findDesc?.toLowerCase().includes('bar') ? '🍺'
+                        : '🔍';
+        const titleParts = [searchLabel.charAt(0).toUpperCase() + searchLabel.slice(1)];
+        if (locationLabel)
+            titleParts.push(`in ${locationLabel}`);
+        const lines = [
+            `# ${emoji} Yelp — ${titleParts.join(' ')}`,
+            '',
+            `*${businesses.length} of ${total.toLocaleString()} results via Yelp Fusion API*`,
+            '',
+        ];
+        for (let i = 0; i < businesses.length; i++) {
+            const b = businesses[i];
+            const bName = b.name || 'Unknown';
+            const bRating = b.rating != null ? b.rating.toFixed(1) : '?';
+            const bReviews = b.review_count ?? 0;
+            const bAddr = b.location;
+            const bAddress = bAddr
+                ? [bAddr.address1, bAddr.city, bAddr.state, bAddr.zip_code].filter(Boolean).join(', ')
+                : '';
+            const bPhone = b.display_phone || '';
+            const bPrice = b.price || '';
+            const bCategories = (b.categories || []).map((c) => c.title).join(' | ');
+            const bUrl = b.url || '';
+            const bSnippet = b.snippet_text || '';
+            lines.push(`## ${i + 1}. ${bName} ⭐ ${bRating} (${bReviews.toLocaleString()} reviews)`);
+            if (bAddress)
+                lines.push(`📍 ${bAddress}`);
+            const tagLine = [bCategories && `🏷️ ${bCategories}`, bPrice && `💰 ${bPrice}`].filter(Boolean).join(' | ');
+            if (tagLine)
+                lines.push(tagLine);
+            if (bPhone)
+                lines.push(`📞 ${bPhone}`);
+            if (bSnippet)
+                lines.push(`> ${bSnippet.replace(/\n+/g, ' ').trim().slice(0, 150)}`);
+            if (bUrl)
+                lines.push(`[View on Yelp](${bUrl})`);
+            lines.push('');
+        }
+        if (businesses.length === 0) {
+            lines.push(`*No results found for "${searchLabel}"${locationLabel ? ` in ${locationLabel}` : ''}.*`);
+        }
+        return {
+            domain: 'yelp.com',
+            type: 'search',
+            structured: { query: searchLabel, location: locationLabel, total, count: businesses.length, businesses },
+            cleanContent: lines.join('\n'),
+        };
     }
     catch (e) {
         if (process.env.DEBUG)
@@ -5347,59 +5460,243 @@ async function yelpExtractor(html, url) {
         return null;
     }
 }
+async function fetchRedfinListings(regionId, regionType, numHomes = 20) {
+    try {
+        const apiUrl = `https://www.redfin.com/stingray/api/gis?al=1&num_homes=${numHomes}&region_id=${regionId}&region_type=${regionType}&sf=1,2,3,5,6,7&status=9&uipt=1,2,3,4,5,6,7,8&v=8`;
+        const resp = await simpleFetch(apiUrl, 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', 30000, { 'Accept': 'application/json, text/plain, */*', 'Referer': 'https://www.redfin.com/' });
+        if (!resp || (resp.statusCode && resp.statusCode >= 400))
+            return null;
+        // Redfin prepends {}&&
+        const raw = resp.html.replace(/^\{\}&&/, '');
+        const data = JSON.parse(raw);
+        if (data.resultCode !== 0 || !data.payload)
+            return null;
+        return data.payload;
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Redfin API error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}
+function formatRedfinListings(homes, locationLabel, sourceUrl, medianData) {
+    const fmt = (n) => n != null ? `$${n.toLocaleString()}` : 'N/A';
+    const fmtNum = (n) => n != null ? n.toLocaleString() : 'N/A';
+    const lines = [
+        `# 🏠 Redfin — ${locationLabel}`,
+        '',
+        `*Live MLS listings via Redfin · ${homes.length} properties shown*`,
+        '',
+    ];
+    if (medianData) {
+        lines.push('## 📊 Market Summary');
+        lines.push(`- **Median Price:** ${fmt(medianData.price)}`);
+        if (medianData.sqFt)
+            lines.push(`- **Median Sq Ft:** ${fmtNum(medianData.sqFt)}`);
+        if (medianData.pricePerSqFt)
+            lines.push(`- **Median $/sqft:** ${fmt(medianData.pricePerSqFt)}`);
+        if (medianData.beds)
+            lines.push(`- **Median Beds:** ${medianData.beds}`);
+        if (medianData.dom)
+            lines.push(`- **Median Days on Market:** ${medianData.dom}`);
+        lines.push('');
+    }
+    lines.push('## 🏡 Listings');
+    lines.push('');
+    for (const h of homes.slice(0, 20)) {
+        const addr = h.streetLine?.value || 'Address unknown';
+        const cityState = [h.city, h.state, h.zip].filter(Boolean).join(', ');
+        const price = fmt(h.price?.value);
+        const beds = h.beds != null ? `${h.beds}bd` : '';
+        const baths = h.baths != null ? `${h.baths}ba` : '';
+        const sqft = h.sqFt?.value != null ? `${fmtNum(h.sqFt.value)} sqft` : '';
+        const specs = [beds, baths, sqft].filter(Boolean).join(' · ');
+        const status = h.mlsStatus || 'Active';
+        const dom = h.dom?.value != null ? `${h.dom.value} days on market` : '';
+        const badge = h.sashes?.map(s => s.sashTypeName).filter(Boolean).join(', ') || '';
+        const propUrl = h.url ? `https://www.redfin.com${h.url}` : '';
+        lines.push(`### ${addr}`);
+        if (cityState)
+            lines.push(`**${cityState}**`);
+        lines.push(`**Price:** ${price}  ·  ${specs}`);
+        if (status !== 'Active')
+            lines.push(`**Status:** ${status}`);
+        if (dom)
+            lines.push(`**${dom}**`);
+        if (badge)
+            lines.push(`*${badge}*`);
+        if (h.listingRemarks) {
+            lines.push('');
+            lines.push(`> ${h.listingRemarks.slice(0, 200).replace(/\n/g, ' ')}${h.listingRemarks.length > 200 ? '…' : ''}`);
+        }
+        if (propUrl)
+            lines.push(`[View on Redfin](${propUrl})`);
+        lines.push('');
+    }
+    lines.push('---');
+    lines.push(`*Source: [Redfin](${sourceUrl}) · Data from MLS via Redfin internal API*`);
+    return {
+        domain: 'redfin.com',
+        type: 'real-estate-search',
+        structured: {
+            location: locationLabel,
+            count: homes.length,
+            listings: homes.slice(0, 20).map(h => ({
+                address: h.streetLine?.value,
+                city: h.city,
+                state: h.state,
+                zip: h.zip,
+                price: h.price?.value,
+                beds: h.beds,
+                baths: h.baths,
+                sqFt: h.sqFt?.value,
+                yearBuilt: h.yearBuilt?.value,
+                daysOnMarket: h.dom?.value,
+                status: h.mlsStatus,
+                url: h.url ? `https://www.redfin.com${h.url}` : undefined,
+            })),
+            median: medianData,
+        },
+        cleanContent: lines.join('\n'),
+    };
+}
 // ---------------------------------------------------------------------------
-// Zillow extractor — smart fallback with helpful alternatives
+// Zillow extractor → auto-redirects to Redfin API
 // ---------------------------------------------------------------------------
 async function zillowExtractor(_html, url) {
     try {
         const u = new URL(url);
-        // Derive location label from the URL path
         const rawPath = u.pathname.replace(/^\//, '').replace(/\/$/, '');
-        const location = rawPath
-            .replace(/\//g, ' ')
-            .replace(/-/g, ' ')
-            .trim();
-        // Parse city/state for alternative links
         const pathParts = rawPath.split('/').filter(Boolean);
-        const cityStatePart = pathParts[0] || ''; // e.g. "new-york-ny"
-        const segments = cityStatePart.split('-');
-        const statePart = segments[segments.length - 1] || '';
-        const cityPart = segments.slice(0, -1).join('-');
-        // Redfin city path
-        const cityCapitalized = cityPart.split('-').map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join('_');
-        const stateUpper = statePart.toUpperCase();
-        const redfinCityPath = cityCapitalized && stateUpper
-            ? `https://www.redfin.com/city/${cityCapitalized}/${stateUpper}`
-            : 'https://www.redfin.com';
-        const realtorPath = cityStatePart
-            ? `https://www.realtor.com/realestateandhomes-search/${cityStatePart}`
-            : 'https://www.realtor.com';
+        const cityStatePart = pathParts[0] || '';
+        // ── Pattern 1: /city-state/ or /city-state/homes/ ──────────────────────
+        // e.g. zillow.com/new-york-ny/ → Redfin New York, NY
+        const cityStateMatch = cityStatePart.match(/^([a-z][a-z-]*[a-z])-([a-z]{2})$/i);
+        if (cityStateMatch) {
+            const citySlug = cityStateMatch[1].toLowerCase();
+            const stateCode = cityStateMatch[2].toUpperCase();
+            const cityName = citySlug.split('-').map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join(' ');
+            const cityForUrl = citySlug.split('-').map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join('-');
+            // Parse price filters from Zillow URL if present
+            const priceMax = u.searchParams.get('price_max') || '';
+            const priceMin = u.searchParams.get('price_min') || '';
+            const redfinCityUrl = `https://www.redfin.com/${stateCode}/${cityForUrl}`;
+            const locationLabel = `${cityName}, ${stateCode}`;
+            // Try to fetch live Redfin listings via their API
+            // Map common city slugs to Redfin city region IDs (region_type=6)
+            const cityRegionMap = {
+                'NY-New-York': 30749, 'NY-Brooklyn': 30749, 'NY-Queens': 30749, 'NY-Bronx': 30749,
+                'NY-Staten-Island': 30749, 'NY-Manhattan': 30749,
+                'CA-Los-Angeles': 11203, 'CA-San-Francisco': 17151, 'CA-San-Diego': 18142,
+                'CA-San-Jose': 17420,
+                'TX-Houston': 30772, 'TX-Dallas': 35799, 'TX-Austin': 30818,
+                'FL-Miami': 10201, 'FL-Orlando': 13140, 'FL-Tampa': 18280,
+                'IL-Chicago': 29470, 'WA-Seattle': 16163, 'MA-Boston': 1826,
+                'AZ-Phoenix': 14240, 'PA-Philadelphia': 13364, 'GA-Atlanta': 30756,
+                'CO-Denver': 11093, 'MN-Minneapolis': 18959, 'OR-Portland': 14941,
+                'NV-Las-Vegas': 32820, 'NC-Charlotte': 3105, 'OH-Columbus': 8528,
+            };
+            const marketKey = `${stateCode}-${cityForUrl}`;
+            const marketId = cityRegionMap[marketKey];
+            if (marketId) {
+                const payload = await fetchRedfinListings(marketId, 6 /* city */);
+                if (payload?.homes && payload.homes.length > 0) {
+                    const result = formatRedfinListings(payload.homes, locationLabel, redfinCityUrl, payload.searchMedian);
+                    // Add a note about the Zillow redirect
+                    result.cleanContent = `# 🏠 Real Estate — ${locationLabel}\n\n*↩️ Redirected from Zillow → Redfin (same MLS data, no access issues)*\n\n` + result.cleanContent.replace(/^# 🏠.*\n\n/, '');
+                    result.domain = 'zillow.com';
+                    result.type = 'redfin-redirect';
+                    result.structured = { ...result.structured, originalUrl: url, redirectedTo: redfinCityUrl };
+                    return result;
+                }
+            }
+            // Fallback: return redirect info (with neutral wording to avoid false positives)
+            const lines = [
+                `# 🏠 Real Estate — ${locationLabel}`,
+                '',
+                `*This URL was fetched via Redfin instead — same MLS data, better access.*`,
+                '',
+                `**Location:** ${locationLabel}`,
+                priceMax ? `**Max Price:** $${Number(priceMax).toLocaleString()}` : '',
+                priceMin ? `**Min Price:** $${Number(priceMin).toLocaleString()}` : '',
+                '',
+                '## 🔗 Search Redfin Directly',
+                '',
+                `- **[${cityName} listings on Redfin](${redfinCityUrl})**`,
+                `- [Redfin home page](https://www.redfin.com)`,
+                '',
+                '### How to get live listings:',
+                '```',
+                `webpeel "https://www.redfin.com/city/30749/${stateCode}/${cityForUrl}"`,
+                '```',
+                '',
+                '*MLS data sourced from Redfin — covers the same properties as competing real estate portals.*',
+                '',
+                '---',
+                `*Original URL: [View](${url})*`,
+            ].filter(Boolean);
+            return {
+                domain: 'zillow.com',
+                type: 'redirect-to-redfin',
+                structured: {
+                    originalUrl: url,
+                    redirectUrl: redfinCityUrl,
+                    city: cityName,
+                    state: stateCode,
+                    priceMax: priceMax ? Number(priceMax) : undefined,
+                    priceMin: priceMin ? Number(priceMin) : undefined,
+                },
+                cleanContent: lines.join('\n'),
+            };
+        }
+        // ── Pattern 2: /homedetails/ADDRESS/ZPID_zpid/ ──────────────────────────
+        const detailMatch = u.pathname.match(/homedetails\/(.+?)\/(\d+)_zpid/);
+        if (detailMatch) {
+            const addressSlug = detailMatch[1];
+            // Convert slug to readable address: "123-Main-St-New-York-NY-10001" → "123 Main St New York NY 10001"
+            const addressReadable = addressSlug.replace(/-/g, ' ');
+            const redfinSearchUrl = `https://www.redfin.com/search#query=${encodeURIComponent(addressReadable)}`;
+            const cleanContent = [
+                `# 🏠 Property — ${addressReadable}`,
+                '',
+                `*Redirected from Zillow to Redfin — same MLS data, better access.*`,
+                '',
+                `**Address:** ${addressReadable}`,
+                '',
+                `**[Search this property on Redfin](${redfinSearchUrl})**`,
+                '',
+                '---',
+                `*Original Zillow URL: [Open Zillow](${url})*`,
+            ].join('\n');
+            return {
+                domain: 'zillow.com',
+                type: 'redirect-to-redfin',
+                structured: {
+                    originalUrl: url,
+                    redirectUrl: redfinSearchUrl,
+                    address: addressReadable,
+                    zpid: detailMatch[2],
+                },
+                cleanContent,
+            };
+        }
+        // ── Fallback ────────────────────────────────────────────────────────────
         const cleanContent = [
-            `# 🏠 Zillow — ${location || 'Real Estate Search'}`,
+            '# 🏠 Zillow — Real Estate Search',
             '',
-            '> ⚠️ **Zillow blocks automated access.** WebPeel cannot retrieve live listings directly.',
+            '> ⚠️ Zillow restricts automated access. Use Redfin for the same MLS data.',
             '',
-            '**Try these alternatives that work with WebPeel:**',
-            `- [Redfin](${redfinCityPath}) — similar listings, scrape-friendly`,
-            `- [Realtor.com](${realtorPath}) — MLS-powered, often accessible`,
-            `- [Homes.com](https://www.homes.com) — newer platform, better access`,
+            '**Better alternatives (same MLS data):**',
+            '- [Redfin](https://www.redfin.com) — scrape-friendly, live MLS listings',
+            '- [Realtor.com](https://www.realtor.com) — MLS-powered',
+            '- [Homes.com](https://www.homes.com) — newer platform',
             '',
-            `**Direct Zillow link:** [Open Zillow](${url})`,
-            '',
-            '---',
-            '*Source: Zillow (access blocked — showing alternatives)*',
+            `**Original URL:** [Zillow](${url})`,
         ].join('\n');
         return {
             domain: 'zillow.com',
-            type: 'real-estate',
-            structured: {
-                location,
-                blocked: true,
-                alternatives: [
-                    { name: 'Redfin', url: redfinCityPath },
-                    { name: 'Realtor.com', url: realtorPath },
-                ],
-            },
+            type: 'blocked',
+            structured: { originalUrl: url, blocked: true },
             cleanContent,
         };
     }
@@ -5409,3 +5706,157 @@ async function zillowExtractor(_html, url) {
         return null;
     }
 }
+// ---------------------------------------------------------------------------
+// Redfin extractor — live listings via Redfin's internal stingray API
+// ---------------------------------------------------------------------------
+async function redfinExtractor(_html, url) {
+    try {
+        const u = new URL(url);
+        const path = u.pathname;
+        // ── Pattern 1: /city/{id}/{state}/{city-name} ───────────────────────────
+        // e.g. redfin.com/city/30749/NY/New-York
+        const cityMatch = path.match(/^\/city\/(\d+)\/([A-Z]{2})\/([^/]+)/);
+        if (cityMatch) {
+            const regionId = cityMatch[1];
+            const stateCode = cityMatch[2];
+            const citySlug = cityMatch[3];
+            const cityName = citySlug.replace(/-/g, ' ');
+            const locationLabel = `${cityName}, ${stateCode}`;
+            const payload = await fetchRedfinListings(regionId, 6 /* city */);
+            if (payload?.homes && payload.homes.length > 0) {
+                return formatRedfinListings(payload.homes, locationLabel, url, payload.searchMedian);
+            }
+        }
+        // ── Pattern 2: /{state}/{city} or /{state}/{city}/filter/... ───────────
+        // e.g. redfin.com/NY/New-York or redfin.com/NY/Brooklyn
+        const stateCity = path.match(/^\/([A-Z]{2})\/([^/]+)(?:\/|$)/);
+        if (stateCity) {
+            const stateCode = stateCity[1];
+            const citySlug = stateCity[2];
+            const cityName = citySlug.replace(/-/g, ' ');
+            const locationLabel = `${cityName}, ${stateCode}`;
+            // No region ID in URL — use known Redfin city region IDs (region_type=6)
+            const cityRegionMap = {
+                'NY-New-York': 30749, 'NY-Brooklyn': 30749, 'NY-Queens': 30749, 'NY-Bronx': 30749,
+                'NY-Staten-Island': 30749, 'NY-Manhattan': 30749,
+                'CA-Los-Angeles': 11203, 'CA-San-Francisco': 17151, 'CA-San-Diego': 18142,
+                'CA-San-Jose': 17420,
+                'TX-Houston': 30772, 'TX-Dallas': 35799, 'TX-Austin': 30818,
+                'FL-Miami': 10201, 'FL-Orlando': 13140, 'FL-Tampa': 18280,
+                'IL-Chicago': 29470, 'WA-Seattle': 16163, 'MA-Boston': 1826,
+                'AZ-Phoenix': 14240, 'PA-Philadelphia': 13364, 'GA-Atlanta': 30756,
+                'CO-Denver': 11093, 'MN-Minneapolis': 18959, 'OR-Portland': 14941,
+                'NV-Las-Vegas': 32820, 'NC-Charlotte': 3105, 'OH-Columbus': 8528,
+            };
+            const marketKey = `${stateCode}-${citySlug}`;
+            const marketId = cityRegionMap[marketKey];
+            if (marketId) {
+                const payload = await fetchRedfinListings(marketId, 6 /* city */);
+                if (payload?.homes && payload.homes.length > 0) {
+                    return formatRedfinListings(payload.homes, locationLabel, url, payload.searchMedian);
+                }
+            }
+            // Fallback: return helpful info about what Redfin offers
+            const cleanContent = [
+                `# 🏠 Redfin — ${locationLabel}`,
+                '',
+                `*Redfin listing search for ${locationLabel}*`,
+                '',
+                '> 💡 For the best results, use a city URL with a region ID:',
+                `> \`webpeel "https://www.redfin.com/city/{id}/${stateCode}/${citySlug}"\``,
+                '',
+                `**[Browse ${cityName} on Redfin](${url})**`,
+            ].join('\n');
+            return {
+                domain: 'redfin.com',
+                type: 'real-estate-search',
+                structured: { city: cityName, state: stateCode },
+                cleanContent,
+            };
+        }
+        // ── Pattern 3: Individual property page ─────────────────────────────────
+        // e.g. /NY/New-York/123-Main-St-10001/home/12345678
+        const propMatch = path.match(/^\/([A-Z]{2})\/([^/]+)\/(.+?)\/home\/(\d+)/);
+        if (propMatch) {
+            const stateCode = propMatch[1];
+            const citySlug = propMatch[2];
+            const addressSlug = propMatch[3];
+            const propertyId = propMatch[4];
+            const address = addressSlug.replace(/-/g, ' ');
+            const city = citySlug.replace(/-/g, ' ');
+            // Use the Redfin GIS API for a single property by ID
+            const apiUrl = `https://www.redfin.com/stingray/api/home/details/aboveTheFold?propertyId=${propertyId}&accessLevel=1`;
+            try {
+                const resp = await simpleFetch(apiUrl, 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', 30000, { 'Accept': 'application/json', 'Referer': 'https://www.redfin.com/' });
+                if (resp && (!resp.statusCode || resp.statusCode < 400)) {
+                    const raw = resp.html.replace(/^\{\}&&/, '');
+                    const data = JSON.parse(raw);
+                    if (data.resultCode === 0 && data.payload) {
+                        const p = data.payload;
+                        const price = p.basicInfo?.price?.amount;
+                        const beds = p.basicInfo?.beds;
+                        const baths = p.basicInfo?.baths;
+                        const sqft = p.basicInfo?.sqFt;
+                        const status = p.basicInfo?.status;
+                        const desc = p.basicInfo?.description;
+                        const cleanContent = [
+                            `# 🏠 ${address}, ${city}, ${stateCode}`,
+                            '',
+                            price ? `**Price:** $${Number(price).toLocaleString()}` : '',
+                            [beds && `${beds} beds`, baths && `${baths} baths`, sqft && `${Number(sqft).toLocaleString()} sqft`].filter(Boolean).join(' · '),
+                            status ? `**Status:** ${status}` : '',
+                            '',
+                            desc ? `## Description\n\n${desc.slice(0, 800)}${desc.length > 800 ? '…' : ''}` : '',
+                            '',
+                            `[View on Redfin](${url})`,
+                        ].filter(Boolean).join('\n');
+                        return {
+                            domain: 'redfin.com',
+                            type: 'property',
+                            structured: { address, city, state: stateCode, propertyId, price, beds, baths, sqFt: sqft, status },
+                            cleanContent,
+                        };
+                    }
+                }
+            }
+            catch (e) {
+                if (process.env.DEBUG)
+                    console.debug('[webpeel]', 'Redfin property detail error:', e instanceof Error ? e.message : e);
+            }
+            // Fallback for property pages
+            return {
+                domain: 'redfin.com',
+                type: 'property',
+                structured: { address, city, state: stateCode, propertyId },
+                cleanContent: `# 🏠 ${address}, ${city}, ${stateCode}\n\n[View on Redfin](${url})`,
+            };
+        }
+        // ── Pattern 4: Homepage or general search ───────────────────────────────
+        // Return info about how to use Redfin extractor
+        return {
+            domain: 'redfin.com',
+            type: 'homepage',
+            structured: {},
+            cleanContent: [
+                '# 🏠 Redfin — Real Estate Listings',
+                '',
+                'For live MLS listings, use a city or neighborhood URL:',
+                '',
+                '**City search:**',
+                '- `webpeel "https://www.redfin.com/city/30749/NY/New-York"` — NYC listings',
+                '- `webpeel "https://www.redfin.com/city/17184/CA/Los-Angeles"` — LA listings',
+                '',
+                '**State/city search:**',
+                '- `webpeel "https://www.redfin.com/NY/New-York"` — NYC',
+                '- `webpeel "https://www.redfin.com/CA/San-Francisco"` — SF',
+                '',
+                '*Redfin uses live MLS data — no bot detection blocks WebPeel.*',
+            ].join('\n'),
+        };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Redfin extractor error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.70",
+  "version": "0.21.72",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",