webpeel 0.21.67 → 0.21.69

package/dist/core/domain-extractors.d.ts

@@ -30,10 +30,19 @@ export type DomainExtractor = (html: string, url: string) => Promise<DomainExtra
 export declare function getDomainExtractor(url: string): DomainExtractor | null;
 /** Clear the extractor response cache (used in tests). */
 export declare function clearExtractorCache(): void;
+/**
+ * Inject a Redis client for shared cross-pod caching.
+ * Called from server startup after Redis is initialized.
+ * Safe to call with null to disable Redis caching (e.g., CLI mode).
+ */
+export declare function setExtractorRedis(redis: any): void;
 /**
  * Convenience: run the extractor for the URL (if one exists).
- * Wraps _extractDomainDataImpl with a 5-minute LRU cache so that
- * rate-limited API responses fall back to cached results instead of
- * garbage browser rendering.
+ * Wraps _extractDomainDataImpl with a two-tier cache:
+ *   1. In-memory LRU (per-pod, fastest)
+ *   2. Redis shared cache (cross-pod, shared across all replicas)
+ *
+ * With multiple API pods, Redis ensures the first pod to fetch a URL
+ * populates cache for all others — eliminating redundant API calls.
  */
 export declare function extractDomainData(html: string, url: string): Promise<DomainExtractResult | null>;

package/dist/core/domain-extractors.js

@@ -112,6 +112,14 @@ const REGISTRY = [
     { match: (h) => h === 'open-meteo.com' || h === 'api.open-meteo.com' || h === 'www.open-meteo.com', extractor: weatherExtractor },
     { match: (h) => h === 'weather.com' || h === 'www.weather.com', extractor: weatherExtractor },
     { match: (h) => h === 'accuweather.com' || h === 'www.accuweather.com', extractor: weatherExtractor },
+    // ── Marketplaces & Shopping ───────────────────────────────────────────────
+    { match: (h) => h === 'facebook.com' || h === 'www.facebook.com', extractor: facebookMarketplaceExtractor },
+    { match: (h) => h === 'etsy.com' || h === 'www.etsy.com', extractor: etsyExtractor },
+    { match: (h) => h === 'cars.com' || h === 'www.cars.com', extractor: carsComExtractor },
+    { match: (h) => h === 'ebay.com' || h === 'www.ebay.com', extractor: ebayExtractor },
+    // ── Local / Real Estate ────────────────────────────────────────────────────
+    { match: (h) => h === 'yelp.com' || h === 'www.yelp.com', extractor: yelpExtractor },
+    { match: (h) => h === 'zillow.com' || h === 'www.zillow.com', extractor: zillowExtractor },
 ];
 /**
  * Returns the domain extractor for a URL, or null if none matches.
@@ -159,6 +167,50 @@ function setCachedExtractorResult(url, result) {
             EXTRACTOR_CACHE.delete(oldest);
     }
 }
+// ── Redis Shared Cache (cross-pod cache for multi-replica deployments) ────────
+// When running multiple API pods, each pod has its own in-memory cache.
+// With 6 pods, 5/6 requests for the same URL miss cache.
+// Redis solves this: all pods share one cache, so the first pod to fetch
+// populates it for all others.
+//
+// Redis is injected from the server startup to keep this core module
+// dependency-free (works in CLI mode without Redis too).
+let _redisClient = null;
+const REDIS_CACHE_PREFIX = 'wp:ext:';
+const REDIS_CACHE_TTL_SECS = 300; // 5 minutes
+/**
+ * Inject a Redis client for shared cross-pod caching.
+ * Called from server startup after Redis is initialized.
+ * Safe to call with null to disable Redis caching (e.g., CLI mode).
+ */
+export function setExtractorRedis(redis) {
+    _redisClient = redis;
+}
+async function getRedisCache(url) {
+    try {
+        if (!_redisClient)
+            return null;
+        const key = REDIS_CACHE_PREFIX + url.replace(/[?#].*$/, '').toLowerCase();
+        const cached = await _redisClient.get(key);
+        if (!cached)
+            return null;
+        return JSON.parse(cached);
+    }
+    catch {
+        return null; // Redis unavailable — fall back to in-memory cache
+    }
+}
+async function setRedisCache(url, result) {
+    try {
+        if (!_redisClient)
+            return;
+        const key = REDIS_CACHE_PREFIX + url.replace(/[?#].*$/, '').toLowerCase();
+        await _redisClient.set(key, JSON.stringify(result), 'EX', REDIS_CACHE_TTL_SECS);
+    }
+    catch {
+        // Redis unavailable — in-memory cache still works, this is non-fatal
+    }
+}
 // ─────────────────────────────────────────────────────────────────────────────
 /**
  * Internal implementation: run the extractor for the URL (if one exists).
@@ -177,28 +229,39 @@ async function _extractDomainDataImpl(html, url) {
 }
 /**
  * Convenience: run the extractor for the URL (if one exists).
- * Wraps _extractDomainDataImpl with a 5-minute LRU cache so that
- * rate-limited API responses fall back to cached results instead of
- * garbage browser rendering.
+ * Wraps _extractDomainDataImpl with a two-tier cache:
+ *   1. In-memory LRU (per-pod, fastest)
+ *   2. Redis shared cache (cross-pod, shared across all replicas)
+ *
+ * With multiple API pods, Redis ensures the first pod to fetch a URL
+ * populates cache for all others — eliminating redundant API calls.
  */
 export async function extractDomainData(html, url) {
-    // 1. Check fresh cache first
+    // 1. Check in-memory cache (fastest — no network)
     const cached = getCachedExtractorResult(url);
     if (cached)
         return cached;
-    // 2. Try the real extractor
+    // 2. Check Redis cache (shared across all pods)
+    const redisCached = await getRedisCache(url);
+    if (redisCached) {
+        // Populate local in-memory cache to avoid Redis round-trips on repeat
+        setCachedExtractorResult(url, redisCached);
+        return redisCached;
+    }
+    // 3. Try the real extractor
     const result = await _extractDomainDataImpl(html, url);
     if (result && result.cleanContent.length > 20) {
-        // 3. Cache the successful result
+        // 4. Cache the successful result in both layers
         setCachedExtractorResult(url, result);
+        void setRedisCache(url, result); // fire-and-forget, non-blocking
         return result;
     }
-    // 4. Extractor failed/returned garbage — check for any stale cache entry
+    // 5. Extractor failed/returned garbage — check for any stale cache entry
     //    (stale structured data beats a browser "Loading…" page)
     const stale = getCachedExtractorResult(url);
     if (stale)
         return stale;
-    // 5. Genuinely nothing — return null so the pipeline falls back to fetch
+    // 6. Genuinely nothing — return null so the pipeline falls back to fetch
     return result;
 }
 // ---------------------------------------------------------------------------
@@ -3139,6 +3202,62 @@ async function bbcExtractor(html, url) {
     return extractNewsArticle(html, url, 'bbc.com');
 }
 async function cnnExtractor(html, url) {
+    try {
+        const u = new URL(url);
+        // For homepage — use CNN Lite which has actual headline links
+        if (u.pathname === '/' || u.pathname === '' || u.hostname === 'lite.cnn.com') {
+            const liteResp = await fetch('https://lite.cnn.com', { headers: { 'User-Agent': 'webpeel/0.21' } });
+            if (liteResp.ok) {
+                const liteHtml = await liteResp.text();
+                const headlines = [];
+                const matches = liteHtml.matchAll(/<a[^>]+href="([^"]*)"[^>]*>([^<]+)<\/a>/g);
+                for (const m of matches) {
+                    const href = m[1].trim();
+                    const text = m[2].trim();
+                    // CNN Lite article links contain year patterns like /2026/
+                    if (/\/20\d\d\//.test(href) && text.length > 10) {
+                        const fullUrl = href.startsWith('http') ? href : `https://www.cnn.com${href}`;
+                        headlines.push(`- [${text}](${fullUrl})`);
+                    }
+                }
+                if (headlines.length > 5) {
+                    return {
+                        domain: 'cnn.com',
+                        type: 'headlines',
+                        structured: { headlines: headlines.length, source: 'cnn-lite' },
+                        cleanContent: `# 📰 CNN — Top Headlines\n\n${headlines.slice(0, 20).join('\n')}\n\n---\n*Source: CNN Lite*`,
+                    };
+                }
+            }
+        }
+        // For article pages — try CNN Lite version of the same URL
+        if (/\/20\d\d\//.test(u.pathname)) {
+            const liteUrl = `https://lite.cnn.com${u.pathname}`;
+            const liteResp = await fetch(liteUrl, { headers: { 'User-Agent': 'webpeel/0.21' } });
+            if (liteResp.ok) {
+                const liteHtml = await liteResp.text();
+                const { load } = await import('cheerio');
+                const $l = load(liteHtml);
+                const title = $l('h1').first().text().trim();
+                const paragraphs = [];
+                $l('p').each((_, el) => {
+                    const text = $l(el).text().trim();
+                    if (text.length > 20)
+                        paragraphs.push(text);
+                });
+                if (title && paragraphs.length > 0) {
+                    return {
+                        domain: 'cnn.com',
+                        type: 'article',
+                        structured: { title, paragraphs: paragraphs.length, source: 'cnn-lite' },
+                        cleanContent: `# ${title}\n\n${paragraphs.join('\n\n')}\n\n---\n*Source: CNN*`,
+                    };
+                }
+            }
+        }
+    }
+    catch { /* fall through to standard extractor */ }
+    // Fallback to standard news article extractor (works if HTML has content)
     return extractNewsArticle(html, url, 'cnn.com');
 }
 // ---------------------------------------------------------------------------
@@ -4846,3 +4965,447 @@ async function weatherExtractor(_html, url) {
         return null;
     }
 }
+// ---------------------------------------------------------------------------
+// Facebook Marketplace extractor (login-wall fallback)
+// ---------------------------------------------------------------------------
+async function facebookMarketplaceExtractor(_html, url) {
+    const u = new URL(url);
+    if (!u.pathname.includes('/marketplace'))
+        return null;
+    const query = u.searchParams.get('query') || '';
+    const maxPrice = u.searchParams.get('maxPrice') || '';
+    const minPrice = u.searchParams.get('minPrice') || '';
+    // Extract location segment: /marketplace/nyc/search → "nyc"
+    const locationMatch = u.pathname.match(/\/marketplace\/([^/]+)(?:\/|$)/);
+    const location = (locationMatch?.[1] && locationMatch[1] !== 'search' && locationMatch[1] !== 'category') ? locationMatch[1] : '';
+    const priceRange = [minPrice && `$${minPrice}`, maxPrice && `$${maxPrice}`].filter(Boolean).join(' – ');
+    const lines = [
+        `# 🛒 Facebook Marketplace`,
+        '',
+        `**Search:** ${query || 'Browse all'}`,
+        ...(location ? [`**Location:** ${location}`] : []),
+        ...(priceRange ? [`**Price range:** ${priceRange}`] : []),
+        '',
+        '> ⚠️ Facebook Marketplace requires authentication. WebPeel cannot access listings directly.',
+        '',
+        '**Alternative searches that work:**',
+    ];
+    if (query) {
+        const clUrl = `https://newyork.craigslist.org/search/sss?query=${encodeURIComponent(query)}${maxPrice ? '&max_price=' + maxPrice : ''}`;
+        const carsUrl = `https://www.cars.com/shopping/results/?keyword=${encodeURIComponent(query)}&list_price_max=${maxPrice || ''}&zip=10001&stock_type=used`;
+        const ebayUrl = `https://www.ebay.com/sch/i.html?_nkw=${encodeURIComponent(query)}${maxPrice ? '&_udhi=' + maxPrice : ''}&LH_BIN=1`;
+        lines.push(`- \`webpeel "${clUrl}"\` — Craigslist`, `- \`webpeel "${carsUrl}"\` — Cars.com`, `- \`webpeel "${ebayUrl}"\` — eBay`);
+    }
+    lines.push('', '*Tip: Craigslist and Cars.com return full structured results with WebPeel.*');
+    return {
+        domain: 'facebook.com',
+        type: 'blocked',
+        structured: {
+            query,
+            location,
+            minPrice,
+            maxPrice,
+            reason: 'authentication required',
+            alternatives: ['craigslist', 'cars.com', 'ebay'],
+        },
+        cleanContent: lines.join('\n'),
+    };
+}
+// ---------------------------------------------------------------------------
+// Etsy extractor (bot-block fallback with Google site-search suggestion)
+// ---------------------------------------------------------------------------
+async function etsyExtractor(_html, url) {
+    const u = new URL(url);
+    // Extract search query from various URL patterns
+    // /search?q=handmade+jewelry  OR  /search/handmade-jewelry
+    let query = u.searchParams.get('q') || '';
+    if (!query) {
+        const pathMatch = u.pathname.match(/\/search\/([^?#]+)/);
+        if (pathMatch)
+            query = decodeURIComponent(pathMatch[1].replace(/-/g, ' '));
+    }
+    // Shop page: /shop/ShopName
+    const shopMatch = u.pathname.match(/^\/shop\/([^/?#]+)/);
+    const shopName = shopMatch?.[1] || '';
+    if (!query && !shopName)
+        return null;
+    const googleUrl = query
+        ? `https://www.google.com/search?q=site:etsy.com+${encodeURIComponent(query)}`
+        : `https://www.google.com/search?q=site:etsy.com+${encodeURIComponent(shopName)}`;
+    const etsySearchUrl = query ? `https://www.etsy.com/search?q=${encodeURIComponent(query)}` : url;
+    const displayTitle = query ? `"${query}"` : `Shop: ${shopName}`;
+    const cleanContent = [
+        `# 🎨 Etsy — ${displayTitle}`,
+        '',
+        '> ⚠️ Etsy blocks automated access. WebPeel cannot scrape listings directly.',
+        '',
+        '**Alternatives that work:**',
+        `- \`webpeel "${googleUrl}"\` — Google site:etsy.com results`,
+        `- Direct link: [etsy.com/search?q=${encodeURIComponent(query || shopName)}](${etsySearchUrl})`,
+        '',
+        ...(query ? [
+            '**Similar items on open marketplaces:**',
+            `- \`webpeel "https://www.ebay.com/sch/i.html?_nkw=${encodeURIComponent(query)}&LH_BIN=1"\` — eBay`,
+            `- \`webpeel "https://newyork.craigslist.org/search/sss?query=${encodeURIComponent(query)}"\` — Craigslist`,
+        ] : []),
+        '',
+        '*Etsy Open API v3 (free key at etsy.com/developers) can unlock direct access.*',
+    ].join('\n');
+    return {
+        domain: 'etsy.com',
+        type: 'blocked',
+        structured: {
+            query,
+            shopName,
+            reason: 'bot-block',
+            googleFallback: googleUrl,
+        },
+        cleanContent,
+    };
+}
+// ---------------------------------------------------------------------------
+// Cars.com extractor — structured parsing via data-vehicle-details JSON attrs
+// ---------------------------------------------------------------------------
+async function carsComExtractor(html, url) {
+    try {
+        const { load } = await import('cheerio');
+        const $ = load(html);
+        const u = new URL(url);
+        const keyword = u.searchParams.get('keyword') || '';
+        const maxPrice = u.searchParams.get('list_price_max') || '';
+        const minPrice = u.searchParams.get('list_price_min') || '';
+        const zip = u.searchParams.get('zip') || '';
+        const stockType = u.searchParams.get('stock_type') || '';
+        // Individual vehicle detail page
+        if (u.pathname.includes('/vehicledetail/')) {
+            const title = $('h1').first().text().trim() ||
+                $('title').text().trim().split(' | ')[0];
+            if (!title)
+                return null;
+            const price = $('[class*="price"]').first().text().trim();
+            const mileage = $('[class*="mileage"]').first().text().trim();
+            return {
+                domain: 'cars.com',
+                type: 'listing',
+                structured: { title, price, mileage, url },
+                cleanContent: [
+                    `# 🚗 ${title}`,
+                    price && `**Price:** ${price}`,
+                    mileage && `**Mileage:** ${mileage}`,
+                    `\n[View listing](${url})`,
+                ].filter(Boolean).join('\n'),
+            };
+        }
+        // Search results page — Cars.com embeds JSON in fuse-card data-vehicle-details
+        const listings = [];
+        $('fuse-card[data-vehicle-details]').each((_, el) => {
+            try {
+                const raw = $(el).attr('data-vehicle-details');
+                if (!raw)
+                    return;
+                const v = JSON.parse(raw);
+                const listingId = v.listingId || $(el).attr('data-listing-id') || '';
+                const cardLink = $(el).find('card-gallery').attr('card-link') || (listingId ? `/vehicledetail/${listingId}/` : '');
+                const title = `${v.stockType || 'Used'} ${v.year} ${v.make} ${v.model}${v.trim ? ' ' + v.trim : ''}`.trim();
+                const price = v.price ? `$${Number(v.price).toLocaleString()}` : '';
+                const mileage = v.mileage ? `${Number(v.mileage).toLocaleString()} mi` : '';
+                const bodyStyle = v.bodyStyle || '';
+                const fuelType = v.fuelType || '';
+                if (title && title !== 'Used  ') {
+                    listings.push({ title, price, mileage, bodyStyle, fuelType, url: cardLink });
+                }
+            }
+            catch { /* skip malformed */ }
+        });
+        if (listings.length === 0)
+            return null; // Let pipeline handle it
+        const priceRange = [minPrice && `$${minPrice}`, maxPrice && `$${maxPrice}`].filter(Boolean).join(' – ');
+        const header = [
+            `# 🚗 Cars.com — ${keyword || 'Vehicle Search'}`,
+            '',
+            keyword && `**Search:** ${keyword}`,
+            zip && `**Location:** ZIP ${zip}`,
+            priceRange && `**Price:** up to $${maxPrice}`,
+            stockType && `**Stock:** ${stockType}`,
+            `**Results:** ${listings.length} listings`,
+            '',
+        ].filter(Boolean).join('\n');
+        const rows = listings.slice(0, 20).map((l, i) => {
+            const parts = [
+                `${i + 1}. **${l.title}**`,
+                l.price,
+                l.mileage,
+                l.bodyStyle,
+                l.url && `[→](https://www.cars.com${l.url})`,
+            ].filter(Boolean);
+            return parts.join(' · ');
+        });
+        return {
+            domain: 'cars.com',
+            type: 'search',
+            structured: { keyword, zip, minPrice, maxPrice, stockType, count: listings.length, listings },
+            cleanContent: header + rows.join('\n'),
+        };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Cars.com extractor error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// eBay extractor — clean up noisy search results
+// ---------------------------------------------------------------------------
+async function ebayExtractor(html, url) {
+    try {
+        const { load } = await import('cheerio');
+        const $ = load(html);
+        const u = new URL(url);
+        // Individual item page
+        if (u.pathname.startsWith('/itm/')) {
+            const title = $('h1').first().text().trim();
+            if (!title)
+                return null;
+            const price = $('[class*="price"]').not('[class*="shipping"]').first().text().trim();
+            const condition = $('[class*="condition"]').first().text().trim();
+            return {
+                domain: 'ebay.com',
+                type: 'listing',
+                structured: { title, price, condition, url },
+                cleanContent: [
+                    `# 🛍 ${title}`,
+                    price && `**Price:** ${price}`,
+                    condition && `**Condition:** ${condition}`,
+                    `\n[View on eBay](${url})`,
+                ].filter(Boolean).join('\n'),
+            };
+        }
+        // Search results page
+        const keyword = u.searchParams.get('_nkw') || '';
+        const maxPrice = u.searchParams.get('_udhi') || '';
+        const minPrice = u.searchParams.get('_udlo') || '';
+        const listings = [];
+        // eBay search results use li[data-listingid] + .s-card__title / .s-card__price
+        $('li[data-listingid]').each((_, el) => {
+            const titleRaw = $(el).find('.s-card__title').text().trim()
+                .replace(/Opens in a new window or tab/g, '')
+                .replace(/^New Listing\s*/i, '')
+                .trim();
+            if (!titleRaw || titleRaw === 'Shop on eBay')
+                return;
+            const title = titleRaw;
+            const price = $(el).find('.s-card__price').first().text().trim();
+            // .s-card__subtitle contains "DealerNameCondition" as merged text — extract condition keyword
+            const subtitleText = $(el).find('.s-card__subtitle').text().trim();
+            const conditionKeywords = ['Pre-Owned', 'Brand New', 'Open Box', 'Refurbished', 'For Parts'];
+            const condition = conditionKeywords.find((k) => subtitleText.includes(k)) || '';
+            // Get clean URL — extract /itm/<id> and strip tracking params
+            let href = '';
+            const itemLink = $(el).find('a[href*="/itm/"]').first().attr('href') || '';
+            const itmMatch = itemLink.match(/(https?:\/\/[^/]*\/itm\/\d+)/);
+            if (itmMatch)
+                href = itmMatch[1];
+            const listingId = $(el).attr('data-listingid') || '';
+            if (!href && listingId)
+                href = `https://www.ebay.com/itm/${listingId}`;
+            listings.push({ title, price, condition, url: href });
+        });
+        if (listings.length === 0)
+            return null; // Let pipeline handle it
+        const priceRange = [minPrice && `$${minPrice}`, maxPrice && `$${maxPrice}`].filter(Boolean).join(' – ');
+        const header = [
+            `# 🛍 eBay — ${keyword || 'Search Results'}`,
+            '',
+            keyword && `**Search:** ${keyword}`,
+            priceRange && `**Price:** up to $${maxPrice}`,
+            `**Results:** ${listings.length} listings`,
+            '',
+        ].filter(Boolean).join('\n');
+        const rows = listings.slice(0, 20).map((l, i) => {
+            const parts = [
+                `${i + 1}. **${l.title}**`,
+                l.price,
+                l.condition && `[${l.condition}]`,
+                l.url && `[→](${l.url})`,
+            ].filter(Boolean);
+            return parts.join(' · ');
+        });
+        return {
+            domain: 'ebay.com',
+            type: 'search',
+            structured: { keyword, minPrice, maxPrice, count: listings.length, listings },
+            cleanContent: header + rows.join('\n'),
+        };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'eBay extractor error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Yelp extractor — parse JSON-LD + meta from stealth-rendered HTML
+// ---------------------------------------------------------------------------
+async function yelpExtractor(html, url) {
+    try {
+        const { load } = await import('cheerio');
+        const $ = load(html);
+        // Try JSON-LD structured data first
+        const jsonLdScripts = $('script[type="application/ld+json"]');
+        let businessData = null;
+        jsonLdScripts.each((_, el) => {
+            const raw = $(el).html() || '';
+            try {
+                const parsed = JSON.parse(raw);
+                const items = Array.isArray(parsed) ? parsed : [parsed];
+                for (const item of items) {
+                    const type = item['@type'];
+                    if (type === 'Restaurant' || type === 'LocalBusiness' || type === 'FoodEstablishment' ||
+                        type === 'BarOrPub' || type === 'CafeOrCoffeeShop') {
+                        businessData = item;
+                    }
+                }
+            }
+            catch { /* ignore malformed JSON-LD */ }
+        });
+        // --- Business page ---
+        if (businessData) {
+            const name = businessData.name || '';
+            const rating = businessData.aggregateRating?.ratingValue;
+            const reviewCount = businessData.aggregateRating?.reviewCount;
+            const addr = businessData.address;
+            const address = addr
+                ? [addr.streetAddress, addr.addressLocality, addr.addressRegion, addr.postalCode].filter(Boolean).join(', ')
+                : '';
+            const phone = businessData.telephone || '';
+            const cuisine = businessData.servesCuisine || '';
+            const priceRange = businessData.priceRange || '';
+            const description = businessData.description || $('meta[property="og:description"]').attr('content') || '';
+            const hours = businessData.openingHours || '';
+            const lines = [
+                `# ⭐ Yelp: ${name}`,
+                '',
+                rating && `**Rating:** ${rating}/5 (${reviewCount} reviews)`,
+                cuisine && `**Cuisine:** ${cuisine}`,
+                priceRange && `**Price:** ${priceRange}`,
+                address && `**Address:** ${address}`,
+                phone && `**Phone:** ${phone}`,
+                hours && `**Hours:** ${Array.isArray(hours) ? hours.join(', ') : hours}`,
+                description && `\n${description.substring(0, 500)}`,
+                '',
+                `**More info:** [View on Yelp](${url})`,
+                '',
+                '---',
+                '*Source: Yelp*',
+            ].filter(Boolean);
+            return {
+                domain: 'yelp.com',
+                type: 'business',
+                structured: { name, rating, reviewCount, address, phone, cuisine, priceRange, description },
+                cleanContent: lines.join('\n'),
+            };
+        }
+        // --- Search page — parse from meta / og tags ---
+        const ogTitle = $('meta[property="og:title"]').attr('content') || '';
+        const ogDescription = $('meta[property="og:description"]').attr('content') || '';
+        // Try to extract listing names from heading tags
+        const listings = [];
+        $('h3, h4').each((_, el) => {
+            const text = $(el).text().trim();
+            if (text && text.length > 2 && text.length < 100) {
+                const anchor = $(el).find('a').first();
+                const href = anchor.attr('href') || '';
+                const fullHref = href.startsWith('/') ? `https://www.yelp.com${href}` : href;
+                listings.push({ name: text, url: fullHref || undefined });
+            }
+        });
+        if (ogTitle || listings.length > 0) {
+            const searchTerm = ogTitle.replace(/\s*-\s*Yelp$/, '').trim();
+            const lines = [
+                `# 🔍 Yelp Search: ${searchTerm || 'Results'}`,
+                ogDescription && `\n${ogDescription}`,
+                listings.length > 0 && `\n**Found ${listings.length} results:**`,
+                ...listings.slice(0, 15).map((l, i) => `${i + 1}. ${l.url ? `[${l.name}](${l.url})` : l.name}`),
+                '',
+                `**Search:** [View on Yelp](${url})`,
+                '',
+                '---',
+                '*Source: Yelp*',
+            ].filter(Boolean);
+            return {
+                domain: 'yelp.com',
+                type: 'search',
+                structured: { query: searchTerm, count: listings.length, listings },
+                cleanContent: lines.join('\n'),
+            };
+        }
+        return null;
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Yelp extractor error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Zillow extractor — smart fallback with helpful alternatives
+// ---------------------------------------------------------------------------
+async function zillowExtractor(_html, url) {
+    try {
+        const u = new URL(url);
+        // Derive location label from the URL path
+        const rawPath = u.pathname.replace(/^\//, '').replace(/\/$/, '');
+        const location = rawPath
+            .replace(/\//g, ' ')
+            .replace(/-/g, ' ')
+            .trim();
+        // Parse city/state for alternative links
+        const pathParts = rawPath.split('/').filter(Boolean);
+        const cityStatePart = pathParts[0] || ''; // e.g. "new-york-ny"
+        const segments = cityStatePart.split('-');
+        const statePart = segments[segments.length - 1] || '';
+        const cityPart = segments.slice(0, -1).join('-');
+        // Redfin city path
+        const cityCapitalized = cityPart.split('-').map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join('_');
+        const stateUpper = statePart.toUpperCase();
+        const redfinCityPath = cityCapitalized && stateUpper
+            ? `https://www.redfin.com/city/${cityCapitalized}/${stateUpper}`
+            : 'https://www.redfin.com';
+        const realtorPath = cityStatePart
+            ? `https://www.realtor.com/realestateandhomes-search/${cityStatePart}`
+            : 'https://www.realtor.com';
+        const cleanContent = [
+            `# 🏠 Zillow — ${location || 'Real Estate Search'}`,
+            '',
+            '> ⚠️ **Zillow blocks automated access.** WebPeel cannot retrieve live listings directly.',
+            '',
+            '**Try these alternatives that work with WebPeel:**',
+            `- [Redfin](${redfinCityPath}) — similar listings, scrape-friendly`,
+            `- [Realtor.com](${realtorPath}) — MLS-powered, often accessible`,
+            `- [Homes.com](https://www.homes.com) — newer platform, better access`,
+            '',
+            `**Direct Zillow link:** [Open Zillow](${url})`,
+            '',
+            '---',
+            '*Source: Zillow (access blocked — showing alternatives)*',
+        ].join('\n');
+        return {
+            domain: 'zillow.com',
+            type: 'real-estate',
+            structured: {
+                location,
+                blocked: true,
+                alternatives: [
+                    { name: 'Redfin', url: redfinCityPath },
+                    { name: 'Realtor.com', url: realtorPath },
+                ],
+            },
+            cleanContent,
+        };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Zillow extractor error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/dist/core/strategies.js

@@ -51,6 +51,7 @@ function shouldForceBrowser(url) {
             'glassdoor.com',
             'bloomberg.com',
             'indeed.com',
+            'yelp.com', // aggressive bot detection
             'amazon.com', // captcha wall on simple/browser fetch
             'zillow.com', // aggressive bot detection
             'ticketmaster.com', // Distil Networks / PerimeterX

package/dist/server/app.js

@@ -54,6 +54,7 @@ import { createSentryHooks } from './sentry.js';
 import { requireScope } from './middleware/scope-guard.js';
 import { createCacheWarmRouter, startCacheWarmer } from './routes/cache-warm.js';
 import { warmup, cleanup as cleanupFetcher } from '../core/fetcher.js';
+import { setExtractorRedis } from '../core/domain-extractors.js';
 import { registerPremiumHooks } from './premium/index.js';
 import { readFileSync } from 'fs';
 import { join, dirname } from 'path';
@@ -421,6 +422,29 @@ export function startServer(config = {}) {
     const port = config.port || parseInt(process.env.PORT || '3000', 10);
     // Activate premium strategy hooks (SWR cache, domain intelligence, race).
     registerPremiumHooks();
+    // Inject Redis into the domain extractor cache for cross-pod cache sharing.
+    // When REDIS_URL is set (multi-pod k8s deployments), all pods share one cache
+    // so the first pod to fetch a URL populates it for all others.
+    if (process.env.REDIS_URL) {
+        // @ts-ignore — ioredis CJS/ESM interop
+        import('ioredis').then((IoRedisModule) => {
+            const IoRedis = IoRedisModule.default ?? IoRedisModule;
+            const url = process.env.REDIS_URL;
+            const parsed = new URL(url);
+            const redis = new IoRedis({
+                host: parsed.hostname,
+                port: parseInt(parsed.port || '6379', 10),
+                db: parseInt(parsed.pathname?.slice(1) || '0', 10) || 0,
+                lazyConnect: true,
+                maxRetriesPerRequest: 3,
+                enableOfflineQueue: false,
+            });
+            setExtractorRedis(redis);
+            log.info('Redis extractor cache initialized (shared cross-pod cache active)');
+        }).catch((err) => {
+            log.warn('Failed to init Redis extractor cache (in-memory only)', { error: err.message });
+        });
+    }
     // Pre-warm browser resources in the background to reduce first-request latency.
     void warmup().catch((error) => {
         log.warn('Browser warmup failed', { error: error instanceof Error ? error.message : String(error) });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.67",
+  "version": "0.21.69",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",