webpeel 0.21.68 → 0.21.70

package/dist/core/domain-extractors.js

@@ -117,6 +117,9 @@ const REGISTRY = [
     { match: (h) => h === 'etsy.com' || h === 'www.etsy.com', extractor: etsyExtractor },
     { match: (h) => h === 'cars.com' || h === 'www.cars.com', extractor: carsComExtractor },
     { match: (h) => h === 'ebay.com' || h === 'www.ebay.com', extractor: ebayExtractor },
+    // ── Local / Real Estate ────────────────────────────────────────────────────
+    { match: (h) => h === 'yelp.com' || h === 'www.yelp.com', extractor: yelpExtractor },
+    { match: (h) => h === 'zillow.com' || h === 'www.zillow.com', extractor: zillowExtractor },
 ];
 /**
  * Returns the domain extractor for a URL, or null if none matches.
@@ -3199,6 +3202,62 @@ async function bbcExtractor(html, url) {
     return extractNewsArticle(html, url, 'bbc.com');
 }
 async function cnnExtractor(html, url) {
+    try {
+        const u = new URL(url);
+        // For homepage — use CNN Lite which has actual headline links
+        if (u.pathname === '/' || u.pathname === '' || u.hostname === 'lite.cnn.com') {
+            const liteResp = await fetch('https://lite.cnn.com', { headers: { 'User-Agent': 'webpeel/0.21' } });
+            if (liteResp.ok) {
+                const liteHtml = await liteResp.text();
+                const headlines = [];
+                const matches = liteHtml.matchAll(/<a[^>]+href="([^"]*)"[^>]*>([^<]+)<\/a>/g);
+                for (const m of matches) {
+                    const href = m[1].trim();
+                    const text = m[2].trim();
+                    // CNN Lite article links contain year patterns like /2026/
+                    if (/\/20\d\d\//.test(href) && text.length > 10) {
+                        const fullUrl = href.startsWith('http') ? href : `https://www.cnn.com${href}`;
+                        headlines.push(`- [${text}](${fullUrl})`);
+                    }
+                }
+                if (headlines.length > 5) {
+                    return {
+                        domain: 'cnn.com',
+                        type: 'headlines',
+                        structured: { headlines: headlines.length, source: 'cnn-lite' },
+                        cleanContent: `# 📰 CNN — Top Headlines\n\n${headlines.slice(0, 20).join('\n')}\n\n---\n*Source: CNN Lite*`,
+                    };
+                }
+            }
+        }
+        // For article pages — try CNN Lite version of the same URL
+        if (/\/20\d\d\//.test(u.pathname)) {
+            const liteUrl = `https://lite.cnn.com${u.pathname}`;
+            const liteResp = await fetch(liteUrl, { headers: { 'User-Agent': 'webpeel/0.21' } });
+            if (liteResp.ok) {
+                const liteHtml = await liteResp.text();
+                const { load } = await import('cheerio');
+                const $l = load(liteHtml);
+                const title = $l('h1').first().text().trim();
+                const paragraphs = [];
+                $l('p').each((_, el) => {
+                    const text = $l(el).text().trim();
+                    if (text.length > 20)
+                        paragraphs.push(text);
+                });
+                if (title && paragraphs.length > 0) {
+                    return {
+                        domain: 'cnn.com',
+                        type: 'article',
+                        structured: { title, paragraphs: paragraphs.length, source: 'cnn-lite' },
+                        cleanContent: `# ${title}\n\n${paragraphs.join('\n\n')}\n\n---\n*Source: CNN*`,
+                    };
+                }
+            }
+        }
+    }
+    catch { /* fall through to standard extractor */ }
+    // Fallback to standard news article extractor (works if HTML has content)
     return extractNewsArticle(html, url, 'cnn.com');
 }
 // ---------------------------------------------------------------------------
@@ -5184,3 +5243,169 @@ async function ebayExtractor(html, url) {
         return null;
     }
 }
+// ---------------------------------------------------------------------------
+// Yelp extractor — parse JSON-LD + meta from stealth-rendered HTML
+// ---------------------------------------------------------------------------
+async function yelpExtractor(html, url) {
+    try {
+        const { load } = await import('cheerio');
+        const $ = load(html);
+        // Try JSON-LD structured data first
+        const jsonLdScripts = $('script[type="application/ld+json"]');
+        let businessData = null;
+        jsonLdScripts.each((_, el) => {
+            const raw = $(el).html() || '';
+            try {
+                const parsed = JSON.parse(raw);
+                const items = Array.isArray(parsed) ? parsed : [parsed];
+                for (const item of items) {
+                    const type = item['@type'];
+                    if (type === 'Restaurant' || type === 'LocalBusiness' || type === 'FoodEstablishment' ||
+                        type === 'BarOrPub' || type === 'CafeOrCoffeeShop') {
+                        businessData = item;
+                    }
+                }
+            }
+            catch { /* ignore malformed JSON-LD */ }
+        });
+        // --- Business page ---
+        if (businessData) {
+            const name = businessData.name || '';
+            const rating = businessData.aggregateRating?.ratingValue;
+            const reviewCount = businessData.aggregateRating?.reviewCount;
+            const addr = businessData.address;
+            const address = addr
+                ? [addr.streetAddress, addr.addressLocality, addr.addressRegion, addr.postalCode].filter(Boolean).join(', ')
+                : '';
+            const phone = businessData.telephone || '';
+            const cuisine = businessData.servesCuisine || '';
+            const priceRange = businessData.priceRange || '';
+            const description = businessData.description || $('meta[property="og:description"]').attr('content') || '';
+            const hours = businessData.openingHours || '';
+            const lines = [
+                `# ⭐ Yelp: ${name}`,
+                '',
+                rating && `**Rating:** ${rating}/5 (${reviewCount} reviews)`,
+                cuisine && `**Cuisine:** ${cuisine}`,
+                priceRange && `**Price:** ${priceRange}`,
+                address && `**Address:** ${address}`,
+                phone && `**Phone:** ${phone}`,
+                hours && `**Hours:** ${Array.isArray(hours) ? hours.join(', ') : hours}`,
+                description && `\n${description.substring(0, 500)}`,
+                '',
+                `**More info:** [View on Yelp](${url})`,
+                '',
+                '---',
+                '*Source: Yelp*',
+            ].filter(Boolean);
+            return {
+                domain: 'yelp.com',
+                type: 'business',
+                structured: { name, rating, reviewCount, address, phone, cuisine, priceRange, description },
+                cleanContent: lines.join('\n'),
+            };
+        }
+        // --- Search page — parse from meta / og tags ---
+        const ogTitle = $('meta[property="og:title"]').attr('content') || '';
+        const ogDescription = $('meta[property="og:description"]').attr('content') || '';
+        // Try to extract listing names from heading tags
+        const listings = [];
+        $('h3, h4').each((_, el) => {
+            const text = $(el).text().trim();
+            if (text && text.length > 2 && text.length < 100) {
+                const anchor = $(el).find('a').first();
+                const href = anchor.attr('href') || '';
+                const fullHref = href.startsWith('/') ? `https://www.yelp.com${href}` : href;
+                listings.push({ name: text, url: fullHref || undefined });
+            }
+        });
+        if (ogTitle || listings.length > 0) {
+            const searchTerm = ogTitle.replace(/\s*-\s*Yelp$/, '').trim();
+            const lines = [
+                `# 🔍 Yelp Search: ${searchTerm || 'Results'}`,
+                ogDescription && `\n${ogDescription}`,
+                listings.length > 0 && `\n**Found ${listings.length} results:**`,
+                ...listings.slice(0, 15).map((l, i) => `${i + 1}. ${l.url ? `[${l.name}](${l.url})` : l.name}`),
+                '',
+                `**Search:** [View on Yelp](${url})`,
+                '',
+                '---',
+                '*Source: Yelp*',
+            ].filter(Boolean);
+            return {
+                domain: 'yelp.com',
+                type: 'search',
+                structured: { query: searchTerm, count: listings.length, listings },
+                cleanContent: lines.join('\n'),
+            };
+        }
+        return null;
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Yelp extractor error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Zillow extractor — smart fallback with helpful alternatives
+// ---------------------------------------------------------------------------
+async function zillowExtractor(_html, url) {
+    try {
+        const u = new URL(url);
+        // Derive location label from the URL path
+        const rawPath = u.pathname.replace(/^\//, '').replace(/\/$/, '');
+        const location = rawPath
+            .replace(/\//g, ' ')
+            .replace(/-/g, ' ')
+            .trim();
+        // Parse city/state for alternative links
+        const pathParts = rawPath.split('/').filter(Boolean);
+        const cityStatePart = pathParts[0] || ''; // e.g. "new-york-ny"
+        const segments = cityStatePart.split('-');
+        const statePart = segments[segments.length - 1] || '';
+        const cityPart = segments.slice(0, -1).join('-');
+        // Redfin city path
+        const cityCapitalized = cityPart.split('-').map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join('_');
+        const stateUpper = statePart.toUpperCase();
+        const redfinCityPath = cityCapitalized && stateUpper
+            ? `https://www.redfin.com/city/${cityCapitalized}/${stateUpper}`
+            : 'https://www.redfin.com';
+        const realtorPath = cityStatePart
+            ? `https://www.realtor.com/realestateandhomes-search/${cityStatePart}`
+            : 'https://www.realtor.com';
+        const cleanContent = [
+            `# 🏠 Zillow — ${location || 'Real Estate Search'}`,
+            '',
+            '> ⚠️ **Zillow blocks automated access.** WebPeel cannot retrieve live listings directly.',
+            '',
+            '**Try these alternatives that work with WebPeel:**',
+            `- [Redfin](${redfinCityPath}) — similar listings, scrape-friendly`,
+            `- [Realtor.com](${realtorPath}) — MLS-powered, often accessible`,
+            `- [Homes.com](https://www.homes.com) — newer platform, better access`,
+            '',
+            `**Direct Zillow link:** [Open Zillow](${url})`,
+            '',
+            '---',
+            '*Source: Zillow (access blocked — showing alternatives)*',
+        ].join('\n');
+        return {
+            domain: 'zillow.com',
+            type: 'real-estate',
+            structured: {
+                location,
+                blocked: true,
+                alternatives: [
+                    { name: 'Redfin', url: redfinCityPath },
+                    { name: 'Realtor.com', url: realtorPath },
+                ],
+            },
+            cleanContent,
+        };
+    }
+    catch (e) {
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'Zillow extractor error:', e instanceof Error ? e.message : e);
+        return null;
+    }
+}

package/dist/core/pipeline.js

@@ -415,9 +415,21 @@ export async function fetchContent(ctx) {
         }
         // Enhance error messages with actionable advice
         if (fetchError instanceof BlockedError) {
-            const actionableMsg = `${fetchError.message}\n\nThis site blocks automated access. Try using \`stealth: true\` and a residential proxy.`;
-            const enhancedError = new BlockedError(actionableMsg);
-            throw enhancedError;
+            // Instead of crashing, return a helpful response with the block info
+            ctx.timer.end('fetch');
+            const host = new URL(ctx.url).hostname.replace('www.', '');
+            ctx.content = `# ⚠️ ${host} — Access Blocked\n\nThis site uses advanced bot protection and blocked our request.\n\n**What you can try:**\n- Use a browser profile with saved login: \`webpeel login ${host}\`\n- Try an alternative site that provides similar data\n\n*Direct link: [Open in browser](${ctx.url})*`;
+            ctx.title = `${host} — Blocked`;
+            ctx.quality = 0.2;
+            ctx.warnings.push('Site blocked automated access. Showing fallback content.');
+            ctx.fetchResult = {
+                html: ctx.content,
+                url: ctx.url,
+                status: 403,
+                contentType: 'text/markdown',
+                method: 'blocked-fallback',
+            };
+            return;
         }
         const errMsg = fetchError instanceof Error ? fetchError.message : String(fetchError);
         if (errMsg.toLowerCase().includes('timeout') || errMsg.toLowerCase().includes('timed out') || errMsg.includes('AbortError')) {

package/dist/core/strategies.js

@@ -16,6 +16,40 @@ import { getStrategyHooks, } from './strategy-hooks.js';
 import { createLogger } from './logger.js';
 const log = createLogger('fetch');
 /* ---------- hardcoded domain rules -------------------------------------- */
+/**
+ * Domains that require a residential proxy to bypass datacenter IP blocks.
+ * These sites don't just need stealth — they fingerprint the IP itself and
+ * block all cloud/datacenter ranges. Webshare residential proxy bypasses this.
+ *
+ * When no explicit proxy is set and Webshare is configured, requests to these
+ * domains skip the direct (datacenter) attempt and go straight to residential proxy.
+ */
+const RESIDENTIAL_PROXY_DOMAINS = [
+    'zillow.com',
+    'yelp.com',
+    'pinterest.com',
+    'ticketmaster.com',
+    'stubhub.com',
+    'cargurus.com',
+    'realtor.com',
+    'redfin.com',
+    'apartments.com',
+    'trulia.com',
+    'homefinder.com',
+];
+/**
+ * Check if a URL matches a domain that requires residential proxy.
+ * Returns true if no explicit proxy is set and Webshare env vars are available.
+ */
+function requiresResidentialProxy(url) {
+    try {
+        const hostname = new URL(url).hostname.toLowerCase();
+        return RESIDENTIAL_PROXY_DOMAINS.some(domain => hostname === domain || hostname.endsWith(`.${domain}`));
+    }
+    catch {
+        return false;
+    }
+}
 function shouldForceBrowser(url) {
     // Hashbang URLs (#!) are always JS-routed SPAs — browser rendering required
     if (url.includes('#!')) {
@@ -51,6 +85,7 @@ function shouldForceBrowser(url) {
             'glassdoor.com',
             'bloomberg.com',
             'indeed.com',
+            'yelp.com', // aggressive bot detection
             'amazon.com', // captcha wall on simple/browser fetch
             'zillow.com', // aggressive bot detection
             'ticketmaster.com', // Distil Networks / PerimeterX
@@ -313,13 +348,21 @@ export async function smartFetch(url, options = {}) {
     const { forceBrowser = false, stealth = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, actions, keepPageOpen = false, noCache = false, raceTimeoutMs = 2000, profileDir, headed = false, storageState, proxy, proxies, device, viewportWidth, viewportHeight, waitUntil, waitSelector, blockResources, cloaked = false, cycle = false, tls = false, noEscalate = false, } = options;
     const usePeelTLS = tls || cycle;
     // Build effective proxy list: explicit proxies array, or single proxy, or empty.
-    // When no explicit proxy is configured and Webshare is available, automatically
-    // add it as a fallback: try direct connection first (fast), then Webshare on block.
+    // For domains that require residential proxies (Zillow, Yelp, Pinterest, etc.),
+    // skip the direct datacenter connection entirely and go straight to Webshare.
+    // For all other domains, try direct first (fast), then Webshare as fallback.
     const effectiveProxies = proxies?.length ? proxies :
         proxy ? [proxy] :
             (() => {
                 const wsUrl = getWebshareProxyUrl();
-                return wsUrl ? [undefined, wsUrl] : [undefined];
+                if (!wsUrl)
+                    return [undefined];
+                // Skip datacenter IP for known residential-proxy-required domains
+                if (requiresResidentialProxy(url)) {
+                    log.debug('Residential proxy domain detected — skipping datacenter IP, using Webshare directly');
+                    return [wsUrl];
+                }
+                return [undefined, wsUrl];
             })();
     const firstProxy = effectiveProxies[0];
     const hooks = getStrategyHooks();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.68",
+  "version": "0.21.70",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",