webpeel 0.21.69 → 0.21.70

package/dist/core/pipeline.js

@@ -415,9 +415,21 @@ export async function fetchContent(ctx) {
         }
         // Enhance error messages with actionable advice
         if (fetchError instanceof BlockedError) {
-            const actionableMsg = `${fetchError.message}\n\nThis site blocks automated access. Try using \`stealth: true\` and a residential proxy.`;
-            const enhancedError = new BlockedError(actionableMsg);
-            throw enhancedError;
+            // Instead of crashing, return a helpful response with the block info
+            ctx.timer.end('fetch');
+            const host = new URL(ctx.url).hostname.replace('www.', '');
+            ctx.content = `# ⚠️ ${host} — Access Blocked\n\nThis site uses advanced bot protection and blocked our request.\n\n**What you can try:**\n- Use a browser profile with saved login: \`webpeel login ${host}\`\n- Try an alternative site that provides similar data\n\n*Direct link: [Open in browser](${ctx.url})*`;
+            ctx.title = `${host} — Blocked`;
+            ctx.quality = 0.2;
+            ctx.warnings.push('Site blocked automated access. Showing fallback content.');
+            ctx.fetchResult = {
+                html: ctx.content,
+                url: ctx.url,
+                status: 403,
+                contentType: 'text/markdown',
+                method: 'blocked-fallback',
+            };
+            return;
         }
         const errMsg = fetchError instanceof Error ? fetchError.message : String(fetchError);
         if (errMsg.toLowerCase().includes('timeout') || errMsg.toLowerCase().includes('timed out') || errMsg.includes('AbortError')) {

package/dist/core/strategies.js

@@ -16,6 +16,40 @@ import { getStrategyHooks, } from './strategy-hooks.js';
 import { createLogger } from './logger.js';
 const log = createLogger('fetch');
 /* ---------- hardcoded domain rules -------------------------------------- */
+/**
+ * Domains that require a residential proxy to bypass datacenter IP blocks.
+ * These sites don't just need stealth — they fingerprint the IP itself and
+ * block all cloud/datacenter ranges. Webshare residential proxy bypasses this.
+ *
+ * When no explicit proxy is set and Webshare is configured, requests to these
+ * domains skip the direct (datacenter) attempt and go straight to residential proxy.
+ */
+const RESIDENTIAL_PROXY_DOMAINS = [
+    'zillow.com',
+    'yelp.com',
+    'pinterest.com',
+    'ticketmaster.com',
+    'stubhub.com',
+    'cargurus.com',
+    'realtor.com',
+    'redfin.com',
+    'apartments.com',
+    'trulia.com',
+    'homefinder.com',
+];
+/**
+ * Check if a URL matches a domain that requires residential proxy.
+ * Returns true if no explicit proxy is set and Webshare env vars are available.
+ */
+function requiresResidentialProxy(url) {
+    try {
+        const hostname = new URL(url).hostname.toLowerCase();
+        return RESIDENTIAL_PROXY_DOMAINS.some(domain => hostname === domain || hostname.endsWith(`.${domain}`));
+    }
+    catch {
+        return false;
+    }
+}
 function shouldForceBrowser(url) {
     // Hashbang URLs (#!) are always JS-routed SPAs — browser rendering required
     if (url.includes('#!')) {
@@ -314,13 +348,21 @@ export async function smartFetch(url, options = {}) {
     const { forceBrowser = false, stealth = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, actions, keepPageOpen = false, noCache = false, raceTimeoutMs = 2000, profileDir, headed = false, storageState, proxy, proxies, device, viewportWidth, viewportHeight, waitUntil, waitSelector, blockResources, cloaked = false, cycle = false, tls = false, noEscalate = false, } = options;
     const usePeelTLS = tls || cycle;
     // Build effective proxy list: explicit proxies array, or single proxy, or empty.
-    // When no explicit proxy is configured and Webshare is available, automatically
-    // add it as a fallback: try direct connection first (fast), then Webshare on block.
+    // For domains that require residential proxies (Zillow, Yelp, Pinterest, etc.),
+    // skip the direct datacenter connection entirely and go straight to Webshare.
+    // For all other domains, try direct first (fast), then Webshare as fallback.
     const effectiveProxies = proxies?.length ? proxies :
         proxy ? [proxy] :
             (() => {
                 const wsUrl = getWebshareProxyUrl();
-                return wsUrl ? [undefined, wsUrl] : [undefined];
+                if (!wsUrl)
+                    return [undefined];
+                // Skip datacenter IP for known residential-proxy-required domains
+                if (requiresResidentialProxy(url)) {
+                    log.debug('Residential proxy domain detected — skipping datacenter IP, using Webshare directly');
+                    return [wsUrl];
+                }
+                return [undefined, wsUrl];
             })();
     const firstProxy = effectiveProxies[0];
     const hooks = getStrategyHooks();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.69",
+  "version": "0.21.70",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",