webpeel 0.21.27 → 0.21.28

package/dist/core/search-provider.js

@@ -628,26 +628,34 @@ export class DuckDuckGoProvider {
         // Use realistic browser headers to avoid DDG bot detection on datacenter IPs
         // Route through residential proxy when available (datacenter IPs are blocked)
         const proxyUrl = getWebshareProxyUrl();
-        const fetchOpts = {
-            headers: {
-                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-                'Accept-Language': 'en-US,en;q=0.9',
-                'Accept-Encoding': 'gzip, deflate, br',
-                'Cache-Control': 'no-cache',
-                'Sec-Fetch-Dest': 'document',
-                'Sec-Fetch-Mode': 'navigate',
-                'Sec-Fetch-Site': 'none',
-                'Sec-Fetch-User': '?1',
-                'Upgrade-Insecure-Requests': '1',
-                'Referer': 'https://duckduckgo.com/',
-            },
-            signal,
+        const baseHeaders = {
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Cache-Control': 'no-cache',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Sec-Fetch-User': '?1',
+            'Upgrade-Insecure-Requests': '1',
+            'Referer': 'https://duckduckgo.com/',
         };
+        // Try with proxy first (bypasses datacenter IP blocks), fall back to direct
+        let response;
         if (proxyUrl) {
-            fetchOpts.dispatcher = new ProxyAgent(proxyUrl);
+            try {
+                const dispatcher = new ProxyAgent(proxyUrl);
+                response = await undiciFetch(searchUrl, { headers: baseHeaders, signal, dispatcher });
+            }
+            catch (proxyErr) {
+                log.debug('DDG proxy fetch failed, falling back to direct:', proxyErr instanceof Error ? proxyErr.message : proxyErr);
+                response = await undiciFetch(searchUrl, { headers: baseHeaders, signal });
+            }
+        }
+        else {
+            response = await undiciFetch(searchUrl, { headers: baseHeaders, signal });
         }
-        const response = await undiciFetch(searchUrl, fetchOpts);
         if (!response.ok) {
             throw new Error(`Search failed: HTTP ${response.status}`);
         }
@@ -708,19 +716,26 @@ export class DuckDuckGoProvider {
         const params = new URLSearchParams();
         params.set('q', query);
         const liteProxyUrl = getWebshareProxyUrl();
-        const liteFetchOpts = {
-            headers: {
-                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-                'Accept-Language': 'en-US,en;q=0.9',
-                'Referer': 'https://lite.duckduckgo.com/',
-            },
-            signal,
+        const liteHeaders = {
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Referer': 'https://lite.duckduckgo.com/',
         };
+        const liteUrl = `https://lite.duckduckgo.com/lite/?${params.toString()}`;
+        let response;
         if (liteProxyUrl) {
-            liteFetchOpts.dispatcher = new ProxyAgent(liteProxyUrl);
+            try {
+                const dispatcher = new ProxyAgent(liteProxyUrl);
+                response = await undiciFetch(liteUrl, { headers: liteHeaders, signal, dispatcher });
+            }
+            catch {
+                response = await undiciFetch(liteUrl, { headers: liteHeaders, signal });
+            }
+        }
+        else {
+            response = await undiciFetch(liteUrl, { headers: liteHeaders, signal });
         }
-        const response = await undiciFetch(`https://lite.duckduckgo.com/lite/?${params.toString()}`, liteFetchOpts);
         if (!response.ok)
             return [];
         const html = await response.text();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.27",
+  "version": "0.21.28",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",