webpeel 0.21.25 → 0.21.26

package/dist/core/search-provider.js

@@ -12,10 +12,10 @@
  * In production with no API keys configured, getBestSearchProvider() returns
  * StealthSearchProvider since DDG HTTP is often blocked on datacenter IPs.
  */
-import { fetch as undiciFetch } from 'undici';
+import { fetch as undiciFetch, ProxyAgent } from 'undici';
 import { load } from 'cheerio';
 import { getStealthBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
-import { getWebshareProxy } from './proxy-config.js';
+import { getWebshareProxy, getWebshareProxyUrl } from './proxy-config.js';
 import { createLogger } from './logger.js';
 const log = createLogger('search');
 function decodeHtmlEntities(input) {
@@ -626,7 +626,9 @@ export class DuckDuckGoProvider {
         const { count, signal } = options;
         const searchUrl = this.buildSearchUrl(query, options);
         // Use realistic browser headers to avoid DDG bot detection on datacenter IPs
-        const response = await undiciFetch(searchUrl, {
+        // Route through residential proxy when available (datacenter IPs are blocked)
+        const proxyUrl = getWebshareProxyUrl();
+        const fetchOpts = {
             headers: {
                 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
                 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
@@ -641,7 +643,11 @@ export class DuckDuckGoProvider {
                 'Referer': 'https://duckduckgo.com/',
             },
             signal,
-        });
+        };
+        if (proxyUrl) {
+            fetchOpts.dispatcher = new ProxyAgent(proxyUrl);
+        }
+        const response = await undiciFetch(searchUrl, fetchOpts);
         if (!response.ok) {
             throw new Error(`Search failed: HTTP ${response.status}`);
         }
@@ -701,7 +707,8 @@ export class DuckDuckGoProvider {
         const { count, signal } = options;
         const params = new URLSearchParams();
         params.set('q', query);
-        const response = await undiciFetch(`https://lite.duckduckgo.com/lite/?${params.toString()}`, {
+        const liteProxyUrl = getWebshareProxyUrl();
+        const liteFetchOpts = {
             headers: {
                 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
                 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
@@ -709,7 +716,11 @@ export class DuckDuckGoProvider {
                 'Referer': 'https://lite.duckduckgo.com/',
             },
             signal,
-        });
+        };
+        if (liteProxyUrl) {
+            liteFetchOpts.dispatcher = new ProxyAgent(liteProxyUrl);
+        }
+        const response = await undiciFetch(`https://lite.duckduckgo.com/lite/?${params.toString()}`, liteFetchOpts);
         if (!response.ok)
             return [];
         const html = await response.text();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.25",
+  "version": "0.21.26",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",