webpeel 0.21.29 → 0.21.31

package/dist/core/search-provider.d.ts

@@ -44,18 +44,21 @@ declare class ProviderStatsTracker {
     private readonly windowSize;
     private readonly failThreshold;
     private readonly minSamples;
-    constructor(windowSize?: number, failThreshold?: number, minSamples?: number);
+    private readonly decayMs;
+    constructor(windowSize?: number, failThreshold?: number, minSamples?: number, decayMs?: number);
     /** Record the outcome of a single attempt for the given source. */
     record(sourceId: string, success: boolean): void;
     /**
      * Returns the failure rate (0–1) for the given source based on
      * the sliding window of recorded attempts.  Returns 0 if fewer
-     * than minSamples have been recorded.
+     * than minSamples have been recorded, or if all samples are older
+     * than decayMs (failures expire so cold-start blips don't permanently
+     * lock out a provider).
      */
     getFailureRate(sourceId: string): number;
     /**
      * Returns true when the source should be skipped (failure rate >=
-     * failThreshold with at least minSamples recorded).
+     * failThreshold with at least minSamples recent recorded).
      */
     shouldSkip(sourceId: string): boolean;
     /** Debug snapshot for a source. */

package/dist/core/search-provider.js

@@ -114,15 +114,17 @@ class ProviderStatsTracker {
     windowSize;
     failThreshold;
     minSamples;
-    constructor(windowSize = 10, failThreshold = 0.8, minSamples = 3) {
+    decayMs; // failures older than this are ignored
+    constructor(windowSize = 10, failThreshold = 0.8, minSamples = 5, decayMs = 5 * 60 * 1000) {
         this.windowSize = windowSize;
         this.failThreshold = failThreshold;
         this.minSamples = minSamples;
+        this.decayMs = decayMs; // default 5 minutes: old failures don't permanently lock a provider
     }
     /** Record the outcome of a single attempt for the given source. */
     record(sourceId, success) {
         const arr = this.history.get(sourceId) ?? [];
-        arr.push({ success });
+        arr.push({ success, ts: Date.now() });
         if (arr.length > this.windowSize)
             arr.splice(0, arr.length - this.windowSize);
         this.history.set(sourceId, arr);
@@ -130,18 +132,24 @@ class ProviderStatsTracker {
     /**
      * Returns the failure rate (0–1) for the given source based on
      * the sliding window of recorded attempts.  Returns 0 if fewer
-     * than minSamples have been recorded.
+     * than minSamples have been recorded, or if all samples are older
+     * than decayMs (failures expire so cold-start blips don't permanently
+     * lock out a provider).
      */
     getFailureRate(sourceId) {
         const arr = this.history.get(sourceId);
         if (!arr || arr.length < this.minSamples)
             return 0;
-        const failures = arr.filter(a => !a.success).length;
-        return failures / arr.length;
+        const cutoff = Date.now() - this.decayMs;
+        const recent = arr.filter(a => a.ts >= cutoff);
+        if (recent.length < this.minSamples)
+            return 0; // not enough recent samples
+        const failures = recent.filter(a => !a.success).length;
+        return failures / recent.length;
     }
     /**
      * Returns true when the source should be skipped (failure rate >=
-     * failThreshold with at least minSamples recorded).
+     * failThreshold with at least minSamples recent recorded).
      */
     shouldSkip(sourceId) {
         return this.getFailureRate(sourceId) >= this.failThreshold;
@@ -676,25 +684,37 @@ export class DuckDuckGoProvider {
             'Upgrade-Insecure-Requests': '1',
             'Referer': 'https://duckduckgo.com/',
         };
-        // Try with proxy first (bypasses datacenter IP blocks), fall back to direct
+        // Try direct first, then proxy as fallback.
+        // Webshare backbone IPs are blocked by DDG (returns empty results).
+        // Render datacenter IPs work intermittently — direct has better odds.
         let response;
-        if (proxyUrl) {
+        let html;
+        // let usedProxy = false;
+        // Attempt 1: Direct fetch (no proxy)
+        try {
+            response = await undiciFetch(searchUrl, { headers: baseHeaders, signal });
+            html = response.ok ? await response.text() : '';
+        }
+        catch (directErr) {
+            log.debug('DDG direct fetch failed:', directErr instanceof Error ? directErr.message : directErr);
+            html = '';
+        }
+        // Check if direct returned actual results (not empty/CAPTCHA)
+        const hasResults = html.includes('class="result"') || html.includes('class="result ');
+        if (!hasResults && proxyUrl) {
+            // Attempt 2: Proxy fallback
+            log.debug('DDG direct returned no results, trying proxy...');
             try {
+                // usedProxy = true;
                 const dispatcher = new ProxyAgent(proxyUrl);
                 response = await undiciFetch(searchUrl, { headers: baseHeaders, signal, dispatcher });
+                if (response.ok)
+                    html = await response.text();
             }
             catch (proxyErr) {
-                log.debug('DDG proxy fetch failed, falling back to direct:', proxyErr instanceof Error ? proxyErr.message : proxyErr);
-                response = await undiciFetch(searchUrl, { headers: baseHeaders, signal });
+                log.debug('DDG proxy also failed:', proxyErr instanceof Error ? proxyErr.message : proxyErr);
             }
         }
-        else {
-            response = await undiciFetch(searchUrl, { headers: baseHeaders, signal });
-        }
-        if (!response.ok) {
-            throw new Error(`Search failed: HTTP ${response.status}`);
-        }
-        const html = await response.text();
         const $ = load(html);
         const results = [];
         const seen = new Set();
@@ -758,22 +778,25 @@ export class DuckDuckGoProvider {
             'Referer': 'https://lite.duckduckgo.com/',
         };
         const liteUrl = `https://lite.duckduckgo.com/lite/?${params.toString()}`;
-        let response;
-        if (liteProxyUrl) {
+        // Direct first, proxy fallback (same reasoning as searchOnce — Webshare IPs blocked by DDG)
+        let html = '';
+        try {
+            const resp = await undiciFetch(liteUrl, { headers: liteHeaders, signal });
+            if (resp.ok)
+                html = await resp.text();
+        }
+        catch { /* direct failed */ }
+        if (!html.includes('result-link') && liteProxyUrl) {
             try {
                 const dispatcher = new ProxyAgent(liteProxyUrl);
-                response = await undiciFetch(liteUrl, { headers: liteHeaders, signal, dispatcher });
+                const resp = await undiciFetch(liteUrl, { headers: liteHeaders, signal, dispatcher });
+                if (resp.ok)
+                    html = await resp.text();
             }
-            catch {
-                response = await undiciFetch(liteUrl, { headers: liteHeaders, signal });
-            }
-        }
-        else {
-            response = await undiciFetch(liteUrl, { headers: liteHeaders, signal });
+            catch { /* proxy also failed */ }
         }
-        if (!response.ok)
+        if (!html)
             return [];
-        const html = await response.text();
         const $ = load(html);
         const results = [];
         const seen = new Set();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.29",
+  "version": "0.21.31",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",