webpeel 0.21.30 → 0.21.32

package/dist/core/search-provider.js

@@ -17,6 +17,7 @@ import { load } from 'cheerio';
 import { getStealthBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
 import { getWebshareProxy, getWebshareProxyUrl } from './proxy-config.js';
 import { createLogger } from './logger.js';
+import { searchViaSearXNG } from './searxng-provider.js';
 const log = createLogger('search');
 function decodeHtmlEntities(input) {
     // Cheerio usually decodes entities when using `.text()`, but keep this as a
@@ -684,25 +685,37 @@ export class DuckDuckGoProvider {
             'Upgrade-Insecure-Requests': '1',
             'Referer': 'https://duckduckgo.com/',
         };
-        // Try with proxy first (bypasses datacenter IP blocks), fall back to direct
+        // Try direct first, then proxy as fallback.
+        // Webshare backbone IPs are blocked by DDG (returns empty results).
+        // Render datacenter IPs work intermittently — direct has better odds.
         let response;
-        if (proxyUrl) {
+        let html;
+        // let usedProxy = false;
+        // Attempt 1: Direct fetch (no proxy)
+        try {
+            response = await undiciFetch(searchUrl, { headers: baseHeaders, signal });
+            html = response.ok ? await response.text() : '';
+        }
+        catch (directErr) {
+            log.debug('DDG direct fetch failed:', directErr instanceof Error ? directErr.message : directErr);
+            html = '';
+        }
+        // Check if direct returned actual results (not empty/CAPTCHA)
+        const hasResults = html.includes('class="result"') || html.includes('class="result ');
+        if (!hasResults && proxyUrl) {
+            // Attempt 2: Proxy fallback
+            log.debug('DDG direct returned no results, trying proxy...');
             try {
+                // usedProxy = true;
                 const dispatcher = new ProxyAgent(proxyUrl);
                 response = await undiciFetch(searchUrl, { headers: baseHeaders, signal, dispatcher });
+                if (response.ok)
+                    html = await response.text();
             }
             catch (proxyErr) {
-                log.debug('DDG proxy fetch failed, falling back to direct:', proxyErr instanceof Error ? proxyErr.message : proxyErr);
-                response = await undiciFetch(searchUrl, { headers: baseHeaders, signal });
+                log.debug('DDG proxy also failed:', proxyErr instanceof Error ? proxyErr.message : proxyErr);
             }
         }
-        else {
-            response = await undiciFetch(searchUrl, { headers: baseHeaders, signal });
-        }
-        if (!response.ok) {
-            throw new Error(`Search failed: HTTP ${response.status}`);
-        }
-        const html = await response.text();
         const $ = load(html);
         const results = [];
         const seen = new Set();
@@ -766,22 +779,25 @@ export class DuckDuckGoProvider {
             'Referer': 'https://lite.duckduckgo.com/',
         };
         const liteUrl = `https://lite.duckduckgo.com/lite/?${params.toString()}`;
-        let response;
-        if (liteProxyUrl) {
+        // Direct first, proxy fallback (same reasoning as searchOnce — Webshare IPs blocked by DDG)
+        let html = '';
+        try {
+            const resp = await undiciFetch(liteUrl, { headers: liteHeaders, signal });
+            if (resp.ok)
+                html = await resp.text();
+        }
+        catch { /* direct failed */ }
+        if (!html.includes('result-link') && liteProxyUrl) {
             try {
                 const dispatcher = new ProxyAgent(liteProxyUrl);
-                response = await undiciFetch(liteUrl, { headers: liteHeaders, signal, dispatcher });
+                const resp = await undiciFetch(liteUrl, { headers: liteHeaders, signal, dispatcher });
+                if (resp.ok)
+                    html = await resp.text();
             }
-            catch {
-                response = await undiciFetch(liteUrl, { headers: liteHeaders, signal });
-            }
-        }
-        else {
-            response = await undiciFetch(liteUrl, { headers: liteHeaders, signal });
+            catch { /* proxy also failed */ }
         }
-        if (!response.ok)
+        if (!html)
             return [];
-        const html = await response.text();
         const $ = load(html);
         const results = [];
         const seen = new Set();
@@ -1035,6 +1051,33 @@ export class DuckDuckGoProvider {
     async searchWeb(query, options) {
         const attempts = this.buildQueryAttempts(query);
         // -----------------------------------------------------------
+        // Stage 0: SearXNG (self-hosted, residential IP — highest reliability)
+        // Uses Mac Mini running SearXNG exposed via Cloudflare Tunnel.
+        // Aggregates Google, Bing, Brave, Startpage — 30-40 results typical.
+        // Env: SEARXNG_URL=https://search.webpeel.dev
+        // -----------------------------------------------------------
+        if (process.env.SEARXNG_URL) {
+            try {
+                const searxResults = await searchViaSearXNG(query, {
+                    count: options.count ?? 10,
+                    signal: options.signal,
+                    timeoutMs: 6000,
+                });
+                if (searxResults.length > 0) {
+                    providerStats.record('searxng', true);
+                    log.debug(`source=searxng returned ${searxResults.length} results`);
+                    const filtered = filterRelevantResults(searxResults, query);
+                    return filtered.length > 0 ? filtered : searxResults;
+                }
+                providerStats.record('searxng', false);
+                log.debug('SearXNG returned 0 results, falling through to DDG');
+            }
+            catch (e) {
+                providerStats.record('searxng', false);
+                log.debug('SearXNG failed:', e instanceof Error ? e.message : e);
+            }
+        }
+        // -----------------------------------------------------------
         // Stage 1: DDG HTTP
         // Skip entirely if the source has a ≥80% failure rate over the
         // last 10 attempts.  When elevated-but-not-skipped, cap the per-

package/dist/core/searxng-provider.d.ts

@@ -0,0 +1,34 @@
+/**
+ * SearXNG Search Provider
+ *
+ * Connects to a self-hosted SearXNG instance (running on Mac Mini with residential IP,
+ * exposed via Cloudflare Tunnel). SearXNG aggregates Google, Bing, Brave, Startpage, etc.
+ * and is not rate-limited or blocked since it runs on a residential IP.
+ *
+ * Config (env vars):
+ *   SEARXNG_URL   — Base URL of SearXNG instance (e.g. https://search.webpeel.dev)
+ *
+ * Falls back gracefully if SEARXNG_URL is not set or instance is unreachable.
+ */
+export interface SearXNGSearchResult {
+    title: string;
+    url: string;
+    description?: string;
+    publishedDate?: string;
+    score?: number;
+}
+/**
+ * Fetches search results from a SearXNG instance.
+ * Returns results compatible with WebSearchResult interface in search-provider.ts.
+ */
+export declare function searchViaSearXNG(query: string, options?: {
+    count?: number;
+    signal?: AbortSignal;
+    timeoutMs?: number;
+    engines?: string;
+    language?: string;
+}): Promise<SearXNGSearchResult[]>;
+/**
+ * Quick health check — true if SearXNG is reachable and returning results.
+ */
+export declare function isSearXNGHealthy(): Promise<boolean>;

package/dist/core/searxng-provider.js

@@ -0,0 +1,104 @@
+/**
+ * SearXNG Search Provider
+ *
+ * Connects to a self-hosted SearXNG instance (running on Mac Mini with residential IP,
+ * exposed via Cloudflare Tunnel). SearXNG aggregates Google, Bing, Brave, Startpage, etc.
+ * and is not rate-limited or blocked since it runs on a residential IP.
+ *
+ * Config (env vars):
+ *   SEARXNG_URL   — Base URL of SearXNG instance (e.g. https://search.webpeel.dev)
+ *
+ * Falls back gracefully if SEARXNG_URL is not set or instance is unreachable.
+ */
+import { fetch as undiciFetch } from 'undici';
+import { createLogger } from './logger.js';
+const log = createLogger('searxng');
+/**
+ * Fetches search results from a SearXNG instance.
+ * Returns results compatible with WebSearchResult interface in search-provider.ts.
+ */
+export async function searchViaSearXNG(query, options = {}) {
+    const baseUrl = process.env.SEARXNG_URL;
+    if (!baseUrl)
+        return [];
+    const { count = 10, signal, timeoutMs = 8000, engines = '', language = 'en', } = options;
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+    if (signal)
+        signal.addEventListener('abort', () => controller.abort());
+    try {
+        const params = new URLSearchParams({
+            q: query,
+            format: 'json',
+            language,
+            safesearch: '0',
+            categories: 'general',
+        });
+        if (engines)
+            params.set('engines', engines);
+        const url = `${baseUrl.replace(/\/$/, '')}/search?${params.toString()}`;
+        const response = await undiciFetch(url, {
+            signal: controller.signal,
+            headers: {
+                'Accept': 'application/json',
+                'User-Agent': 'WebPeel/1.0 (internal search aggregator)',
+            },
+        });
+        if (!response.ok) {
+            log.debug(`HTTP ${response.status}`);
+            return [];
+        }
+        const data = (await response.json());
+        const results = data?.results ?? [];
+        if (results.length === 0) {
+            log.debug('0 results returned');
+            return [];
+        }
+        const seen = new Set();
+        const output = [];
+        for (const r of results) {
+            if (!r.url || !r.title)
+                continue;
+            const normalized = r.url.replace(/\/$/, '').toLowerCase();
+            if (seen.has(normalized))
+                continue;
+            seen.add(normalized);
+            output.push({
+                title: r.title,
+                url: r.url,
+                description: r.content ?? undefined,
+                publishedDate: r.publishedDate ?? undefined,
+                score: r.score ?? undefined,
+            });
+            if (output.length >= count)
+                break;
+        }
+        log.debug(`${output.length} results for "${query.substring(0, 40)}"`);
+        return output;
+    }
+    catch (e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        if (msg.includes('abort') || msg.includes('timeout') || msg.includes('AbortError')) {
+            log.debug(`timed out after ${timeoutMs}ms`);
+        }
+        else {
+            log.debug('fetch error:', msg);
+        }
+        return [];
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}
+/**
+ * Quick health check — true if SearXNG is reachable and returning results.
+ */
+export async function isSearXNGHealthy() {
+    try {
+        const results = await searchViaSearXNG('test', { count: 1, timeoutMs: 5000 });
+        return results.length > 0;
+    }
+    catch {
+        return false;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.30",
+  "version": "0.21.32",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",