webpeel 0.21.31 → 0.21.32

package/dist/core/search-provider.js

@@ -17,6 +17,7 @@ import { load } from 'cheerio';
 import { getStealthBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
 import { getWebshareProxy, getWebshareProxyUrl } from './proxy-config.js';
 import { createLogger } from './logger.js';
+import { searchViaSearXNG } from './searxng-provider.js';
 const log = createLogger('search');
 function decodeHtmlEntities(input) {
     // Cheerio usually decodes entities when using `.text()`, but keep this as a
@@ -1050,6 +1051,33 @@ export class DuckDuckGoProvider {
     async searchWeb(query, options) {
         const attempts = this.buildQueryAttempts(query);
         // -----------------------------------------------------------
+        // Stage 0: SearXNG (self-hosted, residential IP — highest reliability)
+        // Uses Mac Mini running SearXNG exposed via Cloudflare Tunnel.
+        // Aggregates Google, Bing, Brave, Startpage — 30-40 results typical.
+        // Env: SEARXNG_URL=https://search.webpeel.dev
+        // -----------------------------------------------------------
+        if (process.env.SEARXNG_URL) {
+            try {
+                const searxResults = await searchViaSearXNG(query, {
+                    count: options.count ?? 10,
+                    signal: options.signal,
+                    timeoutMs: 6000,
+                });
+                if (searxResults.length > 0) {
+                    providerStats.record('searxng', true);
+                    log.debug(`source=searxng returned ${searxResults.length} results`);
+                    const filtered = filterRelevantResults(searxResults, query);
+                    return filtered.length > 0 ? filtered : searxResults;
+                }
+                providerStats.record('searxng', false);
+                log.debug('SearXNG returned 0 results, falling through to DDG');
+            }
+            catch (e) {
+                providerStats.record('searxng', false);
+                log.debug('SearXNG failed:', e instanceof Error ? e.message : e);
+            }
+        }
+        // -----------------------------------------------------------
         // Stage 1: DDG HTTP
         // Skip entirely if the source has a ≥80% failure rate over the
         // last 10 attempts.  When elevated-but-not-skipped, cap the per-

package/dist/core/searxng-provider.d.ts

@@ -0,0 +1,34 @@
+/**
+ * SearXNG Search Provider
+ *
+ * Connects to a self-hosted SearXNG instance (running on Mac Mini with residential IP,
+ * exposed via Cloudflare Tunnel). SearXNG aggregates Google, Bing, Brave, Startpage, etc.
+ * and is not rate-limited or blocked since it runs on a residential IP.
+ *
+ * Config (env vars):
+ *   SEARXNG_URL   — Base URL of SearXNG instance (e.g. https://search.webpeel.dev)
+ *
+ * Falls back gracefully if SEARXNG_URL is not set or instance is unreachable.
+ */
+export interface SearXNGSearchResult {
+    title: string;
+    url: string;
+    description?: string;
+    publishedDate?: string;
+    score?: number;
+}
+/**
+ * Fetches search results from a SearXNG instance.
+ * Returns results compatible with WebSearchResult interface in search-provider.ts.
+ */
+export declare function searchViaSearXNG(query: string, options?: {
+    count?: number;
+    signal?: AbortSignal;
+    timeoutMs?: number;
+    engines?: string;
+    language?: string;
+}): Promise<SearXNGSearchResult[]>;
+/**
+ * Quick health check — true if SearXNG is reachable and returning results.
+ */
+export declare function isSearXNGHealthy(): Promise<boolean>;

package/dist/core/searxng-provider.js

@@ -0,0 +1,104 @@
+/**
+ * SearXNG Search Provider
+ *
+ * Connects to a self-hosted SearXNG instance (running on Mac Mini with residential IP,
+ * exposed via Cloudflare Tunnel). SearXNG aggregates Google, Bing, Brave, Startpage, etc.
+ * and is not rate-limited or blocked since it runs on a residential IP.
+ *
+ * Config (env vars):
+ *   SEARXNG_URL   — Base URL of SearXNG instance (e.g. https://search.webpeel.dev)
+ *
+ * Falls back gracefully if SEARXNG_URL is not set or instance is unreachable.
+ */
+import { fetch as undiciFetch } from 'undici';
+import { createLogger } from './logger.js';
+const log = createLogger('searxng');
+/**
+ * Fetches search results from a SearXNG instance.
+ * Returns results compatible with WebSearchResult interface in search-provider.ts.
+ */
+export async function searchViaSearXNG(query, options = {}) {
+    const baseUrl = process.env.SEARXNG_URL;
+    if (!baseUrl)
+        return [];
+    const { count = 10, signal, timeoutMs = 8000, engines = '', language = 'en', } = options;
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+    if (signal)
+        signal.addEventListener('abort', () => controller.abort());
+    try {
+        const params = new URLSearchParams({
+            q: query,
+            format: 'json',
+            language,
+            safesearch: '0',
+            categories: 'general',
+        });
+        if (engines)
+            params.set('engines', engines);
+        const url = `${baseUrl.replace(/\/$/, '')}/search?${params.toString()}`;
+        const response = await undiciFetch(url, {
+            signal: controller.signal,
+            headers: {
+                'Accept': 'application/json',
+                'User-Agent': 'WebPeel/1.0 (internal search aggregator)',
+            },
+        });
+        if (!response.ok) {
+            log.debug(`HTTP ${response.status}`);
+            return [];
+        }
+        const data = (await response.json());
+        const results = data?.results ?? [];
+        if (results.length === 0) {
+            log.debug('0 results returned');
+            return [];
+        }
+        const seen = new Set();
+        const output = [];
+        for (const r of results) {
+            if (!r.url || !r.title)
+                continue;
+            const normalized = r.url.replace(/\/$/, '').toLowerCase();
+            if (seen.has(normalized))
+                continue;
+            seen.add(normalized);
+            output.push({
+                title: r.title,
+                url: r.url,
+                description: r.content ?? undefined,
+                publishedDate: r.publishedDate ?? undefined,
+                score: r.score ?? undefined,
+            });
+            if (output.length >= count)
+                break;
+        }
+        log.debug(`${output.length} results for "${query.substring(0, 40)}"`);
+        return output;
+    }
+    catch (e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        if (msg.includes('abort') || msg.includes('timeout') || msg.includes('AbortError')) {
+            log.debug(`timed out after ${timeoutMs}ms`);
+        }
+        else {
+            log.debug('fetch error:', msg);
+        }
+        return [];
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}
+/**
+ * Quick health check — true if SearXNG is reachable and returning results.
+ */
+export async function isSearXNGHealthy() {
+    try {
+        const results = await searchViaSearXNG('test', { count: 1, timeoutMs: 5000 });
+        return results.length > 0;
+    }
+    catch {
+        return false;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.31",
+  "version": "0.21.32",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",