webpeel 0.21.22 → 0.21.24

package/dist/core/search-provider.js

@@ -844,20 +844,27 @@ export class DuckDuckGoProvider {
         // Stage 4: Stealth multi-engine (DDG + Bing + Ecosia in parallel)
         // Bypasses bot-detection on datacenter IPs. This is the reliable
         // last resort — but it spins up a browser so it takes a few seconds.
+        // DISABLED on memory-constrained servers (512MB) — Playwright OOM kills.
+        // Set NO_BROWSER_SEARCH=1 to skip this stage entirely.
         // -----------------------------------------------------------
-        log.debug('Trying stealth browser search (DDG + Bing + Ecosia)...');
-        try {
-            const stealthProvider = new StealthSearchProvider();
-            // StealthSearchProvider already applies filterRelevantResults internally.
-            const stealthResults = await stealthProvider.searchWeb(query, options);
-            if (stealthResults.length > 0) {
-                log.debug(`source=stealth returned ${stealthResults.length} results`);
-                return stealthResults;
+        if (!process.env.NO_BROWSER_SEARCH) {
+            log.debug('Trying stealth browser search (DDG + Bing + Ecosia)...');
+            try {
+                const stealthProvider = new StealthSearchProvider();
+                // StealthSearchProvider already applies filterRelevantResults internally.
+                const stealthResults = await stealthProvider.searchWeb(query, options);
+                if (stealthResults.length > 0) {
+                    log.debug(`source=stealth returned ${stealthResults.length} results`);
+                    return stealthResults;
+                }
+                log.debug('Stealth search returned 0 results');
+            }
+            catch (e) {
+                log.debug('Stealth search failed:', e instanceof Error ? e.message : e);
             }
-            log.debug('Stealth search returned 0 results');
         }
-        catch (e) {
-            log.debug('Stealth search failed:', e instanceof Error ? e.message : e);
+        else {
+            log.debug('Stealth browser search skipped (NO_BROWSER_SEARCH=1)');
         }
         return [];
     }

package/dist/server/routes/fetch.js

@@ -341,6 +341,10 @@ export function createFetchRouter(authStore) {
                 lite: lite === 'true',
                 timeout: timeout ? parseInt(timeout, 10) : undefined,
                 captionImages: captionImages === 'true',
+                // Prevent auto-escalation to browser unless render=true is explicitly requested.
+                // On 512MB containers, surprise browser launches cause OOM kills.
+                // Domain extractors (GitHub, Wikipedia, npm etc.) use HTTP APIs, not the browser.
+                noEscalate: !shouldRender,
             };
             // Auto-budget: default to 4000 tokens for API requests when no budget specified
             // Opt-out: budget=0 explicitly disables. Lite mode disables auto-budget.

package/dist/server/routes/search.js

@@ -6,6 +6,7 @@ import { fetch as undiciFetch } from 'undici';
 import { load } from 'cheerio';
 import { LRUCache } from 'lru-cache';
 import { peel } from '../../index.js';
+import { simpleFetch } from '../../core/fetcher.js';
 import { getSearchProvider, getBestSearchProvider, } from '../../core/search-provider.js';
 export function createSearchRouter(authStore) {
     const router = Router();
@@ -149,25 +150,35 @@ export function createSearchRouter(authStore) {
                         }
                     }
                 }
-                // Enrich top N results in parallel with timeout (fast alternative to scrapeResults)
-                // IMPORTANT: forceBrowser=false, stealth=false to prevent OOM on 512MB containers
+                // Lightweight enrichment — HTTP-only, no browser, no full pipeline
+                // Uses simpleFetch + cheerio to extract text without launching Playwright
+                // This is intentionally minimal to stay within 512MB container memory limit
                 if (enrichCount > 0 && !shouldScrape) {
-                    const ENRICH_TIMEOUT = 4000; // 4s hard timeout per URL
+                    const ENRICH_TIMEOUT = 4000;
                     const toEnrich = results.slice(0, enrichCount);
                     const enrichResults = await Promise.allSettled(toEnrich.map(async (result) => {
-                        const fetchPromise = peel(result.url, {
-                            format: 'markdown',
-                            maxTokens: 1500,
-                            render: false,
-                            stealth: false,
-                        }).then(peelResult => ({
-                            url: result.url,
-                            content: peelResult.content?.substring(0, 1500) || null,
-                            wordCount: peelResult.content?.trim().split(/\s+/).length || 0,
-                            method: peelResult.method || 'unknown',
-                            fetchTimeMs: peelResult.elapsed || 0,
-                        }));
-                        const timeoutPromise = new Promise(resolve => setTimeout(() => resolve({ url: result.url, content: null, wordCount: 0, method: 'timeout', fetchTimeMs: 0 }), ENRICH_TIMEOUT));
+                        const t0 = Date.now();
+                        const fetchPromise = (async () => {
+                            const fetched = await simpleFetch(result.url, undefined, ENRICH_TIMEOUT);
+                            if (!fetched.html)
+                                return { url: result.url, content: null, wordCount: 0, method: 'empty', fetchTimeMs: 0 };
+                            // Extract visible text with cheerio — lightweight, no full pipeline
+                            const $ = load(fetched.html);
+                            $('script, style, nav, header, footer, [aria-hidden="true"], .ad, .advertisement').remove();
+                            // Try main content selectors first, then body
+                            const mainEl = $('main, article, [role="main"], .content, .article-body, #content').first();
+                            const textEl = mainEl.length ? mainEl : $('body');
+                            const text = textEl.text().replace(/\s+/g, ' ').trim().substring(0, 2000);
+                            const wordCount = text.split(/\s+/).filter(Boolean).length;
+                            return {
+                                url: result.url,
+                                content: text.substring(0, 1500) || null,
+                                wordCount,
+                                method: 'simple',
+                                fetchTimeMs: Date.now() - t0,
+                            };
+                        })();
+                        const timeoutPromise = new Promise(resolve => setTimeout(() => resolve({ url: result.url, content: null, wordCount: 0, method: 'timeout', fetchTimeMs: ENRICH_TIMEOUT }), ENRICH_TIMEOUT));
                         return Promise.race([fetchPromise, timeoutPromise]);
                     }));
                     // Merge enrichment data back into results

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.22",
+  "version": "0.21.24",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",