webpeel 0.21.39 → 0.21.41

package/dist/core/search-provider.js

@@ -1061,7 +1061,7 @@ export class DuckDuckGoProvider {
                 const searxResults = await searchViaSearXNG(query, {
                     count: options.count ?? 10,
                     signal: options.signal,
-                    timeoutMs: 6000,
+                    timeoutMs: 12000,
                 });
                 if (searxResults.length > 0) {
                     providerStats.record('searxng', true);

package/dist/core/searxng-provider.js

@@ -21,7 +21,7 @@ export async function searchViaSearXNG(query, options = {}) {
     const baseUrl = process.env.SEARXNG_URL;
     if (!baseUrl)
         return [];
-    const { count = 10, signal, timeoutMs = 8000, engines = '', language = 'en', } = options;
+    const { count = 10, signal, timeoutMs = 15000, engines = '', language = 'en', } = options;
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
     if (signal)
@@ -95,7 +95,7 @@ export async function searchViaSearXNG(query, options = {}) {
  */
 export async function isSearXNGHealthy() {
     try {
-        const results = await searchViaSearXNG('test', { count: 1, timeoutMs: 5000 });
+        const results = await searchViaSearXNG('test', { count: 1, timeoutMs: 10000 });
         return results.length > 0;
     }
     catch {

package/dist/server/routes/research.js

@@ -9,7 +9,8 @@
  * Body: ResearchRequest
  */
 import { Router } from 'express';
-import { peel } from '../../index.js';
+import { simpleFetch } from '../../core/fetcher.js';
+import { load as cheerioLoad } from 'cheerio';
 import { getSearchProvider } from '../../core/search-provider.js';
 import { callLLM, } from '../../core/llm-provider.js';
 import { sanitizeForLLM, hardenSystemPrompt, validateOutput } from '../../core/prompt-guard.js';
@@ -295,21 +296,21 @@ export function createResearchRouter() {
                     break;
                 const fetchStart = Date.now();
                 try {
-                    const result = await Promise.race([
-                        peel(url, {
-                            format: 'markdown',
-                            noEscalate: true, // NEVER launch browser — 512MB container
-                            timeout: urlTimeout,
-                            readable: true,
-                            budget: 3000,
-                        }),
+                    // Use simpleFetch + cheerio (no peel/pipeline) — keeps memory under 512MB
+                    const fetchResult = await Promise.race([
+                        simpleFetch(url, undefined, urlTimeout),
                         new Promise((_, reject) => setTimeout(() => reject(new Error('per-url timeout')), urlTimeout)),
                     ]);
                     const fetchTime = Date.now() - fetchStart;
-                    const content = result.content || '';
+                    // Extract clean text via cheerio (no Readability.js, no markdown pipeline)
+                    const $ = cheerioLoad(fetchResult.html || '');
+                    $('script,style,nav,footer,header,aside,noscript,[aria-hidden]').remove();
+                    const pageTitle = ($('title').text() || $('h1').first().text() || title).trim().slice(0, 200);
+                    const rawText = $('main, article, [role=main], body').first().text()
+                        .replace(/\s+/g, ' ').trim();
+                    const content = rawText.slice(0, 4000); // ~3000 words max
                     const wordCount = content.split(/\s+/).filter(Boolean).length;
-                    const pageTitle = result.title || title;
-                    // Build snippet: prefer LLM-extracted summary, else first 500 chars of content
+                    // Build snippet: first 500 chars of content
                     const sourceSnippet = content.slice(0, 500).replace(/\s+/g, ' ').trim();
                     sources.push({
                         url,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.39",
+  "version": "0.21.41",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",