webpeel 0.21.43 → 0.21.45

package/dist/server/routes/research.js

@@ -153,7 +153,7 @@ const VALID_LLM_PROVIDERS = [
     'cerebras',
     'cloudflare',
 ];
-const MAX_SOURCES_HARD_LIMIT = 8;
+const MAX_SOURCES_HARD_LIMIT = 4; // 512MB container — never fetch more than 4 sources
 const PER_URL_TIMEOUT_MS = 8_000;
 const TOTAL_TIMEOUT_MS = 60_000;
 export function createResearchRouter() {
@@ -175,6 +175,33 @@ export function createResearchRouter() {
             });
             return;
         }
+        // ── Hetzner research worker proxy ────────────────────────────────────
+        // When RESEARCH_WORKER_URL is set, forward the entire request to the
+        // Hetzner VPS worker (local SearXNG + Ollama). Falls back to local if proxy fails.
+        if (process.env.RESEARCH_WORKER_URL) {
+            try {
+                const resp = await fetch(process.env.RESEARCH_WORKER_URL + '/research', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'Authorization': 'Bearer ' + (process.env.OLLAMA_SECRET || ''),
+                    },
+                    body: JSON.stringify(req.body),
+                    signal: AbortSignal.timeout(55_000),
+                });
+                const result = await resp.json();
+                // Attach requestId for consistency
+                if (result && typeof result === 'object') {
+                    result.requestId = req.requestId;
+                }
+                res.json(result);
+                return;
+            }
+            catch (proxyErr) {
+                console.warn('[research] Hetzner proxy failed, falling back to local:', proxyErr.message);
+                // Fall through to local research pipeline
+            }
+        }
         // ── Parse & validate body ─────────────────────────────────────────────
         const body = req.body;
         if (!body.query || typeof body.query !== 'string' || body.query.trim().length === 0) {
@@ -302,8 +329,10 @@ export function createResearchRouter() {
                         new Promise((_, reject) => setTimeout(() => reject(new Error('per-url timeout')), urlTimeout)),
                     ]);
                     const fetchTime = Date.now() - fetchStart;
+                    // Cap HTML at 100KB before parsing — huge pages (Reddit 500KB+) OOM 512MB container
+                    const rawHtml = (fetchResult.html || '').slice(0, 100_000);
                     // Extract clean text via cheerio (no Readability.js, no markdown pipeline)
-                    const $ = cheerioLoad(fetchResult.html || '');
+                    const $ = cheerioLoad(rawHtml);
                     $('script,style,nav,footer,header,aside,noscript,[aria-hidden]').remove();
                     const pageTitle = ($('title').text() || $('h1').first().text() || title).trim().slice(0, 200);
                     const rawText = $('main, article, [role=main], body').first().text()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.43",
+  "version": "0.21.45",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",