webpeel 0.21.36 → 0.21.37

package/dist/core/domain-extractors.js

@@ -153,7 +153,12 @@ async function fetchJson(url, customHeaders) {
         Accept: 'application/json',
         ...customHeaders,
     });
-    return tryParseJson(result.html);
+    const parsed = tryParseJson(result.html);
+    if (parsed === null && result.html.length > 0) {
+        // Log when we get non-JSON back (likely an HTML error page)
+        console.warn(`[webpeel:fetchJson] Non-JSON response from ${url} (${result.html.length} bytes, status: ${result.statusCode}): ${result.html.slice(0, 120)}`);
+    }
+    return parsed;
 }
 /** Fetch JSON with exponential backoff retry on 429 / rate-limit errors. */
 async function fetchJsonWithRetry(url, headers, retries = 2, baseDelayMs = 1000) {
@@ -910,11 +915,17 @@ ${commentsMd || '*No comments.*'}`;
     if (pathParts.length >= 2) {
         // Sequential fetches to avoid secondary rate limits on popular repos
         const repoData = await fetchJsonWithRetry(`https://api.github.com/repos/${owner}/${repo}`, ghHeaders, 2, 1000);
-        if (!repoData || repoData.message === 'Not Found')
-            return null;
-        // Secondary rate limit check
-        if (repoData.message?.includes('secondary rate limit') || repoData.message?.includes('abuse'))
+        if (!repoData) {
+            console.warn(`[webpeel:github] repo API returned null for ${owner}/${repo}`);
             return null;
+        }
+        if (repoData.message) {
+            console.warn(`[webpeel:github] repo API error for ${owner}/${repo}: ${repoData.message}`);
+            if (repoData.message === 'Not Found')
+                return null;
+            if (repoData.message.includes('secondary rate limit') || repoData.message.includes('abuse'))
+                return null;
+        }
         const readmeData = await fetchJsonWithRetry(`https://api.github.com/repos/${owner}/${repo}/readme`, ghHeaders, 1, 500).catch(() => null);
         // README content is base64 encoded
         let readmeText = '';

package/dist/core/pipeline.js

@@ -307,7 +307,7 @@ export async function fetchContent(ctx) {
         }
         catch (e) {
             // Domain API failed — fall through to normal fetch
-            log.debug('domain API first-pass failed, falling back to fetch:', e instanceof Error ? e.message : e);
+            log.warn('domain API first-pass failed, falling back to fetch:', e instanceof Error ? e.message : e);
         }
     }
     ctx.timer.mark('fetch');
@@ -959,7 +959,7 @@ export async function postProcess(ctx) {
         }
         catch (e) {
             // Domain extraction failure is non-fatal; continue with normal content
-            log.debug('domain extraction failed:', e instanceof Error ? e.message : e);
+            log.warn('domain extraction (second pass) failed:', e instanceof Error ? e.message : e);
         }
     }
     // === Challenge / bot-protection page detection ===

package/dist/server/routes/research.js

@@ -153,7 +153,7 @@ const VALID_LLM_PROVIDERS = [
     'cloudflare',
 ];
 const MAX_SOURCES_HARD_LIMIT = 8;
-const PER_URL_TIMEOUT_MS = 15_000;
+const PER_URL_TIMEOUT_MS = 8_000;
 const TOTAL_TIMEOUT_MS = 60_000;
 export function createResearchRouter() {
     const router = Router();
@@ -318,9 +318,14 @@ export function createResearchRouter() {
                         wordCount,
                         fetchTime,
                     });
-                    if (content.length > 0) {
+                    if (wordCount >= 50) {
                         fetchedContents.push({ url, content });
                     }
+                    else if (snippet.length > 20) {
+                        // Content too thin — use search snippet + title as surrogate
+                        const surrogateContent = `${pageTitle}\n\n${snippet}`;
+                        fetchedContents.push({ url, content: surrogateContent });
+                    }
                 }
                 catch {
                     // Skip failed URLs, continue to next
@@ -349,10 +354,15 @@ export function createResearchRouter() {
             const effectiveLLMConfig = llmConfig ?? (process.env.OLLAMA_URL
                 ? { provider: 'ollama', apiKey: process.env.OLLAMA_SECRET || '' }
                 : undefined);
-            if (effectiveLLMConfig && fetchedContents.length > 0 && Date.now() < overallDeadline - 3_000) {
+            if (effectiveLLMConfig && fetchedContents.length > 0 && Date.now() < overallDeadline - 1_000) {
                 try {
+                    // Filter to sources with 30+ words; fall back to all if none pass the threshold
+                    const contentsForLLM = (() => {
+                        const filtered = fetchedContents.filter(fc => fc.content.split(/\s+/).filter(Boolean).length >= 30);
+                        return filtered.length > 0 ? filtered : fetchedContents;
+                    })();
                     // Sanitize web content before sending to LLM (prompt injection defense layer 1)
-                    const sourcesText = fetchedContents
+                    const sourcesText = contentsForLLM
                         .map((fc, i) => {
                         const sanitized = sanitizeForLLM(fc.content.slice(0, 1200));
                         if (sanitized.injectionDetected) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.36",
+  "version": "0.21.37",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",