webpeel 0.21.35 → 0.21.37

package/dist/core/domain-extractors.js

@@ -153,7 +153,12 @@ async function fetchJson(url, customHeaders) {
         Accept: 'application/json',
         ...customHeaders,
     });
-    return tryParseJson(result.html);
+    const parsed = tryParseJson(result.html);
+    if (parsed === null && result.html.length > 0) {
+        // Log when we get non-JSON back (likely an HTML error page)
+        console.warn(`[webpeel:fetchJson] Non-JSON response from ${url} (${result.html.length} bytes, status: ${result.statusCode}): ${result.html.slice(0, 120)}`);
+    }
+    return parsed;
 }
 /** Fetch JSON with exponential backoff retry on 429 / rate-limit errors. */
 async function fetchJsonWithRetry(url, headers, retries = 2, baseDelayMs = 1000) {
@@ -910,11 +915,17 @@ ${commentsMd || '*No comments.*'}`;
     if (pathParts.length >= 2) {
         // Sequential fetches to avoid secondary rate limits on popular repos
         const repoData = await fetchJsonWithRetry(`https://api.github.com/repos/${owner}/${repo}`, ghHeaders, 2, 1000);
-        if (!repoData || repoData.message === 'Not Found')
-            return null;
-        // Secondary rate limit check
-        if (repoData.message?.includes('secondary rate limit') || repoData.message?.includes('abuse'))
+        if (!repoData) {
+            console.warn(`[webpeel:github] repo API returned null for ${owner}/${repo}`);
             return null;
+        }
+        if (repoData.message) {
+            console.warn(`[webpeel:github] repo API error for ${owner}/${repo}: ${repoData.message}`);
+            if (repoData.message === 'Not Found')
+                return null;
+            if (repoData.message.includes('secondary rate limit') || repoData.message.includes('abuse'))
+                return null;
+        }
         const readmeData = await fetchJsonWithRetry(`https://api.github.com/repos/${owner}/${repo}/readme`, ghHeaders, 1, 500).catch(() => null);
         // README content is base64 encoded
         let readmeText = '';

package/dist/core/llm-provider.js

@@ -391,13 +391,52 @@ async function callOllama(config, options) {
     const endpoint = (config.endpoint || process.env.OLLAMA_URL || 'http://localhost:11434').replace(/\/$/, '');
     const model = config.model || process.env.OLLAMA_MODEL || defaultModel('ollama');
     const { messages, stream, onChunk, signal, maxTokens = 4096, temperature = 0.2 } = options;
-    const url = `${endpoint}/v1/chat/completions`;
     // Support bearer token auth (for nginx reverse proxy on Hetzner)
     const headers = { 'Content-Type': 'application/json' };
     const secret = config.apiKey || process.env.OLLAMA_SECRET;
     if (secret)
         headers['Authorization'] = `Bearer ${secret}`;
-    const resp = await fetch(url, {
+    // ── Non-streaming: use /api/generate with think:false for speed ──────
+    // Qwen3 thinking mode wastes 300-400 tokens on CoT and takes 25s+.
+    // With think:false via /api/generate, response comes in ~8s.
+    if (!stream) {
+        // Build a single prompt from messages (system + user)
+        const systemMsg = messages.find((m) => m.role === 'system')?.content || '';
+        const userMsg = messages.filter((m) => m.role === 'user').map((m) => m.content).join('\n\n');
+        const prompt = systemMsg ? `${systemMsg}\n\n${userMsg}` : userMsg;
+        const resp = await fetch(`${endpoint}/api/generate`, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                model,
+                prompt,
+                stream: false,
+                think: false, // Critical: disables Qwen3 CoT thinking (8s vs 25s+)
+                options: {
+                    temperature,
+                    num_predict: maxTokens,
+                },
+            }),
+            signal,
+        });
+        if (!resp.ok) {
+            const text = await resp.text().catch(() => '');
+            throw new Error(`Ollama API error: HTTP ${resp.status}${text ? ` - ${text}` : ''}`);
+        }
+        const json = await resp.json();
+        let text = String(json?.response || '').trim();
+        // Strip any residual <think> tags
+        text = text.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+        return {
+            text,
+            usage: {
+                input: Number(json?.prompt_eval_count || 0),
+                output: Number(json?.eval_count || 0),
+            },
+        };
+    }
+    // ── Streaming: use OpenAI-compatible /v1/chat/completions ────────────
+    const resp = await fetch(`${endpoint}/v1/chat/completions`, {
         method: 'POST',
         headers,
         body: JSON.stringify({
@@ -405,7 +444,7 @@ async function callOllama(config, options) {
             messages,
             temperature,
             max_tokens: maxTokens,
-            stream: stream ?? false,
+            stream: true,
         }),
         signal,
     });
@@ -413,23 +452,6 @@ async function callOllama(config, options) {
         const text = await resp.text().catch(() => '');
         throw new Error(`Ollama API error: HTTP ${resp.status}${text ? ` - ${text}` : ''}`);
     }
-    if (!stream) {
-        const json = await resp.json();
-        const msg = json?.choices?.[0]?.message;
-        // Ollama Qwen3 thinking: content may be empty, CoT goes to `reasoning` field
-        let text = String(msg?.content || '').trim();
-        if (!text && msg?.reasoning)
-            text = String(msg.reasoning).trim();
-        // Strip <think> tags from Qwen3 models
-        text = text.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
-        return {
-            text,
-            usage: {
-                input: Number(json?.usage?.prompt_tokens || 0),
-                output: Number(json?.usage?.completion_tokens || 0),
-            },
-        };
-    }
     if (!resp.body)
         throw new Error('Ollama stream: missing body');
     let out = '';
@@ -449,7 +471,9 @@ async function callOllama(config, options) {
             onChunk?.(delta);
         }
     }, signal);
-    return { text: out.trim(), usage: { input: 0, output: 0 } };
+    // Strip thinking from streamed output
+    out = out.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+    return { text: out, usage: { input: 0, output: 0 } };
 }
 // ---------------------------------------------------------------------------
 // Cerebras (OpenAI-compatible)

package/dist/core/pipeline.js

@@ -307,7 +307,7 @@ export async function fetchContent(ctx) {
         }
         catch (e) {
             // Domain API failed — fall through to normal fetch
-            log.debug('domain API first-pass failed, falling back to fetch:', e instanceof Error ? e.message : e);
+            log.warn('domain API first-pass failed, falling back to fetch:', e instanceof Error ? e.message : e);
         }
     }
     ctx.timer.mark('fetch');
@@ -959,7 +959,7 @@ export async function postProcess(ctx) {
         }
         catch (e) {
             // Domain extraction failure is non-fatal; continue with normal content
-            log.debug('domain extraction failed:', e instanceof Error ? e.message : e);
+            log.warn('domain extraction (second pass) failed:', e instanceof Error ? e.message : e);
         }
     }
     // === Challenge / bot-protection page detection ===

package/dist/server/routes/research.js

@@ -153,7 +153,7 @@ const VALID_LLM_PROVIDERS = [
     'cloudflare',
 ];
 const MAX_SOURCES_HARD_LIMIT = 8;
-const PER_URL_TIMEOUT_MS = 15_000;
+const PER_URL_TIMEOUT_MS = 8_000;
 const TOTAL_TIMEOUT_MS = 60_000;
 export function createResearchRouter() {
     const router = Router();
@@ -318,9 +318,14 @@ export function createResearchRouter() {
                         wordCount,
                         fetchTime,
                     });
-                    if (content.length > 0) {
+                    if (wordCount >= 50) {
                         fetchedContents.push({ url, content });
                     }
+                    else if (snippet.length > 20) {
+                        // Content too thin — use search snippet + title as surrogate
+                        const surrogateContent = `${pageTitle}\n\n${snippet}`;
+                        fetchedContents.push({ url, content: surrogateContent });
+                    }
                 }
                 catch {
                     // Skip failed URLs, continue to next
@@ -349,10 +354,15 @@ export function createResearchRouter() {
             const effectiveLLMConfig = llmConfig ?? (process.env.OLLAMA_URL
                 ? { provider: 'ollama', apiKey: process.env.OLLAMA_SECRET || '' }
                 : undefined);
-            if (effectiveLLMConfig && fetchedContents.length > 0 && Date.now() < overallDeadline - 3_000) {
+            if (effectiveLLMConfig && fetchedContents.length > 0 && Date.now() < overallDeadline - 1_000) {
                 try {
+                    // Filter to sources with 30+ words; fall back to all if none pass the threshold
+                    const contentsForLLM = (() => {
+                        const filtered = fetchedContents.filter(fc => fc.content.split(/\s+/).filter(Boolean).length >= 30);
+                        return filtered.length > 0 ? filtered : fetchedContents;
+                    })();
                     // Sanitize web content before sending to LLM (prompt injection defense layer 1)
-                    const sourcesText = fetchedContents
+                    const sourcesText = contentsForLLM
                         .map((fc, i) => {
                         const sanitized = sanitizeForLLM(fc.content.slice(0, 1200));
                         if (sanitized.injectionDetected) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.35",
+  "version": "0.21.37",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",