webpeel 0.21.24 → 0.21.26

package/dist/core/search-provider.js

@@ -12,10 +12,10 @@
  * In production with no API keys configured, getBestSearchProvider() returns
  * StealthSearchProvider since DDG HTTP is often blocked on datacenter IPs.
  */
-import { fetch as undiciFetch } from 'undici';
+import { fetch as undiciFetch, ProxyAgent } from 'undici';
 import { load } from 'cheerio';
 import { getStealthBrowser, getRandomUserAgent, applyStealthScripts } from './browser-pool.js';
-import { getWebshareProxy } from './proxy-config.js';
+import { getWebshareProxy, getWebshareProxyUrl } from './proxy-config.js';
 import { createLogger } from './logger.js';
 const log = createLogger('search');
 function decodeHtmlEntities(input) {
@@ -626,7 +626,9 @@ export class DuckDuckGoProvider {
         const { count, signal } = options;
         const searchUrl = this.buildSearchUrl(query, options);
         // Use realistic browser headers to avoid DDG bot detection on datacenter IPs
-        const response = await undiciFetch(searchUrl, {
+        // Route through residential proxy when available (datacenter IPs are blocked)
+        const proxyUrl = getWebshareProxyUrl();
+        const fetchOpts = {
             headers: {
                 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
                 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
@@ -641,7 +643,11 @@ export class DuckDuckGoProvider {
                 'Referer': 'https://duckduckgo.com/',
             },
             signal,
-        });
+        };
+        if (proxyUrl) {
+            fetchOpts.dispatcher = new ProxyAgent(proxyUrl);
+        }
+        const response = await undiciFetch(searchUrl, fetchOpts);
         if (!response.ok) {
             throw new Error(`Search failed: HTTP ${response.status}`);
         }
@@ -701,7 +707,8 @@ export class DuckDuckGoProvider {
         const { count, signal } = options;
         const params = new URLSearchParams();
         params.set('q', query);
-        const response = await undiciFetch(`https://lite.duckduckgo.com/lite/?${params.toString()}`, {
+        const liteProxyUrl = getWebshareProxyUrl();
+        const liteFetchOpts = {
             headers: {
                 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
                 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
@@ -709,7 +716,11 @@ export class DuckDuckGoProvider {
                 'Referer': 'https://lite.duckduckgo.com/',
             },
             signal,
-        });
+        };
+        if (liteProxyUrl) {
+            liteFetchOpts.dispatcher = new ProxyAgent(liteProxyUrl);
+        }
+        const response = await undiciFetch(`https://lite.duckduckgo.com/lite/?${params.toString()}`, liteFetchOpts);
         if (!response.ok)
             return [];
         const html = await response.text();

package/dist/core/structured-extract.d.ts

@@ -30,7 +30,7 @@ export interface ExtractionResult {
  * @param llmConfig Optional LLM config (if omitted, uses heuristic fallback)
  * @param prompt    Optional user guidance added to the LLM prompt
  */
-export declare function extractStructured(content: string, schema: ExtractionSchema, llmConfig?: LLMConfig, prompt?: string): Promise<ExtractionResult>;
+export declare function extractStructured(content: string, schema: ExtractionSchema, llmConfig?: LLMConfig, prompt?: string, domainHints?: Record<string, unknown>): Promise<ExtractionResult>;
 /**
  * Convert a shorthand schema `{ field: "string", active: "boolean" }` to a
  * full ExtractionSchema.  Useful for CLI --extract flag.

package/dist/core/structured-extract.js

@@ -422,7 +422,7 @@ async function heuristicExtract(content, schema) {
  * @param llmConfig Optional LLM config (if omitted, uses heuristic fallback)
  * @param prompt    Optional user guidance added to the LLM prompt
  */
-export async function extractStructured(content, schema, llmConfig, prompt) {
+export async function extractStructured(content, schema, llmConfig, prompt, domainHints) {
     // Guard: empty content
     if (!content || content.trim().length === 0) {
         return { data: {}, confidence: 0, tokensUsed: 0 };
@@ -495,7 +495,35 @@ export async function extractStructured(content, schema, llmConfig, prompt) {
         }
     }
     // ── Heuristic extraction ─────────────────────────────────────────────────
-    return heuristicExtract(content, schema);
+    const heuristic = await heuristicExtract(content, schema);
+    // ── Domain hints overlay ─────────────────────────────────────────────────
+    // If domain-api pre-extracted fields (e.g. GitHub stars/language), merge them
+    // into the result. Domain-api data is authoritative — prefer over heuristic.
+    if (domainHints && Object.keys(domainHints).length > 0) {
+        const props = schema.properties;
+        let hintMerged = 0;
+        for (const [field, hintValue] of Object.entries(domainHints)) {
+            if (field in props && hintValue !== null && hintValue !== undefined) {
+                const expected = props[field].type;
+                const actual = typeof hintValue;
+                // Only merge if type matches (or number vs string coercion)
+                if (actual === expected ||
+                    (expected === 'number' && actual === 'string' && !isNaN(Number(hintValue))) ||
+                    (expected === 'string' && actual !== 'object')) {
+                    heuristic.data[field] =
+                        expected === 'number' ? Number(hintValue) : hintValue;
+                    hintMerged++;
+                }
+            }
+        }
+        if (hintMerged > 0) {
+            // Boost confidence since we have authoritative domain-api data
+            const filled = Object.values(heuristic.data).filter(v => v !== null && v !== undefined).length;
+            const total = Object.keys(props).length;
+            heuristic.confidence = parseFloat(Math.min(0.90, 0.65 + (filled / total) * 0.25).toFixed(2));
+        }
+    }
+    return heuristic;
 }
 // ---------------------------------------------------------------------------
 // Helper: convert simple { field: "type" } map → ExtractionSchema

package/dist/server/routes/extract.js

@@ -186,12 +186,24 @@ export function createExtractRouter() {
             const peelResult = await peel(url, {
                 format: 'markdown',
                 render: useRender,
+                noEscalate: !useRender, // prevent OOM: only browser when render=true explicitly
                 timeout: 30000,
                 readable: true,
             });
             const content = peelResult.content || '';
             // ── Extract structured data ─────────────────────────────────────────
-            const extractResult = await extractStructured(content, schema, llmConfig, typeof prompt === 'string' ? prompt : undefined);
+            // Seed hints from domain-api structured data (GitHub stars/language, etc.)
+            // This lets heuristic extraction use pre-parsed structured fields as ground truth.
+            const domainHints = {};
+            const rawDomainData = peelResult.domainData?.structured;
+            if (rawDomainData && typeof rawDomainData === 'object') {
+                for (const [k, v] of Object.entries(rawDomainData)) {
+                    if (v !== null && v !== undefined && v !== '') {
+                        domainHints[k] = v;
+                    }
+                }
+            }
+            const extractResult = await extractStructured(content, schema, llmConfig, typeof prompt === 'string' ? prompt : undefined, Object.keys(domainHints).length > 0 ? domainHints : undefined);
             const method = llmConfig ? 'llm' : 'heuristic';
             res.json({
                 success: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.24",
+  "version": "0.21.26",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",