webpeel 0.21.48 → 0.21.53

package/dist/core/apply.js

@@ -590,6 +590,7 @@ export async function applyToJob(options) {
     try {
         // ── 3. Launch persistent browser ──────────────────────────────
         progress('navigating', 'Launching browser with persistent session...');
+        // @ts-expect-error rebrowser-playwright types diverge from upstream playwright-core
         context = await stealthChromium.launchPersistentContext(sessionDir, {
             headless: false, // visible so user can monitor (or log in on first run)
             viewport: { width: 1440, height: 900 },

package/dist/core/challenge-solver.d.ts

@@ -18,6 +18,32 @@
  *  }
  */
 import type { ChallengeType } from './challenge-detection.js';
+export interface ImageCaptchaResult {
+    solved: boolean;
+    rounds: number;
+    error?: string;
+}
+/**
+ * Ask the moondream vision model which grid cells contain the target object.
+ * Returns an array of 1-indexed grid positions (1–9), or null if the call fails.
+ */
+export declare function askVisionModel(base64Image: string, targetObject: string): Promise<number[] | null>;
+/**
+ * Detect if the page has an image grid CAPTCHA and extract the target object.
+ * Returns the object name (e.g. "traffic lights") or null if not detected.
+ */
+export declare function detectImageCaptchaTarget(page: import('playwright').Page): Promise<string | null>;
+/**
+ * Solve an image grid CAPTCHA using the moondream vision model.
+ *
+ * Flow per round:
+ *  1. Screenshot the CAPTCHA grid element
+ *  2. Send to moondream → get grid positions
+ *  3. Click identified cells
+ *  4. Click Verify button
+ *  5. Check if solved; if a new round appears, repeat (max 3 rounds)
+ */
+export declare function solveImageCaptcha(page: import('playwright').Page, targetObject: string): Promise<ImageCaptchaResult>;
 export interface SolveOptions {
     /** Hard timeout in ms (default: 15 000) */
     timeout?: number;

package/dist/core/challenge-solver.js

@@ -20,6 +20,298 @@
 import { cacheCookiesForUrl } from './cookie-cache.js';
 import { createLogger } from './logger.js';
 const log = createLogger('challenge-solver');
+// ── Image CAPTCHA solver constants ────────────────────────────────────────────
+const OLLAMA_VISION_URL = 'http://178.156.229.86:11435/api/generate';
+const OLLAMA_AUTH_TOKEN = 'c996233de4addb47e4cdec8bc5ff8776397f813ca7bd444e7258e0e2ed251963';
+const OLLAMA_VISION_MODEL = 'moondream';
+/** moondream on the 4GB Hetzner VPS takes ~30s per image */
+const VISION_TIMEOUT_MS = 45_000;
+const IMAGE_CAPTCHA_MAX_ROUNDS = 3;
+/** Grid element selectors to try (reCAPTCHA, hCaptcha, generic) */
+const CAPTCHA_GRID_SELECTORS = [
+    '.rc-imageselect-table',
+    '.task-grid',
+    '.task-image',
+    'table.rc-imageselect-table',
+    '[class*="grid"]:not(body):not(html)',
+    '.captcha-grid',
+];
+/** Verify/Submit button selectors */
+const CAPTCHA_VERIFY_SELECTORS = [
+    '#recaptcha-verify-button',
+    'button[data-action="verify"]',
+    'button[class*="verify"]',
+    'button[class*="submit"]',
+    '.rc-button-default',
+    '[id*="verify"]',
+    '[class*="verify"]',
+];
+/** Instruction text containers to extract the target object from */
+const CAPTCHA_INSTRUCTION_SELECTORS = [
+    '.rc-imageselect-desc-wrapper',
+    '.rc-imageselect-desc',
+    '.prompt-text',
+    '[class*="prompt"]',
+    '[class*="instruction"]',
+    '[class*="task-desc"]',
+    '[aria-label*="select"]',
+    '[aria-label*="click"]',
+];
+/** Patterns to extract the object name from instruction text */
+const CAPTCHA_OBJECT_PATTERNS = [
+    /select all (?:images|squares|tiles) (?:with|containing|that (?:have|contain)) (?:a |an )?(.+?)(?:\.|$)/i,
+    /click (?:all )?(?:images|squares|tiles) (?:containing|with|that (?:have|contain)) (?:a |an )?(.+?)(?:\.|$)/i,
+    /please click each image containing (?:a |an )?(.+?)(?:\.|$)/i,
+    /select all (?:the )?(?:image|picture)s? of (?:a |an )?(.+?)(?:\.|$)/i,
+    /identify all (?:images|pictures|squares) (?:with|showing|of) (?:a |an )?(.+?)(?:\.|$)/i,
+];
+// ── Vision API call ───────────────────────────────────────────────────────────
+/**
+ * Ask the moondream vision model which grid cells contain the target object.
+ * Returns an array of 1-indexed grid positions (1–9), or null if the call fails.
+ */
+export async function askVisionModel(base64Image, targetObject) {
+    const prompt = `This is a 3x3 image grid CAPTCHA. Select all squares containing "${targetObject}". Reply with ONLY the grid positions as numbers 1-9 (left to right, top to bottom), separated by commas. Example: 1,3,7`;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), VISION_TIMEOUT_MS);
+    try {
+        const response = await fetch(OLLAMA_VISION_URL, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${OLLAMA_AUTH_TOKEN}`,
+            },
+            body: JSON.stringify({
+                model: OLLAMA_VISION_MODEL,
+                prompt,
+                images: [base64Image],
+                stream: false,
+                options: { num_predict: 50, temperature: 0.1 },
+            }),
+            signal: controller.signal,
+        });
+        clearTimeout(timer);
+        if (!response.ok) {
+            log.debug(`Vision API returned HTTP ${response.status}`);
+            return null;
+        }
+        const data = await response.json();
+        const text = data.response ?? '';
+        log.debug(`Vision model response: "${text}"`);
+        // Match whole numbers only (not individual digits from multi-digit numbers like 10, 11)
+        const positions = text.match(/\b[1-9]\b/g)?.map(Number) ?? [];
+        if (positions.length === 0) {
+            log.debug('Vision model returned no valid grid positions');
+            return null;
+        }
+        return positions;
+    }
+    catch (err) {
+        clearTimeout(timer);
+        log.debug('Vision model call failed:', err instanceof Error ? err.message : String(err));
+        return null;
+    }
+}
+// ── Target object extraction ──────────────────────────────────────────────────
+/**
+ * Detect if the page has an image grid CAPTCHA and extract the target object.
+ * Returns the object name (e.g. "traffic lights") or null if not detected.
+ */
+export async function detectImageCaptchaTarget(page) {
+    for (const selector of CAPTCHA_INSTRUCTION_SELECTORS) {
+        try {
+            const el = await page.$(selector);
+            if (!el)
+                continue;
+            const text = await el.innerText().catch(() => '');
+            if (!text)
+                continue;
+            const normalized = text.trim().replace(/\s+/g, ' ');
+            for (const pattern of CAPTCHA_OBJECT_PATTERNS) {
+                const match = normalized.match(pattern);
+                if (match?.[1]) {
+                    const target = match[1].trim().toLowerCase();
+                    log.debug(`Detected image CAPTCHA target: "${target}" from selector ${selector}`);
+                    return target;
+                }
+            }
+        }
+        catch {
+            // Continue to next selector
+        }
+    }
+    return null;
+}
+// ── Image CAPTCHA solver ──────────────────────────────────────────────────────
+/**
+ * Solve an image grid CAPTCHA using the moondream vision model.
+ *
+ * Flow per round:
+ *  1. Screenshot the CAPTCHA grid element
+ *  2. Send to moondream → get grid positions
+ *  3. Click identified cells
+ *  4. Click Verify button
+ *  5. Check if solved; if a new round appears, repeat (max 3 rounds)
+ */
+export async function solveImageCaptcha(page, targetObject) {
+    // Guard: only run when explicitly enabled or remote worker configured
+    const enabled = process.env.ENABLE_LOCAL_CHALLENGE_SOLVE === 'true' || !!process.env.BROWSER_WORKER_URL;
+    if (!enabled) {
+        return { solved: false, rounds: 0, error: 'Image CAPTCHA solving not enabled (set ENABLE_LOCAL_CHALLENGE_SOLVE=true)' };
+    }
+    let rounds = 0;
+    for (let attempt = 0; attempt < IMAGE_CAPTCHA_MAX_ROUNDS; attempt++) {
+        rounds++;
+        // ── 1. Screenshot the grid element ─────────────────────────────────────
+        let base64Screenshot = null;
+        for (const selector of CAPTCHA_GRID_SELECTORS) {
+            try {
+                const gridEl = await page.$(selector);
+                if (!gridEl)
+                    continue;
+                const screenshot = await gridEl.screenshot({ type: 'png' });
+                base64Screenshot = screenshot.toString('base64');
+                log.debug(`Captured CAPTCHA grid with selector: ${selector}`);
+                break;
+            }
+            catch {
+                // Try next selector
+            }
+        }
+        if (!base64Screenshot) {
+            // Fall back to a viewport screenshot if no grid element found
+            try {
+                const fullshot = await page.screenshot({ type: 'png' });
+                base64Screenshot = fullshot.toString('base64');
+                log.debug('Fell back to full-page screenshot for CAPTCHA');
+            }
+            catch (err) {
+                return {
+                    solved: false,
+                    rounds,
+                    error: `Screenshot failed: ${err instanceof Error ? err.message : String(err)}`,
+                };
+            }
+        }
+        // ── 2. Ask vision model ────────────────────────────────────────────────
+        log.debug(`Round ${rounds}: asking moondream to find "${targetObject}"…`);
+        const positions = await askVisionModel(base64Screenshot, targetObject);
+        if (!positions || positions.length === 0) {
+            log.debug(`Round ${rounds}: vision model returned no positions — stopping`);
+            return { solved: false, rounds, error: 'Vision model returned no valid positions' };
+        }
+        log.debug(`Round ${rounds}: vision model selected positions: ${positions.join(',')}`);
+        // ── 3. Click grid cells ────────────────────────────────────────────────
+        let clickedCount = 0;
+        for (const pos of positions) {
+            for (const gridSelector of CAPTCHA_GRID_SELECTORS) {
+                try {
+                    const gridEl = await page.$(gridSelector);
+                    if (!gridEl)
+                        continue;
+                    // Each grid cell: nth-child or direct child
+                    const cells = await gridEl.$$('td, div[class*="cell"], div[class*="tile"], div[class*="image"]');
+                    if (cells.length === 0) {
+                        // Try direct children
+                        const children = await gridEl.$$(':scope > *');
+                        const target = children[pos - 1];
+                        if (target) {
+                            await target.click({ timeout: 5000 });
+                            clickedCount++;
+                        }
+                    }
+                    else {
+                        const target = cells[pos - 1];
+                        if (target) {
+                            await target.click({ timeout: 5000 });
+                            clickedCount++;
+                        }
+                    }
+                    break;
+                }
+                catch {
+                    // Try next selector
+                }
+            }
+        }
+        log.debug(`Round ${rounds}: clicked ${clickedCount}/${positions.length} cells`);
+        // Short delay before verify (let animation/state settle)
+        await page.waitForTimeout(500);
+        // ── 4. Click Verify button ─────────────────────────────────────────────
+        let clicked = false;
+        for (const btnSelector of CAPTCHA_VERIFY_SELECTORS) {
+            try {
+                const btn = await page.$(btnSelector);
+                if (btn) {
+                    await btn.click({ timeout: 3000 });
+                    clicked = true;
+                    log.debug(`Round ${rounds}: clicked verify button (${btnSelector})`);
+                    break;
+                }
+            }
+            catch {
+                // Try next
+            }
+        }
+        if (!clicked) {
+            log.debug(`Round ${rounds}: could not find verify button`);
+        }
+        // ── 5. Check if solved ─────────────────────────────────────────────────
+        await page.waitForTimeout(2000);
+        // Check for success indicators
+        const solved = await checkCaptchaSolved(page);
+        if (solved) {
+            log.debug(`Round ${rounds}: CAPTCHA solved!`);
+            return { solved: true, rounds };
+        }
+        // Check if a new round appeared (grid refreshed)
+        const newTarget = await detectImageCaptchaTarget(page);
+        if (!newTarget) {
+            // No more instructions — likely solved or error
+            log.debug(`Round ${rounds}: no more instruction text — assuming solved`);
+            return { solved: true, rounds };
+        }
+        // Update target object for next round (may change between rounds)
+        // eslint-disable-next-line no-param-reassign
+        targetObject = newTarget;
+        log.debug(`Round ${rounds}: new target for next round: "${targetObject}"`);
+    }
+    return { solved: false, rounds, error: `Reached max rounds (${IMAGE_CAPTCHA_MAX_ROUNDS}) without solving` };
+}
+/**
+ * Check if the CAPTCHA appears to have been solved (challenge gone, success message, etc.)
+ */
+async function checkCaptchaSolved(page) {
+    // Check for reCAPTCHA success state
+    try {
+        const successEl = await page.$('.recaptcha-checkbox-checked, .rc-anchor-normal-footer, [aria-checked="true"]');
+        if (successEl)
+            return true;
+    }
+    catch { /* ignore */ }
+    // Check if CAPTCHA challenge overlay disappeared (grid gone)
+    try {
+        const gridEl = await page.$('.rc-imageselect-table, .task-grid');
+        // If we were on a CAPTCHA page and the grid is now gone, it was likely solved
+        if (!gridEl) {
+            // Only count as solved if we're no longer on a CAPTCHA title page
+            const title = await page.title().catch(() => '');
+            const isCaptchaTitle = title.toLowerCase().includes('captcha') || title.toLowerCase().includes('robot');
+            if (!isCaptchaTitle)
+                return true;
+        }
+    }
+    catch { /* ignore */ }
+    // Check page URL changed (successful solve often triggers redirect)
+    try {
+        const title = await page.title();
+        const isCaptchaPage = title.toLowerCase().includes('captcha') || title.toLowerCase().includes('robot');
+        if (!isCaptchaPage)
+            return true;
+    }
+    catch { /* ignore */ }
+    return false;
+}
 // ── Constants ─────────────────────────────────────────────────────────────────
 const DEFAULT_TIMEOUT_MS = 15_000;
 /** Cloudflare challenge title before it's solved */
@@ -55,8 +347,7 @@ export async function solveChallenge(url, challengeType, html, options = {}) {
         case 'cloudflare':
             return solveCloudflare(url, html, timeout, options.proxy);
         case 'captcha':
-            // TODO: hCaptcha accessibility bypass — see comment below
-            return { solved: false, html, error: 'No free captcha solver available for generic captcha' };
+            return solveCaptchaWithVision(url, html, timeout, options.proxy);
         case 'datadome':
             // DataDome can sometimes be bypassed with a stealth browser
             return solveWithStealthBrowser(url, html, timeout, options.proxy, 'datadome');
@@ -73,6 +364,68 @@ export async function solveChallenge(url, challengeType, html, options = {}) {
             return { solved: false, html, error: `Unknown challenge type: ${challengeType}` };
     }
 }
+// ── Image CAPTCHA orchestrator ────────────────────────────────────────────────
+/**
+ * Solve an image CAPTCHA by opening a stealth browser, detecting the target
+ * object from the page instructions, and calling solveImageCaptcha().
+ */
+async function solveCaptchaWithVision(url, _html, timeoutMs, proxy) {
+    let page = null;
+    try {
+        const { getStealthBrowser, getRandomUserAgent, getRandomViewport, applyStealthScripts } = await import('./browser-pool.js');
+        const browser = await getStealthBrowser();
+        const vp = getRandomViewport();
+        const ctx = await browser.newContext({
+            userAgent: getRandomUserAgent(),
+            viewport: { width: vp.width, height: vp.height },
+            ...(proxy ? { proxy: { server: proxy } } : {}),
+            locale: 'en-US',
+            timezoneId: 'America/New_York',
+        });
+        page = await ctx.newPage();
+        await applyStealthScripts(page);
+        await page.goto(url, {
+            waitUntil: 'domcontentloaded',
+            timeout: timeoutMs,
+        });
+        // Wait for CAPTCHA to render
+        await page.waitForTimeout(2000);
+        // Detect the target object from the CAPTCHA instructions
+        const targetObject = await detectImageCaptchaTarget(page);
+        if (!targetObject) {
+            const html = await page.content().catch(() => _html);
+            await ctx.close().catch(() => { });
+            return { solved: false, html, error: 'Could not detect image CAPTCHA target object from page' };
+        }
+        log.debug(`Image CAPTCHA target: "${targetObject}"`);
+        // Solve the CAPTCHA — may take up to VISION_TIMEOUT_MS * IMAGE_CAPTCHA_MAX_ROUNDS
+        const captchaResult = await solveImageCaptcha(page, targetObject);
+        const html = await page.content().catch(() => _html);
+        const cookies = await ctx.cookies();
+        const cookieStrings = cookies.map(c => `${c.name}=${c.value}; Path=${c.path || '/'}${c.domain ? `; Domain=${c.domain}` : ''}`);
+        if (cookieStrings.length > 0) {
+            cacheCookiesForUrl(url, cookieStrings);
+        }
+        await ctx.close().catch(() => { });
+        if (captchaResult.solved) {
+            console.log(`[challenge-solver] Image CAPTCHA solved for ${getDomain(url)} in ${captchaResult.rounds} round(s)`);
+            return { solved: true, html, cookies: cookieStrings, method: 'local-browser' };
+        }
+        return {
+            solved: false,
+            html,
+            error: captchaResult.error ?? `Image CAPTCHA not solved after ${captchaResult.rounds} round(s)`,
+        };
+    }
+    catch (err) {
+        const error = err instanceof Error ? err.message : String(err);
+        log.debug('Image CAPTCHA solve failed:', error);
+        return { solved: false, html: _html, error };
+    }
+    finally {
+        page = null;
+    }
+}
 // ── Cloudflare solver ─────────────────────────────────────────────────────────
 /**
  * Solve Cloudflare JS challenge by rendering the page in a stealth browser.

package/dist/core/fetch-cache.d.ts

@@ -0,0 +1,57 @@
+/**
+ * In-memory LRU fetch cache for WebPeel
+ *
+ * Caches pipeline results to avoid redundant fetches for identical requests.
+ * Supports TTL-based expiry and LRU eviction when maxEntries is exceeded.
+ * Exported as a singleton: import { fetchCache } from './fetch-cache.js'
+ */
+export interface FetchCacheEntry {
+    content: string;
+    title: string;
+    metadata: any;
+    method: string;
+    tokens: number;
+    links?: any[];
+    timestamp: number;
+}
+export interface FetchCacheStats {
+    size: number;
+    hits: number;
+    misses: number;
+    hitRate: number;
+}
+export declare class FetchCache {
+    private cache;
+    private maxEntries;
+    private defaultTTL;
+    private hits;
+    private misses;
+    constructor(maxEntries?: number, defaultTTLSeconds?: number);
+    /**
+     * Generate a stable cache key from url + relevant fetch options.
+     * Different option combinations produce different cache entries.
+     */
+    getKey(url: string, options?: {
+        render?: boolean;
+        stealth?: boolean;
+        budget?: number;
+    }): string;
+    /**
+     * Retrieve a cached entry. Returns null if missing or expired.
+     * On hit: entry is moved to the end of the Map (LRU refresh).
+     */
+    get(key: string): FetchCacheEntry | null;
+    /**
+     * Store an entry in the cache.
+     * If the cache is at capacity, the least recently used entry is evicted.
+     */
+    set(key: string, entry: FetchCacheEntry): void;
+    /** Clear all entries and reset stats. */
+    clear(): void;
+    /** Return cache stats. hitRate is in [0, 1]. */
+    stats(): FetchCacheStats;
+}
+/** Singleton fetch cache — shared across all requests (5 min TTL, 500 entries). */
+export declare const fetchCache: FetchCache;
+/** Singleton search cache — shorter TTL since results change faster (60 s). */
+export declare const searchCache: FetchCache;

package/dist/core/fetch-cache.js

@@ -0,0 +1,95 @@
+/**
+ * In-memory LRU fetch cache for WebPeel
+ *
+ * Caches pipeline results to avoid redundant fetches for identical requests.
+ * Supports TTL-based expiry and LRU eviction when maxEntries is exceeded.
+ * Exported as a singleton: import { fetchCache } from './fetch-cache.js'
+ */
+export class FetchCache {
+    cache;
+    maxEntries;
+    defaultTTL; // ms
+    hits;
+    misses;
+    constructor(maxEntries = 500, defaultTTLSeconds = 300) {
+        this.cache = new Map();
+        this.maxEntries = maxEntries;
+        this.defaultTTL = defaultTTLSeconds * 1000;
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /**
+     * Generate a stable cache key from url + relevant fetch options.
+     * Different option combinations produce different cache entries.
+     */
+    getKey(url, options = {}) {
+        const render = options.render ? '1' : '0';
+        const stealth = options.stealth ? '1' : '0';
+        const budget = options.budget !== undefined ? String(options.budget) : '';
+        return `${url}|r:${render}|s:${stealth}|b:${budget}`;
+    }
+    /**
+     * Retrieve a cached entry. Returns null if missing or expired.
+     * On hit: entry is moved to the end of the Map (LRU refresh).
+     */
+    get(key) {
+        const entry = this.cache.get(key);
+        if (!entry) {
+            this.misses++;
+            return null;
+        }
+        const ageMs = Date.now() - entry.timestamp;
+        if (ageMs > this.defaultTTL) {
+            // Expired — evict and return null
+            this.cache.delete(key);
+            this.misses++;
+            return null;
+        }
+        // LRU touch: move to end
+        this.cache.delete(key);
+        this.cache.set(key, entry);
+        this.hits++;
+        return entry;
+    }
+    /**
+     * Store an entry in the cache.
+     * If the cache is at capacity, the least recently used entry is evicted.
+     */
+    set(key, entry) {
+        // Remove existing to refresh position
+        if (this.cache.has(key)) {
+            this.cache.delete(key);
+        }
+        this.cache.set(key, entry);
+        // LRU eviction: remove oldest entry (first in Map iteration order)
+        while (this.cache.size > this.maxEntries) {
+            const oldestKey = this.cache.keys().next().value;
+            if (oldestKey !== undefined) {
+                this.cache.delete(oldestKey);
+            }
+            else {
+                break;
+            }
+        }
+    }
+    /** Clear all entries and reset stats. */
+    clear() {
+        this.cache.clear();
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /** Return cache stats. hitRate is in [0, 1]. */
+    stats() {
+        const total = this.hits + this.misses;
+        return {
+            size: this.cache.size,
+            hits: this.hits,
+            misses: this.misses,
+            hitRate: total === 0 ? 0 : Math.round((this.hits / total) * 100) / 100,
+        };
+    }
+}
+/** Singleton fetch cache — shared across all requests (5 min TTL, 500 entries). */
+export const fetchCache = new FetchCache(500, 300);
+/** Singleton search cache — shorter TTL since results change faster (60 s). */
+export const searchCache = new FetchCache(500, 60);

package/dist/core/pipeline.js

@@ -1217,6 +1217,52 @@ export async function finalize(ctx) {
             log.error('Change tracking failed:', error);
         }
     }
+    // ── Auto-escalation: retry thin content with browser rendering ──────────────
+    // If simple fetch returned very little content and user didn't explicitly disable render,
+    // automatically retry with browser rendering to handle JS-heavy/paywalled sites.
+    const preEscalationWords = ctx.content.trim().split(/\s+/).filter((w) => w.length > 0).length;
+    const escalationFetchMethod = fetchResult?.method || 'unknown';
+    const alreadyTriedBrowser = escalationFetchMethod === 'browser' || escalationFetchMethod === 'stealth'
+        || options.render || options.stealth;
+    const userDisabledRender = options.render === false;
+    const escalationCandidate = preEscalationWords < 200 && preEscalationWords > 0
+        && escalationFetchMethod === 'simple' && !alreadyTriedBrowser && !userDisabledRender
+        && !ctx._escalated;
+    if (escalationCandidate) {
+        log.info(`thin content (${preEscalationWords}w) from simple fetch, auto-escalating to browser render for ${ctx.url}`);
+        ctx._escalated = true;
+        try {
+            const { smartFetch } = await import('./strategies.js');
+            const browserResult = await smartFetch(ctx.url, {
+                forceBrowser: true,
+                stealth: false,
+                timeoutMs: options.timeout || 15000,
+                proxy: options.proxy,
+            });
+            if (browserResult.html && browserResult.html.length > (fetchResult?.html?.length || 0)) {
+                const { htmlToMarkdown } = await import('./markdown.js');
+                const browserContent = htmlToMarkdown(browserResult.html);
+                const browserWords = browserContent.trim().split(/\s+/).filter((w) => w.length > 0).length;
+                if (browserWords > preEscalationWords) {
+                    log.info(`browser escalation improved content: ${preEscalationWords}w → ${browserWords}w`);
+                    ctx.content = browserContent;
+                    ctx.fetchResult = browserResult;
+                    ctx.fetchResult.method = 'browser-escalation';
+                }
+                else {
+                    log.debug(`browser escalation did not improve (${browserWords}w vs ${preEscalationWords}w)`);
+                }
+                // Always clean up browser resources
+                if (browserResult.page)
+                    await browserResult.page.close().catch(() => { });
+                if (browserResult.browser)
+                    await browserResult.browser.close().catch(() => { });
+            }
+        }
+        catch (e) {
+            log.debug('browser escalation failed:', e instanceof Error ? e.message : e);
+        }
+    }
     // Generate AI summary if requested
     if (options.summary && options.llm) {
         try {

package/dist/server/routes/health.js

@@ -7,6 +7,7 @@ import { Router } from 'express';
 import { readFileSync } from 'fs';
 import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
+import { fetchCache, searchCache } from '../../core/fetch-cache.js';
 const startTime = Date.now();
 // Read version once at startup
 let version = 'unknown';
@@ -26,11 +27,23 @@ export function createHealthRouter() {
     const router = Router();
     router.get('/health', (_req, res) => {
         const uptime = Math.floor((Date.now() - startTime) / 1000);
+        const fetchStats = fetchCache.stats();
+        const searchStats = searchCache.stats();
         res.json({
             status: 'healthy',
             version,
             uptime,
             timestamp: new Date().toISOString(),
+            cache: {
+                fetch: {
+                    size: fetchStats.size,
+                    hitRate: fetchStats.hitRate,
+                },
+                search: {
+                    size: searchStats.size,
+                    hitRate: searchStats.hitRate,
+                },
+            },
         });
     });
     return router;

package/dist/server/routes/search.js

@@ -7,6 +7,7 @@ import { load } from 'cheerio';
 import { LRUCache } from 'lru-cache';
 import { peel } from '../../index.js';
 import { simpleFetch } from '../../core/fetcher.js';
+import { searchCache } from '../../core/fetch-cache.js';
 import { getSearchProvider, getBestSearchProvider, } from '../../core/search-provider.js';
 export function createSearchRouter(authStore) {
     const router = Router();
@@ -47,9 +48,9 @@ export function createSearchRouter(authStore) {
                 return;
             }
             // Parse and validate count
-            const resultCount = count ? parseInt(count, 10) : 5;
-            if (isNaN(resultCount) || resultCount < 1 || resultCount > 10) {
-                res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "count" parameter: must be between 1 and 10', hint: 'Use a count value between 1 and 10', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+            const resultCount = count ? parseInt(count, 10) : 10;
+            if (isNaN(resultCount) || resultCount < 1 || resultCount > 20) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "count" parameter: must be between 1 and 20', hint: 'Use a count value between 1 and 20', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
                 return;
             }
             // Parse sources parameter (comma-separated: web,news,images)
@@ -64,10 +65,12 @@ export function createSearchRouter(authStore) {
             // Build cache key (include all parameters)
             const enrichCount = enrich ? Math.min(Math.max(parseInt(enrich, 10) || 0, 0), 5) : 0;
             const cacheKey = `search:${providerId}:${q}:${resultCount}:${sourcesStr}:${shouldScrape}:${enrichCount}:${categoriesStr}:${tbsStr}:${countryStr}:${locationStr}`;
-            // Check cache
+            const sharedCacheKey = searchCache.getKey(cacheKey, {});
+            // Check cache (local LRU first, then shared singleton)
             const cached = cache.get(cacheKey);
             if (cached) {
                 res.setHeader('X-Cache', 'HIT');
+                res.setHeader('X-Cache-Status', 'HIT');
                 res.setHeader('X-Cache-Age', Math.floor((Date.now() - cached.timestamp) / 1000).toString());
                 res.json({
                     success: true,
@@ -75,6 +78,19 @@ export function createSearchRouter(authStore) {
                 });
                 return;
             }
+            // Also check shared searchCache singleton (used for /health stats)
+            const sharedCached = searchCache.get(sharedCacheKey);
+            if (sharedCached) {
+                const age = Math.floor((Date.now() - sharedCached.timestamp) / 1000);
+                res.setHeader('X-Cache', 'HIT');
+                res.setHeader('X-Cache-Status', 'HIT');
+                res.setHeader('X-Cache-Age', age.toString());
+                res.json({
+                    success: true,
+                    data: sharedCached.content ? JSON.parse(sharedCached.content) : {},
+                });
+                return;
+            }
             const startTime = Date.now();
             const data = {};
             // Fetch web results via the search-provider abstraction
@@ -317,13 +333,22 @@ export function createSearchRouter(authStore) {
                     await pgStore.trackUsage(req.auth.keyInfo.key, 'search');
                 }
             }
-            // Cache results
+            // Cache results (local LRU + shared singleton for /health stats)
             cache.set(cacheKey, {
                 data,
                 timestamp: Date.now(),
             });
+            searchCache.set(sharedCacheKey, {
+                content: JSON.stringify(data),
+                title: q,
+                metadata: {},
+                method: 'search',
+                tokens: 0,
+                timestamp: Date.now(),
+            });
             // Add headers
             res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Cache-Status', 'MISS');
             res.setHeader('X-Credits-Used', '1');
             res.setHeader('X-Processing-Time', elapsed.toString());
             res.setHeader('X-Fetch-Type', 'search');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.48",
+  "version": "0.21.53",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",