webpeel 0.21.52 → 0.21.53

package/dist/core/fetch-cache.d.ts

@@ -0,0 +1,57 @@
+/**
+ * In-memory LRU fetch cache for WebPeel
+ *
+ * Caches pipeline results to avoid redundant fetches for identical requests.
+ * Supports TTL-based expiry and LRU eviction when maxEntries is exceeded.
+ * Exported as a singleton: import { fetchCache } from './fetch-cache.js'
+ */
+export interface FetchCacheEntry {
+    content: string;
+    title: string;
+    metadata: any;
+    method: string;
+    tokens: number;
+    links?: any[];
+    timestamp: number;
+}
+export interface FetchCacheStats {
+    size: number;
+    hits: number;
+    misses: number;
+    hitRate: number;
+}
+export declare class FetchCache {
+    private cache;
+    private maxEntries;
+    private defaultTTL;
+    private hits;
+    private misses;
+    constructor(maxEntries?: number, defaultTTLSeconds?: number);
+    /**
+     * Generate a stable cache key from url + relevant fetch options.
+     * Different option combinations produce different cache entries.
+     */
+    getKey(url: string, options?: {
+        render?: boolean;
+        stealth?: boolean;
+        budget?: number;
+    }): string;
+    /**
+     * Retrieve a cached entry. Returns null if missing or expired.
+     * On hit: entry is moved to the end of the Map (LRU refresh).
+     */
+    get(key: string): FetchCacheEntry | null;
+    /**
+     * Store an entry in the cache.
+     * If the cache is at capacity, the least recently used entry is evicted.
+     */
+    set(key: string, entry: FetchCacheEntry): void;
+    /** Clear all entries and reset stats. */
+    clear(): void;
+    /** Return cache stats. hitRate is in [0, 1]. */
+    stats(): FetchCacheStats;
+}
+/** Singleton fetch cache — shared across all requests (5 min TTL, 500 entries). */
+export declare const fetchCache: FetchCache;
+/** Singleton search cache — shorter TTL since results change faster (60 s). */
+export declare const searchCache: FetchCache;

package/dist/core/fetch-cache.js

@@ -0,0 +1,95 @@
+/**
+ * In-memory LRU fetch cache for WebPeel
+ *
+ * Caches pipeline results to avoid redundant fetches for identical requests.
+ * Supports TTL-based expiry and LRU eviction when maxEntries is exceeded.
+ * Exported as a singleton: import { fetchCache } from './fetch-cache.js'
+ */
+export class FetchCache {
+    cache;
+    maxEntries;
+    defaultTTL; // ms
+    hits;
+    misses;
+    constructor(maxEntries = 500, defaultTTLSeconds = 300) {
+        this.cache = new Map();
+        this.maxEntries = maxEntries;
+        this.defaultTTL = defaultTTLSeconds * 1000;
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /**
+     * Generate a stable cache key from url + relevant fetch options.
+     * Different option combinations produce different cache entries.
+     */
+    getKey(url, options = {}) {
+        const render = options.render ? '1' : '0';
+        const stealth = options.stealth ? '1' : '0';
+        const budget = options.budget !== undefined ? String(options.budget) : '';
+        return `${url}|r:${render}|s:${stealth}|b:${budget}`;
+    }
+    /**
+     * Retrieve a cached entry. Returns null if missing or expired.
+     * On hit: entry is moved to the end of the Map (LRU refresh).
+     */
+    get(key) {
+        const entry = this.cache.get(key);
+        if (!entry) {
+            this.misses++;
+            return null;
+        }
+        const ageMs = Date.now() - entry.timestamp;
+        if (ageMs > this.defaultTTL) {
+            // Expired — evict and return null
+            this.cache.delete(key);
+            this.misses++;
+            return null;
+        }
+        // LRU touch: move to end
+        this.cache.delete(key);
+        this.cache.set(key, entry);
+        this.hits++;
+        return entry;
+    }
+    /**
+     * Store an entry in the cache.
+     * If the cache is at capacity, the least recently used entry is evicted.
+     */
+    set(key, entry) {
+        // Remove existing to refresh position
+        if (this.cache.has(key)) {
+            this.cache.delete(key);
+        }
+        this.cache.set(key, entry);
+        // LRU eviction: remove oldest entry (first in Map iteration order)
+        while (this.cache.size > this.maxEntries) {
+            const oldestKey = this.cache.keys().next().value;
+            if (oldestKey !== undefined) {
+                this.cache.delete(oldestKey);
+            }
+            else {
+                break;
+            }
+        }
+    }
+    /** Clear all entries and reset stats. */
+    clear() {
+        this.cache.clear();
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /** Return cache stats. hitRate is in [0, 1]. */
+    stats() {
+        const total = this.hits + this.misses;
+        return {
+            size: this.cache.size,
+            hits: this.hits,
+            misses: this.misses,
+            hitRate: total === 0 ? 0 : Math.round((this.hits / total) * 100) / 100,
+        };
+    }
+}
+/** Singleton fetch cache — shared across all requests (5 min TTL, 500 entries). */
+export const fetchCache = new FetchCache(500, 300);
+/** Singleton search cache — shorter TTL since results change faster (60 s). */
+export const searchCache = new FetchCache(500, 60);

package/dist/core/pipeline.js

@@ -1217,6 +1217,52 @@ export async function finalize(ctx) {
             log.error('Change tracking failed:', error);
         }
     }
+    // ── Auto-escalation: retry thin content with browser rendering ──────────────
+    // If simple fetch returned very little content and user didn't explicitly disable render,
+    // automatically retry with browser rendering to handle JS-heavy/paywalled sites.
+    const preEscalationWords = ctx.content.trim().split(/\s+/).filter((w) => w.length > 0).length;
+    const escalationFetchMethod = fetchResult?.method || 'unknown';
+    const alreadyTriedBrowser = escalationFetchMethod === 'browser' || escalationFetchMethod === 'stealth'
+        || options.render || options.stealth;
+    const userDisabledRender = options.render === false;
+    const escalationCandidate = preEscalationWords < 200 && preEscalationWords > 0
+        && escalationFetchMethod === 'simple' && !alreadyTriedBrowser && !userDisabledRender
+        && !ctx._escalated;
+    if (escalationCandidate) {
+        log.info(`thin content (${preEscalationWords}w) from simple fetch, auto-escalating to browser render for ${ctx.url}`);
+        ctx._escalated = true;
+        try {
+            const { smartFetch } = await import('./strategies.js');
+            const browserResult = await smartFetch(ctx.url, {
+                forceBrowser: true,
+                stealth: false,
+                timeoutMs: options.timeout || 15000,
+                proxy: options.proxy,
+            });
+            if (browserResult.html && browserResult.html.length > (fetchResult?.html?.length || 0)) {
+                const { htmlToMarkdown } = await import('./markdown.js');
+                const browserContent = htmlToMarkdown(browserResult.html);
+                const browserWords = browserContent.trim().split(/\s+/).filter((w) => w.length > 0).length;
+                if (browserWords > preEscalationWords) {
+                    log.info(`browser escalation improved content: ${preEscalationWords}w → ${browserWords}w`);
+                    ctx.content = browserContent;
+                    ctx.fetchResult = browserResult;
+                    ctx.fetchResult.method = 'browser-escalation';
+                }
+                else {
+                    log.debug(`browser escalation did not improve (${browserWords}w vs ${preEscalationWords}w)`);
+                }
+                // Always clean up browser resources
+                if (browserResult.page)
+                    await browserResult.page.close().catch(() => { });
+                if (browserResult.browser)
+                    await browserResult.browser.close().catch(() => { });
+            }
+        }
+        catch (e) {
+            log.debug('browser escalation failed:', e instanceof Error ? e.message : e);
+        }
+    }
     // Generate AI summary if requested
     if (options.summary && options.llm) {
         try {

package/dist/server/routes/health.js

@@ -7,6 +7,7 @@ import { Router } from 'express';
 import { readFileSync } from 'fs';
 import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
+import { fetchCache, searchCache } from '../../core/fetch-cache.js';
 const startTime = Date.now();
 // Read version once at startup
 let version = 'unknown';
@@ -26,11 +27,23 @@ export function createHealthRouter() {
     const router = Router();
     router.get('/health', (_req, res) => {
         const uptime = Math.floor((Date.now() - startTime) / 1000);
+        const fetchStats = fetchCache.stats();
+        const searchStats = searchCache.stats();
         res.json({
             status: 'healthy',
             version,
             uptime,
             timestamp: new Date().toISOString(),
+            cache: {
+                fetch: {
+                    size: fetchStats.size,
+                    hitRate: fetchStats.hitRate,
+                },
+                search: {
+                    size: searchStats.size,
+                    hitRate: searchStats.hitRate,
+                },
+            },
         });
     });
     return router;

package/dist/server/routes/search.js

@@ -7,6 +7,7 @@ import { load } from 'cheerio';
 import { LRUCache } from 'lru-cache';
 import { peel } from '../../index.js';
 import { simpleFetch } from '../../core/fetcher.js';
+import { searchCache } from '../../core/fetch-cache.js';
 import { getSearchProvider, getBestSearchProvider, } from '../../core/search-provider.js';
 export function createSearchRouter(authStore) {
     const router = Router();
@@ -47,9 +48,9 @@ export function createSearchRouter(authStore) {
                 return;
             }
             // Parse and validate count
-            const resultCount = count ? parseInt(count, 10) : 5;
-            if (isNaN(resultCount) || resultCount < 1 || resultCount > 10) {
-                res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "count" parameter: must be between 1 and 10', hint: 'Use a count value between 1 and 10', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+            const resultCount = count ? parseInt(count, 10) : 10;
+            if (isNaN(resultCount) || resultCount < 1 || resultCount > 20) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "count" parameter: must be between 1 and 20', hint: 'Use a count value between 1 and 20', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
                 return;
             }
             // Parse sources parameter (comma-separated: web,news,images)
@@ -64,10 +65,12 @@ export function createSearchRouter(authStore) {
             // Build cache key (include all parameters)
             const enrichCount = enrich ? Math.min(Math.max(parseInt(enrich, 10) || 0, 0), 5) : 0;
             const cacheKey = `search:${providerId}:${q}:${resultCount}:${sourcesStr}:${shouldScrape}:${enrichCount}:${categoriesStr}:${tbsStr}:${countryStr}:${locationStr}`;
-            // Check cache
+            const sharedCacheKey = searchCache.getKey(cacheKey, {});
+            // Check cache (local LRU first, then shared singleton)
             const cached = cache.get(cacheKey);
             if (cached) {
                 res.setHeader('X-Cache', 'HIT');
+                res.setHeader('X-Cache-Status', 'HIT');
                 res.setHeader('X-Cache-Age', Math.floor((Date.now() - cached.timestamp) / 1000).toString());
                 res.json({
                     success: true,
@@ -75,6 +78,19 @@ export function createSearchRouter(authStore) {
                 });
                 return;
             }
+            // Also check shared searchCache singleton (used for /health stats)
+            const sharedCached = searchCache.get(sharedCacheKey);
+            if (sharedCached) {
+                const age = Math.floor((Date.now() - sharedCached.timestamp) / 1000);
+                res.setHeader('X-Cache', 'HIT');
+                res.setHeader('X-Cache-Status', 'HIT');
+                res.setHeader('X-Cache-Age', age.toString());
+                res.json({
+                    success: true,
+                    data: sharedCached.content ? JSON.parse(sharedCached.content) : {},
+                });
+                return;
+            }
             const startTime = Date.now();
             const data = {};
             // Fetch web results via the search-provider abstraction
@@ -317,13 +333,22 @@ export function createSearchRouter(authStore) {
                     await pgStore.trackUsage(req.auth.keyInfo.key, 'search');
                 }
             }
-            // Cache results
+            // Cache results (local LRU + shared singleton for /health stats)
             cache.set(cacheKey, {
                 data,
                 timestamp: Date.now(),
             });
+            searchCache.set(sharedCacheKey, {
+                content: JSON.stringify(data),
+                title: q,
+                metadata: {},
+                method: 'search',
+                tokens: 0,
+                timestamp: Date.now(),
+            });
             // Add headers
             res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Cache-Status', 'MISS');
             res.setHeader('X-Credits-Used', '1');
             res.setHeader('X-Processing-Time', elapsed.toString());
             res.setHeader('X-Fetch-Type', 'search');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.52",
+  "version": "0.21.53",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",