webpeel 0.21.52 → 0.21.56

package/dist/core/fetch-cache.d.ts

@@ -0,0 +1,57 @@
+/**
+ * In-memory LRU fetch cache for WebPeel
+ *
+ * Caches pipeline results to avoid redundant fetches for identical requests.
+ * Supports TTL-based expiry and LRU eviction when maxEntries is exceeded.
+ * Exported as a singleton: import { fetchCache } from './fetch-cache.js'
+ */
+export interface FetchCacheEntry {
+    content: string;
+    title: string;
+    metadata: any;
+    method: string;
+    tokens: number;
+    links?: any[];
+    timestamp: number;
+}
+export interface FetchCacheStats {
+    size: number;
+    hits: number;
+    misses: number;
+    hitRate: number;
+}
+export declare class FetchCache {
+    private cache;
+    private maxEntries;
+    private defaultTTL;
+    private hits;
+    private misses;
+    constructor(maxEntries?: number, defaultTTLSeconds?: number);
+    /**
+     * Generate a stable cache key from url + relevant fetch options.
+     * Different option combinations produce different cache entries.
+     */
+    getKey(url: string, options?: {
+        render?: boolean;
+        stealth?: boolean;
+        budget?: number;
+    }): string;
+    /**
+     * Retrieve a cached entry. Returns null if missing or expired.
+     * On hit: entry is moved to the end of the Map (LRU refresh).
+     */
+    get(key: string): FetchCacheEntry | null;
+    /**
+     * Store an entry in the cache.
+     * If the cache is at capacity, the least recently used entry is evicted.
+     */
+    set(key: string, entry: FetchCacheEntry): void;
+    /** Clear all entries and reset stats. */
+    clear(): void;
+    /** Return cache stats. hitRate is in [0, 1]. */
+    stats(): FetchCacheStats;
+}
+/** Singleton fetch cache — shared across all requests (5 min TTL, 500 entries). */
+export declare const fetchCache: FetchCache;
+/** Singleton search cache — shorter TTL since results change faster (60 s). */
+export declare const searchCache: FetchCache;

package/dist/core/fetch-cache.js

@@ -0,0 +1,95 @@
+/**
+ * In-memory LRU fetch cache for WebPeel
+ *
+ * Caches pipeline results to avoid redundant fetches for identical requests.
+ * Supports TTL-based expiry and LRU eviction when maxEntries is exceeded.
+ * Exported as a singleton: import { fetchCache } from './fetch-cache.js'
+ */
+export class FetchCache {
+    cache;
+    maxEntries;
+    defaultTTL; // ms
+    hits;
+    misses;
+    constructor(maxEntries = 500, defaultTTLSeconds = 300) {
+        this.cache = new Map();
+        this.maxEntries = maxEntries;
+        this.defaultTTL = defaultTTLSeconds * 1000;
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /**
+     * Generate a stable cache key from url + relevant fetch options.
+     * Different option combinations produce different cache entries.
+     */
+    getKey(url, options = {}) {
+        const render = options.render ? '1' : '0';
+        const stealth = options.stealth ? '1' : '0';
+        const budget = options.budget !== undefined ? String(options.budget) : '';
+        return `${url}|r:${render}|s:${stealth}|b:${budget}`;
+    }
+    /**
+     * Retrieve a cached entry. Returns null if missing or expired.
+     * On hit: entry is moved to the end of the Map (LRU refresh).
+     */
+    get(key) {
+        const entry = this.cache.get(key);
+        if (!entry) {
+            this.misses++;
+            return null;
+        }
+        const ageMs = Date.now() - entry.timestamp;
+        if (ageMs > this.defaultTTL) {
+            // Expired — evict and return null
+            this.cache.delete(key);
+            this.misses++;
+            return null;
+        }
+        // LRU touch: move to end
+        this.cache.delete(key);
+        this.cache.set(key, entry);
+        this.hits++;
+        return entry;
+    }
+    /**
+     * Store an entry in the cache.
+     * If the cache is at capacity, the least recently used entry is evicted.
+     */
+    set(key, entry) {
+        // Remove existing to refresh position
+        if (this.cache.has(key)) {
+            this.cache.delete(key);
+        }
+        this.cache.set(key, entry);
+        // LRU eviction: remove oldest entry (first in Map iteration order)
+        while (this.cache.size > this.maxEntries) {
+            const oldestKey = this.cache.keys().next().value;
+            if (oldestKey !== undefined) {
+                this.cache.delete(oldestKey);
+            }
+            else {
+                break;
+            }
+        }
+    }
+    /** Clear all entries and reset stats. */
+    clear() {
+        this.cache.clear();
+        this.hits = 0;
+        this.misses = 0;
+    }
+    /** Return cache stats. hitRate is in [0, 1]. */
+    stats() {
+        const total = this.hits + this.misses;
+        return {
+            size: this.cache.size,
+            hits: this.hits,
+            misses: this.misses,
+            hitRate: total === 0 ? 0 : Math.round((this.hits / total) * 100) / 100,
+        };
+    }
+}
+/** Singleton fetch cache — shared across all requests (5 min TTL, 500 entries). */
+export const fetchCache = new FetchCache(500, 300);
+/** Singleton search cache — shorter TTL since results change faster (60 s). */
+export const searchCache = new FetchCache(500, 60);

package/dist/core/source-credibility.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Source credibility scoring — lightweight, zero dependencies.
+ *
+ * Classifies URLs by trustworthiness:
+ *   - Official (★★★): .gov, .edu, .mil, WHO, NIH, academic journals
+ *   - Verified (★★):  Wikipedia, Reuters, BBC, GitHub, StackOverflow
+ *   - General (★):    Everything else
+ */
+export interface SourceCredibility {
+    tier: 'official' | 'verified' | 'general';
+    stars: number;
+    label: string;
+}
+/**
+ * Assess the credibility of a source URL.
+ */
+export declare function getSourceCredibility(url: string): SourceCredibility;

package/dist/core/source-credibility.js

@@ -0,0 +1,83 @@
+/**
+ * Source credibility scoring — lightweight, zero dependencies.
+ *
+ * Classifies URLs by trustworthiness:
+ *   - Official (★★★): .gov, .edu, .mil, WHO, NIH, academic journals
+ *   - Verified (★★):  Wikipedia, Reuters, BBC, GitHub, StackOverflow
+ *   - General (★):    Everything else
+ */
+/** Official TLDs and hostnames that indicate high-authority sources */
+const OFFICIAL_TLDS = new Set(['.gov', '.edu', '.mil']);
+const OFFICIAL_HOSTNAMES = new Set([
+    // Academic / research
+    'arxiv.org', 'scholar.google.com', 'pubmed.ncbi.nlm.nih.gov', 'ncbi.nlm.nih.gov',
+    'jstor.org', 'nature.com', 'science.org', 'cell.com', 'nejm.org', 'bmj.com',
+    'thelancet.com', 'plos.org', 'springer.com', 'elsevier.com',
+    // International organisations
+    'who.int', 'un.org', 'worldbank.org', 'imf.org', 'oecd.org', 'europa.eu',
+    // Official tech documentation
+    'docs.python.org', 'developer.mozilla.org', 'nodejs.org', 'rust-lang.org',
+    'docs.microsoft.com', 'learn.microsoft.com', 'developer.apple.com',
+    'developer.android.com', 'php.net', 'ruby-lang.org', 'golang.org', 'go.dev',
+    // Health / medicine
+    'cdc.gov', 'nih.gov', 'fda.gov', 'mayoclinic.org', 'clevelandclinic.org',
+    'webmd.com', 'medlineplus.gov',
+    // Standards / specs
+    'w3.org', 'ietf.org', 'rfc-editor.org', 'iso.org',
+]);
+const VERIFIED_HOSTNAMES = new Set([
+    // Encyclopaedia / reference
+    'wikipedia.org', 'en.wikipedia.org', 'britannica.com',
+    // Reputable news agencies
+    'reuters.com', 'apnews.com', 'bbc.com', 'bbc.co.uk', 'nytimes.com',
+    'washingtonpost.com', 'theguardian.com', 'economist.com', 'ft.com',
+    'cnn.com', 'npr.org', 'pbs.org',
+    // Developer resources
+    'github.com', 'stackoverflow.com', 'npmjs.com', 'pypi.org',
+    'crates.io', 'docs.rs', 'packagist.org', 'rubygems.org',
+    // Official cloud / vendor docs
+    'docs.aws.amazon.com', 'cloud.google.com', 'docs.github.com',
+    'azure.microsoft.com', 'registry.terraform.io',
+    // Reputable tech publications
+    'arstechnica.com', 'wired.com', 'techcrunch.com', 'theverge.com',
+    // National Geographic, Smithsonian
+    'nationalgeographic.com', 'smithsonianmag.com',
+]);
+/**
+ * Assess the credibility of a source URL.
+ */
+export function getSourceCredibility(url) {
+    try {
+        const hostname = new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+        // Check official TLDs
+        for (const tld of OFFICIAL_TLDS) {
+            if (hostname.endsWith(tld)) {
+                return { tier: 'official', stars: 3, label: 'OFFICIAL SOURCE' };
+            }
+        }
+        // Check known official hostnames  
+        if (OFFICIAL_HOSTNAMES.has(hostname)) {
+            return { tier: 'official', stars: 3, label: 'OFFICIAL SOURCE' };
+        }
+        // Check parent domain (e.g. en.wikipedia.org → wikipedia.org)
+        const parts = hostname.split('.');
+        if (parts.length > 2) {
+            const parentDomain = parts.slice(-2).join('.');
+            if (OFFICIAL_HOSTNAMES.has(parentDomain)) {
+                return { tier: 'official', stars: 3, label: 'OFFICIAL SOURCE' };
+            }
+            if (VERIFIED_HOSTNAMES.has(parentDomain)) {
+                return { tier: 'verified', stars: 2, label: 'VERIFIED' };
+            }
+        }
+        // Check known verified hostnames
+        if (VERIFIED_HOSTNAMES.has(hostname)) {
+            return { tier: 'verified', stars: 2, label: 'VERIFIED' };
+        }
+        // Everything else
+        return { tier: 'general', stars: 1, label: 'UNVERIFIED' };
+    }
+    catch {
+        return { tier: 'general', stars: 1, label: 'UNVERIFIED' };
+    }
+}

package/dist/server/routes/health.js

@@ -7,6 +7,7 @@ import { Router } from 'express';
 import { readFileSync } from 'fs';
 import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
+import { fetchCache, searchCache } from '../../core/fetch-cache.js';
 const startTime = Date.now();
 // Read version once at startup
 let version = 'unknown';
@@ -26,11 +27,23 @@ export function createHealthRouter() {
     const router = Router();
     router.get('/health', (_req, res) => {
         const uptime = Math.floor((Date.now() - startTime) / 1000);
+        const fetchStats = fetchCache.stats();
+        const searchStats = searchCache.stats();
         res.json({
             status: 'healthy',
             version,
             uptime,
             timestamp: new Date().toISOString(),
+            cache: {
+                fetch: {
+                    size: fetchStats.size,
+                    hitRate: fetchStats.hitRate,
+                },
+                search: {
+                    size: searchStats.size,
+                    hitRate: searchStats.hitRate,
+                },
+            },
         });
     });
     return router;

package/dist/server/routes/search.js

@@ -7,7 +7,9 @@ import { load } from 'cheerio';
 import { LRUCache } from 'lru-cache';
 import { peel } from '../../index.js';
 import { simpleFetch } from '../../core/fetcher.js';
+import { searchCache } from '../../core/fetch-cache.js';
 import { getSearchProvider, getBestSearchProvider, } from '../../core/search-provider.js';
+import { getSourceCredibility } from '../../core/source-credibility.js';
 export function createSearchRouter(authStore) {
     const router = Router();
     // LRU cache: 15 minute TTL, max 500 entries, 50MB total size
@@ -47,9 +49,9 @@ export function createSearchRouter(authStore) {
                 return;
             }
             // Parse and validate count
-            const resultCount = count ? parseInt(count, 10) : 5;
-            if (isNaN(resultCount) || resultCount < 1 || resultCount > 10) {
-                res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "count" parameter: must be between 1 and 10', hint: 'Use a count value between 1 and 10', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+            const resultCount = count ? parseInt(count, 10) : 10;
+            if (isNaN(resultCount) || resultCount < 1 || resultCount > 20) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "count" parameter: must be between 1 and 20', hint: 'Use a count value between 1 and 20', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
                 return;
             }
             // Parse sources parameter (comma-separated: web,news,images)
@@ -64,10 +66,12 @@ export function createSearchRouter(authStore) {
             // Build cache key (include all parameters)
             const enrichCount = enrich ? Math.min(Math.max(parseInt(enrich, 10) || 0, 0), 5) : 0;
             const cacheKey = `search:${providerId}:${q}:${resultCount}:${sourcesStr}:${shouldScrape}:${enrichCount}:${categoriesStr}:${tbsStr}:${countryStr}:${locationStr}`;
-            // Check cache
+            const sharedCacheKey = searchCache.getKey(cacheKey, {});
+            // Check cache (local LRU first, then shared singleton)
             const cached = cache.get(cacheKey);
             if (cached) {
                 res.setHeader('X-Cache', 'HIT');
+                res.setHeader('X-Cache-Status', 'HIT');
                 res.setHeader('X-Cache-Age', Math.floor((Date.now() - cached.timestamp) / 1000).toString());
                 res.json({
                     success: true,
@@ -75,6 +79,19 @@ export function createSearchRouter(authStore) {
                 });
                 return;
             }
+            // Also check shared searchCache singleton (used for /health stats)
+            const sharedCached = searchCache.get(sharedCacheKey);
+            if (sharedCached) {
+                const age = Math.floor((Date.now() - sharedCached.timestamp) / 1000);
+                res.setHeader('X-Cache', 'HIT');
+                res.setHeader('X-Cache-Status', 'HIT');
+                res.setHeader('X-Cache-Age', age.toString());
+                res.json({
+                    success: true,
+                    data: sharedCached.content ? JSON.parse(sharedCached.content) : {},
+                });
+                return;
+            }
             const startTime = Date.now();
             const data = {};
             // Fetch web results via the search-provider abstraction
@@ -194,6 +211,19 @@ export function createSearchRouter(authStore) {
                         }
                     }
                 }
+                // Add credibility scores and sort by trustworthiness
+                const tierOrder = { official: 0, verified: 1, general: 2 };
+                results = results
+                    .map(r => {
+                    const cred = getSourceCredibility(r.url);
+                    return { ...r, credibility: cred };
+                })
+                    .sort((a, b) => {
+                    const aTier = tierOrder[a.credibility?.tier || 'general'] ?? 2;
+                    const bTier = tierOrder[b.credibility?.tier || 'general'] ?? 2;
+                    return aTier - bTier; // Official first, then verified, then general
+                })
+                    .map((r, i) => ({ ...r, rank: i + 1 }));
                 data.web = results;
             }
             // Fetch news results (DDG only — Brave news is not supported via HTML scraping)
@@ -317,13 +347,22 @@ export function createSearchRouter(authStore) {
                     await pgStore.trackUsage(req.auth.keyInfo.key, 'search');
                 }
             }
-            // Cache results
+            // Cache results (local LRU + shared singleton for /health stats)
             cache.set(cacheKey, {
                 data,
                 timestamp: Date.now(),
             });
+            searchCache.set(sharedCacheKey, {
+                content: JSON.stringify(data),
+                title: q,
+                metadata: {},
+                method: 'search',
+                tokens: 0,
+                timestamp: Date.now(),
+            });
             // Add headers
             res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Cache-Status', 'MISS');
             res.setHeader('X-Credits-Used', '1');
             res.setHeader('X-Processing-Time', elapsed.toString());
             res.setHeader('X-Fetch-Type', 'search');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.52",
+  "version": "0.21.56",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",