webpeel 0.21.0 → 0.21.2

package/dist/core/dns-cache.js

@@ -40,6 +40,21 @@ const DNS_WARMUP_DOMAINS = [
     'tools.ietf.org',
     'unicode.org',
     'www.bbc.com',
+    'bbc.co.uk',
+    'stripe.com',
+    'docs.stripe.com',
+    'vuejs.org',
+    'angular.io',
+    'www.washingtonpost.com',
+    'www.theguardian.com',
+    'techcrunch.com',
+    'www.wired.com',
+    'arstechnica.com',
+    'docs.google.com',
+    'drive.google.com',
+    'www.notion.so',
+    'www.producthunt.com',
+    'www.crunchbase.com',
     'news.google.com',
     'www.youtube.com',
     'example.com',

package/dist/core/http-fetch.js

@@ -42,11 +42,13 @@ const HTTP_STATUS_TEXT = {
 // ── HTTP connection pool ──────────────────────────────────────────────────────
 function createHttpPool() {
     return new Agent({
-        connections: 20,
-        pipelining: 6,
+        connections: 50,
+        pipelining: 10,
         keepAliveTimeout: 60000,
         keepAliveMaxTimeout: 60000,
         allowH2: true,
+        headersTimeout: 10000,
+        bodyTimeout: 30000,
         connect: {
             lookup: cachedLookup,
         },

package/dist/core/pipeline.d.ts

@@ -75,6 +75,8 @@ export interface PipelineContext {
     domainApiHandled?: boolean;
     /** True when server returned pre-rendered markdown (Content-Type: text/markdown) */
     serverMarkdown?: boolean;
+    /** True when HTTP fetch completed in < 500ms — enables fast path (skip challenge detection) */
+    fastPath?: boolean;
     /** Non-fatal warnings accumulated during the pipeline run */
     warnings: string[];
     /** Raw HTML size in characters (measured from fetched content before any conversion) */

package/dist/core/pipeline.js

@@ -399,7 +399,17 @@ export async function fetchContent(ctx) {
         }
         throw fetchError;
     }
-    ctx.timer.end('fetch');
+    const fetchDuration = ctx.timer.end('fetch');
+    // Fast path: if a plain HTTP fetch completed quickly with real HTML content,
+    // mark it so post-processing can skip expensive heuristics (challenge detection).
+    // Only applies to non-browser fetches that succeeded with HTML content.
+    if (fetchDuration < 500 &&
+        !ctx.render &&
+        fetchResult.statusCode === 200 &&
+        (fetchResult.contentType || '').includes('html') &&
+        (fetchResult.html?.length || 0) > 200) {
+        ctx.fastPath = true;
+    }
     // Auto-scroll to load lazy content, then grab fresh HTML
     if (needsAutoScroll && fetchResult.page) {
         try {
@@ -927,7 +937,9 @@ export async function postProcess(ctx) {
     // === Challenge / bot-protection page detection ===
     // If the extracted content looks like a challenge page (not real content),
     // mark it and try the search-as-proxy fallback to get the real info.
-    if (ctx.content && ctx.content.length < 2000) {
+    // Fast path: skip this check for HTTP fetches that completed in < 500ms —
+    // a fast successful response is virtually never a challenge page.
+    if (!ctx.fastPath && ctx.content && ctx.content.length < 2000) {
         const lowerContent = ctx.content.toLowerCase();
         const challengeSignals = [
             'please verify you are a human',

package/dist/server/app.js

@@ -49,6 +49,7 @@ import { createAgentRouter } from './routes/agent.js';
 import { createSessionRouter } from './routes/session.js';
 import { createSentryHooks } from './sentry.js';
 import { requireScope } from './middleware/scope-guard.js';
+import { createCacheWarmRouter, startCacheWarmer } from './routes/cache-warm.js';
 import { warmup, cleanup as cleanupFetcher } from '../core/fetcher.js';
 import { registerPremiumHooks } from './premium/index.js';
 import { readFileSync } from 'fs';
@@ -238,6 +239,9 @@ export function createApp(config = {}) {
     app.get('/docs/api', (_req, res) => {
         res.redirect('/openapi.yaml');
     });
+    // Internal cache-warming endpoints — unauthenticated (self-auth via bearer token)
+    // Must be BEFORE auth middleware so the CF Worker can call without an API key
+    app.use(createCacheWarmRouter(pool));
     // Demo endpoint — unauthenticated, must be before auth middleware
     app.use(createDemoRouter());
     // Playground endpoint — unauthenticated, CORS-locked to webpeel.dev/localhost
@@ -398,9 +402,22 @@ export function startServer(config = {}) {
     void warmup().catch((error) => {
         log.warn('Browser warmup failed', { error: error instanceof Error ? error.message : String(error) });
     });
+    // Build a dedicated pool for the cache warmer (separate from the app pool inside createApp)
+    const warmerPool = process.env.DATABASE_URL
+        ? new pg.Pool({
+            connectionString: process.env.DATABASE_URL,
+            ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: true } : false,
+            max: 2, // small pool — warmer only needs occasional queries
+        })
+        : null;
     const server = app.listen(port, () => {
         log.info(`WebPeel API server listening on port ${port}`);
         log.info(`Health: http://localhost:${port}/health  Fetch: /v1/fetch  Search: /v1/search`);
+        // Start cache warmer only when opted-in
+        if (process.env.ENABLE_CACHE_WARM === 'true') {
+            log.info('Cache warming enabled (ENABLE_CACHE_WARM=true)');
+            startCacheWarmer(warmerPool);
+        }
     });
     // Graceful shutdown
     const shutdown = () => {

package/dist/server/routes/cache-warm.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Cache Pre-Warming Routes
+ *
+ * GET /internal/popular-urls  — Top N URLs fetched in the last 24h (for CF Worker)
+ * GET /internal/cache-status  — Current warmer state (warmed URLs, last run time)
+ *
+ * Both routes are mounted BEFORE auth middleware so they're accessible internally.
+ * /internal/popular-urls is protected by CACHE_WARM_SECRET bearer token when set.
+ *
+ * startCacheWarmer() — server-side self-warming (opt-in via ENABLE_CACHE_WARM=true)
+ */
+import { Router } from 'express';
+import pg from 'pg';
+export declare function createCacheWarmRouter(pool: pg.Pool | null): Router;
+/**
+ * startCacheWarmer — server-side self-warming (fallback when no CF Worker).
+ *
+ * Every `intervalMs` (default 2 min):
+ *  1. Queries /internal/popular-urls (via the DB, not HTTP)
+ *  2. Fetches each URL through /r/<url> with concurrency 5
+ *  3. Updates warmerState for /internal/cache-status
+ *
+ * Only started if ENABLE_CACHE_WARM=true.
+ */
+export declare function startCacheWarmer(pool: pg.Pool | null, intervalMs?: number): void;

package/dist/server/routes/cache-warm.js

@@ -0,0 +1,210 @@
+/**
+ * Cache Pre-Warming Routes
+ *
+ * GET /internal/popular-urls  — Top N URLs fetched in the last 24h (for CF Worker)
+ * GET /internal/cache-status  — Current warmer state (warmed URLs, last run time)
+ *
+ * Both routes are mounted BEFORE auth middleware so they're accessible internally.
+ * /internal/popular-urls is protected by CACHE_WARM_SECRET bearer token when set.
+ *
+ * startCacheWarmer() — server-side self-warming (opt-in via ENABLE_CACHE_WARM=true)
+ */
+import { Router } from 'express';
+import { createLogger } from '../logger.js';
+const log = createLogger('cache-warm');
+// ─── Fallback URLs ────────────────────────────────────────────────────────────
+// Used when the DB has no usage data yet (new deployment / empty DB).
+const FALLBACK_URLS = [
+    'https://www.bbc.com/news',
+    'https://news.ycombinator.com',
+    'https://github.com',
+    'https://en.wikipedia.org/wiki/Main_Page',
+    'https://www.reuters.com',
+    'https://techcrunch.com',
+    'https://stripe.com/docs',
+    'https://developer.mozilla.org',
+    'https://react.dev',
+    'https://docs.python.org/3/',
+    'https://nodejs.org/en/docs',
+    'https://www.npmjs.com',
+    'https://vercel.com/docs',
+    'https://nextjs.org/docs',
+    'https://tailwindcss.com/docs',
+    'https://www.typescriptlang.org/docs/',
+    'https://docs.render.com',
+    'https://cloudflare.com/docs',
+    'https://aws.amazon.com/documentation/',
+    'https://docs.github.com',
+    'https://www.nytimes.com',
+    'https://www.theguardian.com',
+    'https://arstechnica.com',
+    'https://www.wired.com',
+    'https://www.bloomberg.com/technology',
+    'https://lobste.rs',
+    'https://www.producthunt.com',
+    'https://stackoverflow.com',
+    'https://css-tricks.com',
+    'https://web.dev',
+];
+// ─── In-memory warmer state ───────────────────────────────────────────────────
+const warmerState = {
+    warmedUrls: new Set(),
+    lastWarmTime: null,
+};
+// ─── Router ──────────────────────────────────────────────────────────────────
+export function createCacheWarmRouter(pool) {
+    const router = Router();
+    // GET /internal/popular-urls
+    router.get('/internal/popular-urls', async (req, res) => {
+        // Auth check — if CACHE_WARM_SECRET is set, require it
+        const secret = process.env.CACHE_WARM_SECRET;
+        if (secret) {
+            const authHeader = req.headers['authorization'] || '';
+            const token = authHeader.startsWith('Bearer ') ? authHeader.slice(7) : '';
+            if (token !== secret) {
+                res.status(401).json({
+                    success: false,
+                    error: { type: 'unauthorized', message: 'Invalid or missing bearer token.' },
+                });
+                return;
+            }
+        }
+        let urls = [];
+        // Query DB if available
+        if (pool) {
+            try {
+                const result = await pool.query(`
+          SELECT url, COUNT(*) as fetch_count
+          FROM usage_logs
+          WHERE created_at > NOW() - INTERVAL '24 hours'
+            AND url IS NOT NULL
+            AND status_code >= 200 AND status_code < 300
+            AND url NOT LIKE '%localhost%'
+            AND url NOT LIKE '%127.0.0.1%'
+            AND url NOT LIKE '%169.254%'
+          GROUP BY url
+          ORDER BY fetch_count DESC
+          LIMIT 50
+        `);
+                urls = result.rows.map((row) => ({
+                    url: row.url,
+                    count: parseInt(row.fetch_count, 10),
+                }));
+            }
+            catch (err) {
+                log.warn('Failed to query usage_logs, falling back to static list', {
+                    error: err?.message,
+                });
+            }
+        }
+        // Fall back to static list if no DB data
+        if (urls.length === 0) {
+            log.info('No usage data found, using fallback URL list');
+            urls = FALLBACK_URLS.map((url) => ({ url, count: 0 }));
+        }
+        res.json({
+            urls,
+            total: urls.length,
+            generatedAt: new Date().toISOString(),
+        });
+    });
+    // GET /internal/cache-status
+    router.get('/internal/cache-status', (_req, res) => {
+        res.json({
+            warmedUrls: Array.from(warmerState.warmedUrls),
+            urlCount: warmerState.warmedUrls.size,
+            lastWarmTime: warmerState.lastWarmTime?.toISOString() ?? null,
+        });
+    });
+    return router;
+}
+// ─── Self-Warming ─────────────────────────────────────────────────────────────
+/**
+ * startCacheWarmer — server-side self-warming (fallback when no CF Worker).
+ *
+ * Every `intervalMs` (default 2 min):
+ *  1. Queries /internal/popular-urls (via the DB, not HTTP)
+ *  2. Fetches each URL through /r/<url> with concurrency 5
+ *  3. Updates warmerState for /internal/cache-status
+ *
+ * Only started if ENABLE_CACHE_WARM=true.
+ */
+export function startCacheWarmer(pool, intervalMs = 120_000) {
+    log.info('Cache warmer started', { intervalMs });
+    const runWarm = async () => {
+        const t0 = Date.now();
+        log.info('Cache warm cycle starting');
+        // Determine base URL
+        const base = process.env.RENDER_EXTERNAL_URL?.replace(/\/$/, '') ||
+            `http://localhost:${process.env.PORT || 3000}`;
+        // Step 1: Fetch popular URLs (same logic as the endpoint)
+        let urls = [];
+        if (pool) {
+            try {
+                const result = await pool.query(`
+          SELECT url, COUNT(*) as fetch_count
+          FROM usage_logs
+          WHERE created_at > NOW() - INTERVAL '24 hours'
+            AND url IS NOT NULL
+            AND status_code >= 200 AND status_code < 300
+            AND url NOT LIKE '%localhost%'
+            AND url NOT LIKE '%127.0.0.1%'
+            AND url NOT LIKE '%169.254%'
+          GROUP BY url
+          ORDER BY fetch_count DESC
+          LIMIT 50
+        `);
+                urls = result.rows.map((row) => ({
+                    url: row.url,
+                    count: parseInt(row.fetch_count, 10),
+                }));
+            }
+            catch (err) {
+                log.warn('Warm cycle: DB query failed, using fallback', { error: err?.message });
+            }
+        }
+        if (urls.length === 0) {
+            urls = FALLBACK_URLS.map((u) => ({ url: u, count: 0 }));
+        }
+        // Step 2: Warm each URL with concurrency 5
+        const concurrency = 5;
+        let warmed = 0;
+        let failed = 0;
+        const newWarmedSet = new Set();
+        for (let i = 0; i < urls.length; i += concurrency) {
+            const batch = urls.slice(i, i + concurrency);
+            const results = await Promise.allSettled(batch.map(({ url }) => fetch(`${base}/r/${encodeURIComponent(url)}`, {
+                headers: { 'User-Agent': 'WebPeel-CacheWarmer/1.0' },
+                signal: AbortSignal.timeout(15_000),
+            }).then((r) => {
+                if (r.ok) {
+                    newWarmedSet.add(url);
+                    warmed++;
+                }
+                else {
+                    failed++;
+                }
+            })));
+            // Count settled rejections as failures
+            results.forEach((r) => {
+                if (r.status === 'rejected') {
+                    failed++;
+                }
+            });
+        }
+        // Step 3: Update state
+        warmerState.warmedUrls = newWarmedSet;
+        warmerState.lastWarmTime = new Date();
+        const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
+        log.info(`Warmed ${warmed}/${urls.length} URLs in ${elapsed}s`, { failed });
+    };
+    // Run once immediately, then on interval
+    void runWarm().catch((err) => {
+        log.error('Cache warm cycle error', { error: err?.message });
+    });
+    setInterval(() => {
+        void runWarm().catch((err) => {
+            log.error('Cache warm cycle error', { error: err?.message });
+        });
+    }, intervalMs);
+}

package/dist/server/routes/fetch.js

@@ -229,7 +229,10 @@ export function createFetchRouter(authStore) {
                     const cacheAge = Date.now() - cached.timestamp;
                     if (cacheAge < maxAgeMs && cacheAge < cacheTtlMs) {
                         res.setHeader('X-Cache', 'HIT');
+                        res.setHeader('X-Cache-Status', 'HIT');
                         res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
+                        // Cache-Control: allow Cloudflare edge to cache successful GET responses
+                        res.setHeader('Cache-Control', 'public, s-maxage=60, stale-while-revalidate=300');
                         if (wantsEnvelope(req)) {
                             successResponse(res, cached.result, {
                                 requestId: req.requestId,
@@ -467,9 +470,19 @@ export function createFetchRouter(authStore) {
                 : undefined;
             // Add usage headers (kept for backward compat; also surfaced in envelope metadata)
             res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Cache-Status', 'MISS');
             res.setHeader('X-Credits-Used', '1');
             res.setHeader('X-Processing-Time', elapsed.toString());
             res.setHeader('X-Fetch-Type', fetchType);
+            // Cache-Control: allow Cloudflare edge to cache successful GET responses for 60s
+            res.setHeader('Cache-Control', 'public, s-maxage=60, stale-while-revalidate=300');
+            // Response timing headers — let customers see exactly where time is spent
+            const timingFetch = result.timing?.fetch ?? 0;
+            const timingParse = (result.timing?.convert ?? 0) + (result.timing?.metadata ?? 0) + (result.timing?.prune ?? 0);
+            res.setHeader('X-Response-Time', `${elapsed}ms`);
+            res.setHeader('X-Fetch-Time', `${timingFetch}ms`);
+            res.setHeader('X-Parse-Time', `${timingParse}ms`);
+            res.setHeader('Server-Timing', `fetch;dur=${timingFetch}, parse;dur=${timingParse}, total;dur=${elapsed}`);
             // Build response — extend result with optional answer/summary fields
             const getResponseBody = { ...result };
             if (getAnswerResult !== undefined)
@@ -676,6 +689,7 @@ export function createFetchRouter(authStore) {
                     const cacheAge = Date.now() - cached.timestamp;
                     if (cacheAge < postCacheTtlMs) {
                         res.setHeader('X-Cache', 'HIT');
+                        res.setHeader('X-Cache-Status', 'HIT');
                         res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
                         if (wantsEnvelope(req)) {
                             successResponse(res, cached.result, {
@@ -932,9 +946,17 @@ export function createFetchRouter(authStore) {
             // --- Build response ------------------------------------------------------
             // Headers kept for backward compat; also surfaced in envelope metadata.
             res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Cache-Status', 'MISS');
             res.setHeader('X-Credits-Used', '1');
             res.setHeader('X-Processing-Time', elapsed.toString());
             res.setHeader('X-Fetch-Type', fetchType);
+            // Response timing headers — let customers see exactly where time is spent
+            const postTimingFetch = result.timing?.fetch ?? 0;
+            const postTimingParse = (result.timing?.convert ?? 0) + (result.timing?.metadata ?? 0) + (result.timing?.prune ?? 0);
+            res.setHeader('X-Response-Time', `${elapsed}ms`);
+            res.setHeader('X-Fetch-Time', `${postTimingFetch}ms`);
+            res.setHeader('X-Parse-Time', `${postTimingParse}ms`);
+            res.setHeader('Server-Timing', `fetch;dur=${postTimingFetch}, parse;dur=${postTimingParse}, total;dur=${elapsed}`);
             const responseBody = { ...result };
             if (jsonData !== undefined) {
                 responseBody.json = jsonData;

package/dist/server/routes/reader.js

@@ -96,6 +96,11 @@ export function createReaderRouter() {
                 selector: targetSelector,
                 waitSelector: waitForSelector,
             });
+            // Cache-Control: this endpoint is public and heavily cacheable.
+            // Cloudflare edge caches for 2 min; serves stale for up to 10 min while revalidating.
+            res.setHeader('Cache-Control', 'public, s-maxage=120, stale-while-revalidate=600');
+            // Vary on Accept so different content-type representations are cached separately.
+            res.setHeader('Vary', 'Accept');
             // Return based on format
             const responseFormat = format.toLowerCase();
             if (responseFormat === 'text') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.0",
+  "version": "0.21.2",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",