webpeel 0.21.1 → 0.21.2

package/dist/server/app.js

@@ -49,6 +49,7 @@ import { createAgentRouter } from './routes/agent.js';
 import { createSessionRouter } from './routes/session.js';
 import { createSentryHooks } from './sentry.js';
 import { requireScope } from './middleware/scope-guard.js';
+import { createCacheWarmRouter, startCacheWarmer } from './routes/cache-warm.js';
 import { warmup, cleanup as cleanupFetcher } from '../core/fetcher.js';
 import { registerPremiumHooks } from './premium/index.js';
 import { readFileSync } from 'fs';
@@ -238,6 +239,9 @@ export function createApp(config = {}) {
     app.get('/docs/api', (_req, res) => {
         res.redirect('/openapi.yaml');
     });
+    // Internal cache-warming endpoints — unauthenticated (self-auth via bearer token)
+    // Must be BEFORE auth middleware so the CF Worker can call without an API key
+    app.use(createCacheWarmRouter(pool));
     // Demo endpoint — unauthenticated, must be before auth middleware
     app.use(createDemoRouter());
     // Playground endpoint — unauthenticated, CORS-locked to webpeel.dev/localhost
@@ -398,9 +402,22 @@ export function startServer(config = {}) {
     void warmup().catch((error) => {
         log.warn('Browser warmup failed', { error: error instanceof Error ? error.message : String(error) });
     });
+    // Build a dedicated pool for the cache warmer (separate from the app pool inside createApp)
+    const warmerPool = process.env.DATABASE_URL
+        ? new pg.Pool({
+            connectionString: process.env.DATABASE_URL,
+            ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: true } : false,
+            max: 2, // small pool — warmer only needs occasional queries
+        })
+        : null;
     const server = app.listen(port, () => {
         log.info(`WebPeel API server listening on port ${port}`);
         log.info(`Health: http://localhost:${port}/health  Fetch: /v1/fetch  Search: /v1/search`);
+        // Start cache warmer only when opted-in
+        if (process.env.ENABLE_CACHE_WARM === 'true') {
+            log.info('Cache warming enabled (ENABLE_CACHE_WARM=true)');
+            startCacheWarmer(warmerPool);
+        }
     });
     // Graceful shutdown
     const shutdown = () => {

package/dist/server/routes/cache-warm.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Cache Pre-Warming Routes
+ *
+ * GET /internal/popular-urls  — Top N URLs fetched in the last 24h (for CF Worker)
+ * GET /internal/cache-status  — Current warmer state (warmed URLs, last run time)
+ *
+ * Both routes are mounted BEFORE auth middleware so they're accessible internally.
+ * /internal/popular-urls is protected by CACHE_WARM_SECRET bearer token when set.
+ *
+ * startCacheWarmer() — server-side self-warming (opt-in via ENABLE_CACHE_WARM=true)
+ */
+import { Router } from 'express';
+import pg from 'pg';
+export declare function createCacheWarmRouter(pool: pg.Pool | null): Router;
+/**
+ * startCacheWarmer — server-side self-warming (fallback when no CF Worker).
+ *
+ * Every `intervalMs` (default 2 min):
+ *  1. Queries /internal/popular-urls (via the DB, not HTTP)
+ *  2. Fetches each URL through /r/<url> with concurrency 5
+ *  3. Updates warmerState for /internal/cache-status
+ *
+ * Only started if ENABLE_CACHE_WARM=true.
+ */
+export declare function startCacheWarmer(pool: pg.Pool | null, intervalMs?: number): void;

package/dist/server/routes/cache-warm.js

@@ -0,0 +1,210 @@
+/**
+ * Cache Pre-Warming Routes
+ *
+ * GET /internal/popular-urls  — Top N URLs fetched in the last 24h (for CF Worker)
+ * GET /internal/cache-status  — Current warmer state (warmed URLs, last run time)
+ *
+ * Both routes are mounted BEFORE auth middleware so they're accessible internally.
+ * /internal/popular-urls is protected by CACHE_WARM_SECRET bearer token when set.
+ *
+ * startCacheWarmer() — server-side self-warming (opt-in via ENABLE_CACHE_WARM=true)
+ */
+import { Router } from 'express';
+import { createLogger } from '../logger.js';
+const log = createLogger('cache-warm');
+// ─── Fallback URLs ────────────────────────────────────────────────────────────
+// Used when the DB has no usage data yet (new deployment / empty DB).
+const FALLBACK_URLS = [
+    'https://www.bbc.com/news',
+    'https://news.ycombinator.com',
+    'https://github.com',
+    'https://en.wikipedia.org/wiki/Main_Page',
+    'https://www.reuters.com',
+    'https://techcrunch.com',
+    'https://stripe.com/docs',
+    'https://developer.mozilla.org',
+    'https://react.dev',
+    'https://docs.python.org/3/',
+    'https://nodejs.org/en/docs',
+    'https://www.npmjs.com',
+    'https://vercel.com/docs',
+    'https://nextjs.org/docs',
+    'https://tailwindcss.com/docs',
+    'https://www.typescriptlang.org/docs/',
+    'https://docs.render.com',
+    'https://cloudflare.com/docs',
+    'https://aws.amazon.com/documentation/',
+    'https://docs.github.com',
+    'https://www.nytimes.com',
+    'https://www.theguardian.com',
+    'https://arstechnica.com',
+    'https://www.wired.com',
+    'https://www.bloomberg.com/technology',
+    'https://lobste.rs',
+    'https://www.producthunt.com',
+    'https://stackoverflow.com',
+    'https://css-tricks.com',
+    'https://web.dev',
+];
+// ─── In-memory warmer state ───────────────────────────────────────────────────
+const warmerState = {
+    warmedUrls: new Set(),
+    lastWarmTime: null,
+};
+// ─── Router ──────────────────────────────────────────────────────────────────
+export function createCacheWarmRouter(pool) {
+    const router = Router();
+    // GET /internal/popular-urls
+    router.get('/internal/popular-urls', async (req, res) => {
+        // Auth check — if CACHE_WARM_SECRET is set, require it
+        const secret = process.env.CACHE_WARM_SECRET;
+        if (secret) {
+            const authHeader = req.headers['authorization'] || '';
+            const token = authHeader.startsWith('Bearer ') ? authHeader.slice(7) : '';
+            if (token !== secret) {
+                res.status(401).json({
+                    success: false,
+                    error: { type: 'unauthorized', message: 'Invalid or missing bearer token.' },
+                });
+                return;
+            }
+        }
+        let urls = [];
+        // Query DB if available
+        if (pool) {
+            try {
+                const result = await pool.query(`
+          SELECT url, COUNT(*) as fetch_count
+          FROM usage_logs
+          WHERE created_at > NOW() - INTERVAL '24 hours'
+            AND url IS NOT NULL
+            AND status_code >= 200 AND status_code < 300
+            AND url NOT LIKE '%localhost%'
+            AND url NOT LIKE '%127.0.0.1%'
+            AND url NOT LIKE '%169.254%'
+          GROUP BY url
+          ORDER BY fetch_count DESC
+          LIMIT 50
+        `);
+                urls = result.rows.map((row) => ({
+                    url: row.url,
+                    count: parseInt(row.fetch_count, 10),
+                }));
+            }
+            catch (err) {
+                log.warn('Failed to query usage_logs, falling back to static list', {
+                    error: err?.message,
+                });
+            }
+        }
+        // Fall back to static list if no DB data
+        if (urls.length === 0) {
+            log.info('No usage data found, using fallback URL list');
+            urls = FALLBACK_URLS.map((url) => ({ url, count: 0 }));
+        }
+        res.json({
+            urls,
+            total: urls.length,
+            generatedAt: new Date().toISOString(),
+        });
+    });
+    // GET /internal/cache-status
+    router.get('/internal/cache-status', (_req, res) => {
+        res.json({
+            warmedUrls: Array.from(warmerState.warmedUrls),
+            urlCount: warmerState.warmedUrls.size,
+            lastWarmTime: warmerState.lastWarmTime?.toISOString() ?? null,
+        });
+    });
+    return router;
+}
+// ─── Self-Warming ─────────────────────────────────────────────────────────────
+/**
+ * startCacheWarmer — server-side self-warming (fallback when no CF Worker).
+ *
+ * Every `intervalMs` (default 2 min):
+ *  1. Queries /internal/popular-urls (via the DB, not HTTP)
+ *  2. Fetches each URL through /r/<url> with concurrency 5
+ *  3. Updates warmerState for /internal/cache-status
+ *
+ * Only started if ENABLE_CACHE_WARM=true.
+ */
+export function startCacheWarmer(pool, intervalMs = 120_000) {
+    log.info('Cache warmer started', { intervalMs });
+    const runWarm = async () => {
+        const t0 = Date.now();
+        log.info('Cache warm cycle starting');
+        // Determine base URL
+        const base = process.env.RENDER_EXTERNAL_URL?.replace(/\/$/, '') ||
+            `http://localhost:${process.env.PORT || 3000}`;
+        // Step 1: Fetch popular URLs (same logic as the endpoint)
+        let urls = [];
+        if (pool) {
+            try {
+                const result = await pool.query(`
+          SELECT url, COUNT(*) as fetch_count
+          FROM usage_logs
+          WHERE created_at > NOW() - INTERVAL '24 hours'
+            AND url IS NOT NULL
+            AND status_code >= 200 AND status_code < 300
+            AND url NOT LIKE '%localhost%'
+            AND url NOT LIKE '%127.0.0.1%'
+            AND url NOT LIKE '%169.254%'
+          GROUP BY url
+          ORDER BY fetch_count DESC
+          LIMIT 50
+        `);
+                urls = result.rows.map((row) => ({
+                    url: row.url,
+                    count: parseInt(row.fetch_count, 10),
+                }));
+            }
+            catch (err) {
+                log.warn('Warm cycle: DB query failed, using fallback', { error: err?.message });
+            }
+        }
+        if (urls.length === 0) {
+            urls = FALLBACK_URLS.map((u) => ({ url: u, count: 0 }));
+        }
+        // Step 2: Warm each URL with concurrency 5
+        const concurrency = 5;
+        let warmed = 0;
+        let failed = 0;
+        const newWarmedSet = new Set();
+        for (let i = 0; i < urls.length; i += concurrency) {
+            const batch = urls.slice(i, i + concurrency);
+            const results = await Promise.allSettled(batch.map(({ url }) => fetch(`${base}/r/${encodeURIComponent(url)}`, {
+                headers: { 'User-Agent': 'WebPeel-CacheWarmer/1.0' },
+                signal: AbortSignal.timeout(15_000),
+            }).then((r) => {
+                if (r.ok) {
+                    newWarmedSet.add(url);
+                    warmed++;
+                }
+                else {
+                    failed++;
+                }
+            })));
+            // Count settled rejections as failures
+            results.forEach((r) => {
+                if (r.status === 'rejected') {
+                    failed++;
+                }
+            });
+        }
+        // Step 3: Update state
+        warmerState.warmedUrls = newWarmedSet;
+        warmerState.lastWarmTime = new Date();
+        const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
+        log.info(`Warmed ${warmed}/${urls.length} URLs in ${elapsed}s`, { failed });
+    };
+    // Run once immediately, then on interval
+    void runWarm().catch((err) => {
+        log.error('Cache warm cycle error', { error: err?.message });
+    });
+    setInterval(() => {
+        void runWarm().catch((err) => {
+            log.error('Cache warm cycle error', { error: err?.message });
+        });
+    }, intervalMs);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.1",
+  "version": "0.21.2",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",