webpeel 0.20.2 → 0.20.3

package/dist/server/routes/fetch.js

@@ -0,0 +1,999 @@
+/**
+ * Fetch endpoint with caching
+ */
+import { Router } from 'express';
+import '../types.js'; // Augments Express.Request with requestId
+import { peel } from '../../index.js';
+import { normalizeActions } from '../../core/actions.js';
+import { extractInlineJson } from '../../core/extract-inline.js';
+import { LRUCache } from 'lru-cache';
+import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
+import { wantsEnvelope, successResponse } from '../utils/response.js';
+import { getSchemaTemplate } from '../../core/schema-templates.js';
+import { quickAnswer } from '../../core/quick-answer.js';
+import { sendUsageAlertEmail } from '../email-service.js';
+// ── Helper: extractive summarizer (TF-IDF-like sentence scoring) ─────────────
+function extractSummary(content, maxWords = 150) {
+    if (!content)
+        return '';
+    const sentences = content
+        .split(/(?<=[.!?])\s+/)
+        .map(s => s.trim())
+        .filter(s => s.length > 40 && s.length < 600);
+    if (sentences.length === 0) {
+        const words = content.split(/\s+/);
+        return words.slice(0, maxWords).join(' ') + (words.length > maxWords ? '\u2026' : '');
+    }
+    if (sentences.length <= 3)
+        return sentences.join(' ');
+    const allWords = content.toLowerCase().split(/\W+/).filter(w => w.length > 3);
+    const wordFreq = {};
+    for (const w of allWords)
+        wordFreq[w] = (wordFreq[w] || 0) + 1;
+    const maxFreq = Math.max(1, ...Object.values(wordFreq));
+    const scored = sentences.map((sentence, idx) => {
+        const words = sentence.toLowerCase().split(/\W+/).filter(w => w.length > 3);
+        const score = words.reduce((sum, w) => sum + (wordFreq[w] || 0) / maxFreq, 0) / Math.max(1, words.length);
+        const posBonus = idx === 0 ? 0.3 : idx === sentences.length - 1 ? 0.1 : 0;
+        return { sentence, score: score + posBonus, idx };
+    });
+    scored.sort((a, b) => b.score - a.score);
+    const selected = [];
+    let wc = 0;
+    for (const item of scored) {
+        const itemWc = item.sentence.split(/\s+/).length;
+        if (wc + itemWc > maxWords * 1.3)
+            break;
+        selected.push(item);
+        wc += itemWc;
+        if (selected.length >= 5)
+            break;
+    }
+    selected.sort((a, b) => a.idx - b.idx);
+    return selected.map(s => s.sentence).join(' ');
+}
+// ── Helper: check usage and determine if alert email should be sent ───────────
+async function checkAndTriggerAlert(pgStore, userId) {
+    const getCurrentWeek = () => {
+        const now = new Date();
+        const year = now.getUTCFullYear();
+        const jan4 = new Date(Date.UTC(year, 0, 4));
+        const weekNum = Math.ceil(((now.getTime() - jan4.getTime()) / 86400000 + jan4.getUTCDay() + 1) / 7);
+        return `${year}-W${String(weekNum).padStart(2, '0')}`;
+    };
+    const currentWeek = getCurrentWeek();
+    const result = await pgStore.pool.query(`SELECT u.email, u.name, u.tier, u.alert_threshold, u.alert_email, u.alert_sent_at,
+            u.weekly_limit,
+            COALESCE(SUM(wu.total_count), 0) AS total_used,
+            u.weekly_limit + COALESCE(MAX(wu.rollover_credits), 0) AS total_available
+     FROM users u
+     LEFT JOIN api_keys ak ON ak.user_id = u.id
+     LEFT JOIN weekly_usage wu ON wu.api_key_id = ak.id AND wu.week = $2
+     WHERE u.id = $1
+     GROUP BY u.id, u.email, u.name, u.tier, u.alert_threshold, u.alert_email, u.alert_sent_at, u.weekly_limit`, [userId, currentWeek]);
+    const row = result.rows[0];
+    if (!row || !row.alert_threshold)
+        return { shouldSendAlert: false };
+    const used = parseInt(row.total_used, 10) || 0;
+    const total = parseInt(row.total_available, 10) || row.weekly_limit || 999;
+    const usagePercent = total > 0 ? Math.round((used / total) * 100) : 0;
+    // Only alert if: crosses threshold AND haven't sent alert this week
+    const lastAlert = row.alert_sent_at ? new Date(row.alert_sent_at) : null;
+    const oneWeekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
+    const alreadySentThisWeek = lastAlert !== null && lastAlert > oneWeekAgo;
+    return {
+        shouldSendAlert: usagePercent >= row.alert_threshold && !alreadySentThisWeek,
+        usagePercent,
+        used,
+        total,
+        userEmail: row.email,
+        userName: row.name || undefined,
+        userTier: row.tier,
+        alertEmail: row.alert_email || undefined,
+    };
+}
+const VALID_LLM_PROVIDERS = ['openai', 'anthropic', 'google'];
+export function createFetchRouter(authStore) {
+    const router = Router();
+    // LRU cache: 5 minute TTL, max 500 entries, 100MB total size
+    const cache = new LRUCache({
+        max: 500,
+        ttl: 5 * 60 * 1000, // 5 minutes default
+        maxSize: 100 * 1024 * 1024, // 100MB
+        sizeCalculation: (entry) => {
+            return JSON.stringify(entry).length;
+        },
+    });
+    router.get('/v1/fetch', async (req, res) => {
+        try {
+            // Require authentication — API key or JWT session
+            const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
+            if (!userId) {
+                res.status(401).json({
+                    success: false,
+                    error: {
+                        type: 'unauthorized',
+                        message: 'API key required. Get one free at https://app.webpeel.dev/keys',
+                        hint: 'Get a free API key at https://app.webpeel.dev/keys',
+                        docs: 'https://webpeel.dev/docs/errors#unauthorized',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const { url, render, wait, format, includeTags, excludeTags, images, location, languages, onlyMainContent, actions, maxAge, storeInCache, stream, noCache, cacheTtl, budget, question, summary, readable, stealth, screenshot, maxTokens, selector, exclude, fullPage, raw, lite, timeout, schema, detail, } = req.query;
+            const detailMode = detail || 'standard';
+            // Validate URL parameter
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Missing or invalid "url" parameter.',
+                        hint: 'Pass a URL as a query parameter: GET /v1/fetch?url=https://example.com',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // SECURITY: Validate URL format and length
+            if (url.length > 2048) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_url',
+                        message: 'URL too long (max 2048 characters)',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            try {
+                const parsed = new URL(url);
+                // Normalize URL for consistent caching
+                const normalizedUrl = parsed.href;
+                // Use normalized URL for cache key
+                if (normalizedUrl !== url) {
+                    // URL was normalized, update for caching
+                }
+            }
+            catch {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_url',
+                        message: 'Invalid URL format',
+                        hint: 'Ensure the URL includes a scheme (https://) and a valid hostname',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // SECURITY: Validate URL to prevent SSRF attacks
+            try {
+                validateUrlForSSRF(url);
+            }
+            catch (error) {
+                if (error instanceof SSRFError) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'forbidden_url',
+                            message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
+                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+                throw error;
+            }
+            // Parse actions query param (JSON-encoded array)
+            let parsedActions;
+            if (actions && typeof actions === 'string') {
+                try {
+                    const raw = JSON.parse(actions);
+                    parsedActions = normalizeActions(raw);
+                }
+                catch (e) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_request',
+                            message: 'Invalid "actions" parameter: must be a valid JSON array',
+                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+            }
+            // Build cache key (include new parameters)
+            const actionsKey = parsedActions ? JSON.stringify(parsedActions) : '';
+            const cacheKey = `fetch:${url}:${render}:${wait}:${format}:${includeTags}:${excludeTags}:${images}:${location}:${languages}:${onlyMainContent}:${stream}:${actionsKey}:${budget}:${question}:${summary}:${readable}:${stealth}:${screenshot}:${maxTokens}:${selector}:${exclude}:${fullPage}:${raw}`;
+            // Cache bypass: ?noCache=true or Cache-Control: no-cache header
+            const bypassCache = noCache === 'true' || req.headers['cache-control'] === 'no-cache';
+            // Per-request TTL (cacheTtl in seconds, default 300s = 5 min)
+            const cacheTtlMs = cacheTtl !== undefined
+                ? parseInt(cacheTtl, 10) * 1000
+                : 5 * 60 * 1000;
+            // Check cache (with maxAge support)
+            const maxAgeMs = maxAge !== undefined ? parseInt(maxAge, 10) : 172800000; // Default 2 days
+            if (!bypassCache) {
+                const cached = cache.get(cacheKey);
+                if (cached && maxAgeMs > 0) {
+                    const cacheAge = Date.now() - cached.timestamp;
+                    if (cacheAge < maxAgeMs && cacheAge < cacheTtlMs) {
+                        res.setHeader('X-Cache', 'HIT');
+                        res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
+                        if (wantsEnvelope(req)) {
+                            successResponse(res, cached.result, {
+                                requestId: req.requestId,
+                                cached: true,
+                            });
+                        }
+                        else {
+                            res.json(cached.result);
+                        }
+                        return;
+                    }
+                }
+            }
+            // Parse options
+            const isSoftLimited = req.auth?.softLimited === true;
+            const hasExtraUsage = req.auth?.extraUsageAvailable === true;
+            // Parse tag arrays from comma-separated strings
+            const includeTagsArray = includeTags
+                ? includeTags.split(',').map(t => t.trim()).filter(Boolean)
+                : undefined;
+            const excludeTagsArray = excludeTags
+                ? excludeTags.split(',').map(t => t.trim()).filter(Boolean)
+                : undefined;
+            const languagesArray = languages
+                ? languages.split(',').map(l => l.trim()).filter(Boolean)
+                : undefined;
+            // onlyMainContent is a shortcut for common include tags
+            const finalIncludeTags = onlyMainContent === 'true'
+                ? ['main', 'article', '.content', '#content']
+                : includeTagsArray;
+            // When actions are present, force browser mode (skip HTTP fast path)
+            const hasActions = parsedActions && parsedActions.length > 0;
+            const shouldRender = hasActions || render === 'true';
+            const options = {
+                // SOFT LIMIT: When over quota AND no extra usage, force HTTP-only
+                // If extra usage is available, allow full functionality
+                // Exception: actions always require render
+                render: (isSoftLimited && !hasExtraUsage && !hasActions) ? false : shouldRender,
+                wait: (isSoftLimited && !hasExtraUsage) ? 0 : (wait ? parseInt(wait, 10) : undefined),
+                format: format || 'markdown',
+                stream: stream === 'true',
+                includeTags: finalIncludeTags,
+                excludeTags: excludeTagsArray,
+                images: images === 'true',
+                actions: parsedActions,
+                location: location || languagesArray ? {
+                    country: location,
+                    languages: languagesArray,
+                } : undefined,
+                budget: budget ? parseInt(budget, 10) : undefined,
+                question: question,
+                readable: readable === 'true',
+                stealth: (isSoftLimited && !hasExtraUsage) ? false : stealth === 'true',
+                screenshot: (isSoftLimited && !hasExtraUsage) ? false : screenshot === 'true',
+                maxTokens: maxTokens ? parseInt(maxTokens, 10) : undefined,
+                selector: selector,
+                exclude: exclude ? exclude.split(',').map(s => s.trim()).filter(Boolean) : undefined,
+                fullPage: fullPage === 'true',
+                raw: raw === 'true',
+                lite: lite === 'true',
+                timeout: timeout ? parseInt(timeout, 10) : undefined,
+            };
+            // Auto-budget: default to 4000 tokens for API requests when no budget specified
+            // Opt-out: budget=0 explicitly disables. Lite mode disables auto-budget.
+            if (options.budget === undefined && !options.lite) {
+                options.budget = 4000;
+                res.setHeader('X-Auto-Budget', '4000');
+            }
+            // Inform the user if their request was degraded
+            if (isSoftLimited && !hasExtraUsage && render === 'true' && !hasActions) {
+                res.setHeader('X-Degraded', 'render=true downgraded to HTTP-only (quota exceeded)');
+            }
+            if (isSoftLimited && !hasExtraUsage && stealth === 'true') {
+                res.setHeader('X-Degraded', 'stealth=true downgraded (quota exceeded)');
+            }
+            if (isSoftLimited && !hasExtraUsage && screenshot === 'true') {
+                res.setHeader('X-Degraded', 'screenshot=true downgraded (quota exceeded)');
+            }
+            // Validate wait parameter
+            if (options.wait !== undefined && (isNaN(options.wait) || options.wait < 0 || options.wait > 60000)) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // Validate format parameter
+            if (!['markdown', 'text', 'html', 'clean'].includes(options.format || '')) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Invalid "format" parameter: must be "markdown", "text", "html", or "clean"',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const shouldStream = options.stream === true;
+            if (shouldStream) {
+                res.setHeader('X-Stream', 'true');
+                if (typeof res.flushHeaders === 'function') {
+                    res.flushHeaders();
+                }
+            }
+            // Fetch content
+            const startTime = Date.now();
+            const result = await peel(url, options);
+            const elapsed = Date.now() - startTime;
+            // --- BM25 Schema Template Extraction (GET, no LLM needed) ---
+            if (schema && typeof schema === 'string' && result.content) {
+                const template = getSchemaTemplate(schema);
+                if (template) {
+                    const { quickAnswer } = await import('../../core/quick-answer.js');
+                    const { smartExtractSchemaFields } = await import('../../core/schema-postprocess.js');
+                    const extracted = smartExtractSchemaFields(result.content, template.fields, quickAnswer, {
+                        pageTitle: result.title,
+                        pageUrl: result.url,
+                        metadata: result.metadata,
+                    });
+                    result.extracted = extracted;
+                }
+            }
+            // Determine fetch type from the result method
+            const fetchType = result.method === 'stealth' ? 'stealth' :
+                result.method === 'browser' ? 'stealth' : 'basic';
+            // Log request to database (PostgreSQL only)
+            const pgStore = authStore;
+            // Log usage for BOTH API key auth AND JWT session auth
+            const logUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
+            if (logUserId && typeof pgStore.pool !== 'undefined') {
+                pgStore.pool.query(`INSERT INTO usage_logs 
+            (user_id, endpoint, url, method, processing_time_ms, status_code, ip_address, user_agent, tokens_used)
+          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
+                    logUserId,
+                    'fetch',
+                    url,
+                    fetchType,
+                    elapsed,
+                    200,
+                    req.ip || req.socket.remoteAddress,
+                    req.get('user-agent'),
+                    result?.tokens || null,
+                ]).catch((err) => {
+                    console.error('Failed to log request to usage_logs:', err);
+                });
+            }
+            // Track usage (check for trackBurstUsage method to detect PostgresAuthStore)
+            if (req.auth?.keyInfo?.key && typeof pgStore.trackBurstUsage === 'function') {
+                // Track burst usage (always)
+                await pgStore.trackBurstUsage(req.auth.keyInfo.key);
+                // If soft-limited with extra usage available, charge to extra usage
+                if (isSoftLimited && hasExtraUsage) {
+                    const extraResult = await pgStore.trackExtraUsage(req.auth.keyInfo.key, fetchType, url, elapsed, 200 // PeelResult doesn't include statusCode, assume success
+                    );
+                    if (extraResult.success) {
+                        res.setHeader('X-Extra-Usage-Charged', `$${extraResult.cost.toFixed(4)}`);
+                        res.setHeader('X-Extra-Usage-New-Balance', extraResult.newBalance.toFixed(2));
+                    }
+                    else {
+                        // Extra usage failed - fall back to soft limit
+                        res.setHeader('X-Degraded', 'Extra usage insufficient, degraded to soft limit');
+                    }
+                }
+                else if (!isSoftLimited) {
+                    // Normal weekly usage tracking
+                    await pgStore.trackUsage(req.auth.keyInfo.key, fetchType);
+                }
+                // If soft-limited WITHOUT extra usage, don't track (already over quota)
+            }
+            // Check usage alert (fire-and-forget, never block the response)
+            if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
+                try {
+                    const alertResult = await checkAndTriggerAlert(pgStore, req.auth.keyInfo.accountId);
+                    if (alertResult.shouldSendAlert && alertResult.usagePercent !== undefined) {
+                        await sendUsageAlertEmail({
+                            toEmail: alertResult.alertEmail || alertResult.userEmail,
+                            userName: alertResult.userName,
+                            usagePercent: alertResult.usagePercent,
+                            used: alertResult.used,
+                            total: alertResult.total,
+                            tier: alertResult.userTier,
+                        });
+                        // Mark alert as sent so we don't spam (rate-limited to once/week)
+                        await pgStore.pool.query('UPDATE users SET alert_sent_at = NOW() WHERE id = $1', [req.auth.keyInfo.accountId]);
+                    }
+                }
+                catch (alertErr) {
+                    // Never let alert errors affect the main response
+                    console.warn('[alert] Failed to check/send alert:', alertErr);
+                }
+            }
+            // Cache result (unless storeInCache is explicitly false or cache bypass requested)
+            if (storeInCache !== 'false' && !bypassCache) {
+                cache.set(cacheKey, {
+                    result,
+                    timestamp: Date.now(),
+                }, { ttl: cacheTtlMs });
+            }
+            // Apply ?detail=brief mode: truncate content and prepend TL;DR
+            if (detailMode === 'brief' && result.content) {
+                const words = result.content.split(/\s+/);
+                const truncatedWords = words.slice(0, 500);
+                const truncated = truncatedWords.join(' ');
+                // Extract TL;DR from first non-empty paragraph
+                const firstPara = result.content
+                    .split(/\n{2,}/)
+                    .map((p) => p.replace(/^#+\s*/, '').trim())
+                    .find((p) => p.length > 40 && !p.startsWith('!') && !p.startsWith('['));
+                const tldr = firstPara
+                    ? firstPara.replace(/\s+/g, ' ').slice(0, 300) + (firstPara.length > 300 ? '...' : '')
+                    : truncated.slice(0, 200) + '...';
+                result.content = `**TL;DR:** ${tldr}\n\n---\n\n${truncated}${words.length > 500 ? '\n\n*[Content truncated — use ?detail=full for complete output]*' : ''}`;
+                const tokenEstimate = Math.round(truncatedWords.length * 0.75);
+                res.setHeader('X-Detail-Mode', 'brief');
+                res.setHeader('X-Token-Estimate', tokenEstimate.toString());
+            }
+            // --- question → answer field (GET) ---
+            // When ?question= is provided, run quickAnswer() on the fetched content
+            // and expose the result as an `answer` field in the response.
+            const getAnswerResult = (question && typeof question === 'string' && result.content)
+                ? quickAnswer({ question, content: result.content, url: result.url })
+                : undefined;
+            // --- summary field (GET) ---
+            // When ?summary=true, return a truncated 500-word summary in a `summary` field.
+            const getSummaryText = (summary === 'true' && result.content)
+                ? extractSummary(result.content)
+                : undefined;
+            // Add usage headers (kept for backward compat; also surfaced in envelope metadata)
+            res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Credits-Used', '1');
+            res.setHeader('X-Processing-Time', elapsed.toString());
+            res.setHeader('X-Fetch-Type', fetchType);
+            // Build response — extend result with optional answer/summary fields
+            const getResponseBody = { ...result };
+            if (getAnswerResult !== undefined)
+                getResponseBody.answer = getAnswerResult.answer;
+            if (getSummaryText !== undefined)
+                getResponseBody.summary = getSummaryText;
+            if (wantsEnvelope(req)) {
+                successResponse(res, getResponseBody, {
+                    requestId: req.requestId,
+                    processingTimeMs: elapsed,
+                    creditsUsed: 1,
+                    cached: false,
+                    fetchType,
+                });
+            }
+            else {
+                res.json(getResponseBody);
+            }
+        }
+        catch (error) {
+            const err = error;
+            // Log error to database (PostgreSQL only)
+            const pgStore = authStore;
+            if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
+                const url = req.query.url;
+                const render = req.query.render === 'true';
+                const fetchType = render ? 'stealth' : 'basic';
+                pgStore.pool.query(`INSERT INTO usage_logs 
+            (user_id, endpoint, url, method, status_code, error, ip_address, user_agent, tokens_used)
+          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
+                    req.auth.keyInfo.accountId,
+                    'fetch',
+                    url,
+                    fetchType,
+                    500,
+                    err.message || 'Unknown error',
+                    req.ip || req.socket.remoteAddress,
+                    req.get('user-agent'),
+                    null,
+                ]).catch((logErr) => {
+                    console.error('Failed to log error to usage_logs:', logErr);
+                });
+            }
+            // SECURITY: Sanitize error messages to prevent information disclosure
+            if (err.code) {
+                // WebPeelError from core library - safe to expose with helpful context
+                const safeMessage = err.message.replace(/[<>"']/g, ''); // Remove HTML chars
+                const statusCode = err.code === 'TIMEOUT' ? 504
+                    : err.code === 'BLOCKED' ? 403
+                        : err.code === 'NETWORK' ? 502
+                            : 500;
+                const hints = {
+                    TIMEOUT: 'Try increasing timeout with ?wait=10000, or use render=true for JS-heavy sites.',
+                    BLOCKED: 'This site blocks automated requests. Try adding render=true or use stealth mode (costs 5 credits).',
+                    NETWORK: 'Could not reach the target URL. Verify the URL is correct and the site is online.',
+                };
+                res.status(statusCode).json({
+                    success: false,
+                    error: {
+                        type: err.code,
+                        message: safeMessage,
+                        hint: hints[err.code] || undefined,
+                        docs: 'https://webpeel.dev/docs/api-reference#errors',
+                    },
+                    requestId: req.requestId,
+                });
+            }
+            else {
+                // Unexpected error - generic message only
+                console.error('Fetch error:', err); // Log full error server-side
+                res.status(500).json({
+                    success: false,
+                    error: {
+                        type: 'internal_error',
+                        message: 'An unexpected error occurred while fetching the URL. If this persists, check https://webpeel.dev/status',
+                        docs: 'https://webpeel.dev/docs/api-reference#errors',
+                    },
+                    requestId: req.requestId,
+                });
+            }
+        }
+    });
+    // -----------------------------------------------------------------------
+    // POST /v1/fetch — same as GET but accepts JSON body with extract param
+    // POST /v2/scrape — alias with identical behaviour
+    // -----------------------------------------------------------------------
+    async function handlePostFetch(req, res) {
+        try {
+            // Require authentication — API key or JWT session
+            const postUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
+            if (!postUserId) {
+                res.status(401).json({
+                    success: false,
+                    error: {
+                        type: 'unauthorized',
+                        message: 'API key required. Get one free at https://app.webpeel.dev/keys',
+                        hint: 'Get a free API key at https://app.webpeel.dev/keys',
+                        docs: 'https://webpeel.dev/docs/errors#unauthorized',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const { url, render, wait, format, includeTags, excludeTags, images, location, languages, onlyMainContent, actions: rawActions, storeInCache: storeFlag, 
+            // Cache control
+            noCache: noCacheBody, cacheTtl: cacheTtlBody, 
+            // Inline extraction (BYOK)
+            extract, llmProvider, llmApiKey, llmModel, 
+            // Firecrawl-compatible formats array
+            formats, stream, 
+            // Extended peel options
+            budget, question, summary: summaryParam, readable, stealth, screenshot, maxTokens, selector, exclude, fullPage, raw, lite, timeout, proxies, chunk, device, viewportWidth, viewportHeight, waitUntil, waitSelector, blockResources, cloaked, schema: bodySchema, } = req.body;
+            // --- Validate URL -------------------------------------------------------
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Missing or invalid "url" in request body.',
+                        hint: 'Send JSON: { "url": "https://example.com" }',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            if (url.length > 2048) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_url',
+                        message: 'URL too long (max 2048 characters)',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            try {
+                new URL(url);
+            }
+            catch {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_url',
+                        message: 'Invalid URL format',
+                        hint: 'Ensure the URL includes a scheme (https://) and a valid hostname',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            try {
+                validateUrlForSSRF(url);
+            }
+            catch (error) {
+                if (error instanceof SSRFError) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'forbidden_url',
+                            message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
+                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+                throw error;
+            }
+            // --- Parse and normalize actions -----------------------------------------
+            let postActions;
+            if (rawActions !== undefined) {
+                try {
+                    postActions = normalizeActions(rawActions);
+                }
+                catch (e) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_request',
+                            message: `Invalid "actions" parameter: ${e.message}`,
+                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+            }
+            // --- Cache bypass and lookup -------------------------------------------
+            const postBypassCache = noCacheBody === true || req.headers['cache-control'] === 'no-cache';
+            const postCacheTtlMs = typeof cacheTtlBody === 'number' ? cacheTtlBody * 1000 : 5 * 60 * 1000;
+            const postActionsKey = postActions ? JSON.stringify(postActions) : '';
+            const postCacheKey = `fetch:${url}:${render}:${wait}:${format}:${JSON.stringify(includeTags)}:${JSON.stringify(excludeTags)}:${images}:${location}:${JSON.stringify(languages)}:${onlyMainContent}:${stream}:${postActionsKey}:${budget}:${question}:${summaryParam}:${readable}:${stealth}:${screenshot}:${maxTokens}:${selector}:${JSON.stringify(exclude)}:${fullPage}:${raw}`;
+            if (!postBypassCache && !extract) {
+                const cached = cache.get(postCacheKey);
+                if (cached) {
+                    const cacheAge = Date.now() - cached.timestamp;
+                    if (cacheAge < postCacheTtlMs) {
+                        res.setHeader('X-Cache', 'HIT');
+                        res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
+                        if (wantsEnvelope(req)) {
+                            successResponse(res, cached.result, {
+                                requestId: req.requestId,
+                                cached: true,
+                            });
+                        }
+                        else {
+                            res.json(cached.result);
+                        }
+                        return;
+                    }
+                }
+            }
+            // --- Resolve inline extract from body or Firecrawl-compatible formats ---
+            let resolvedExtract = extract;
+            if (!resolvedExtract && Array.isArray(formats)) {
+                const jsonFormat = formats.find((f) => (typeof f === 'object' && f !== null && f.type === 'json') ||
+                    (typeof f === 'string' && f === 'json'));
+                if (jsonFormat && typeof jsonFormat === 'object' && (jsonFormat.schema || jsonFormat.prompt)) {
+                    resolvedExtract = {
+                        schema: jsonFormat.schema,
+                        prompt: jsonFormat.prompt,
+                    };
+                }
+            }
+            // Resolve schema template names (e.g. "product", "article") to field objects
+            if (resolvedExtract && typeof resolvedExtract.schema === 'string') {
+                const tmpl = getSchemaTemplate(resolvedExtract.schema);
+                if (tmpl) {
+                    resolvedExtract = { ...resolvedExtract, schema: tmpl.fields };
+                }
+                else {
+                    // Try parsing as JSON string
+                    try {
+                        resolvedExtract = { ...resolvedExtract, schema: JSON.parse(resolvedExtract.schema) };
+                    }
+                    catch { /* leave as-is */ }
+                }
+            }
+            // Validate LLM params if extraction is requested
+            if (resolvedExtract && (resolvedExtract.schema || resolvedExtract.prompt)) {
+                if (!llmProvider || !VALID_LLM_PROVIDERS.includes(llmProvider)) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_request',
+                            message: `"llmProvider" is required for inline extraction and must be one of: ${VALID_LLM_PROVIDERS.join(', ')}`,
+                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+                if (!llmApiKey || typeof llmApiKey !== 'string' || llmApiKey.trim().length === 0) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_request',
+                            message: 'Missing or invalid "llmApiKey" (BYOK required for inline extraction)',
+                            hint: 'Pass your LLM provider API key in the "llmApiKey" field',
+                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                        },
+                        requestId: req.requestId,
+                    });
+                    return;
+                }
+            }
+            // --- Build PeelOptions ---------------------------------------------------
+            const isSoftLimited = req.auth?.softLimited === true;
+            const hasExtraUsage = req.auth?.extraUsageAvailable === true;
+            const includeTagsArray = Array.isArray(includeTags) ? includeTags : undefined;
+            const excludeTagsArray = Array.isArray(excludeTags) ? excludeTags : undefined;
+            const languagesArray = Array.isArray(languages) ? languages : undefined;
+            const finalIncludeTags = onlyMainContent === true
+                ? ['main', 'article', '.content', '#content']
+                : includeTagsArray;
+            const resolvedFormat = format || 'markdown';
+            if (!['markdown', 'text', 'html', 'clean'].includes(resolvedFormat)) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Invalid "format" parameter: must be "markdown", "text", "html", or "clean"',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            const resolvedWait = typeof wait === 'number' ? wait : undefined;
+            if (resolvedWait !== undefined && (isNaN(resolvedWait) || resolvedWait < 0 || resolvedWait > 60000)) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
+                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // When actions are present, force browser mode
+            const postHasActions = postActions && postActions.length > 0;
+            const postShouldRender = postHasActions || render === true;
+            // Normalize exclude: accept string (comma-separated) or string array
+            const excludeArray = exclude
+                ? (Array.isArray(exclude) ? exclude : exclude.split(',').map(s => s.trim()).filter(Boolean))
+                : undefined;
+            const options = {
+                render: (isSoftLimited && !hasExtraUsage && !postHasActions) ? false : postShouldRender,
+                wait: (isSoftLimited && !hasExtraUsage) ? 0 : resolvedWait,
+                format: resolvedFormat,
+                stream: stream === true,
+                includeTags: finalIncludeTags,
+                excludeTags: excludeTagsArray,
+                images: images === true,
+                actions: postActions,
+                location: location || languagesArray ? {
+                    country: location,
+                    languages: languagesArray,
+                } : undefined,
+                budget: typeof budget === 'number' ? budget : undefined,
+                question: question,
+                readable: readable === true,
+                stealth: (isSoftLimited && !hasExtraUsage) ? false : stealth === true,
+                screenshot: (isSoftLimited && !hasExtraUsage) ? false : screenshot === true,
+                maxTokens: typeof maxTokens === 'number' ? maxTokens : undefined,
+                selector: selector,
+                exclude: excludeArray,
+                fullPage: fullPage === true,
+                raw: raw === true,
+                lite: lite === true,
+                timeout: typeof timeout === 'number' ? timeout : undefined,
+                proxies: Array.isArray(proxies) ? proxies : undefined,
+                device: device,
+                viewportWidth: typeof viewportWidth === 'number' ? viewportWidth : undefined,
+                viewportHeight: typeof viewportHeight === 'number' ? viewportHeight : undefined,
+                waitUntil: waitUntil,
+                waitSelector: waitSelector,
+                blockResources: Array.isArray(blockResources) ? blockResources : undefined,
+            };
+            if (cloaked)
+                options.cloaked = cloaked;
+            if (chunk)
+                options.chunk = chunk === true ? true : chunk;
+            // Auto-budget: default to 4000 tokens for API requests when no budget specified
+            // Opt-out: budget=0 explicitly disables. Lite mode disables auto-budget.
+            if (options.budget === undefined && !options.lite) {
+                options.budget = 4000;
+                res.setHeader('X-Auto-Budget', '4000');
+            }
+            if (isSoftLimited && !hasExtraUsage && render === true && !postHasActions) {
+                res.setHeader('X-Degraded', 'render=true downgraded to HTTP-only (quota exceeded)');
+            }
+            if (isSoftLimited && !hasExtraUsage && stealth === true) {
+                res.setHeader('X-Degraded', 'stealth=true downgraded (quota exceeded)');
+            }
+            if (isSoftLimited && !hasExtraUsage && screenshot === true) {
+                res.setHeader('X-Degraded', 'screenshot=true downgraded (quota exceeded)');
+            }
+            const shouldStream = options.stream === true;
+            if (shouldStream) {
+                res.setHeader('X-Stream', 'true');
+                if (typeof res.flushHeaders === 'function') {
+                    res.flushHeaders();
+                }
+            }
+            // --- Fetch content -------------------------------------------------------
+            const startTime = Date.now();
+            const result = await peel(url, options);
+            const elapsed = Date.now() - startTime;
+            // --- BM25 Schema Template Extraction (POST, no LLM needed) ---
+            if (bodySchema && typeof bodySchema === 'string' && result.content) {
+                const template = getSchemaTemplate(bodySchema);
+                if (template) {
+                    const { quickAnswer } = await import('../../core/quick-answer.js');
+                    const { smartExtractSchemaFields } = await import('../../core/schema-postprocess.js');
+                    const extracted = smartExtractSchemaFields(result.content, template.fields, quickAnswer, {
+                        pageTitle: result.title,
+                        pageUrl: result.url,
+                        metadata: result.metadata,
+                    });
+                    result.extracted = extracted;
+                }
+            }
+            // --- Inline extraction (post-fetch) -------------------------------------
+            let jsonData;
+            let extractTokensUsed;
+            if (resolvedExtract && (resolvedExtract.schema || resolvedExtract.prompt) && llmApiKey) {
+                const extractResult = await extractInlineJson(result.content, {
+                    schema: resolvedExtract.schema,
+                    prompt: resolvedExtract.prompt,
+                    llmProvider: llmProvider,
+                    llmApiKey: llmApiKey.trim(),
+                    llmModel,
+                });
+                jsonData = extractResult.data;
+                extractTokensUsed = extractResult.tokensUsed;
+            }
+            // --- Usage tracking (same as GET) ----------------------------------------
+            const fetchType = result.method === 'stealth' ? 'stealth' :
+                result.method === 'browser' ? 'stealth' : 'basic';
+            const pgStore = authStore;
+            if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
+                pgStore.pool.query(`INSERT INTO usage_logs
+            (user_id, endpoint, url, method, processing_time_ms, status_code, ip_address, user_agent, tokens_used)
+          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
+                    req.auth.keyInfo.accountId,
+                    'fetch',
+                    url,
+                    fetchType,
+                    elapsed,
+                    200,
+                    req.ip || req.socket.remoteAddress,
+                    req.get('user-agent'),
+                    result?.tokens || null,
+                ]).catch((err) => {
+                    console.error('Failed to log request to usage_logs:', err);
+                });
+            }
+            if (req.auth?.keyInfo?.key && typeof pgStore.trackBurstUsage === 'function') {
+                await pgStore.trackBurstUsage(req.auth.keyInfo.key);
+                if (isSoftLimited && hasExtraUsage) {
+                    const extraResult = await pgStore.trackExtraUsage(req.auth.keyInfo.key, fetchType, url, elapsed, 200);
+                    if (extraResult.success) {
+                        res.setHeader('X-Extra-Usage-Charged', `$${extraResult.cost.toFixed(4)}`);
+                        res.setHeader('X-Extra-Usage-New-Balance', extraResult.newBalance.toFixed(2));
+                    }
+                    else {
+                        res.setHeader('X-Degraded', 'Extra usage insufficient, degraded to soft limit');
+                    }
+                }
+                else if (!isSoftLimited) {
+                    await pgStore.trackUsage(req.auth.keyInfo.key, fetchType);
+                }
+            }
+            // Cache result (skip extraction results — they depend on user's LLM keys)
+            if (storeFlag !== false && !postBypassCache && !resolvedExtract) {
+                cache.set(postCacheKey, { result, timestamp: Date.now() }, { ttl: postCacheTtlMs });
+            }
+            // --- question → answer field (POST) ---
+            // When question is provided, run quickAnswer() on the fetched content
+            // and expose the result as an `answer` field in the response.
+            const postAnswerResult = (question && typeof question === 'string' && result.content)
+                ? quickAnswer({ question, content: result.content, url: result.url })
+                : undefined;
+            // --- summary field (POST) ---
+            // When summary: true, return a truncated 500-word summary in a `summary` field.
+            const postSummaryText = (summaryParam === true && result.content)
+                ? extractSummary(result.content)
+                : undefined;
+            // --- Build response ------------------------------------------------------
+            // Headers kept for backward compat; also surfaced in envelope metadata.
+            res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Credits-Used', '1');
+            res.setHeader('X-Processing-Time', elapsed.toString());
+            res.setHeader('X-Fetch-Type', fetchType);
+            const responseBody = { ...result };
+            if (jsonData !== undefined) {
+                responseBody.json = jsonData;
+            }
+            if (extractTokensUsed) {
+                responseBody.extractTokensUsed = extractTokensUsed;
+            }
+            if (postAnswerResult !== undefined) {
+                responseBody.answer = postAnswerResult.answer;
+            }
+            if (postSummaryText !== undefined) {
+                responseBody.summary = postSummaryText;
+            }
+            if (wantsEnvelope(req)) {
+                successResponse(res, responseBody, {
+                    requestId: req.requestId,
+                    processingTimeMs: elapsed,
+                    creditsUsed: 1,
+                    cached: false,
+                    fetchType,
+                });
+            }
+            else {
+                res.json(responseBody);
+            }
+        }
+        catch (error) {
+            const err = error;
+            console.error('POST fetch/scrape error:', err);
+            if (err.code) {
+                const safeMessage = err.message.replace(/[<>"']/g, '');
+                const statusCode = err.code === 'TIMEOUT' ? 504
+                    : err.code === 'BLOCKED' ? 403
+                        : err.code === 'NETWORK' ? 502
+                            : 500;
+                const hints = {
+                    TIMEOUT: 'Try increasing timeout, or set render:true for JS-heavy sites.',
+                    BLOCKED: 'Site blocks automated requests. Try render:true or stealth mode.',
+                    NETWORK: 'Could not reach the target URL. Verify it is correct and online.',
+                };
+                res.status(statusCode).json({
+                    success: false,
+                    error: {
+                        type: err.code,
+                        message: safeMessage,
+                        hint: hints[err.code] || undefined,
+                        docs: 'https://webpeel.dev/docs/api-reference#errors',
+                    },
+                    requestId: req.requestId,
+                });
+            }
+            else {
+                res.status(500).json({
+                    success: false,
+                    error: {
+                        type: 'internal_error',
+                        message: 'An unexpected error occurred. If this persists, check https://webpeel.dev/status',
+                        docs: 'https://webpeel.dev/docs/api-reference#errors',
+                    },
+                    requestId: req.requestId,
+                });
+            }
+        }
+    }
+    router.post('/v1/fetch', handlePostFetch);
+    router.post('/v2/scrape', handlePostFetch);
+    return router;
+}