webpeel 0.19.4 → 0.20.0

package/dist/server/routes/fetch.js

@@ -1,988 +0,0 @@
-/**
- * Fetch endpoint with caching
- */
-import { Router } from 'express';
-import '../types.js'; // Augments Express.Request with requestId
-import { peel } from '../../index.js';
-import { normalizeActions } from '../../core/actions.js';
-import { extractInlineJson } from '../../core/extract-inline.js';
-import { LRUCache } from 'lru-cache';
-import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
-import { wantsEnvelope, successResponse } from '../utils/response.js';
-import { getSchemaTemplate } from '../../core/schema-templates.js';
-import { quickAnswer } from '../../core/quick-answer.js';
-import { sendUsageAlertEmail } from '../email-service.js';
-// ── Helper: extractive summarizer (TF-IDF-like sentence scoring) ─────────────
-function extractSummary(content, maxWords = 150) {
-    if (!content)
-        return '';
-    const sentences = content
-        .split(/(?<=[.!?])\s+/)
-        .map(s => s.trim())
-        .filter(s => s.length > 40 && s.length < 600);
-    if (sentences.length === 0) {
-        const words = content.split(/\s+/);
-        return words.slice(0, maxWords).join(' ') + (words.length > maxWords ? '\u2026' : '');
-    }
-    if (sentences.length <= 3)
-        return sentences.join(' ');
-    const allWords = content.toLowerCase().split(/\W+/).filter(w => w.length > 3);
-    const wordFreq = {};
-    for (const w of allWords)
-        wordFreq[w] = (wordFreq[w] || 0) + 1;
-    const maxFreq = Math.max(1, ...Object.values(wordFreq));
-    const scored = sentences.map((sentence, idx) => {
-        const words = sentence.toLowerCase().split(/\W+/).filter(w => w.length > 3);
-        const score = words.reduce((sum, w) => sum + (wordFreq[w] || 0) / maxFreq, 0) / Math.max(1, words.length);
-        const posBonus = idx === 0 ? 0.3 : idx === sentences.length - 1 ? 0.1 : 0;
-        return { sentence, score: score + posBonus, idx };
-    });
-    scored.sort((a, b) => b.score - a.score);
-    const selected = [];
-    let wc = 0;
-    for (const item of scored) {
-        const itemWc = item.sentence.split(/\s+/).length;
-        if (wc + itemWc > maxWords * 1.3)
-            break;
-        selected.push(item);
-        wc += itemWc;
-        if (selected.length >= 5)
-            break;
-    }
-    selected.sort((a, b) => a.idx - b.idx);
-    return selected.map(s => s.sentence).join(' ');
-}
-// ── Helper: check usage and determine if alert email should be sent ───────────
-async function checkAndTriggerAlert(pgStore, userId) {
-    const getCurrentWeek = () => {
-        const now = new Date();
-        const year = now.getUTCFullYear();
-        const jan4 = new Date(Date.UTC(year, 0, 4));
-        const weekNum = Math.ceil(((now.getTime() - jan4.getTime()) / 86400000 + jan4.getUTCDay() + 1) / 7);
-        return `${year}-W${String(weekNum).padStart(2, '0')}`;
-    };
-    const currentWeek = getCurrentWeek();
-    const result = await pgStore.pool.query(`SELECT u.email, u.name, u.tier, u.alert_threshold, u.alert_email, u.alert_sent_at,
-            u.weekly_limit,
-            COALESCE(SUM(wu.total_count), 0) AS total_used,
-            u.weekly_limit + COALESCE(MAX(wu.rollover_credits), 0) AS total_available
-     FROM users u
-     LEFT JOIN api_keys ak ON ak.user_id = u.id
-     LEFT JOIN weekly_usage wu ON wu.api_key_id = ak.id AND wu.week = $2
-     WHERE u.id = $1
-     GROUP BY u.id, u.email, u.name, u.tier, u.alert_threshold, u.alert_email, u.alert_sent_at, u.weekly_limit`, [userId, currentWeek]);
-    const row = result.rows[0];
-    if (!row || !row.alert_threshold)
-        return { shouldSendAlert: false };
-    const used = parseInt(row.total_used, 10) || 0;
-    const total = parseInt(row.total_available, 10) || row.weekly_limit || 999;
-    const usagePercent = total > 0 ? Math.round((used / total) * 100) : 0;
-    // Only alert if: crosses threshold AND haven't sent alert this week
-    const lastAlert = row.alert_sent_at ? new Date(row.alert_sent_at) : null;
-    const oneWeekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
-    const alreadySentThisWeek = lastAlert !== null && lastAlert > oneWeekAgo;
-    return {
-        shouldSendAlert: usagePercent >= row.alert_threshold && !alreadySentThisWeek,
-        usagePercent,
-        used,
-        total,
-        userEmail: row.email,
-        userName: row.name || undefined,
-        userTier: row.tier,
-        alertEmail: row.alert_email || undefined,
-    };
-}
-const VALID_LLM_PROVIDERS = ['openai', 'anthropic', 'google'];
-export function createFetchRouter(authStore) {
-    const router = Router();
-    // LRU cache: 5 minute TTL, max 500 entries, 100MB total size
-    const cache = new LRUCache({
-        max: 500,
-        ttl: 5 * 60 * 1000, // 5 minutes default
-        maxSize: 100 * 1024 * 1024, // 100MB
-        sizeCalculation: (entry) => {
-            return JSON.stringify(entry).length;
-        },
-    });
-    router.get('/v1/fetch', async (req, res) => {
-        try {
-            // Require authentication — API key or JWT session
-            const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
-            if (!userId) {
-                res.status(401).json({
-                    error: 'unauthorized',
-                    message: 'API key required. Get one free at https://app.webpeel.dev/keys',
-                });
-                return;
-            }
-            const { url, render, wait, format, includeTags, excludeTags, images, location, languages, onlyMainContent, actions, maxAge, storeInCache, stream, noCache, cacheTtl, budget, question, summary, readable, stealth, screenshot, maxTokens, selector, exclude, fullPage, raw, lite, timeout, schema, detail, } = req.query;
-            const detailMode = detail || 'standard';
-            // Validate URL parameter
-            if (!url || typeof url !== 'string') {
-                res.status(400).json({
-                    success: false,
-                    error: {
-                        type: 'invalid_request',
-                        message: 'Missing or invalid "url" parameter.',
-                        hint: 'Pass a URL as a query parameter: GET /v1/fetch?url=https://example.com',
-                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                    },
-                    requestId: req.requestId,
-                });
-                return;
-            }
-            // SECURITY: Validate URL format and length
-            if (url.length > 2048) {
-                res.status(400).json({
-                    success: false,
-                    error: {
-                        type: 'invalid_url',
-                        message: 'URL too long (max 2048 characters)',
-                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                    },
-                    requestId: req.requestId,
-                });
-                return;
-            }
-            try {
-                const parsed = new URL(url);
-                // Normalize URL for consistent caching
-                const normalizedUrl = parsed.href;
-                // Use normalized URL for cache key
-                if (normalizedUrl !== url) {
-                    // URL was normalized, update for caching
-                }
-            }
-            catch {
-                res.status(400).json({
-                    success: false,
-                    error: {
-                        type: 'invalid_url',
-                        message: 'Invalid URL format',
-                        hint: 'Ensure the URL includes a scheme (https://) and a valid hostname',
-                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                    },
-                    requestId: req.requestId,
-                });
-                return;
-            }
-            // SECURITY: Validate URL to prevent SSRF attacks
-            try {
-                validateUrlForSSRF(url);
-            }
-            catch (error) {
-                if (error instanceof SSRFError) {
-                    res.status(400).json({
-                        success: false,
-                        error: {
-                            type: 'forbidden_url',
-                            message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
-                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                        },
-                        requestId: req.requestId,
-                    });
-                    return;
-                }
-                throw error;
-            }
-            // Parse actions query param (JSON-encoded array)
-            let parsedActions;
-            if (actions && typeof actions === 'string') {
-                try {
-                    const raw = JSON.parse(actions);
-                    parsedActions = normalizeActions(raw);
-                }
-                catch (e) {
-                    res.status(400).json({
-                        success: false,
-                        error: {
-                            type: 'invalid_request',
-                            message: 'Invalid "actions" parameter: must be a valid JSON array',
-                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                        },
-                        requestId: req.requestId,
-                    });
-                    return;
-                }
-            }
-            // Build cache key (include new parameters)
-            const actionsKey = parsedActions ? JSON.stringify(parsedActions) : '';
-            const cacheKey = `fetch:${url}:${render}:${wait}:${format}:${includeTags}:${excludeTags}:${images}:${location}:${languages}:${onlyMainContent}:${stream}:${actionsKey}:${budget}:${question}:${summary}:${readable}:${stealth}:${screenshot}:${maxTokens}:${selector}:${exclude}:${fullPage}:${raw}`;
-            // Cache bypass: ?noCache=true or Cache-Control: no-cache header
-            const bypassCache = noCache === 'true' || req.headers['cache-control'] === 'no-cache';
-            // Per-request TTL (cacheTtl in seconds, default 300s = 5 min)
-            const cacheTtlMs = cacheTtl !== undefined
-                ? parseInt(cacheTtl, 10) * 1000
-                : 5 * 60 * 1000;
-            // Check cache (with maxAge support)
-            const maxAgeMs = maxAge !== undefined ? parseInt(maxAge, 10) : 172800000; // Default 2 days
-            if (!bypassCache) {
-                const cached = cache.get(cacheKey);
-                if (cached && maxAgeMs > 0) {
-                    const cacheAge = Date.now() - cached.timestamp;
-                    if (cacheAge < maxAgeMs && cacheAge < cacheTtlMs) {
-                        res.setHeader('X-Cache', 'HIT');
-                        res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
-                        if (wantsEnvelope(req)) {
-                            successResponse(res, cached.result, {
-                                requestId: req.requestId,
-                                cached: true,
-                            });
-                        }
-                        else {
-                            res.json(cached.result);
-                        }
-                        return;
-                    }
-                }
-            }
-            // Parse options
-            const isSoftLimited = req.auth?.softLimited === true;
-            const hasExtraUsage = req.auth?.extraUsageAvailable === true;
-            // Parse tag arrays from comma-separated strings
-            const includeTagsArray = includeTags
-                ? includeTags.split(',').map(t => t.trim()).filter(Boolean)
-                : undefined;
-            const excludeTagsArray = excludeTags
-                ? excludeTags.split(',').map(t => t.trim()).filter(Boolean)
-                : undefined;
-            const languagesArray = languages
-                ? languages.split(',').map(l => l.trim()).filter(Boolean)
-                : undefined;
-            // onlyMainContent is a shortcut for common include tags
-            const finalIncludeTags = onlyMainContent === 'true'
-                ? ['main', 'article', '.content', '#content']
-                : includeTagsArray;
-            // When actions are present, force browser mode (skip HTTP fast path)
-            const hasActions = parsedActions && parsedActions.length > 0;
-            const shouldRender = hasActions || render === 'true';
-            const options = {
-                // SOFT LIMIT: When over quota AND no extra usage, force HTTP-only
-                // If extra usage is available, allow full functionality
-                // Exception: actions always require render
-                render: (isSoftLimited && !hasExtraUsage && !hasActions) ? false : shouldRender,
-                wait: (isSoftLimited && !hasExtraUsage) ? 0 : (wait ? parseInt(wait, 10) : undefined),
-                format: format || 'markdown',
-                stream: stream === 'true',
-                includeTags: finalIncludeTags,
-                excludeTags: excludeTagsArray,
-                images: images === 'true',
-                actions: parsedActions,
-                location: location || languagesArray ? {
-                    country: location,
-                    languages: languagesArray,
-                } : undefined,
-                budget: budget ? parseInt(budget, 10) : undefined,
-                question: question,
-                readable: readable === 'true',
-                stealth: (isSoftLimited && !hasExtraUsage) ? false : stealth === 'true',
-                screenshot: (isSoftLimited && !hasExtraUsage) ? false : screenshot === 'true',
-                maxTokens: maxTokens ? parseInt(maxTokens, 10) : undefined,
-                selector: selector,
-                exclude: exclude ? exclude.split(',').map(s => s.trim()).filter(Boolean) : undefined,
-                fullPage: fullPage === 'true',
-                raw: raw === 'true',
-                lite: lite === 'true',
-                timeout: timeout ? parseInt(timeout, 10) : undefined,
-            };
-            // Auto-budget: default to 4000 tokens for API requests when no budget specified
-            // Opt-out: budget=0 explicitly disables. Lite mode disables auto-budget.
-            if (options.budget === undefined && !options.lite) {
-                options.budget = 4000;
-                res.setHeader('X-Auto-Budget', '4000');
-            }
-            // Inform the user if their request was degraded
-            if (isSoftLimited && !hasExtraUsage && render === 'true' && !hasActions) {
-                res.setHeader('X-Degraded', 'render=true downgraded to HTTP-only (quota exceeded)');
-            }
-            if (isSoftLimited && !hasExtraUsage && stealth === 'true') {
-                res.setHeader('X-Degraded', 'stealth=true downgraded (quota exceeded)');
-            }
-            if (isSoftLimited && !hasExtraUsage && screenshot === 'true') {
-                res.setHeader('X-Degraded', 'screenshot=true downgraded (quota exceeded)');
-            }
-            // Validate wait parameter
-            if (options.wait !== undefined && (isNaN(options.wait) || options.wait < 0 || options.wait > 60000)) {
-                res.status(400).json({
-                    success: false,
-                    error: {
-                        type: 'invalid_request',
-                        message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
-                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                    },
-                    requestId: req.requestId,
-                });
-                return;
-            }
-            // Validate format parameter
-            if (!['markdown', 'text', 'html', 'clean'].includes(options.format || '')) {
-                res.status(400).json({
-                    success: false,
-                    error: {
-                        type: 'invalid_request',
-                        message: 'Invalid "format" parameter: must be "markdown", "text", "html", or "clean"',
-                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                    },
-                    requestId: req.requestId,
-                });
-                return;
-            }
-            const shouldStream = options.stream === true;
-            if (shouldStream) {
-                res.setHeader('X-Stream', 'true');
-                if (typeof res.flushHeaders === 'function') {
-                    res.flushHeaders();
-                }
-            }
-            // Fetch content
-            const startTime = Date.now();
-            const result = await peel(url, options);
-            const elapsed = Date.now() - startTime;
-            // --- BM25 Schema Template Extraction (GET, no LLM needed) ---
-            if (schema && typeof schema === 'string' && result.content) {
-                const template = getSchemaTemplate(schema);
-                if (template) {
-                    const { quickAnswer } = await import('../../core/quick-answer.js');
-                    const { smartExtractSchemaFields } = await import('../../core/schema-postprocess.js');
-                    const extracted = smartExtractSchemaFields(result.content, template.fields, quickAnswer, {
-                        pageTitle: result.title,
-                        pageUrl: result.url,
-                        metadata: result.metadata,
-                    });
-                    result.extracted = extracted;
-                }
-            }
-            // Determine fetch type from the result method
-            const fetchType = result.method === 'stealth' ? 'stealth' :
-                result.method === 'browser' ? 'stealth' : 'basic';
-            // Log request to database (PostgreSQL only)
-            const pgStore = authStore;
-            // Log usage for BOTH API key auth AND JWT session auth
-            const logUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
-            if (logUserId && typeof pgStore.pool !== 'undefined') {
-                pgStore.pool.query(`INSERT INTO usage_logs 
-            (user_id, endpoint, url, method, processing_time_ms, status_code, ip_address, user_agent, tokens_used)
-          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
-                    logUserId,
-                    'fetch',
-                    url,
-                    fetchType,
-                    elapsed,
-                    200,
-                    req.ip || req.socket.remoteAddress,
-                    req.get('user-agent'),
-                    result?.tokens || null,
-                ]).catch((err) => {
-                    console.error('Failed to log request to usage_logs:', err);
-                });
-            }
-            // Track usage (check for trackBurstUsage method to detect PostgresAuthStore)
-            if (req.auth?.keyInfo?.key && typeof pgStore.trackBurstUsage === 'function') {
-                // Track burst usage (always)
-                await pgStore.trackBurstUsage(req.auth.keyInfo.key);
-                // If soft-limited with extra usage available, charge to extra usage
-                if (isSoftLimited && hasExtraUsage) {
-                    const extraResult = await pgStore.trackExtraUsage(req.auth.keyInfo.key, fetchType, url, elapsed, 200 // PeelResult doesn't include statusCode, assume success
-                    );
-                    if (extraResult.success) {
-                        res.setHeader('X-Extra-Usage-Charged', `$${extraResult.cost.toFixed(4)}`);
-                        res.setHeader('X-Extra-Usage-New-Balance', extraResult.newBalance.toFixed(2));
-                    }
-                    else {
-                        // Extra usage failed - fall back to soft limit
-                        res.setHeader('X-Degraded', 'Extra usage insufficient, degraded to soft limit');
-                    }
-                }
-                else if (!isSoftLimited) {
-                    // Normal weekly usage tracking
-                    await pgStore.trackUsage(req.auth.keyInfo.key, fetchType);
-                }
-                // If soft-limited WITHOUT extra usage, don't track (already over quota)
-            }
-            // Check usage alert (fire-and-forget, never block the response)
-            if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
-                try {
-                    const alertResult = await checkAndTriggerAlert(pgStore, req.auth.keyInfo.accountId);
-                    if (alertResult.shouldSendAlert && alertResult.usagePercent !== undefined) {
-                        await sendUsageAlertEmail({
-                            toEmail: alertResult.alertEmail || alertResult.userEmail,
-                            userName: alertResult.userName,
-                            usagePercent: alertResult.usagePercent,
-                            used: alertResult.used,
-                            total: alertResult.total,
-                            tier: alertResult.userTier,
-                        });
-                        // Mark alert as sent so we don't spam (rate-limited to once/week)
-                        await pgStore.pool.query('UPDATE users SET alert_sent_at = NOW() WHERE id = $1', [req.auth.keyInfo.accountId]);
-                    }
-                }
-                catch (alertErr) {
-                    // Never let alert errors affect the main response
-                    console.warn('[alert] Failed to check/send alert:', alertErr);
-                }
-            }
-            // Cache result (unless storeInCache is explicitly false or cache bypass requested)
-            if (storeInCache !== 'false' && !bypassCache) {
-                cache.set(cacheKey, {
-                    result,
-                    timestamp: Date.now(),
-                }, { ttl: cacheTtlMs });
-            }
-            // Apply ?detail=brief mode: truncate content and prepend TL;DR
-            if (detailMode === 'brief' && result.content) {
-                const words = result.content.split(/\s+/);
-                const truncatedWords = words.slice(0, 500);
-                const truncated = truncatedWords.join(' ');
-                // Extract TL;DR from first non-empty paragraph
-                const firstPara = result.content
-                    .split(/\n{2,}/)
-                    .map((p) => p.replace(/^#+\s*/, '').trim())
-                    .find((p) => p.length > 40 && !p.startsWith('!') && !p.startsWith('['));
-                const tldr = firstPara
-                    ? firstPara.replace(/\s+/g, ' ').slice(0, 300) + (firstPara.length > 300 ? '...' : '')
-                    : truncated.slice(0, 200) + '...';
-                result.content = `**TL;DR:** ${tldr}\n\n---\n\n${truncated}${words.length > 500 ? '\n\n*[Content truncated — use ?detail=full for complete output]*' : ''}`;
-                const tokenEstimate = Math.round(truncatedWords.length * 0.75);
-                res.setHeader('X-Detail-Mode', 'brief');
-                res.setHeader('X-Token-Estimate', tokenEstimate.toString());
-            }
-            // --- question → answer field (GET) ---
-            // When ?question= is provided, run quickAnswer() on the fetched content
-            // and expose the result as an `answer` field in the response.
-            const getAnswerResult = (question && typeof question === 'string' && result.content)
-                ? quickAnswer({ question, content: result.content, url: result.url })
-                : undefined;
-            // --- summary field (GET) ---
-            // When ?summary=true, return a truncated 500-word summary in a `summary` field.
-            const getSummaryText = (summary === 'true' && result.content)
-                ? extractSummary(result.content)
-                : undefined;
-            // Add usage headers (kept for backward compat; also surfaced in envelope metadata)
-            res.setHeader('X-Cache', 'MISS');
-            res.setHeader('X-Credits-Used', '1');
-            res.setHeader('X-Processing-Time', elapsed.toString());
-            res.setHeader('X-Fetch-Type', fetchType);
-            // Build response — extend result with optional answer/summary fields
-            const getResponseBody = { ...result };
-            if (getAnswerResult !== undefined)
-                getResponseBody.answer = getAnswerResult.answer;
-            if (getSummaryText !== undefined)
-                getResponseBody.summary = getSummaryText;
-            if (wantsEnvelope(req)) {
-                successResponse(res, getResponseBody, {
-                    requestId: req.requestId,
-                    processingTimeMs: elapsed,
-                    creditsUsed: 1,
-                    cached: false,
-                    fetchType,
-                });
-            }
-            else {
-                res.json(getResponseBody);
-            }
-        }
-        catch (error) {
-            const err = error;
-            // Log error to database (PostgreSQL only)
-            const pgStore = authStore;
-            if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
-                const url = req.query.url;
-                const render = req.query.render === 'true';
-                const fetchType = render ? 'stealth' : 'basic';
-                pgStore.pool.query(`INSERT INTO usage_logs 
-            (user_id, endpoint, url, method, status_code, error, ip_address, user_agent, tokens_used)
-          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
-                    req.auth.keyInfo.accountId,
-                    'fetch',
-                    url,
-                    fetchType,
-                    500,
-                    err.message || 'Unknown error',
-                    req.ip || req.socket.remoteAddress,
-                    req.get('user-agent'),
-                    null,
-                ]).catch((logErr) => {
-                    console.error('Failed to log error to usage_logs:', logErr);
-                });
-            }
-            // SECURITY: Sanitize error messages to prevent information disclosure
-            if (err.code) {
-                // WebPeelError from core library - safe to expose with helpful context
-                const safeMessage = err.message.replace(/[<>"']/g, ''); // Remove HTML chars
-                const statusCode = err.code === 'TIMEOUT' ? 504
-                    : err.code === 'BLOCKED' ? 403
-                        : err.code === 'NETWORK' ? 502
-                            : 500;
-                const hints = {
-                    TIMEOUT: 'Try increasing timeout with ?wait=10000, or use render=true for JS-heavy sites.',
-                    BLOCKED: 'This site blocks automated requests. Try adding render=true or use stealth mode (costs 5 credits).',
-                    NETWORK: 'Could not reach the target URL. Verify the URL is correct and the site is online.',
-                };
-                res.status(statusCode).json({
-                    success: false,
-                    error: {
-                        type: err.code,
-                        message: safeMessage,
-                        hint: hints[err.code] || undefined,
-                        docs: 'https://webpeel.dev/docs/api-reference#errors',
-                    },
-                    requestId: req.requestId,
-                });
-            }
-            else {
-                // Unexpected error - generic message only
-                console.error('Fetch error:', err); // Log full error server-side
-                res.status(500).json({
-                    success: false,
-                    error: {
-                        type: 'internal_error',
-                        message: 'An unexpected error occurred while fetching the URL. If this persists, check https://webpeel.dev/status',
-                        docs: 'https://webpeel.dev/docs/api-reference#errors',
-                    },
-                    requestId: req.requestId,
-                });
-            }
-        }
-    });
-    // -----------------------------------------------------------------------
-    // POST /v1/fetch — same as GET but accepts JSON body with extract param
-    // POST /v2/scrape — alias with identical behaviour
-    // -----------------------------------------------------------------------
-    async function handlePostFetch(req, res) {
-        try {
-            // Require authentication — API key or JWT session
-            const postUserId = req.auth?.keyInfo?.accountId || req.user?.userId;
-            if (!postUserId) {
-                res.status(401).json({
-                    error: 'unauthorized',
-                    message: 'API key required. Get one free at https://app.webpeel.dev/keys',
-                });
-                return;
-            }
-            const { url, render, wait, format, includeTags, excludeTags, images, location, languages, onlyMainContent, actions: rawActions, storeInCache: storeFlag, 
-            // Cache control
-            noCache: noCacheBody, cacheTtl: cacheTtlBody, 
-            // Inline extraction (BYOK)
-            extract, llmProvider, llmApiKey, llmModel, 
-            // Firecrawl-compatible formats array
-            formats, stream, 
-            // Extended peel options
-            budget, question, summary: summaryParam, readable, stealth, screenshot, maxTokens, selector, exclude, fullPage, raw, lite, timeout, proxies, chunk, device, viewportWidth, viewportHeight, waitUntil, waitSelector, blockResources, cloaked, schema: bodySchema, } = req.body;
-            // --- Validate URL -------------------------------------------------------
-            if (!url || typeof url !== 'string') {
-                res.status(400).json({
-                    success: false,
-                    error: {
-                        type: 'invalid_request',
-                        message: 'Missing or invalid "url" in request body.',
-                        hint: 'Send JSON: { "url": "https://example.com" }',
-                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                    },
-                    requestId: req.requestId,
-                });
-                return;
-            }
-            if (url.length > 2048) {
-                res.status(400).json({
-                    success: false,
-                    error: {
-                        type: 'invalid_url',
-                        message: 'URL too long (max 2048 characters)',
-                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                    },
-                    requestId: req.requestId,
-                });
-                return;
-            }
-            try {
-                new URL(url);
-            }
-            catch {
-                res.status(400).json({
-                    success: false,
-                    error: {
-                        type: 'invalid_url',
-                        message: 'Invalid URL format',
-                        hint: 'Ensure the URL includes a scheme (https://) and a valid hostname',
-                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                    },
-                    requestId: req.requestId,
-                });
-                return;
-            }
-            try {
-                validateUrlForSSRF(url);
-            }
-            catch (error) {
-                if (error instanceof SSRFError) {
-                    res.status(400).json({
-                        success: false,
-                        error: {
-                            type: 'forbidden_url',
-                            message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
-                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                        },
-                        requestId: req.requestId,
-                    });
-                    return;
-                }
-                throw error;
-            }
-            // --- Parse and normalize actions -----------------------------------------
-            let postActions;
-            if (rawActions !== undefined) {
-                try {
-                    postActions = normalizeActions(rawActions);
-                }
-                catch (e) {
-                    res.status(400).json({
-                        success: false,
-                        error: {
-                            type: 'invalid_request',
-                            message: `Invalid "actions" parameter: ${e.message}`,
-                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                        },
-                        requestId: req.requestId,
-                    });
-                    return;
-                }
-            }
-            // --- Cache bypass and lookup -------------------------------------------
-            const postBypassCache = noCacheBody === true || req.headers['cache-control'] === 'no-cache';
-            const postCacheTtlMs = typeof cacheTtlBody === 'number' ? cacheTtlBody * 1000 : 5 * 60 * 1000;
-            const postActionsKey = postActions ? JSON.stringify(postActions) : '';
-            const postCacheKey = `fetch:${url}:${render}:${wait}:${format}:${JSON.stringify(includeTags)}:${JSON.stringify(excludeTags)}:${images}:${location}:${JSON.stringify(languages)}:${onlyMainContent}:${stream}:${postActionsKey}:${budget}:${question}:${summaryParam}:${readable}:${stealth}:${screenshot}:${maxTokens}:${selector}:${JSON.stringify(exclude)}:${fullPage}:${raw}`;
-            if (!postBypassCache && !extract) {
-                const cached = cache.get(postCacheKey);
-                if (cached) {
-                    const cacheAge = Date.now() - cached.timestamp;
-                    if (cacheAge < postCacheTtlMs) {
-                        res.setHeader('X-Cache', 'HIT');
-                        res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
-                        if (wantsEnvelope(req)) {
-                            successResponse(res, cached.result, {
-                                requestId: req.requestId,
-                                cached: true,
-                            });
-                        }
-                        else {
-                            res.json(cached.result);
-                        }
-                        return;
-                    }
-                }
-            }
-            // --- Resolve inline extract from body or Firecrawl-compatible formats ---
-            let resolvedExtract = extract;
-            if (!resolvedExtract && Array.isArray(formats)) {
-                const jsonFormat = formats.find((f) => (typeof f === 'object' && f !== null && f.type === 'json') ||
-                    (typeof f === 'string' && f === 'json'));
-                if (jsonFormat && typeof jsonFormat === 'object' && (jsonFormat.schema || jsonFormat.prompt)) {
-                    resolvedExtract = {
-                        schema: jsonFormat.schema,
-                        prompt: jsonFormat.prompt,
-                    };
-                }
-            }
-            // Resolve schema template names (e.g. "product", "article") to field objects
-            if (resolvedExtract && typeof resolvedExtract.schema === 'string') {
-                const tmpl = getSchemaTemplate(resolvedExtract.schema);
-                if (tmpl) {
-                    resolvedExtract = { ...resolvedExtract, schema: tmpl.fields };
-                }
-                else {
-                    // Try parsing as JSON string
-                    try {
-                        resolvedExtract = { ...resolvedExtract, schema: JSON.parse(resolvedExtract.schema) };
-                    }
-                    catch { /* leave as-is */ }
-                }
-            }
-            // Validate LLM params if extraction is requested
-            if (resolvedExtract && (resolvedExtract.schema || resolvedExtract.prompt)) {
-                if (!llmProvider || !VALID_LLM_PROVIDERS.includes(llmProvider)) {
-                    res.status(400).json({
-                        success: false,
-                        error: {
-                            type: 'invalid_request',
-                            message: `"llmProvider" is required for inline extraction and must be one of: ${VALID_LLM_PROVIDERS.join(', ')}`,
-                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                        },
-                        requestId: req.requestId,
-                    });
-                    return;
-                }
-                if (!llmApiKey || typeof llmApiKey !== 'string' || llmApiKey.trim().length === 0) {
-                    res.status(400).json({
-                        success: false,
-                        error: {
-                            type: 'invalid_request',
-                            message: 'Missing or invalid "llmApiKey" (BYOK required for inline extraction)',
-                            hint: 'Pass your LLM provider API key in the "llmApiKey" field',
-                            docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                        },
-                        requestId: req.requestId,
-                    });
-                    return;
-                }
-            }
-            // --- Build PeelOptions ---------------------------------------------------
-            const isSoftLimited = req.auth?.softLimited === true;
-            const hasExtraUsage = req.auth?.extraUsageAvailable === true;
-            const includeTagsArray = Array.isArray(includeTags) ? includeTags : undefined;
-            const excludeTagsArray = Array.isArray(excludeTags) ? excludeTags : undefined;
-            const languagesArray = Array.isArray(languages) ? languages : undefined;
-            const finalIncludeTags = onlyMainContent === true
-                ? ['main', 'article', '.content', '#content']
-                : includeTagsArray;
-            const resolvedFormat = format || 'markdown';
-            if (!['markdown', 'text', 'html', 'clean'].includes(resolvedFormat)) {
-                res.status(400).json({
-                    success: false,
-                    error: {
-                        type: 'invalid_request',
-                        message: 'Invalid "format" parameter: must be "markdown", "text", "html", or "clean"',
-                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                    },
-                    requestId: req.requestId,
-                });
-                return;
-            }
-            const resolvedWait = typeof wait === 'number' ? wait : undefined;
-            if (resolvedWait !== undefined && (isNaN(resolvedWait) || resolvedWait < 0 || resolvedWait > 60000)) {
-                res.status(400).json({
-                    success: false,
-                    error: {
-                        type: 'invalid_request',
-                        message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
-                        docs: 'https://webpeel.dev/docs/api-reference#fetch',
-                    },
-                    requestId: req.requestId,
-                });
-                return;
-            }
-            // When actions are present, force browser mode
-            const postHasActions = postActions && postActions.length > 0;
-            const postShouldRender = postHasActions || render === true;
-            // Normalize exclude: accept string (comma-separated) or string array
-            const excludeArray = exclude
-                ? (Array.isArray(exclude) ? exclude : exclude.split(',').map(s => s.trim()).filter(Boolean))
-                : undefined;
-            const options = {
-                render: (isSoftLimited && !hasExtraUsage && !postHasActions) ? false : postShouldRender,
-                wait: (isSoftLimited && !hasExtraUsage) ? 0 : resolvedWait,
-                format: resolvedFormat,
-                stream: stream === true,
-                includeTags: finalIncludeTags,
-                excludeTags: excludeTagsArray,
-                images: images === true,
-                actions: postActions,
-                location: location || languagesArray ? {
-                    country: location,
-                    languages: languagesArray,
-                } : undefined,
-                budget: typeof budget === 'number' ? budget : undefined,
-                question: question,
-                readable: readable === true,
-                stealth: (isSoftLimited && !hasExtraUsage) ? false : stealth === true,
-                screenshot: (isSoftLimited && !hasExtraUsage) ? false : screenshot === true,
-                maxTokens: typeof maxTokens === 'number' ? maxTokens : undefined,
-                selector: selector,
-                exclude: excludeArray,
-                fullPage: fullPage === true,
-                raw: raw === true,
-                lite: lite === true,
-                timeout: typeof timeout === 'number' ? timeout : undefined,
-                proxies: Array.isArray(proxies) ? proxies : undefined,
-                device: device,
-                viewportWidth: typeof viewportWidth === 'number' ? viewportWidth : undefined,
-                viewportHeight: typeof viewportHeight === 'number' ? viewportHeight : undefined,
-                waitUntil: waitUntil,
-                waitSelector: waitSelector,
-                blockResources: Array.isArray(blockResources) ? blockResources : undefined,
-            };
-            if (cloaked)
-                options.cloaked = cloaked;
-            if (chunk)
-                options.chunk = chunk === true ? true : chunk;
-            // Auto-budget: default to 4000 tokens for API requests when no budget specified
-            // Opt-out: budget=0 explicitly disables. Lite mode disables auto-budget.
-            if (options.budget === undefined && !options.lite) {
-                options.budget = 4000;
-                res.setHeader('X-Auto-Budget', '4000');
-            }
-            if (isSoftLimited && !hasExtraUsage && render === true && !postHasActions) {
-                res.setHeader('X-Degraded', 'render=true downgraded to HTTP-only (quota exceeded)');
-            }
-            if (isSoftLimited && !hasExtraUsage && stealth === true) {
-                res.setHeader('X-Degraded', 'stealth=true downgraded (quota exceeded)');
-            }
-            if (isSoftLimited && !hasExtraUsage && screenshot === true) {
-                res.setHeader('X-Degraded', 'screenshot=true downgraded (quota exceeded)');
-            }
-            const shouldStream = options.stream === true;
-            if (shouldStream) {
-                res.setHeader('X-Stream', 'true');
-                if (typeof res.flushHeaders === 'function') {
-                    res.flushHeaders();
-                }
-            }
-            // --- Fetch content -------------------------------------------------------
-            const startTime = Date.now();
-            const result = await peel(url, options);
-            const elapsed = Date.now() - startTime;
-            // --- BM25 Schema Template Extraction (POST, no LLM needed) ---
-            if (bodySchema && typeof bodySchema === 'string' && result.content) {
-                const template = getSchemaTemplate(bodySchema);
-                if (template) {
-                    const { quickAnswer } = await import('../../core/quick-answer.js');
-                    const { smartExtractSchemaFields } = await import('../../core/schema-postprocess.js');
-                    const extracted = smartExtractSchemaFields(result.content, template.fields, quickAnswer, {
-                        pageTitle: result.title,
-                        pageUrl: result.url,
-                        metadata: result.metadata,
-                    });
-                    result.extracted = extracted;
-                }
-            }
-            // --- Inline extraction (post-fetch) -------------------------------------
-            let jsonData;
-            let extractTokensUsed;
-            if (resolvedExtract && (resolvedExtract.schema || resolvedExtract.prompt) && llmApiKey) {
-                const extractResult = await extractInlineJson(result.content, {
-                    schema: resolvedExtract.schema,
-                    prompt: resolvedExtract.prompt,
-                    llmProvider: llmProvider,
-                    llmApiKey: llmApiKey.trim(),
-                    llmModel,
-                });
-                jsonData = extractResult.data;
-                extractTokensUsed = extractResult.tokensUsed;
-            }
-            // --- Usage tracking (same as GET) ----------------------------------------
-            const fetchType = result.method === 'stealth' ? 'stealth' :
-                result.method === 'browser' ? 'stealth' : 'basic';
-            const pgStore = authStore;
-            if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
-                pgStore.pool.query(`INSERT INTO usage_logs
-            (user_id, endpoint, url, method, processing_time_ms, status_code, ip_address, user_agent, tokens_used)
-          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [
-                    req.auth.keyInfo.accountId,
-                    'fetch',
-                    url,
-                    fetchType,
-                    elapsed,
-                    200,
-                    req.ip || req.socket.remoteAddress,
-                    req.get('user-agent'),
-                    result?.tokens || null,
-                ]).catch((err) => {
-                    console.error('Failed to log request to usage_logs:', err);
-                });
-            }
-            if (req.auth?.keyInfo?.key && typeof pgStore.trackBurstUsage === 'function') {
-                await pgStore.trackBurstUsage(req.auth.keyInfo.key);
-                if (isSoftLimited && hasExtraUsage) {
-                    const extraResult = await pgStore.trackExtraUsage(req.auth.keyInfo.key, fetchType, url, elapsed, 200);
-                    if (extraResult.success) {
-                        res.setHeader('X-Extra-Usage-Charged', `$${extraResult.cost.toFixed(4)}`);
-                        res.setHeader('X-Extra-Usage-New-Balance', extraResult.newBalance.toFixed(2));
-                    }
-                    else {
-                        res.setHeader('X-Degraded', 'Extra usage insufficient, degraded to soft limit');
-                    }
-                }
-                else if (!isSoftLimited) {
-                    await pgStore.trackUsage(req.auth.keyInfo.key, fetchType);
-                }
-            }
-            // Cache result (skip extraction results — they depend on user's LLM keys)
-            if (storeFlag !== false && !postBypassCache && !resolvedExtract) {
-                cache.set(postCacheKey, { result, timestamp: Date.now() }, { ttl: postCacheTtlMs });
-            }
-            // --- question → answer field (POST) ---
-            // When question is provided, run quickAnswer() on the fetched content
-            // and expose the result as an `answer` field in the response.
-            const postAnswerResult = (question && typeof question === 'string' && result.content)
-                ? quickAnswer({ question, content: result.content, url: result.url })
-                : undefined;
-            // --- summary field (POST) ---
-            // When summary: true, return a truncated 500-word summary in a `summary` field.
-            const postSummaryText = (summaryParam === true && result.content)
-                ? extractSummary(result.content)
-                : undefined;
-            // --- Build response ------------------------------------------------------
-            // Headers kept for backward compat; also surfaced in envelope metadata.
-            res.setHeader('X-Cache', 'MISS');
-            res.setHeader('X-Credits-Used', '1');
-            res.setHeader('X-Processing-Time', elapsed.toString());
-            res.setHeader('X-Fetch-Type', fetchType);
-            const responseBody = { ...result };
-            if (jsonData !== undefined) {
-                responseBody.json = jsonData;
-            }
-            if (extractTokensUsed) {
-                responseBody.extractTokensUsed = extractTokensUsed;
-            }
-            if (postAnswerResult !== undefined) {
-                responseBody.answer = postAnswerResult.answer;
-            }
-            if (postSummaryText !== undefined) {
-                responseBody.summary = postSummaryText;
-            }
-            if (wantsEnvelope(req)) {
-                successResponse(res, responseBody, {
-                    requestId: req.requestId,
-                    processingTimeMs: elapsed,
-                    creditsUsed: 1,
-                    cached: false,
-                    fetchType,
-                });
-            }
-            else {
-                res.json(responseBody);
-            }
-        }
-        catch (error) {
-            const err = error;
-            console.error('POST fetch/scrape error:', err);
-            if (err.code) {
-                const safeMessage = err.message.replace(/[<>"']/g, '');
-                const statusCode = err.code === 'TIMEOUT' ? 504
-                    : err.code === 'BLOCKED' ? 403
-                        : err.code === 'NETWORK' ? 502
-                            : 500;
-                const hints = {
-                    TIMEOUT: 'Try increasing timeout, or set render:true for JS-heavy sites.',
-                    BLOCKED: 'Site blocks automated requests. Try render:true or stealth mode.',
-                    NETWORK: 'Could not reach the target URL. Verify it is correct and online.',
-                };
-                res.status(statusCode).json({
-                    success: false,
-                    error: {
-                        type: err.code,
-                        message: safeMessage,
-                        hint: hints[err.code] || undefined,
-                        docs: 'https://webpeel.dev/docs/api-reference#errors',
-                    },
-                    requestId: req.requestId,
-                });
-            }
-            else {
-                res.status(500).json({
-                    success: false,
-                    error: {
-                        type: 'internal_error',
-                        message: 'An unexpected error occurred. If this persists, check https://webpeel.dev/status',
-                        docs: 'https://webpeel.dev/docs/api-reference#errors',
-                    },
-                    requestId: req.requestId,
-                });
-            }
-        }
-    }
-    router.post('/v1/fetch', handlePostFetch);
-    router.post('/v2/scrape', handlePostFetch);
-    return router;
-}
-//# sourceMappingURL=fetch.js.map