webpeel 0.20.2 → 0.20.3

package/dist/server/routes/batch.js

@@ -0,0 +1,493 @@
+/**
+ * Batch scrape API - process multiple URLs concurrently
+ */
+import { Router } from 'express';
+import crypto from 'crypto';
+import { peel } from '../../index.js';
+import { sendWebhook, normalizeWebhook } from './webhooks.js';
+import { initSSE, sendSSE, endSSE, wantsSSE } from '../utils/sse.js';
+export function createBatchRouter(jobQueue) {
+    const router = Router();
+    /**
+     * POST /v1/batch/scrape - Submit batch of URLs
+     */
+    router.post('/v1/batch/scrape', async (req, res) => {
+        try {
+            const { urls, formats, extract, maxTokens, webhook, concurrency } = req.body;
+            // Validate required parameters
+            if (!urls || !Array.isArray(urls) || urls.length === 0) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Missing or invalid "urls" parameter (must be non-empty array)',
+                        hint: 'Pass a non-empty array of URL strings in the "urls" field.',
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            // Limit batch size
+            if (urls.length > 100) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Batch size too large (max 100 URLs)',
+                        hint: 'Split your request into batches of 100 URLs or fewer.',
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            // Validate URLs
+            for (const url of urls) {
+                if (typeof url !== 'string') {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_request',
+                            message: 'All URLs must be strings',
+                            hint: 'Each element in the "urls" array must be a string.',
+                            docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                        },
+                        requestId: crypto.randomUUID(),
+                    });
+                    return;
+                }
+                try {
+                    new URL(url);
+                }
+                catch {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_url',
+                            message: `Invalid URL format: ${url}`,
+                            hint: 'Ensure each URL includes a scheme (https://) and a valid hostname.',
+                            docs: 'https://webpeel.dev/docs/errors#invalid_url',
+                        },
+                        requestId: crypto.randomUUID(),
+                    });
+                    return;
+                }
+            }
+            const ownerId = req.auth?.keyInfo?.accountId;
+            // ── SSE streaming path ────────────────────────────────────────────────
+            if (wantsSSE(req)) {
+                const normalizedWebhook = webhook ? normalizeWebhook(webhook) : undefined;
+                const job = await jobQueue.createJob('batch', normalizedWebhook, ownerId);
+                await jobQueue.updateJob(job.id, { total: urls.length });
+                // Set SSE headers (X-Request-Id already set by global middleware)
+                initSSE(res);
+                sendSSE(res, 'started', {
+                    batchId: job.id,
+                    totalUrls: urls.length,
+                });
+                let closed = false;
+                const heartbeat = setInterval(() => {
+                    if (!closed)
+                        res.write('event: ping\ndata: {}\n\n');
+                }, 15_000);
+                req.on('close', () => {
+                    closed = true;
+                    clearInterval(heartbeat);
+                });
+                const startTime = Date.now();
+                let completedCount = 0;
+                let failedCount = 0;
+                try {
+                    jobQueue.updateJob(job.id, { status: 'processing' });
+                    const peelOptions = {
+                        format: formats?.[0] || 'markdown',
+                        extract,
+                        maxTokens,
+                    };
+                    // Process URLs with bounded concurrency (caller-specified, max 10)
+                    const results = new Array(urls.length);
+                    const maxConcurrent = Math.min(Math.max(parseInt(concurrency) || 5, 1), 10);
+                    let activeCount = 0;
+                    let urlIndex = 0;
+                    const runBatch = async () => {
+                        while (urlIndex < urls.length) {
+                            // Check cancellation
+                            const currentJob = await jobQueue.getJob(job.id);
+                            if (currentJob?.status === 'cancelled')
+                                break;
+                            while (activeCount >= maxConcurrent) {
+                                await new Promise(resolve => setTimeout(resolve, 50));
+                            }
+                            const url = urls[urlIndex];
+                            const index = urlIndex;
+                            urlIndex++;
+                            activeCount++;
+                            (async () => {
+                                try {
+                                    const result = await peel(url, peelOptions);
+                                    results[index] = result;
+                                    completedCount++;
+                                    jobQueue.updateJob(job.id, {
+                                        completed: completedCount + failedCount,
+                                        creditsUsed: completedCount + failedCount,
+                                    });
+                                    if (!closed) {
+                                        sendSSE(res, 'result', {
+                                            url,
+                                            content: result.content,
+                                            metadata: result.metadata,
+                                            index,
+                                        });
+                                    }
+                                }
+                                catch (err) {
+                                    failedCount++;
+                                    results[index] = { url, error: err.message || 'Unknown error' };
+                                    jobQueue.updateJob(job.id, {
+                                        completed: completedCount + failedCount,
+                                        creditsUsed: completedCount + failedCount,
+                                    });
+                                    if (!closed) {
+                                        sendSSE(res, 'error', {
+                                            url,
+                                            error: 'FETCH_ERROR',
+                                            message: err.message || 'Unknown error',
+                                            index,
+                                        });
+                                    }
+                                }
+                                finally {
+                                    activeCount--;
+                                }
+                            })();
+                        }
+                        // Wait for all in-flight requests
+                        while (activeCount > 0) {
+                            await new Promise(resolve => setTimeout(resolve, 50));
+                        }
+                    };
+                    await runBatch();
+                    jobQueue.updateJob(job.id, {
+                        status: 'completed',
+                        data: results,
+                    });
+                    if (!closed) {
+                        sendSSE(res, 'done', {
+                            batchId: job.id,
+                            completed: completedCount,
+                            failed: failedCount,
+                            duration: Date.now() - startTime,
+                        });
+                    }
+                }
+                catch (error) {
+                    jobQueue.updateJob(job.id, {
+                        status: 'failed',
+                        error: error.message || 'Unknown error',
+                    });
+                    if (!closed) {
+                        sendSSE(res, 'error', {
+                            error: 'BATCH_FAILED',
+                            message: error.message || 'Unknown error',
+                        });
+                    }
+                }
+                finally {
+                    clearInterval(heartbeat);
+                    if (!closed)
+                        endSSE(res);
+                }
+                return;
+            }
+            // ── Regular async job path (backward compat) ─────────────────────────
+            const normalizedWebhook2 = webhook ? normalizeWebhook(webhook) : undefined;
+            const job = await jobQueue.createJob('batch', normalizedWebhook2, ownerId);
+            await jobQueue.updateJob(job.id, {
+                total: urls.length,
+            });
+            // Start batch processing in background
+            setImmediate(async () => {
+                try {
+                    // Update job to processing
+                    jobQueue.updateJob(job.id, { status: 'processing' });
+                    // Send started webhook
+                    if (normalizedWebhook2) {
+                        await sendWebhook(normalizedWebhook2, 'started', {
+                            jobId: job.id,
+                            total: urls.length,
+                        });
+                    }
+                    // Build peel options
+                    const peelOptions = {
+                        format: formats?.[0] || 'markdown',
+                        extract,
+                        maxTokens,
+                    };
+                    // Process URLs with semaphore (caller-specified, max 10 concurrent)
+                    const results = [];
+                    const maxConcurrent = Math.min(Math.max(parseInt(concurrency) || 5, 1), 10);
+                    let activeCount = 0;
+                    let urlIndex = 0;
+                    const processBatch = async () => {
+                        while (urlIndex < urls.length) {
+                            // Check if job was cancelled
+                            const currentJob = await jobQueue.getJob(job.id);
+                            if (currentJob?.status === 'cancelled') {
+                                return;
+                            }
+                            // Wait for available slot
+                            while (activeCount >= maxConcurrent) {
+                                await new Promise(resolve => setTimeout(resolve, 100));
+                            }
+                            const url = urls[urlIndex];
+                            const index = urlIndex;
+                            urlIndex++;
+                            activeCount++;
+                            // Process URL
+                            (async () => {
+                                try {
+                                    const result = await peel(url, peelOptions);
+                                    results[index] = result;
+                                    // Update progress
+                                    const completed = results.filter(r => r !== undefined).length;
+                                    jobQueue.updateJob(job.id, {
+                                        completed,
+                                        creditsUsed: completed,
+                                    });
+                                    // Send page webhook
+                                    if (normalizedWebhook2) {
+                                        sendWebhook(normalizedWebhook2, 'page', {
+                                            jobId: job.id,
+                                            url,
+                                            completed,
+                                            total: urls.length,
+                                        }).catch(() => { }); // Fire and forget
+                                    }
+                                }
+                                catch (error) {
+                                    // Store error as result
+                                    results[index] = {
+                                        url,
+                                        error: error.message || 'Unknown error',
+                                    };
+                                    // Update progress
+                                    const completed = results.filter(r => r !== undefined).length;
+                                    jobQueue.updateJob(job.id, {
+                                        completed,
+                                        creditsUsed: completed,
+                                    });
+                                }
+                                finally {
+                                    activeCount--;
+                                }
+                            })();
+                        }
+                        // Wait for all tasks to complete
+                        while (activeCount > 0) {
+                            await new Promise(resolve => setTimeout(resolve, 100));
+                        }
+                    };
+                    await processBatch();
+                    // Update job with results
+                    jobQueue.updateJob(job.id, {
+                        status: 'completed',
+                        data: results,
+                    });
+                    // Send completed webhook and store delivery result
+                    if (normalizedWebhook2) {
+                        const delivery = await sendWebhook(normalizedWebhook2, 'completed', {
+                            jobId: job.id,
+                            total: urls.length,
+                            completed: results.length,
+                        });
+                        if (delivery) {
+                            jobQueue.updateJob(job.id, { webhookDelivery: delivery });
+                        }
+                    }
+                }
+                catch (error) {
+                    // Update job with error
+                    jobQueue.updateJob(job.id, {
+                        status: 'failed',
+                        error: error.message || 'Unknown error',
+                    });
+                    // Send failed webhook
+                    if (normalizedWebhook2) {
+                        await sendWebhook(normalizedWebhook2, 'failed', {
+                            jobId: job.id,
+                            error: error.message || 'Unknown error',
+                        });
+                    }
+                }
+            });
+            // Return job ID immediately
+            res.status(202).json({
+                success: true,
+                id: job.id,
+                url: `/v1/batch/scrape/${job.id}`,
+            });
+        }
+        catch (error) {
+            console.error('Batch scrape creation error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Failed to create batch scrape job',
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: crypto.randomUUID(),
+            });
+        }
+    });
+    /**
+     * GET /v1/batch/scrape/:id - Get batch scrape status + results
+     */
+    router.get('/v1/batch/scrape/:id', async (req, res) => {
+        try {
+            const id = req.params.id;
+            const job = await jobQueue.getJob(id);
+            if (!job) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: 'Job not found',
+                        docs: 'https://webpeel.dev/docs/errors#not_found',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            // SECURITY: Verify the requester owns this job
+            const requestOwnerId = req.auth?.keyInfo?.accountId;
+            if (job.ownerId && requestOwnerId && job.ownerId !== requestOwnerId) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: 'Job not found',
+                        docs: 'https://webpeel.dev/docs/errors#not_found',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            if (job.type !== 'batch') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Job is not a batch scrape job',
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            res.json({
+                success: true,
+                status: job.status,
+                progress: job.progress,
+                total: job.total,
+                completed: job.completed,
+                creditsUsed: job.creditsUsed,
+                data: job.data,
+                error: job.error,
+                expiresAt: job.expiresAt,
+                ...(job.webhookDelivery ? { webhook: job.webhookDelivery } : {}),
+            });
+        }
+        catch (error) {
+            console.error('Get batch scrape error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Failed to retrieve job',
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: crypto.randomUUID(),
+            });
+        }
+    });
+    /**
+     * DELETE /v1/batch/scrape/:id - Cancel batch scrape job
+     */
+    router.delete('/v1/batch/scrape/:id', async (req, res) => {
+        try {
+            const id = req.params.id;
+            const job = await jobQueue.getJob(id);
+            if (!job) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: 'Job not found',
+                        docs: 'https://webpeel.dev/docs/errors#not_found',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            // SECURITY: Verify the requester owns this job
+            const requestOwnerId = req.auth?.keyInfo?.accountId;
+            if (job.ownerId && requestOwnerId && job.ownerId !== requestOwnerId) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: 'Job not found',
+                        docs: 'https://webpeel.dev/docs/errors#not_found',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            if (job.type !== 'batch') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Job is not a batch scrape job',
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            const cancelled = await jobQueue.cancelJob(id);
+            if (!cancelled) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Job cannot be cancelled (already completed or failed)',
+                        hint: 'Only pending or processing jobs can be cancelled.',
+                        docs: 'https://webpeel.dev/docs/errors#invalid_request',
+                    },
+                    requestId: crypto.randomUUID(),
+                });
+                return;
+            }
+            res.json({
+                success: true,
+                message: 'Job cancelled',
+            });
+        }
+        catch (error) {
+            console.error('Cancel batch scrape error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Failed to cancel job',
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: crypto.randomUUID(),
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/cli-usage.d.ts

@@ -0,0 +1,6 @@
+/**
+ * CLI Usage endpoint — works with API key auth (not JWT)
+ * Used by the `webpeel usage` command and pre-fetch usage checks
+ */
+import { Router } from 'express';
+export declare function createCLIUsageRouter(): Router;

package/dist/server/routes/cli-usage.js

@@ -0,0 +1,127 @@
+/**
+ * CLI Usage endpoint — works with API key auth (not JWT)
+ * Used by the `webpeel usage` command and pre-fetch usage checks
+ */
+import { Router } from 'express';
+import pg from 'pg';
+const { Pool } = pg;
+export function createCLIUsageRouter() {
+    const router = Router();
+    const dbUrl = process.env.DATABASE_URL;
+    if (!dbUrl) {
+        // If no DB, return a stub router
+        router.get('/v1/cli/usage', (req, res) => {
+            res.status(501).json({
+                success: false,
+                error: {
+                    type: 'not_configured',
+                    message: 'Usage tracking requires PostgreSQL backend',
+                    docs: 'https://webpeel.dev/docs/errors#not_configured',
+                },
+                requestId: req.requestId,
+            });
+        });
+        return router;
+    }
+    const pool = new Pool({
+        connectionString: dbUrl,
+        // TLS: enabled when DATABASE_URL contains sslmode=require.
+        // Secure by default (rejectUnauthorized: true); set PG_REJECT_UNAUTHORIZED=false
+        // only for managed DBs (Render/Neon/Supabase) that use self-signed certs.
+        ssl: process.env.DATABASE_URL?.includes('sslmode=require')
+            ? { rejectUnauthorized: process.env.PG_REJECT_UNAUTHORIZED !== 'false' }
+            : undefined,
+    });
+    /**
+     * GET /v1/cli/usage
+     * Returns usage info for the authenticated API key's owner
+     * Auth: API key via Authorization: Bearer <key> or X-API-Key header
+     */
+    router.get('/v1/cli/usage', async (req, res) => {
+        try {
+            // Require API key auth (set by global auth middleware)
+            if (!req.auth?.keyInfo?.accountId) {
+                res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'Valid API key required. Run `webpeel login` to authenticate.', docs: 'https://webpeel.dev/docs/authentication' }, requestId: req.requestId });
+                return;
+            }
+            const userId = req.auth.keyInfo.accountId;
+            // Get user plan info
+            const planResult = await pool.query('SELECT tier, weekly_limit, burst_limit FROM users WHERE id = $1', [userId]);
+            if (planResult.rows.length === 0) {
+                res.status(404).json({ success: false, error: { type: 'user_not_found', message: 'User not found', docs: 'https://webpeel.dev/docs/errors#user_not_found' }, requestId: req.requestId });
+                return;
+            }
+            const plan = planResult.rows[0];
+            // Current week (ISO format)
+            const now = new Date();
+            const year = now.getUTCFullYear();
+            const jan4 = new Date(Date.UTC(year, 0, 4));
+            const weekNum = Math.ceil(((now.getTime() - jan4.getTime()) / 86400000 + jan4.getUTCDay() + 1) / 7);
+            const currentWeek = `${year}-W${String(weekNum).padStart(2, '0')}`;
+            // Current hour bucket
+            const currentHour = now.toISOString().substring(0, 13);
+            // Get weekly usage
+            const weeklyResult = await pool.query(`SELECT 
+          COALESCE(SUM(wu.total_count), 0) as total_used,
+          COALESCE(SUM(wu.basic_count), 0) as basic_used,
+          COALESCE(SUM(wu.stealth_count), 0) as stealth_used,
+          COALESCE(SUM(wu.search_count), 0) as search_used
+        FROM api_keys ak
+        LEFT JOIN weekly_usage wu ON wu.api_key_id = ak.id AND wu.week = $2
+        WHERE ak.user_id = $1 AND ak.is_active = true`, [userId, currentWeek]);
+            const weekly = weeklyResult.rows[0];
+            const totalUsed = parseInt(weekly.total_used) || 0;
+            const weeklyLimit = plan.weekly_limit || 125;
+            const remaining = Math.max(0, weeklyLimit - totalUsed);
+            // Get burst usage
+            const burstResult = await pool.query(`SELECT COALESCE(SUM(bu.count), 0) as burst_used
+        FROM api_keys ak
+        LEFT JOIN burst_usage bu ON bu.api_key_id = ak.id AND bu.hour_bucket = $2
+        WHERE ak.user_id = $1 AND ak.is_active = true`, [userId, currentHour]);
+            const burstUsed = parseInt(burstResult.rows[0]?.burst_used) || 0;
+            const burstLimit = plan.burst_limit || 25;
+            const minutesRemaining = 59 - now.getUTCMinutes();
+            // Get next Monday reset time
+            const dayOfWeek = now.getUTCDay();
+            const daysUntilMonday = dayOfWeek === 0 ? 1 : 8 - dayOfWeek;
+            const nextMonday = new Date(now);
+            nextMonday.setUTCDate(now.getUTCDate() + daysUntilMonday);
+            nextMonday.setUTCHours(0, 0, 0, 0);
+            res.json({
+                plan: {
+                    tier: plan.tier,
+                    weeklyLimit,
+                    burstLimit,
+                },
+                weekly: {
+                    used: totalUsed,
+                    limit: weeklyLimit,
+                    remaining,
+                    resetsAt: nextMonday.toISOString(),
+                    percentUsed: Math.round((totalUsed / weeklyLimit) * 100),
+                },
+                burst: {
+                    used: burstUsed,
+                    limit: burstLimit,
+                    resetsIn: minutesRemaining <= 0 ? '< 1 min' : `${minutesRemaining}m`,
+                },
+                // Simple boolean flags for CLI to check quickly
+                canFetch: remaining > 0 && burstUsed < burstLimit,
+                upgradeUrl: 'https://webpeel.dev/pricing',
+            });
+        }
+        catch (error) {
+            console.error('CLI usage error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Failed to retrieve usage',
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/compat.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Firecrawl API Compatibility Layer
+ *
+ * Drop-in replacement for Firecrawl's API - users can switch by ONLY changing the base URL.
+ * This is our killer acquisition feature.
+ *
+ * NOTE: Error responses in this file intentionally use Firecrawl's format:
+ *   { success: false, error: "Human-readable message" }
+ * This is required for Firecrawl drop-in compatibility and differs from the
+ * standard WebPeel API error format: { error: "error_code", message: "description" }.
+ * Do NOT change this format — it would break Firecrawl-compatible integrations.
+ *
+ * Implements Firecrawl endpoints:
+ * - POST /v1/scrape
+ * - POST /v2/scrape  (v2 with formats: ["screenshot"] support)
+ * - POST /v1/crawl
+ * - GET /v1/crawl/:id
+ * - POST /v1/search
+ * - POST /v1/map
+ */
+import { Router } from 'express';
+import type { IJobQueue } from '../job-queue.js';
+export declare function createCompatRouter(jobQueue: IJobQueue): Router;