webpeel 0.12.0 → 0.12.2

package/dist/server/routes/batch.js

@@ -0,0 +1,287 @@
+/**
+ * Batch scrape API - process multiple URLs concurrently
+ */
+import { Router } from 'express';
+import { peel } from '../../index.js';
+import { sendWebhook } from './webhooks.js';
+export function createBatchRouter(jobQueue) {
+    const router = Router();
+    /**
+     * POST /v1/batch/scrape - Submit batch of URLs
+     */
+    router.post('/v1/batch/scrape', async (req, res) => {
+        try {
+            const { urls, formats, extract, maxTokens, webhook } = req.body;
+            // Validate required parameters
+            if (!urls || !Array.isArray(urls) || urls.length === 0) {
+                res.status(400).json({
+                    error: 'invalid_request',
+                    message: 'Missing or invalid "urls" parameter (must be non-empty array)',
+                });
+                return;
+            }
+            // Limit batch size
+            if (urls.length > 100) {
+                res.status(400).json({
+                    error: 'invalid_request',
+                    message: 'Batch size too large (max 100 URLs)',
+                });
+                return;
+            }
+            // Validate URLs
+            for (const url of urls) {
+                if (typeof url !== 'string') {
+                    res.status(400).json({
+                        error: 'invalid_request',
+                        message: 'All URLs must be strings',
+                    });
+                    return;
+                }
+                try {
+                    new URL(url);
+                }
+                catch {
+                    res.status(400).json({
+                        error: 'invalid_url',
+                        message: `Invalid URL format: ${url}`,
+                    });
+                    return;
+                }
+            }
+            // Create job (with owner for authorization)
+            const ownerId = req.auth?.keyInfo?.accountId;
+            const job = await jobQueue.createJob('batch', webhook, ownerId);
+            await jobQueue.updateJob(job.id, {
+                total: urls.length,
+            });
+            // Start batch processing in background
+            setImmediate(async () => {
+                try {
+                    // Update job to processing
+                    jobQueue.updateJob(job.id, { status: 'processing' });
+                    // Send started webhook
+                    if (webhook) {
+                        await sendWebhook(webhook, 'started', {
+                            jobId: job.id,
+                            total: urls.length,
+                        });
+                    }
+                    // Build peel options
+                    const peelOptions = {
+                        format: formats?.[0] || 'markdown',
+                        extract,
+                        maxTokens,
+                    };
+                    // Process URLs with semaphore (max 5 concurrent)
+                    const results = [];
+                    const maxConcurrent = 5;
+                    let activeCount = 0;
+                    let urlIndex = 0;
+                    const processBatch = async () => {
+                        while (urlIndex < urls.length) {
+                            // Check if job was cancelled
+                            const currentJob = await jobQueue.getJob(job.id);
+                            if (currentJob?.status === 'cancelled') {
+                                return;
+                            }
+                            // Wait for available slot
+                            while (activeCount >= maxConcurrent) {
+                                await new Promise(resolve => setTimeout(resolve, 100));
+                            }
+                            const url = urls[urlIndex];
+                            const index = urlIndex;
+                            urlIndex++;
+                            activeCount++;
+                            // Process URL
+                            (async () => {
+                                try {
+                                    const result = await peel(url, peelOptions);
+                                    results[index] = result;
+                                    // Update progress
+                                    const completed = results.filter(r => r !== undefined).length;
+                                    jobQueue.updateJob(job.id, {
+                                        completed,
+                                        creditsUsed: completed,
+                                    });
+                                    // Send page webhook
+                                    if (webhook) {
+                                        sendWebhook(webhook, 'page', {
+                                            jobId: job.id,
+                                            url,
+                                            completed,
+                                            total: urls.length,
+                                        }).catch(() => { }); // Fire and forget
+                                    }
+                                }
+                                catch (error) {
+                                    // Store error as result
+                                    results[index] = {
+                                        url,
+                                        error: error.message || 'Unknown error',
+                                    };
+                                    // Update progress
+                                    const completed = results.filter(r => r !== undefined).length;
+                                    jobQueue.updateJob(job.id, {
+                                        completed,
+                                        creditsUsed: completed,
+                                    });
+                                }
+                                finally {
+                                    activeCount--;
+                                }
+                            })();
+                        }
+                        // Wait for all tasks to complete
+                        while (activeCount > 0) {
+                            await new Promise(resolve => setTimeout(resolve, 100));
+                        }
+                    };
+                    await processBatch();
+                    // Update job with results
+                    jobQueue.updateJob(job.id, {
+                        status: 'completed',
+                        data: results,
+                    });
+                    // Send completed webhook
+                    if (webhook) {
+                        await sendWebhook(webhook, 'completed', {
+                            jobId: job.id,
+                            total: urls.length,
+                            completed: results.length,
+                        });
+                    }
+                }
+                catch (error) {
+                    // Update job with error
+                    jobQueue.updateJob(job.id, {
+                        status: 'failed',
+                        error: error.message || 'Unknown error',
+                    });
+                    // Send failed webhook
+                    if (webhook) {
+                        await sendWebhook(webhook, 'failed', {
+                            jobId: job.id,
+                            error: error.message || 'Unknown error',
+                        });
+                    }
+                }
+            });
+            // Return job ID immediately
+            res.status(202).json({
+                success: true,
+                id: job.id,
+                url: `/v1/batch/scrape/${job.id}`,
+            });
+        }
+        catch (error) {
+            console.error('Batch scrape creation error:', error);
+            res.status(500).json({
+                error: 'internal_error',
+                message: 'Failed to create batch scrape job',
+            });
+        }
+    });
+    /**
+     * GET /v1/batch/scrape/:id - Get batch scrape status + results
+     */
+    router.get('/v1/batch/scrape/:id', async (req, res) => {
+        try {
+            const id = req.params.id;
+            const job = await jobQueue.getJob(id);
+            if (!job) {
+                res.status(404).json({
+                    error: 'not_found',
+                    message: 'Job not found',
+                });
+                return;
+            }
+            // SECURITY: Verify the requester owns this job
+            const requestOwnerId = req.auth?.keyInfo?.accountId;
+            if (job.ownerId && requestOwnerId && job.ownerId !== requestOwnerId) {
+                res.status(404).json({
+                    error: 'not_found',
+                    message: 'Job not found',
+                });
+                return;
+            }
+            if (job.type !== 'batch') {
+                res.status(400).json({
+                    error: 'invalid_request',
+                    message: 'Job is not a batch scrape job',
+                });
+                return;
+            }
+            res.json({
+                success: true,
+                status: job.status,
+                progress: job.progress,
+                total: job.total,
+                completed: job.completed,
+                creditsUsed: job.creditsUsed,
+                data: job.data,
+                error: job.error,
+                expiresAt: job.expiresAt,
+            });
+        }
+        catch (error) {
+            console.error('Get batch scrape error:', error);
+            res.status(500).json({
+                error: 'internal_error',
+                message: 'Failed to retrieve job',
+            });
+        }
+    });
+    /**
+     * DELETE /v1/batch/scrape/:id - Cancel batch scrape job
+     */
+    router.delete('/v1/batch/scrape/:id', async (req, res) => {
+        try {
+            const id = req.params.id;
+            const job = await jobQueue.getJob(id);
+            if (!job) {
+                res.status(404).json({
+                    error: 'not_found',
+                    message: 'Job not found',
+                });
+                return;
+            }
+            // SECURITY: Verify the requester owns this job
+            const requestOwnerId = req.auth?.keyInfo?.accountId;
+            if (job.ownerId && requestOwnerId && job.ownerId !== requestOwnerId) {
+                res.status(404).json({
+                    error: 'not_found',
+                    message: 'Job not found',
+                });
+                return;
+            }
+            if (job.type !== 'batch') {
+                res.status(400).json({
+                    error: 'invalid_request',
+                    message: 'Job is not a batch scrape job',
+                });
+                return;
+            }
+            const cancelled = await jobQueue.cancelJob(id);
+            if (!cancelled) {
+                res.status(400).json({
+                    error: 'invalid_request',
+                    message: 'Job cannot be cancelled (already completed or failed)',
+                });
+                return;
+            }
+            res.json({
+                success: true,
+                message: 'Job cancelled',
+            });
+        }
+        catch (error) {
+            console.error('Cancel batch scrape error:', error);
+            res.status(500).json({
+                error: 'internal_error',
+                message: 'Failed to cancel job',
+            });
+        }
+    });
+    return router;
+}
+//# sourceMappingURL=batch.js.map

package/dist/server/routes/batch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch.js","sourceRoot":"","sources":["../../../src/server/routes/batch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAGtC,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,UAAU,iBAAiB,CAAC,QAAmB;IACnD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;YAEhE,+BAA+B;YAC/B,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,+DAA+D;iBACzE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,mBAAmB;YACnB,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACtB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,qCAAqC;iBAC/C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,gBAAgB;YAChB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC5B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,iBAAiB;wBACxB,OAAO,EAAE,0BAA0B;qBACpC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,IAAI,CAAC;oBACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC;gBAAC,MAAM,CAAC;oBACP,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,aAAa;wBACpB,OAAO,EAAE,uBAAuB,GAAG,EAAE;qBACtC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;YACH,CAAC;YAED,4CAA4C;YAC5C,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC;YAC7C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;gBAC/B,KAAK,EAAE,IAAI,CAAC,MAAM;aACnB,CAAC,CAAC;YAEH,uCAAuC;YACvC,YAAY,CAAC,KAAK,IAAI,EAAE;gBACtB,IAAI,CAAC;oBACH,2BAA2B;oBAC3B,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;oBAErD,uBAAuB;oBACvB,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,WAAW,CAAC,OAAO,EAAE,SAAS,EAAE;4BACpC,KAAK,EAAE,GAAG,CAAC,EAAE;4BACb,KAAK,EAAE,IAAI,CAAC,MAAM;yBACnB,CAAC,CAAC;oBACL,CAAC;oBAED,qBAAqB;oBACrB,MAAM,WAAW,GAAgB;wBAC/B,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU;wBAClC,OAAO;wBACP,SAAS;qBACV,CAAC;oBAEF,iDAAiD;oBACjD,MAAM,OAAO,GAAU,EAAE,CAAC;oBAC1B,MAAM,aAAa,GAAG,CAAC,CAAC;oBACxB,IAAI,WAAW,GAAG,CAAC,CAAC;oBACpB,IAAI,QAAQ,GAAG,CAAC,CAAC;oBAEjB,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;wBAC9B,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;4BAC9B,6BAA6B;4BAC7B,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;4BACjD,IAAI,UAAU,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;gCACvC,OAAO;4BACT,CAAC;4BAED,0BAA0B;4BAC1B,OAAO,WAAW,IAAI,aAAa,EAAE,CAAC;gCACpC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;4BACzD,CAAC;4BAED,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;4BAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC;4BACvB,QAAQ,EAAE,CAAC;4BACX,WAAW,EAAE,CAAC;4BAEd,cAAc;4BACd,CAAC,KAAK,IAAI,EAAE;gCACV,IAAI,CAAC;oCACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;oCAC5C,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;oCAExB,kBAAkB;oCAClB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;oCAC9D,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wCACzB,SAAS;wCACT,WAAW,EAAE,SAAS;qCACvB,CAAC,CAAC;oCAEH,oBAAoB;oCACpB,IAAI,OAAO,EAAE,CAAC;wCACZ,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE;4CAC3B,KAAK,EAAE,GAAG,CAAC,EAAE;4CACb,GAAG;4CACH,SAAS;4CACT,KAAK,EAAE,IAAI,CAAC,MAAM;yCACnB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,kBAAkB;oCACxC,CAAC;gCACH,CAAC;gCAAC,OAAO,KAAU,EAAE,CAAC;oCACpB,wBAAwB;oCACxB,OAAO,CAAC,KAAK,CAAC,GAAG;wCACf,GAAG;wCACH,KAAK,EAAE,KAAK,CAAC,OAAO,IAAI,eAAe;qCACxC,CAAC;oCAEF,kBAAkB;oCAClB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;oCAC9D,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wCACzB,SAAS;wCACT,WAAW,EAAE,SAAS;qCACvB,CAAC,CAAC;gCACL,CAAC;wCAAS,CAAC;oCACT,WAAW,EAAE,CAAC;gCAChB,CAAC;4BACH,CAAC,CAAC,EAAE,CAAC;wBACP,CAAC;wBAED,iCAAiC;wBACjC,OAAO,WAAW,GAAG,CAAC,EAAE,CAAC;4BACvB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;wBACzD,CAAC;oBACH,CAAC,CAAC;oBAEF,MAAM,YAAY,EAAE,CAAC;oBAErB,0BAA0B;oBAC1B,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wBACzB,MAAM,EAAE,WAAW;wBACnB,IAAI,EAAE,OAAO;qBACd,CAAC,CAAC;oBAEH,yBAAyB;oBACzB,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,WAAW,CAAC,OAAO,EAAE,WAAW,EAAE;4BACtC,KAAK,EAAE,GAAG,CAAC,EAAE;4BACb,KAAK,EAAE,IAAI,CAAC,MAAM;4BAClB,SAAS,EAAE,OAAO,CAAC,MAAM;yBAC1B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,wBAAwB;oBACxB,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wBACzB,MAAM,EAAE,QAAQ;wBAChB,KAAK,EAAE,KAAK,CAAC,OAAO,IAAI,eAAe;qBACxC,CAAC,CAAC;oBAEH,sBAAsB;oBACtB,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE;4BACnC,KAAK,EAAE,GAAG,CAAC,EAAE;4BACb,KAAK,EAAE,KAAK,CAAC,OAAO,IAAI,eAAe;yBACxC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,4BAA4B;YAC5B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,OAAO,EAAE,IAAI;gBACb,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,GAAG,EAAE,oBAAoB,GAAG,CAAC,EAAE,EAAE;aAClC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC,CAAC;YACrD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,gBAAgB;gBACvB,OAAO,EAAE,mCAAmC;aAC7C,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH;;OAEG;IACH,MAAM,CAAC,GAAG,CAAC,sBAAsB,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QACvE,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,EAAY,CAAC;YACnC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,WAAW;oBAClB,OAAO,EAAE,eAAe;iBACzB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,+CAA+C;YAC/C,MAAM,cAAc,GAAG,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC;YACpD,IAAI,GAAG,CAAC,OAAO,IAAI,cAAc,IAAI,GAAG,CAAC,OAAO,KAAK,cAAc,EAAE,CAAC;gBACpE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,WAAW;oBAClB,OAAO,EAAE,eAAe;iBACzB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBACzB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,+BAA+B;iBACzC,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,GAAG,CAAC,IAAI,CAAC;gBACP,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,SAAS,EAAE,GAAG,CAAC,SAAS;aACzB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;YAChD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,gBAAgB;gBACvB,OAAO,EAAE,wBAAwB;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH;;OAEG;IACH,MAAM,CAAC,MAAM,CAAC,sBAAsB,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAC1E,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,EAAY,CAAC;YACnC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,WAAW;oBAClB,OAAO,EAAE,eAAe;iBACzB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,+CAA+C;YAC/C,MAAM,cAAc,GAAG,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC;YACpD,IAAI,GAAG,CAAC,OAAO,IAAI,cAAc,IAAI,GAAG,CAAC,OAAO,KAAK,cAAc,EAAE,CAAC;gBACpE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,WAAW;oBAClB,OAAO,EAAE,eAAe;iBACzB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBACzB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,+BAA+B;iBACzC,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAE/C,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,uDAAuD;iBACjE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,GAAG,CAAC,IAAI,CAAC;gBACP,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,eAAe;aACzB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAC;YACnD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,gBAAgB;gBACvB,OAAO,EAAE,sBAAsB;aAChC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/server/routes/cli-usage.d.ts

@@ -0,0 +1,7 @@
+/**
+ * CLI Usage endpoint — works with API key auth (not JWT)
+ * Used by the `webpeel usage` command and pre-fetch usage checks
+ */
+import { Router } from 'express';
+export declare function createCLIUsageRouter(): Router;
+//# sourceMappingURL=cli-usage.d.ts.map

package/dist/server/routes/cli-usage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-usage.d.ts","sourceRoot":"","sources":["../../../src/server/routes/cli-usage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAKpD,wBAAgB,oBAAoB,IAAI,MAAM,CAwI7C"}

package/dist/server/routes/cli-usage.js

@@ -0,0 +1,121 @@
+/**
+ * CLI Usage endpoint — works with API key auth (not JWT)
+ * Used by the `webpeel usage` command and pre-fetch usage checks
+ */
+import { Router } from 'express';
+import pg from 'pg';
+const { Pool } = pg;
+export function createCLIUsageRouter() {
+    const router = Router();
+    const dbUrl = process.env.DATABASE_URL;
+    if (!dbUrl) {
+        // If no DB, return a stub router
+        router.get('/v1/cli/usage', (_req, res) => {
+            res.status(501).json({
+                error: 'not_configured',
+                message: 'Usage tracking requires PostgreSQL backend',
+            });
+        });
+        return router;
+    }
+    const pool = new Pool({
+        connectionString: dbUrl,
+        // TLS: enabled when DATABASE_URL contains sslmode=require.
+        // Secure by default (rejectUnauthorized: true); set PG_REJECT_UNAUTHORIZED=false
+        // only for managed DBs (Render/Neon/Supabase) that use self-signed certs.
+        ssl: process.env.DATABASE_URL?.includes('sslmode=require')
+            ? { rejectUnauthorized: process.env.PG_REJECT_UNAUTHORIZED !== 'false' }
+            : undefined,
+    });
+    /**
+     * GET /v1/cli/usage
+     * Returns usage info for the authenticated API key's owner
+     * Auth: API key via Authorization: Bearer <key> or X-API-Key header
+     */
+    router.get('/v1/cli/usage', async (req, res) => {
+        try {
+            // Require API key auth (set by global auth middleware)
+            if (!req.auth?.keyInfo?.accountId) {
+                res.status(401).json({
+                    error: 'unauthorized',
+                    message: 'Valid API key required. Run `webpeel login` to authenticate.',
+                });
+                return;
+            }
+            const userId = req.auth.keyInfo.accountId;
+            // Get user plan info
+            const planResult = await pool.query('SELECT tier, weekly_limit, burst_limit FROM users WHERE id = $1', [userId]);
+            if (planResult.rows.length === 0) {
+                res.status(404).json({ error: 'user_not_found', message: 'User not found' });
+                return;
+            }
+            const plan = planResult.rows[0];
+            // Current week (ISO format)
+            const now = new Date();
+            const year = now.getUTCFullYear();
+            const jan4 = new Date(Date.UTC(year, 0, 4));
+            const weekNum = Math.ceil(((now.getTime() - jan4.getTime()) / 86400000 + jan4.getUTCDay() + 1) / 7);
+            const currentWeek = `${year}-W${String(weekNum).padStart(2, '0')}`;
+            // Current hour bucket
+            const currentHour = now.toISOString().substring(0, 13);
+            // Get weekly usage
+            const weeklyResult = await pool.query(`SELECT 
+          COALESCE(SUM(wu.total_count), 0) as total_used,
+          COALESCE(SUM(wu.basic_count), 0) as basic_used,
+          COALESCE(SUM(wu.stealth_count), 0) as stealth_used,
+          COALESCE(SUM(wu.search_count), 0) as search_used
+        FROM api_keys ak
+        LEFT JOIN weekly_usage wu ON wu.api_key_id = ak.id AND wu.week = $2
+        WHERE ak.user_id = $1 AND ak.is_active = true`, [userId, currentWeek]);
+            const weekly = weeklyResult.rows[0];
+            const totalUsed = parseInt(weekly.total_used) || 0;
+            const weeklyLimit = plan.weekly_limit || 125;
+            const remaining = Math.max(0, weeklyLimit - totalUsed);
+            // Get burst usage
+            const burstResult = await pool.query(`SELECT COALESCE(SUM(bu.count), 0) as burst_used
+        FROM api_keys ak
+        LEFT JOIN burst_usage bu ON bu.api_key_id = ak.id AND bu.hour_bucket = $2
+        WHERE ak.user_id = $1 AND ak.is_active = true`, [userId, currentHour]);
+            const burstUsed = parseInt(burstResult.rows[0]?.burst_used) || 0;
+            const burstLimit = plan.burst_limit || 25;
+            const minutesRemaining = 59 - now.getUTCMinutes();
+            // Get next Monday reset time
+            const dayOfWeek = now.getUTCDay();
+            const daysUntilMonday = dayOfWeek === 0 ? 1 : 8 - dayOfWeek;
+            const nextMonday = new Date(now);
+            nextMonday.setUTCDate(now.getUTCDate() + daysUntilMonday);
+            nextMonday.setUTCHours(0, 0, 0, 0);
+            res.json({
+                plan: {
+                    tier: plan.tier,
+                    weeklyLimit,
+                    burstLimit,
+                },
+                weekly: {
+                    used: totalUsed,
+                    limit: weeklyLimit,
+                    remaining,
+                    resetsAt: nextMonday.toISOString(),
+                    percentUsed: Math.round((totalUsed / weeklyLimit) * 100),
+                },
+                burst: {
+                    used: burstUsed,
+                    limit: burstLimit,
+                    resetsIn: minutesRemaining <= 0 ? '< 1 min' : `${minutesRemaining}m`,
+                },
+                // Simple boolean flags for CLI to check quickly
+                canFetch: remaining > 0 && burstUsed < burstLimit,
+                upgradeUrl: 'https://webpeel.dev/#pricing',
+            });
+        }
+        catch (error) {
+            console.error('CLI usage error:', error);
+            res.status(500).json({
+                error: 'internal_error',
+                message: 'Failed to retrieve usage',
+            });
+        }
+    });
+    return router;
+}
+//# sourceMappingURL=cli-usage.js.map

package/dist/server/routes/cli-usage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-usage.js","sourceRoot":"","sources":["../../../src/server/routes/cli-usage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;AAEpB,MAAM,UAAU,oBAAoB;IAClC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IACvC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,iCAAiC;QACjC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,IAAa,EAAE,GAAa,EAAE,EAAE;YAC3D,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,gBAAgB;gBACvB,OAAO,EAAE,4CAA4C;aACtD,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC;QACpB,gBAAgB,EAAE,KAAK;QACvB,2DAA2D;QAC3D,iFAAiF;QACjF,0EAA0E;QAC1E,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,QAAQ,CAAC,iBAAiB,CAAC;YACxD,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,KAAK,OAAO,EAAE;YACxE,CAAC,CAAC,SAAS;KACd,CAAC,CAAC;IAEH;;;;OAIG;IACH,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAChE,IAAI,CAAC;YACH,uDAAuD;YACvD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;gBAClC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,cAAc;oBACrB,OAAO,EAAE,8DAA8D;iBACxE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;YAE1C,qBAAqB;YACrB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CACjC,iEAAiE,EACjE,CAAC,MAAM,CAAC,CACT,CAAC;YAEF,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC7E,OAAO;YACT,CAAC;YAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAEhC,4BAA4B;YAC5B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,GAAG,CAAC,cAAc,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACpG,MAAM,WAAW,GAAG,GAAG,IAAI,KAAK,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAEnE,sBAAsB;YACtB,MAAM,WAAW,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAEvD,mBAAmB;YACnB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,KAAK,CACnC;;;;;;;sDAO8C,EAC9C,CAAC,MAAM,EAAE,WAAW,CAAC,CACtB,CAAC;YAEF,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACnD,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,IAAI,GAAG,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC,CAAC;YAEvD,kBAAkB;YAClB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAClC;;;sDAG8C,EAC9C,CAAC,MAAM,EAAE,WAAW,CAAC,CACtB,CAAC;YAEF,MAAM,SAAS,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;YACjE,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;YAC1C,MAAM,gBAAgB,GAAG,EAAE,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;YAElD,6BAA6B;YAC7B,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,eAAe,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;YAC5D,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;YACjC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,eAAe,CAAC,CAAC;YAC1D,UAAU,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAEnC,GAAG,CAAC,IAAI,CAAC;gBACP,IAAI,EAAE;oBACJ,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,WAAW;oBACX,UAAU;iBACX;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,WAAW;oBAClB,SAAS;oBACT,QAAQ,EAAE,UAAU,CAAC,WAAW,EAAE;oBAClC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC;iBACzD;gBACD,KAAK,EAAE;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,UAAU;oBACjB,QAAQ,EAAE,gBAAgB,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG;iBACrE;gBACD,gDAAgD;gBAChD,QAAQ,EAAE,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,UAAU;gBACjD,UAAU,EAAE,8BAA8B;aAC3C,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAC;YACzC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,gBAAgB;gBACvB,OAAO,EAAE,0BAA0B;aACpC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/server/routes/compat.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Firecrawl API Compatibility Layer
+ *
+ * Drop-in replacement for Firecrawl's API - users can switch by ONLY changing the base URL.
+ * This is our killer acquisition feature.
+ *
+ * NOTE: Error responses in this file intentionally use Firecrawl's format:
+ *   { success: false, error: "Human-readable message" }
+ * This is required for Firecrawl drop-in compatibility and differs from the
+ * standard WebPeel API error format: { error: "error_code", message: "description" }.
+ * Do NOT change this format — it would break Firecrawl-compatible integrations.
+ *
+ * Implements Firecrawl endpoints:
+ * - POST /v1/scrape
+ * - POST /v2/scrape  (v2 with formats: ["screenshot"] support)
+ * - POST /v1/crawl
+ * - GET /v1/crawl/:id
+ * - POST /v1/search
+ * - POST /v1/map
+ */
+import { Router } from 'express';
+import type { IJobQueue } from '../job-queue.js';
+export declare function createCompatRouter(jobQueue: IJobQueue): Router;
+//# sourceMappingURL=compat.d.ts.map

package/dist/server/routes/compat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compat.d.ts","sourceRoot":"","sources":["../../../src/server/routes/compat.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAKpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAkBjD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,SAAS,GAAG,MAAM,CA+sB9D"}