webpeel 0.20.2 → 0.20.3

package/dist/server/routes/compat.js

@@ -0,0 +1,652 @@
+/**
+ * Firecrawl API Compatibility Layer
+ *
+ * Drop-in replacement for Firecrawl's API - users can switch by ONLY changing the base URL.
+ * This is our killer acquisition feature.
+ *
+ * NOTE: Error responses in this file intentionally use Firecrawl's format:
+ *   { success: false, error: "Human-readable message" }
+ * This is required for Firecrawl drop-in compatibility and differs from the
+ * standard WebPeel API error format: { error: "error_code", message: "description" }.
+ * Do NOT change this format — it would break Firecrawl-compatible integrations.
+ *
+ * Implements Firecrawl endpoints:
+ * - POST /v1/scrape
+ * - POST /v2/scrape  (v2 with formats: ["screenshot"] support)
+ * - POST /v1/crawl
+ * - GET /v1/crawl/:id
+ * - POST /v1/search
+ * - POST /v1/map
+ */
+import { Router } from 'express';
+import { peel } from '../../index.js';
+import { crawl } from '../../core/crawler.js';
+import { mapDomain } from '../../core/map.js';
+import { takeScreenshot } from '../../core/screenshot.js';
+import { normalizeActions } from '../../core/actions.js';
+import { extractInlineJson } from '../../core/extract-inline.js';
+import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
+const VALID_LLM_PROVIDERS = ['openai', 'anthropic', 'google'];
+/**
+ * Map Firecrawl's action format to our PageAction format.
+ * Delegates to the shared normalizeActions helper so behaviour stays
+ * consistent across all API surfaces.
+ */
+function mapFirecrawlActions(actions) {
+    if (!actions || !Array.isArray(actions))
+        return undefined;
+    return normalizeActions(actions);
+}
+export function createCompatRouter(jobQueue) {
+    const router = Router();
+    /**
+     * POST /v1/scrape - Firecrawl's main scrape endpoint
+     *
+     * Maps to our peel() function
+     */
+    router.post('/v1/scrape', async (req, res) => {
+        try {
+            const { url, formats = ['markdown'], onlyMainContent = true, // Firecrawl defaults to true
+            includeTags, excludeTags, waitFor, timeout, actions, headers, location, 
+            // Inline extraction (BYOK)
+            extract: extractParam, llmProvider, llmApiKey, llmModel, stream, } = req.body;
+            // Validate URL
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: 'Missing or invalid "url" parameter',
+                });
+                return;
+            }
+            // SECURITY: Validate URL to prevent SSRF attacks
+            try {
+                validateUrlForSSRF(url);
+            }
+            catch (error) {
+                if (error instanceof SSRFError) {
+                    res.status(400).json({
+                        success: false,
+                        error: 'blocked_url',
+                        message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
+                    });
+                    return;
+                }
+                throw error;
+            }
+            // Determine if we need to render based on Firecrawl params
+            const needsRender = waitFor !== undefined || actions !== undefined;
+            // Map Firecrawl parameters to our PeelOptions
+            // onlyMainContent=true (default) → raw=false (use smart extraction)
+            // onlyMainContent=false → raw=true (return everything)
+            const options = {
+                render: needsRender,
+                wait: waitFor,
+                timeout: timeout || 30000,
+                stream: stream === true,
+                includeTags: Array.isArray(includeTags) ? includeTags : undefined,
+                excludeTags: Array.isArray(excludeTags) ? excludeTags : undefined,
+                raw: onlyMainContent === false,
+                actions: mapFirecrawlActions(actions),
+                headers,
+                screenshot: formats.includes('screenshot'),
+                images: formats.includes('images'),
+                format: 'markdown', // Always use markdown as base
+            };
+            // If location is provided, map it
+            if (location) {
+                options.location = {
+                    country: location.country,
+                    languages: location.languages,
+                };
+            }
+            if (options.stream) {
+                res.setHeader('X-Stream', 'true');
+                if (typeof res.flushHeaders === 'function') {
+                    res.flushHeaders();
+                }
+            }
+            // Execute peel
+            const result = await peel(url, options);
+            // Build Firecrawl-compatible response
+            const data = {
+                markdown: result.content,
+                metadata: {
+                    ...result.metadata,
+                    title: result.title,
+                    description: result.metadata?.description || '',
+                    language: result.metadata?.language || 'en',
+                    sourceURL: result.url,
+                    statusCode: 200,
+                },
+            };
+            // Add optional formats
+            if (formats.includes('html')) {
+                // Re-fetch with HTML format if requested
+                const htmlResult = await peel(url, { ...options, format: 'html' });
+                data.html = htmlResult.content;
+            }
+            if (formats.includes('rawHtml')) {
+                const rawResult = await peel(url, { ...options, format: 'html', raw: true });
+                data.rawHtml = rawResult.content;
+            }
+            if (formats.includes('links')) {
+                data.links = result.links;
+            }
+            if (formats.includes('screenshot') && result.screenshot) {
+                data.screenshot = `data:image/png;base64,${result.screenshot}`;
+            }
+            if (formats.includes('images') && result.images) {
+                data.images = result.images;
+            }
+            // --- Inline JSON extraction via LLM (BYOK) ---
+            // Resolve extract from: (1) top-level extract param, (2) formats array object
+            let resolvedExtract;
+            if (extractParam && typeof extractParam === 'object' && (extractParam.schema || extractParam.prompt)) {
+                resolvedExtract = extractParam;
+            }
+            if (!resolvedExtract) {
+                const jsonFormatObj = formats.find((f) => typeof f === 'object' && f !== null && f.type === 'json' && (f.schema || f.prompt));
+                if (jsonFormatObj) {
+                    resolvedExtract = { schema: jsonFormatObj.schema, prompt: jsonFormatObj.prompt };
+                }
+            }
+            if (resolvedExtract && llmApiKey && llmProvider && VALID_LLM_PROVIDERS.includes(llmProvider)) {
+                const extractResult = await extractInlineJson(result.content, {
+                    schema: resolvedExtract.schema,
+                    prompt: resolvedExtract.prompt,
+                    llmProvider: llmProvider,
+                    llmApiKey: llmApiKey.trim(),
+                    llmModel,
+                });
+                data.json = extractResult.data;
+                data.extractTokensUsed = extractResult.tokensUsed;
+            }
+            else if (formats.includes('json')) {
+                // Fallback: return structured metadata as JSON (no LLM)
+                data.json = result.extracted || result.metadata;
+            }
+            if (formats.includes('branding')) {
+                data.branding = result.branding;
+            }
+            if (formats.includes('summary')) {
+                data.summary = result.summary;
+            }
+            res.json({
+                success: true,
+                data,
+            });
+        }
+        catch (error) {
+            console.error('Firecrawl /v1/scrape error:', error);
+            res.status(500).json({
+                success: false,
+                error: 'An unexpected error occurred. Please try again.',
+            });
+        }
+    });
+    /**
+     * POST /v1/crawl - Firecrawl's crawl endpoint (async)
+     *
+     * Maps to our crawl() function with job queue
+     */
+    router.post('/v1/crawl', async (req, res) => {
+        try {
+            const { url, limit = 100, maxDepth = 3, includePaths = [], excludePaths = [], scrapeOptions = {}, webhook, } = req.body;
+            // Validate URL
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: 'Missing or invalid "url" parameter',
+                });
+                return;
+            }
+            try {
+                new URL(url);
+            }
+            catch {
+                res.status(400).json({
+                    success: false,
+                    error: 'Invalid URL format',
+                });
+                return;
+            }
+            // SECURITY: Validate URL to prevent SSRF attacks
+            try {
+                validateUrlForSSRF(url);
+            }
+            catch (error) {
+                if (error instanceof SSRFError) {
+                    res.status(400).json({
+                        success: false,
+                        error: 'blocked_url',
+                        message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
+                    });
+                    return;
+                }
+                throw error;
+            }
+            // Create job (with owner for authorization)
+            const ownerId = req.auth?.keyInfo?.accountId;
+            const job = await jobQueue.createJob('crawl', webhook, ownerId);
+            // Start crawl in background
+            setImmediate(async () => {
+                try {
+                    jobQueue.updateJob(job.id, { status: 'processing' });
+                    // Build crawl options
+                    const crawlOptions = {
+                        maxPages: limit,
+                        maxDepth,
+                        tier: req.auth?.tier,
+                        onProgress: (progress) => {
+                            const total = progress.crawled + progress.queued;
+                            jobQueue.updateJob(job.id, {
+                                total,
+                                completed: progress.crawled,
+                                creditsUsed: progress.crawled,
+                            });
+                        },
+                        // Map scrapeOptions to PeelOptions
+                        ...scrapeOptions,
+                    };
+                    // Add path filters if provided
+                    if (includePaths.length > 0) {
+                        crawlOptions.includePatterns = includePaths;
+                    }
+                    if (excludePaths.length > 0) {
+                        crawlOptions.excludePatterns = excludePaths;
+                    }
+                    // Run crawl
+                    const results = await crawl(url, crawlOptions);
+                    // Map results to Firecrawl format
+                    const firecrawlResults = results.map(r => ({
+                        url: r.url,
+                        markdown: r.markdown,
+                        metadata: {
+                            title: r.title,
+                            description: '',
+                            sourceURL: r.url,
+                            statusCode: 200,
+                        },
+                        links: r.links,
+                    }));
+                    // Update job with results
+                    jobQueue.updateJob(job.id, {
+                        status: 'completed',
+                        data: firecrawlResults,
+                        total: results.length,
+                        completed: results.length,
+                        creditsUsed: results.length,
+                    });
+                }
+                catch (error) {
+                    jobQueue.updateJob(job.id, {
+                        status: 'failed',
+                        error: error.message || 'Unknown error',
+                    });
+                }
+            });
+            // Return job ID immediately (Firecrawl format)
+            res.json({
+                success: true,
+                id: job.id,
+            });
+        }
+        catch (error) {
+            console.error('Firecrawl /v1/crawl error:', error);
+            res.status(500).json({
+                success: false,
+                error: 'An unexpected error occurred. Please try again.',
+            });
+        }
+    });
+    /**
+     * GET /v1/crawl/:id - Get crawl job status (Firecrawl format)
+     */
+    router.get('/v1/crawl/:id', async (req, res) => {
+        try {
+            const id = req.params.id;
+            const job = await jobQueue.getJob(id);
+            if (!job) {
+                res.status(404).json({
+                    success: false,
+                    error: 'Job not found',
+                });
+                return;
+            }
+            // SECURITY: Verify the requester owns this job
+            const requestOwnerId = req.auth?.keyInfo?.accountId;
+            if (job.ownerId && requestOwnerId && job.ownerId !== requestOwnerId) {
+                res.status(404).json({
+                    success: false,
+                    error: 'Job not found',
+                });
+                return;
+            }
+            // Map our job status to Firecrawl's status format
+            const firecrawlStatus = job.status === 'processing' ? 'scraping' : job.status;
+            res.json({
+                success: true,
+                status: firecrawlStatus,
+                completed: job.completed || 0,
+                total: job.total || 0,
+                creditsUsed: job.creditsUsed || 0,
+                expiresAt: job.expiresAt,
+                data: job.data || [],
+            });
+        }
+        catch (error) {
+            console.error('Firecrawl GET /v1/crawl/:id error:', error);
+            res.status(500).json({
+                success: false,
+                error: 'An unexpected error occurred. Please try again.',
+            });
+        }
+    });
+    /**
+     * POST /v1/search - Firecrawl's search endpoint
+     *
+     * Uses DuckDuckGo search with optional scraping
+     */
+    router.post('/v1/search', async (req, res) => {
+        try {
+            const { query, limit = 5, scrapeOptions = {}, } = req.body;
+            // Validate query
+            if (!query || typeof query !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: 'Missing or invalid "query" parameter',
+                });
+                return;
+            }
+            // Use our search route logic (DuckDuckGo HTML scraping)
+            const { fetch: undiciFetch } = await import('undici');
+            const { load } = await import('cheerio');
+            const searchUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
+            const response = await undiciFetch(searchUrl, {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
+                },
+            });
+            if (!response.ok) {
+                throw new Error(`Search failed: HTTP ${response.status}`);
+            }
+            const html = await response.text();
+            const $ = load(html);
+            const results = [];
+            $('.result').each((_i, elem) => {
+                if (results.length >= limit)
+                    return;
+                const $result = $(elem);
+                let title = $result.find('.result__title').text().trim();
+                const rawUrl = $result.find('.result__a').attr('href') || '';
+                let snippet = $result.find('.result__snippet').text().trim();
+                if (!title || !rawUrl)
+                    return;
+                // Extract actual URL from DuckDuckGo redirect
+                let url = rawUrl;
+                try {
+                    const ddgUrl = new URL(rawUrl, 'https://duckduckgo.com');
+                    const uddg = ddgUrl.searchParams.get('uddg');
+                    if (uddg) {
+                        url = decodeURIComponent(uddg);
+                    }
+                }
+                catch (e) {
+                    if (process.env.DEBUG)
+                        console.debug('[webpeel]', 'ddg url parse failed:', e instanceof Error ? e.message : e);
+                }
+                // Validate URL
+                try {
+                    const parsed = new URL(url);
+                    if (!['http:', 'https:'].includes(parsed.protocol)) {
+                        return;
+                    }
+                    url = parsed.href;
+                }
+                catch {
+                    return;
+                }
+                results.push({ title, url, snippet });
+            });
+            // If scraping is requested, fetch each result
+            const firecrawlResults = await Promise.all(results.map(async (result) => {
+                try {
+                    // Scrape the URL with provided options
+                    const peelResult = await peel(result.url, {
+                        format: 'markdown',
+                        timeout: 10000,
+                        ...scrapeOptions,
+                    });
+                    return {
+                        url: result.url,
+                        markdown: peelResult.content,
+                        metadata: {
+                            title: peelResult.title || result.title,
+                            description: result.snippet,
+                            sourceURL: result.url,
+                            statusCode: 200,
+                            ...peelResult.metadata,
+                        },
+                    };
+                }
+                catch (error) {
+                    // Return basic result if scraping fails
+                    return {
+                        url: result.url,
+                        markdown: '',
+                        metadata: {
+                            title: result.title,
+                            description: result.snippet,
+                            sourceURL: result.url,
+                            error: error.message,
+                        },
+                    };
+                }
+            }));
+            res.json({
+                success: true,
+                data: firecrawlResults,
+            });
+        }
+        catch (error) {
+            console.error('Firecrawl /v1/search error:', error);
+            res.status(500).json({
+                success: false,
+                error: 'An unexpected error occurred. Please try again.',
+            });
+        }
+    });
+    /**
+     * POST /v1/map - Firecrawl's map endpoint
+     *
+     * Maps to our mapDomain() function
+     */
+    router.post('/v1/map', async (req, res) => {
+        try {
+            const { url, limit = 5000, search, } = req.body;
+            // Validate URL
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: 'Missing or invalid "url" parameter',
+                });
+                return;
+            }
+            try {
+                new URL(url);
+            }
+            catch {
+                res.status(400).json({
+                    success: false,
+                    error: 'Invalid URL format',
+                });
+                return;
+            }
+            // SECURITY: Validate URL to prevent SSRF attacks
+            try {
+                validateUrlForSSRF(url);
+            }
+            catch (error) {
+                if (error instanceof SSRFError) {
+                    res.status(400).json({
+                        success: false,
+                        error: 'blocked_url',
+                        message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
+                    });
+                    return;
+                }
+                throw error;
+            }
+            // Run mapDomain
+            const result = await mapDomain(url, {
+                maxUrls: limit,
+                search,
+            });
+            res.json({
+                success: true,
+                links: result.urls,
+            });
+        }
+        catch (error) {
+            console.error('Firecrawl /v1/map error:', error);
+            res.status(500).json({
+                success: false,
+                error: 'An unexpected error occurred. Please try again.',
+            });
+        }
+    });
+    /**
+     * POST /v2/scrape - Firecrawl v2-compatible scrape with screenshot support
+     *
+     * Same as /v1/scrape but adds first-class screenshot support.
+     * When formats includes "screenshot" (and nothing else), returns
+     * a screenshot directly; otherwise falls through to peel() like v1.
+     */
+    router.post('/v2/scrape', async (req, res) => {
+        try {
+            const { url, formats = ['markdown'], onlyMainContent = true, includeTags, excludeTags, waitFor, timeout, actions, headers, location, 
+            // Screenshot-specific v2 options
+            fullPage, width, height, screenshotFormat, quality, stream, } = req.body;
+            // Validate URL
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: 'Missing or invalid "url" parameter',
+                });
+                return;
+            }
+            // SECURITY: Validate URL to prevent SSRF attacks
+            try {
+                validateUrlForSSRF(url);
+            }
+            catch (error) {
+                if (error instanceof SSRFError) {
+                    res.status(400).json({
+                        success: false,
+                        error: 'blocked_url',
+                        message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
+                    });
+                    return;
+                }
+                throw error;
+            }
+            const wantsScreenshot = formats.includes('screenshot') || formats.includes('screenshot@fullPage');
+            // If screenshot-only request, use the dedicated screenshot function
+            if (wantsScreenshot && formats.length === 1) {
+                const result = await takeScreenshot(url, {
+                    fullPage: fullPage === true || formats[0] === 'screenshot@fullPage',
+                    width: typeof width === 'number' ? width : undefined,
+                    height: typeof height === 'number' ? height : undefined,
+                    format: screenshotFormat || 'png',
+                    quality: typeof quality === 'number' ? quality : undefined,
+                    waitFor: typeof waitFor === 'number' ? waitFor : undefined,
+                    timeout: typeof timeout === 'number' ? timeout : 30000,
+                    actions: mapFirecrawlActions(actions),
+                    headers,
+                });
+                res.json({
+                    success: true,
+                    data: {
+                        screenshot: `data:${result.contentType};base64,${result.screenshot}`,
+                        metadata: {
+                            sourceURL: result.url,
+                            statusCode: 200,
+                            format: result.format,
+                        },
+                    },
+                });
+                return;
+            }
+            // Otherwise, fall through to peel() like v1/scrape
+            const needsRender = waitFor !== undefined || actions !== undefined || wantsScreenshot;
+            const options = {
+                render: needsRender,
+                wait: waitFor,
+                timeout: timeout || 30000,
+                stream: stream === true,
+                includeTags: Array.isArray(includeTags) ? includeTags : undefined,
+                excludeTags: Array.isArray(excludeTags) ? excludeTags : undefined,
+                raw: onlyMainContent === false,
+                actions: mapFirecrawlActions(actions),
+                headers,
+                screenshot: wantsScreenshot,
+                screenshotFullPage: fullPage === true,
+                images: formats.includes('images'),
+                format: 'markdown',
+            };
+            if (location) {
+                options.location = {
+                    country: location.country,
+                    languages: location.languages,
+                };
+            }
+            if (options.stream) {
+                res.setHeader('X-Stream', 'true');
+                if (typeof res.flushHeaders === 'function') {
+                    res.flushHeaders();
+                }
+            }
+            const result = await peel(url, options);
+            const data = {
+                markdown: result.content,
+                metadata: {
+                    title: result.title,
+                    description: result.metadata.description || '',
+                    language: 'en',
+                    sourceURL: result.url,
+                    statusCode: 200,
+                    ...result.metadata,
+                },
+            };
+            if (formats.includes('html')) {
+                const htmlResult = await peel(url, { ...options, format: 'html' });
+                data.html = htmlResult.content;
+            }
+            if (formats.includes('rawHtml')) {
+                const rawResult = await peel(url, { ...options, format: 'html', raw: true });
+                data.rawHtml = rawResult.content;
+            }
+            if (formats.includes('links')) {
+                data.links = result.links;
+            }
+            if (wantsScreenshot && result.screenshot) {
+                data.screenshot = `data:image/png;base64,${result.screenshot}`;
+            }
+            if (formats.includes('images') && result.images) {
+                data.images = result.images;
+            }
+            res.json({
+                success: true,
+                data,
+            });
+        }
+        catch (error) {
+            console.error('Firecrawl /v2/scrape error:', error);
+            res.status(500).json({
+                success: false,
+                error: 'An unexpected error occurred. Please try again.',
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/deep-fetch.d.ts

@@ -0,0 +1,8 @@
+/**
+ * POST /v1/deep-fetch
+ *
+ * Deep web intelligence endpoint: search + fetch + synthesize + structure.
+ * Body: { query, count?, format?, maxChars? }
+ */
+import { Router } from 'express';
+export declare function createDeepFetchRouter(): Router;