webpeel 0.19.4 → 0.20.0

package/dist/server/routes/extract.js

@@ -1,150 +0,0 @@
-/**
- * POST /v1/extract — Firecrawl-compatible JSON Schema extraction endpoint.
- *
- * Body: { url: string, schema?: object, prompt?: string, llmApiKey?: string, model?: string }
- * Returns: { success: true, data: <extracted data> }
- */
-import { Router } from 'express';
-import { peel } from '../../index.js';
-import { extractWithLLM } from '../../core/llm-extract.js';
-export function createExtractRouter() {
-    const router = Router();
-    router.post('/v1/extract', async (req, res) => {
-        try {
-            const { url, schema, prompt, llmApiKey, model, baseUrl, } = req.body;
-            // Validate URL
-            if (!url || typeof url !== 'string') {
-                res.status(400).json({
-                    success: false,
-                    error: 'invalid_request',
-                    message: 'Missing or invalid "url" field in request body.',
-                    example: '{ "url": "https://example.com", "schema": { "type": "object", "properties": { "title": { "type": "string" } } } }',
-                });
-                return;
-            }
-            if (url.length > 2048) {
-                res.status(400).json({
-                    success: false,
-                    error: 'invalid_url',
-                    message: 'URL too long (max 2048 characters)',
-                });
-                return;
-            }
-            // Validate URL format
-            let parsedUrl;
-            try {
-                parsedUrl = new URL(url);
-                if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
-                    res.status(400).json({
-                        success: false,
-                        error: 'invalid_url',
-                        message: 'Only HTTP and HTTPS URLs are supported',
-                    });
-                    return;
-                }
-            }
-            catch {
-                res.status(400).json({
-                    success: false,
-                    error: 'invalid_url',
-                    message: `Invalid URL format: ${url}`,
-                });
-                return;
-            }
-            // Require at least schema or prompt
-            if (!schema && !prompt) {
-                res.status(400).json({
-                    success: false,
-                    error: 'invalid_request',
-                    message: 'Either "schema" or "prompt" is required for structured extraction.',
-                });
-                return;
-            }
-            // Resolve API key from request body or environment
-            const resolvedApiKey = llmApiKey || process.env.OPENAI_API_KEY;
-            if (!resolvedApiKey) {
-                res.status(400).json({
-                    success: false,
-                    error: 'missing_api_key',
-                    message: 'LLM API key required. Provide "llmApiKey" in the request body or set OPENAI_API_KEY on the server.',
-                });
-                return;
-            }
-            // Fetch the page content
-            const peelResult = await peel(url, {
-                format: 'markdown',
-                timeout: 30000,
-            });
-            // Extract structured data with LLM
-            const extractResult = await extractWithLLM({
-                content: peelResult.content,
-                instruction: prompt,
-                schema,
-                apiKey: resolvedApiKey,
-                model: model || process.env.WEBPEEL_LLM_MODEL || 'gpt-4o-mini',
-                baseUrl: baseUrl || process.env.WEBPEEL_LLM_BASE_URL || 'https://api.openai.com/v1',
-            });
-            // Return in Firecrawl-compatible format
-            res.json({
-                success: true,
-                data: extractResult.items.length === 1 ? extractResult.items[0] : extractResult.items,
-                metadata: {
-                    url: peelResult.url,
-                    title: peelResult.title,
-                    tokensUsed: extractResult.tokensUsed,
-                    model: extractResult.model,
-                    cost: extractResult.cost,
-                    elapsed: peelResult.elapsed,
-                },
-            });
-        }
-        catch (error) {
-            console.error('[/v1/extract] Error:', error instanceof Error ? error.message : String(error));
-            const msg = error instanceof Error ? error.message : 'Unknown error';
-            if (msg.includes('authentication failed') || msg.includes('401')) {
-                res.status(401).json({ success: false, error: 'llm_auth_failed', message: msg });
-                return;
-            }
-            if (msg.includes('rate limit') || msg.includes('429')) {
-                res.status(429).json({ success: false, error: 'llm_rate_limited', message: msg });
-                return;
-            }
-            res.status(500).json({
-                success: false,
-                error: 'extraction_failed',
-                message: msg,
-            });
-        }
-    });
-    router.get('/v1/extract/auto', async (req, res) => {
-        const url = req.query.url;
-        if (!url) {
-            res.status(400).json({ error: 'Missing url parameter' });
-            return;
-        }
-        const { autoExtract } = await import('../../core/auto-extract.js');
-        const result = await peel(url, { format: 'html' });
-        const extracted = autoExtract(result.content || '', url);
-        res.json({ url, pageType: extracted.type, structured: extracted });
-    });
-    router.post('/v1/extract/auto', async (req, res) => {
-        const { url, ...rest } = req.body;
-        if (!url || typeof url !== 'string') {
-            res.status(400).json({ error: 'Missing or invalid url field in request body' });
-            return;
-        }
-        try {
-            const { autoExtract } = await import('../../core/auto-extract.js');
-            const result = await peel(url, { format: 'html', ...rest });
-            const extracted = autoExtract(result.content || '', url);
-            res.json({ url, pageType: extracted.type, structured: extracted });
-        }
-        catch (error) {
-            const msg = error instanceof Error ? error.message : 'Unknown error';
-            console.error('[/v1/extract/auto POST] Error:', msg);
-            res.status(500).json({ error: 'extraction_failed', message: msg });
-        }
-    });
-    return router;
-}
-//# sourceMappingURL=extract.js.map

package/dist/server/routes/extract.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../src/server/routes/extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAE3D,MAAM,UAAU,mBAAmB;IACjC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAC/D,IAAI,CAAC;YACH,MAAM,EACJ,GAAG,EACH,MAAM,EACN,MAAM,EACN,SAAS,EACT,KAAK,EACL,OAAO,GACR,GAAG,GAAG,CAAC,IAOP,CAAC;YAEF,eAAe;YACf,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACpC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,iDAAiD;oBAC1D,OAAO,EAAE,mHAAmH;iBAC7H,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBACtB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oCAAoC;iBAC9C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,sBAAsB;YACtB,IAAI,SAAc,CAAC;YACnB,IAAI,CAAC;gBACH,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBACzB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACtD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,aAAa;wBACpB,OAAO,EAAE,wCAAwC;qBAClD,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,uBAAuB,GAAG,EAAE;iBACtC,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,oCAAoC;YACpC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;gBACvB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,oEAAoE;iBAC9E,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,mDAAmD;YACnD,MAAM,cAAc,GAAG,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YAC/D,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,oGAAoG;iBAC9G,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,yBAAyB;YACzB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE;gBACjC,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YAEH,mCAAmC;YACnC,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC;gBACzC,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,WAAW,EAAE,MAAM;gBACnB,MAAM;gBACN,MAAM,EAAE,cAAc;gBACtB,KAAK,EAAE,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa;gBAC9D,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,2BAA2B;aACpF,CAAC,CAAC;YAEH,wCAAwC;YACxC,GAAG,CAAC,IAAI,CAAC;gBACP,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK;gBACrF,QAAQ,EAAE;oBACR,GAAG,EAAE,UAAU,CAAC,GAAG;oBACnB,KAAK,EAAE,UAAU,CAAC,KAAK;oBACvB,UAAU,EAAE,aAAa,CAAC,UAAU;oBACpC,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;oBACxB,OAAO,EAAE,UAAU,CAAC,OAAO;iBAC5B;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAE9F,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAErE,IAAI,GAAG,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,iBAAiB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;gBACjF,OAAO;YACT,CAAC;YACD,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,kBAAkB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;gBAClF,OAAO;YACT,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB;gBAC1B,OAAO,EAAE,GAAG;aACb,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,GAAG,CAAC,kBAAkB,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QACnE,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAa,CAAC;QACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;YACzD,OAAO;QACT,CAAC;QACD,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,4BAA4B,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACnD,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;QACzD,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QACpE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC,IAAgD,CAAC;QAC9E,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YACpC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,8CAA8C,EAAE,CAAC,CAAC;YAChF,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,4BAA4B,CAAC,CAAC;YACnE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;YAC5D,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;YACzD,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YACrE,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;YACrD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/server/routes/fetch.d.ts

@@ -1,8 +0,0 @@
-/**
- * Fetch endpoint with caching
- */
-import { Router } from 'express';
-import '../types.js';
-import { AuthStore } from '../auth-store.js';
-export declare function createFetchRouter(authStore: AuthStore): Router;
-//# sourceMappingURL=fetch.d.ts.map

package/dist/server/routes/fetch.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../../src/server/routes/fetch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,aAAa,CAAC;AAMrB,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AA4G7C,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM,CAwlC9D"}