webpeel 0.20.2 → 0.20.3

package/dist/server/routes/do.js

@@ -0,0 +1,72 @@
+/**
+ * /v1/do — Intent-based endpoint.
+ * One endpoint that understands natural language and routes internally.
+ * POST /v1/do { task: "find Stripe fees" }
+ * GET  /v1/do?task=find+Stripe+fees
+ */
+import { Router } from 'express';
+import { parseIntent } from '../../mcp/smart-router.js';
+import { getHandler } from '../../mcp/handlers/index.js';
+export function createDoRouter() {
+    const router = Router();
+    async function handleDo(req, res) {
+        const task = req.body?.task || req.query?.task;
+        if (!task?.trim()) {
+            res.status(400).json({ success: false, error: { type: 'missing_task', message: 'Provide task= parameter or {"task": "..."}', hint: 'POST {"task": "search for X"} or GET /v1/do?task=search+for+X', docs: 'https://webpeel.dev/docs/errors#missing_task' }, requestId: req.requestId });
+            return;
+        }
+        const startMs = Date.now();
+        const intent = parseIntent(task);
+        // Map intent to handler
+        const toolName = `webpeel_${intent.intent}`;
+        const handler = getHandler(toolName);
+        if (!handler) {
+            res.status(400).json({ success: false, error: { type: 'unknown_intent', message: `Could not understand: "${task}"`, hint: 'Try phrasing as: "fetch https://example.com", "search for AI news", or "screenshot https://example.com"', docs: 'https://webpeel.dev/docs/errors#unknown_intent' }, requestId: req.requestId });
+            return;
+        }
+        // Build args from parsed intent
+        const args = { ...intent.params };
+        if (intent.url)
+            args.url = intent.url;
+        if (intent.query)
+            args.query = intent.query;
+        try {
+            const ctx = {
+                accountId: req.auth?.keyInfo?.accountId || req.user?.userId,
+            };
+            const result = await handler(args, ctx);
+            // Extract text content from MCP result format
+            const firstItem = result.content?.[0];
+            const content = firstItem?.type === 'text' ? firstItem.text : undefined;
+            let parsed;
+            try {
+                parsed = JSON.parse(content || '{}');
+            }
+            catch {
+                parsed = { raw: content };
+            }
+            res.json({
+                task,
+                intent: intent.intent,
+                ...(intent.url ? { url: intent.url } : {}),
+                ...(intent.query ? { query: intent.query } : {}),
+                result: parsed,
+                elapsed: Date.now() - startMs,
+            });
+        }
+        catch (err) {
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'execution_failed',
+                    message: err.message,
+                    docs: 'https://webpeel.dev/docs/errors#execution_failed',
+                },
+                requestId: req.requestId,
+            });
+        }
+    }
+    router.get('/', handleDo);
+    router.post('/', handleDo);
+    return router;
+}

package/dist/server/routes/extract.d.ts

@@ -0,0 +1,8 @@
+/**
+ * POST /v1/extract — Firecrawl-compatible JSON Schema extraction endpoint.
+ *
+ * Body: { url: string, schema?: object, prompt?: string, llmApiKey?: string, model?: string }
+ * Returns: { success: true, data: <extracted data> }
+ */
+import { Router } from 'express';
+export declare function createExtractRouter(): Router;

package/dist/server/routes/extract.js

@@ -0,0 +1,235 @@
+/**
+ * POST /v1/extract — Firecrawl-compatible JSON Schema extraction endpoint.
+ *
+ * Body: { url: string, schema?: object, prompt?: string, llmApiKey?: string, model?: string }
+ * Returns: { success: true, data: <extracted data> }
+ */
+import { Router } from 'express';
+import crypto from 'crypto';
+import { peel } from '../../index.js';
+import { extractWithLLM } from '../../core/llm-extract.js';
+export function createExtractRouter() {
+    const router = Router();
+    router.post('/v1/extract', async (req, res) => {
+        try {
+            const { url, schema, prompt, llmApiKey, llmProvider, model, baseUrl, } = req.body;
+            // Validate URL
+            if (!url || typeof url !== 'string') {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Missing or invalid "url" field in request body.',
+                        hint: 'Pass a URL in the request body: { "url": "https://example.com", "schema": { ... } }',
+                        docs: 'https://webpeel.dev/docs/errors#invalid-request',
+                    },
+                    requestId: req.requestId || crypto.randomUUID(),
+                });
+                return;
+            }
+            if (url.length > 2048) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_url',
+                        message: 'URL too long (max 2048 characters)',
+                        hint: 'Shorten the URL to under 2048 characters.',
+                        docs: 'https://webpeel.dev/docs/errors#invalid-url',
+                    },
+                    requestId: req.requestId || crypto.randomUUID(),
+                });
+                return;
+            }
+            // Validate URL format
+            let parsedUrl;
+            try {
+                parsedUrl = new URL(url);
+                if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
+                    res.status(400).json({
+                        success: false,
+                        error: {
+                            type: 'invalid_url',
+                            message: 'Only HTTP and HTTPS URLs are supported',
+                            hint: 'Ensure the URL starts with http:// or https://',
+                            docs: 'https://webpeel.dev/docs/errors#invalid-url',
+                        },
+                        requestId: req.requestId || crypto.randomUUID(),
+                    });
+                    return;
+                }
+            }
+            catch {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_url',
+                        message: `Invalid URL format: ${url}`,
+                        hint: 'Ensure the URL is well-formed: https://example.com',
+                        docs: 'https://webpeel.dev/docs/errors#invalid-url',
+                    },
+                    requestId: req.requestId || crypto.randomUUID(),
+                });
+                return;
+            }
+            // Require at least schema or prompt
+            if (!schema && !prompt) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'invalid_request',
+                        message: 'Either "schema" or "prompt" is required for structured extraction.',
+                        hint: 'Include a JSON schema or a natural language prompt in the request body.',
+                        docs: 'https://webpeel.dev/docs/errors#invalid-request',
+                    },
+                    requestId: req.requestId || crypto.randomUUID(),
+                });
+                return;
+            }
+            // Resolve provider and API key
+            const resolvedProvider = (['openai', 'anthropic', 'google'].includes(llmProvider || ''))
+                ? llmProvider
+                : 'openai';
+            // Resolve API key from request body or environment
+            const resolvedApiKey = llmApiKey || process.env.OPENAI_API_KEY;
+            if (!resolvedApiKey) {
+                res.status(400).json({
+                    success: false,
+                    error: {
+                        type: 'missing_api_key',
+                        message: 'LLM API key required. Provide "llmApiKey" in the request body or set OPENAI_API_KEY on the server.',
+                        hint: 'Pass your API key: { "llmApiKey": "sk-...", "llmProvider": "openai" }',
+                        docs: 'https://webpeel.dev/docs/errors#missing-api-key',
+                    },
+                    requestId: req.requestId || crypto.randomUUID(),
+                });
+                return;
+            }
+            // Fetch the page content
+            const peelResult = await peel(url, {
+                format: 'markdown',
+                timeout: 30000,
+            });
+            const startTime = Date.now();
+            // Extract structured data with LLM
+            const extractResult = await extractWithLLM({
+                content: peelResult.content,
+                instruction: prompt,
+                prompt,
+                schema,
+                apiKey: resolvedApiKey,
+                llmApiKey: resolvedApiKey,
+                llmProvider: resolvedProvider,
+                model: model || process.env.WEBPEEL_LLM_MODEL || undefined,
+                llmModel: model || process.env.WEBPEEL_LLM_MODEL || undefined,
+                baseUrl: baseUrl || process.env.WEBPEEL_LLM_BASE_URL || 'https://api.openai.com/v1',
+            });
+            const elapsed = Date.now() - startTime;
+            // Return in Firecrawl-compatible format with llm metadata
+            res.json({
+                success: true,
+                data: extractResult.items.length === 1 ? extractResult.items[0] : extractResult.items,
+                llm: {
+                    provider: extractResult.provider || resolvedProvider,
+                    model: extractResult.model,
+                    tokens: extractResult.tokensUsed,
+                },
+                url: peelResult.url,
+                elapsed,
+                metadata: {
+                    url: peelResult.url,
+                    title: peelResult.title,
+                    tokensUsed: extractResult.tokensUsed,
+                    model: extractResult.model,
+                    cost: extractResult.cost,
+                    elapsed: peelResult.elapsed,
+                },
+            });
+        }
+        catch (error) {
+            console.error('[/v1/extract] Error:', error instanceof Error ? error.message : String(error));
+            const msg = error instanceof Error ? error.message : 'Unknown error';
+            if (msg.includes('authentication failed') || msg.includes('401')) {
+                res.status(401).json({ success: false, error: { type: 'llm_auth_failed', message: msg }, requestId: req.requestId });
+                return;
+            }
+            if (msg.includes('rate limit') || msg.includes('429')) {
+                res.status(429).json({
+                    success: false,
+                    error: {
+                        type: 'llm_rate_limited',
+                        message: msg,
+                        hint: 'You have hit the LLM provider rate limit. Try again in a moment.',
+                        docs: 'https://webpeel.dev/docs/errors#llm-rate-limited',
+                    },
+                    requestId: req.requestId || crypto.randomUUID(),
+                });
+                return;
+            }
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'extraction_failed',
+                    message: msg,
+                    docs: 'https://webpeel.dev/docs/errors#extraction-failed',
+                },
+                requestId: req.requestId || crypto.randomUUID(),
+            });
+        }
+    });
+    router.get('/v1/extract/auto', async (req, res) => {
+        const url = req.query.url;
+        if (!url) {
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'missing_url',
+                    message: 'Missing url parameter',
+                    hint: 'Pass a URL: GET /v1/extract/auto?url=https://example.com',
+                    docs: 'https://webpeel.dev/docs/errors#missing-url',
+                },
+                requestId: req.requestId || crypto.randomUUID(),
+            });
+            return;
+        }
+        const { autoExtract } = await import('../../core/auto-extract.js');
+        const result = await peel(url, { format: 'html' });
+        const extracted = autoExtract(result.content || '', url);
+        res.json({ url, pageType: extracted.type, structured: extracted });
+    });
+    router.post('/v1/extract/auto', async (req, res) => {
+        const { url, ...rest } = req.body;
+        if (!url || typeof url !== 'string') {
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'missing_url',
+                    message: 'Missing or invalid url field in request body',
+                    hint: 'Pass a URL in the request body: { "url": "https://example.com" }',
+                    docs: 'https://webpeel.dev/docs/errors#missing-url',
+                },
+                requestId: req.requestId || crypto.randomUUID(),
+            });
+            return;
+        }
+        try {
+            const { autoExtract } = await import('../../core/auto-extract.js');
+            const result = await peel(url, { format: 'html', ...rest });
+            const extracted = autoExtract(result.content || '', url);
+            res.json({ url, pageType: extracted.type, structured: extracted });
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : 'Unknown error';
+            console.error('[/v1/extract/auto POST] Error:', msg);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'extraction_failed',
+                    message: msg,
+                    docs: 'https://webpeel.dev/docs/errors#extraction-failed',
+                },
+                requestId: req.requestId || crypto.randomUUID(),
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/fetch.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Fetch endpoint with caching
+ */
+import { Router } from 'express';
+import '../types.js';
+import { AuthStore } from '../auth-store.js';
+export declare function createFetchRouter(authStore: AuthStore): Router;