webpeel 0.19.4 → 0.20.0

package/dist/mcp/handlers/legacy.js

@@ -0,0 +1,450 @@
+/**
+ * Legacy tool handlers — backward-compatible implementations for all
+ * pre-consolidated tool names. Each handler delegates to the appropriate
+ * core module or consolidated handler.
+ */
+import { peel, peelBatch } from '../../index.js';
+import { getBestSearchProvider } from '../../core/search-provider.js';
+import { quickAnswer } from '../../core/quick-answer.js';
+import { answerQuestion } from '../../core/answer.js';
+import { runAgent } from '../../core/agent.js';
+import { textResult, safeStringify, timeout } from './types.js';
+import { handleExtract } from './extract.js';
+import { handleMonitor } from './monitor.js';
+import { handleFind } from './find.js';
+// ── webpeel_youtube ──────────────────────────────────────────────────────────
+export const handleYoutube = async (args, _ctx) => {
+    const { getYouTubeTranscript } = await import('../../core/youtube.js');
+    const url = args['url'];
+    if (!url || typeof url !== 'string')
+        throw new Error('url is required');
+    const language = args['language'] || 'en';
+    const transcript = await Promise.race([
+        getYouTubeTranscript(url, { language }),
+        timeout(60000, 'YouTube transcript'),
+    ]);
+    return textResult(safeStringify(transcript));
+};
+// ── webpeel_screenshot ───────────────────────────────────────────────────────
+export const handleScreenshot = async (args, _ctx) => {
+    const { takeScreenshot } = await import('../../core/screenshot.js');
+    const { url, fullPage, width, height, format, quality, waitFor, stealth, actions, } = args;
+    if (!url)
+        throw new Error('url is required');
+    const result = await Promise.race([
+        takeScreenshot(url, {
+            fullPage: fullPage || false,
+            width,
+            height,
+            format: format || 'png',
+            quality,
+            waitFor,
+            stealth: stealth || false,
+            actions,
+        }),
+        timeout(60000, 'Screenshot'),
+    ]);
+    return textResult(safeStringify({
+        url: result.url,
+        format: result.format,
+        contentType: result.contentType,
+        screenshot: result.screenshot,
+    }));
+};
+// ── webpeel_search ───────────────────────────────────────────────────────────
+export const handleSearch = async (args, ctx) => {
+    return handleFind({ ...args, depth: 'quick' }, ctx);
+};
+// ── webpeel_research ─────────────────────────────────────────────────────────
+export const handleResearch = async (args, _ctx) => {
+    const { query, maxSources, maxDepth, llmApiKey, llmModel, llmBaseUrl, outputFormat, timeout: resTimeout, } = args;
+    if (!query)
+        throw new Error('query is required');
+    const { research } = await import('../../core/research.js');
+    const result = await Promise.race([
+        research({
+            query,
+            maxSources: maxSources ?? 5,
+            maxDepth: maxDepth ?? 1,
+            apiKey: llmApiKey,
+            model: llmModel,
+            baseUrl: llmBaseUrl,
+            outputFormat: outputFormat ?? 'report',
+            timeout: resTimeout ?? 60000,
+        }),
+        timeout(180000, 'Research'),
+    ]);
+    return textResult(safeStringify({
+        report: result.report,
+        sources: result.sources,
+        totalSourcesFound: result.totalSourcesFound,
+        sourcesConsulted: result.sourcesConsulted,
+        elapsed: result.elapsed,
+        tokensUsed: result.tokensUsed,
+        cost: result.cost,
+    }));
+};
+// ── webpeel_crawl ────────────────────────────────────────────────────────────
+export const handleCrawl = async (args, _ctx) => {
+    const { crawl } = await import('../../core/crawler.js');
+    const { url, maxPages, maxDepth, allowedDomains, excludePatterns, respectRobotsTxt, rateLimitMs, sitemapFirst, render, stealth, } = args;
+    if (!url)
+        throw new Error('url is required');
+    const results = await Promise.race([
+        crawl(url, {
+            maxPages,
+            maxDepth,
+            allowedDomains,
+            excludePatterns,
+            respectRobotsTxt,
+            rateLimitMs,
+            sitemapFirst,
+            render,
+            stealth,
+        }),
+        timeout(600000, 'Crawl'),
+    ]);
+    return textResult(safeStringify(results));
+};
+// ── webpeel_map ──────────────────────────────────────────────────────────────
+export const handleMap = async (args, _ctx) => {
+    const { mapDomain } = await import('../../core/map.js');
+    const { url, maxUrls, includePatterns, excludePatterns } = args;
+    if (!url)
+        throw new Error('url is required');
+    const results = await Promise.race([
+        mapDomain(url, { maxUrls, includePatterns, excludePatterns }),
+        timeout(600000, 'Map'),
+    ]);
+    return textResult(safeStringify(results));
+};
+// ── webpeel_batch ────────────────────────────────────────────────────────────
+export const handleBatch = async (args, _ctx) => {
+    const { urls, concurrency, render, format, selector } = args;
+    if (!urls || !Array.isArray(urls))
+        throw new Error('urls must be an array');
+    if (urls.length === 0)
+        throw new Error('urls cannot be empty');
+    if (urls.length > 50)
+        throw new Error('Too many URLs (max 50)');
+    const options = {
+        concurrency: concurrency || 3,
+        render: render || false,
+        format: format || 'markdown',
+        selector,
+    };
+    const results = await Promise.race([
+        peelBatch(urls, options),
+        timeout(300000, 'Batch'),
+    ]);
+    return textResult(safeStringify(results));
+};
+// ── webpeel_deep_fetch ───────────────────────────────────────────────────────
+export const handleDeepFetch = async (args, ctx) => {
+    const { query, count: countArg, format: formatArg } = args;
+    if (!query)
+        throw new Error('query is required');
+    return handleFind({ query, depth: 'deep', limit: Math.min(Math.max(countArg ?? 5, 1), 10), format: formatArg }, ctx);
+};
+// ── webpeel_summarize ────────────────────────────────────────────────────────
+export const handleSummarize = async (args, _ctx) => {
+    const { url, llmApiKey, prompt, llmModel, llmBaseUrl, render } = args;
+    if (!url)
+        throw new Error('url is required');
+    if (!llmApiKey)
+        throw new Error('llmApiKey is required');
+    const options = {
+        render: render || false,
+        extract: {
+            prompt: prompt || 'Summarize this webpage in 2-3 sentences.',
+            llmApiKey,
+            llmModel: llmModel || 'gpt-4o-mini',
+            llmBaseUrl: llmBaseUrl || 'https://api.openai.com/v1',
+        },
+    };
+    const result = await Promise.race([
+        peel(url, options),
+        timeout(60000, 'Summarize'),
+    ]);
+    return textResult(safeStringify({ url: result.url, title: result.title, summary: result.extracted }));
+};
+// ── webpeel_answer ───────────────────────────────────────────────────────────
+export const handleAnswer = async (args, _ctx) => {
+    const { question: q, searchProvider: sp, searchApiKey: sak, llmProvider: lp, llmApiKey: lak, llmModel: lm, maxSources: ms, } = args;
+    if (!q)
+        throw new Error('question is required');
+    if (!lp)
+        throw new Error('llmProvider is required');
+    if (!lak)
+        throw new Error('llmApiKey is required');
+    const validLlm = ['openai', 'anthropic', 'google'];
+    if (!validLlm.includes(lp))
+        throw new Error('Invalid llmProvider');
+    const validSp = ['duckduckgo', 'brave', 'stealth', 'google'];
+    const resolvedSp = (sp && validSp.includes(sp) ? sp : 'duckduckgo');
+    const result = await Promise.race([
+        answerQuestion({
+            question: q,
+            searchProvider: resolvedSp,
+            searchApiKey: sak,
+            llmProvider: lp,
+            llmApiKey: lak,
+            llmModel: lm,
+            maxSources: Math.min(Math.max(ms ?? 5, 1), 10),
+            stream: false,
+        }),
+        timeout(180000, 'Answer'),
+    ]);
+    return textResult(safeStringify(result));
+};
+// ── webpeel_quick_answer ─────────────────────────────────────────────────────
+export const handleQuickAnswer = async (args, _ctx) => {
+    const { url, question: q, maxPassages: mpArg, render } = args;
+    if (!url)
+        throw new Error('url is required');
+    if (!q)
+        throw new Error('question is required');
+    const maxPassages = typeof mpArg === 'number' ? Math.min(Math.max(mpArg, 1), 10) : 3;
+    const peelResult = await Promise.race([
+        peel(url, { render: render || false, format: 'markdown', budget: 8000 }),
+        timeout(60000, 'Quick answer fetch'),
+    ]);
+    const qa = quickAnswer({
+        question: q,
+        content: peelResult.content || '',
+        url: peelResult.url || url,
+        maxPassages,
+    });
+    return textResult(safeStringify({
+        url: peelResult.url || url,
+        title: peelResult.title,
+        question: qa.question,
+        answer: qa.answer,
+        confidence: qa.confidence,
+        passages: qa.passages,
+        method: qa.method,
+    }));
+};
+// ── webpeel_brand ────────────────────────────────────────────────────────────
+export const handleBrand = async (args, ctx) => {
+    return handleExtract({ ...args, _brand: true }, ctx);
+};
+// ── webpeel_change_track ─────────────────────────────────────────────────────
+export const handleChangeTrack = async (args, ctx) => {
+    return handleMonitor(args, ctx);
+};
+// ── webpeel_watch ────────────────────────────────────────────────────────────
+export const handleWatch = async (args, ctx) => {
+    const action = args['action'];
+    const pool = ctx?.pool;
+    // If pool is available (HTTP route), use WatchManager
+    if (pool) {
+        const { WatchManager } = await import('../../core/watch-manager.js');
+        const wm = new WatchManager(pool);
+        const accountId = ctx?.accountId || 'anonymous';
+        if (action === 'create') {
+            const watch = await wm.create(accountId, args['url'], {
+                webhookUrl: args['webhookUrl'],
+                checkIntervalMinutes: args['intervalMinutes'] || 60,
+                selector: args['selector'],
+            });
+            return textResult(safeStringify(watch));
+        }
+        if (action === 'list') {
+            const watches = await wm.list(accountId);
+            return textResult(safeStringify(watches));
+        }
+        if (action === 'check') {
+            const result = await wm.check(args['id']);
+            return textResult(safeStringify(result));
+        }
+        if (action === 'delete') {
+            await wm.delete(args['id']);
+            return textResult(safeStringify({ success: true }));
+        }
+        return textResult(safeStringify({ error: `Unknown watch action: ${action}` }));
+    }
+    // Standalone fallback (no pool)
+    if (!action || action === 'list' || action === 'check' || action === 'delete') {
+        return textResult(safeStringify({
+            message: 'URL watching requires the hosted API (api.webpeel.dev). ' +
+                'Use webpeel_monitor for one-time change detection.',
+        }));
+    }
+    if (action === 'create') {
+        return handleMonitor({
+            url: args['url'],
+            webhook: args['webhookUrl'],
+        }, ctx);
+    }
+    return textResult(safeStringify({
+        message: 'URL watching requires the hosted API (api.webpeel.dev).',
+    }));
+};
+// ── webpeel_hotels ───────────────────────────────────────────────────────────
+export const handleHotels = async (args, _ctx) => {
+    const { searchHotels, parseDate, addDays } = await import('../../core/hotel-search.js');
+    const destination = args['destination'];
+    if (!destination) {
+        return textResult(safeStringify({ error: 'Missing destination' }));
+    }
+    const checkin = args['checkin']
+        ? parseDate(args['checkin'])
+        : parseDate('tomorrow');
+    const checkout = args['checkout']
+        ? parseDate(args['checkout'])
+        : addDays(checkin, 1);
+    const sort = (['price', 'rating', 'value'].includes(args['sort'])
+        ? args['sort']
+        : 'price');
+    const limit = Math.max(1, Math.min(50, args['limit'] || 20));
+    const result = await searchHotels({ destination, checkin, checkout, sort, limit, stealth: true });
+    return textResult(safeStringify({
+        destination,
+        checkin,
+        checkout,
+        sources: result.sources,
+        count: result.results.length,
+        results: result.results.slice(0, limit),
+    }));
+};
+// ── webpeel_design_analysis ──────────────────────────────────────────────────
+export const handleDesignAnalysis = async (args, _ctx) => {
+    const { takeDesignAnalysis } = await import('../../core/screenshot.js');
+    const url = args['url'];
+    if (!url || typeof url !== 'string')
+        throw new Error('url is required');
+    if (url.length > 2048)
+        throw new Error('URL too long (max 2048 characters)');
+    const result = await Promise.race([
+        takeDesignAnalysis(url, {}),
+        timeout(90000, 'Design analysis'),
+    ]);
+    return textResult(safeStringify({ url: result.url, analysis: result.analysis }));
+};
+// ── webpeel_design_compare ───────────────────────────────────────────────────
+export const handleDesignCompare = async (args, _ctx) => {
+    const { takeDesignComparison } = await import('../../core/screenshot.js');
+    const url1 = args['url1'];
+    const url2 = args['url2'];
+    if (!url1 || typeof url1 !== 'string')
+        throw new Error('url1 is required');
+    if (!url2 || typeof url2 !== 'string')
+        throw new Error('url2 is required');
+    if (url1.length > 2048)
+        throw new Error('url1 too long');
+    if (url2.length > 2048)
+        throw new Error('url2 too long');
+    if (url1 === url2)
+        throw new Error('url1 and url2 must be different URLs');
+    const result = await Promise.race([
+        takeDesignComparison(url1, url2, {}),
+        timeout(120000, 'Design comparison'),
+    ]);
+    return textResult(safeStringify({
+        subjectUrl: result.subjectUrl,
+        referenceUrl: result.referenceUrl,
+        score: result.comparison.score,
+        summary: result.comparison.summary,
+        gaps: result.comparison.gaps,
+        subjectAnalysis: result.comparison.subjectAnalysis,
+        referenceAnalysis: result.comparison.referenceAnalysis,
+    }));
+};
+// ── webpeel_auto_extract ─────────────────────────────────────────────────────
+export const handleAutoExtract = async (args, ctx) => {
+    return handleExtract(args, ctx);
+};
+// ── webpeel_agent / agent ────────────────────────────────────────────────────
+export const handleAgent = async (args, _ctx) => {
+    const llmApiKey = args['llmApiKey'];
+    // LLM mode — delegate to runAgent
+    if (llmApiKey) {
+        const prompt = args['prompt'];
+        if (!prompt)
+            throw new Error('prompt is required');
+        const result = await Promise.race([
+            runAgent({
+                prompt,
+                llmApiKey,
+                llmModel: args['llmModel'],
+                depth: (args['depth'] || 'basic'),
+                topic: (args['topic'] || 'general'),
+                urls: args['urls'],
+                maxSources: args['maxSources'] || args['maxResults'] || undefined,
+                outputSchema: args['outputSchema'],
+            }),
+            timeout(180000, 'Agent'),
+        ]);
+        return textResult(safeStringify(result));
+    }
+    // LLM-free mode — search + fetch + BM25 extraction
+    const urlsArg = args['urls'] || [];
+    const searchArg = args['search'];
+    if (urlsArg.length === 0 && !searchArg) {
+        return textResult(safeStringify({
+            message: 'This tool has been deprecated. Use the webpeel tool instead.',
+        }));
+    }
+    const promptArg = args['prompt'];
+    const schema = args['schema'];
+    const budgetArg = args['budget'] || 4000;
+    const maxResults = Math.min(args['maxResults'] || 5, 20);
+    const targetUrls = [...urlsArg];
+    if (searchArg) {
+        try {
+            const { provider, apiKey } = getBestSearchProvider();
+            const searchResults = await provider.searchWeb(searchArg, {
+                count: Math.max(maxResults, 5),
+                apiKey,
+            });
+            for (const r of searchResults) {
+                if (!targetUrls.includes(r.url))
+                    targetUrls.push(r.url);
+            }
+        }
+        catch { /* continue with provided URLs */ }
+    }
+    const urlsToFetch = targetUrls.slice(0, maxResults);
+    const agentResults = [];
+    await Promise.all(urlsToFetch.map(async (u) => {
+        try {
+            const page = (await peel(u, { budget: budgetArg, format: 'markdown' }));
+            const content = page.content || '';
+            const title = page.title || u;
+            let extracted = null;
+            let confidence = 0;
+            if (schema && Object.keys(schema).length > 0) {
+                extracted = {};
+                let total = 0;
+                for (const [field] of Object.entries(schema)) {
+                    const question = promptArg
+                        ? `${promptArg} — specifically: what is the ${field}?`
+                        : `What is the ${field}?`;
+                    const qa = quickAnswer({ question, content, maxPassages: 1, url: u });
+                    extracted[field] = qa.answer || '';
+                    total += qa.confidence;
+                }
+                if ('source' in schema)
+                    extracted['source'] = u;
+                confidence = Object.keys(schema).length > 0 ? total / Object.keys(schema).length : 0;
+            }
+            else if (promptArg) {
+                const qa = quickAnswer({ question: promptArg, content, maxPassages: 3, url: u });
+                confidence = qa.confidence;
+            }
+            agentResults.push({
+                url: u,
+                title,
+                extracted,
+                content: content.slice(0, 500) + (content.length > 500 ? '…' : ''),
+                confidence,
+            });
+        }
+        catch { /* skip failed URLs */ }
+    }));
+    return textResult(safeStringify({
+        success: true,
+        data: { results: agentResults, totalSources: agentResults.length },
+    }));
+};

package/dist/mcp/handlers/meta.d.ts

@@ -0,0 +1,6 @@
+/**
+ * handleMeta — the 'webpeel' meta tool.
+ * Parses plain-language intent and routes to the appropriate handler.
+ */
+import type { McpHandler } from './types.js';
+export declare const handleMeta: McpHandler;

package/dist/mcp/handlers/meta.js

@@ -0,0 +1,31 @@
+/**
+ * handleMeta — the 'webpeel' meta tool.
+ * Parses plain-language intent and routes to the appropriate handler.
+ */
+import { parseIntent } from '../smart-router.js';
+import { handleRead } from './read.js';
+import { handleSee } from './see.js';
+import { handleFind } from './find.js';
+import { handleExtract } from './extract.js';
+import { handleMonitor } from './monitor.js';
+import { handleAct } from './act.js';
+export const handleMeta = async (args, ctx) => {
+    const task = args['task'];
+    if (!task || typeof task !== 'string')
+        throw new Error('task is required');
+    const parsed = parseIntent(task);
+    const routedArgs = { ...parsed.params };
+    if (parsed.url)
+        routedArgs['url'] = parsed.url;
+    if (parsed.query)
+        routedArgs['query'] = parsed.query;
+    switch (parsed.intent) {
+        case 'read': return handleRead(routedArgs, ctx);
+        case 'see': return handleSee(routedArgs, ctx);
+        case 'find': return handleFind(routedArgs, ctx);
+        case 'extract': return handleExtract(routedArgs, ctx);
+        case 'monitor': return handleMonitor(routedArgs, ctx);
+        case 'act': return handleAct(routedArgs, ctx);
+        default: return handleRead(routedArgs, ctx);
+    }
+};

package/dist/mcp/handlers/monitor.d.ts

@@ -0,0 +1,5 @@
+/**
+ * handleMonitor — watch a URL for changes, with optional webhook.
+ */
+import { type McpHandler } from './types.js';
+export declare const handleMonitor: McpHandler;

package/dist/mcp/handlers/monitor.js

@@ -0,0 +1,41 @@
+/**
+ * handleMonitor — watch a URL for changes, with optional webhook.
+ */
+import { peel } from '../../index.js';
+import { textResult, safeStringify, timeout } from './types.js';
+export const handleMonitor = async (args, _ctx) => {
+    const url = args['url'];
+    if (!url || typeof url !== 'string')
+        throw new Error('url is required');
+    if (url.length > 2048)
+        throw new Error('URL too long (max 2048 characters)');
+    const webhook = args['webhook'];
+    const selector = args['selector'];
+    const render = args['render'] || false;
+    if (webhook) {
+        // Webhook-based persistent monitoring requires hosted API
+        return textResult(safeStringify({
+            message: 'Persistent webhook monitoring requires the hosted API (api.webpeel.dev). ' +
+                'Use webpeel_monitor without webhook= for one-time change detection.',
+            url,
+            webhook,
+        }));
+    }
+    // One-time change snapshot (change_track logic)
+    const options = {
+        render: render || false,
+        ...(selector ? { selector } : {}),
+    };
+    const result = await Promise.race([
+        peel(url, options),
+        timeout(60000, 'Monitor'),
+    ]);
+    return textResult(safeStringify({
+        url: result.url,
+        title: result.title,
+        fingerprint: result.fingerprint,
+        tokens: result.tokens,
+        contentType: result.contentType,
+        lastChecked: new Date().toISOString(),
+    }));
+};

package/dist/mcp/handlers/read.d.ts

@@ -0,0 +1,6 @@
+/**
+ * handleRead — fetch any URL as clean markdown.
+ * Auto-detects YouTube URLs and extracts transcripts.
+ */
+import { type McpHandler } from './types.js';
+export declare const handleRead: McpHandler;

package/dist/mcp/handlers/read.js

@@ -0,0 +1,63 @@
+/**
+ * handleRead — fetch any URL as clean markdown.
+ * Auto-detects YouTube URLs and extracts transcripts.
+ */
+import { peel } from '../../index.js';
+import { textResult, safeStringify, timeout } from './types.js';
+function isYouTubeUrl(url) {
+    return /(?:youtube\.com|youtu\.be)/i.test(url);
+}
+export const handleRead = async (args, _ctx) => {
+    const url = args['url'];
+    if (!url || typeof url !== 'string')
+        throw new Error('url is required');
+    if (url.length > 2048)
+        throw new Error('URL too long (max 2048 characters)');
+    const format = args['format'] || 'markdown';
+    const render = args['render'] || false;
+    const question = args['question'];
+    const summary = args['summary'] || false;
+    const budgetArg = args['budget'];
+    const readable = args['readable'] || false;
+    // YouTube auto-detection
+    if (isYouTubeUrl(url)) {
+        const { getYouTubeTranscript } = await import('../../core/youtube.js');
+        const language = args['language'] || 'en';
+        const transcript = await Promise.race([
+            getYouTubeTranscript(url, { language }),
+            timeout(60000, 'YouTube transcript'),
+        ]);
+        return textResult(safeStringify(transcript));
+    }
+    // Build summary extraction options if requested
+    const extractOpts = summary
+        ? { prompt: 'Summarize this webpage in 2-3 concise sentences.' }
+        : undefined;
+    const options = {
+        render,
+        format: format,
+        question,
+        budget: budgetArg ?? 4000,
+        readable,
+        ...(extractOpts ? { extract: extractOpts } : {}),
+    };
+    const result = await Promise.race([
+        peel(url, options),
+        timeout(60000, 'MCP read'),
+    ]);
+    const out = {
+        url: result.url || url,
+        title: result.title || '',
+        tokens: result.tokens || 0,
+        content: result.content,
+    };
+    if (result.metadata)
+        out['metadata'] = result.metadata;
+    if (result.quickAnswer)
+        out['quickAnswer'] = result.quickAnswer;
+    if (result.extracted)
+        out['extracted'] = result.extracted;
+    if (result.images)
+        out['images'] = result.images;
+    return textResult(safeStringify(out));
+};

package/dist/mcp/handlers/see.d.ts

@@ -0,0 +1,5 @@
+/**
+ * handleSee — take a screenshot, optionally with design analysis or comparison.
+ */
+import { type McpHandler } from './types.js';
+export declare const handleSee: McpHandler;