webpeel 0.19.4 → 0.20.0

package/dist/mcp/handlers/find.js

@@ -0,0 +1,137 @@
+/**
+ * handleFind — search the web, discover domain URLs, or do deep research.
+ */
+import { peel, peelBatch } from '../../index.js';
+import { getBestSearchProvider, getSearchProvider } from '../../core/search-provider.js';
+import { quickAnswer } from '../../core/quick-answer.js';
+import { textResult, safeStringify, timeout } from './types.js';
+export const handleFind = async (args, _ctx) => {
+    const query = args['query'];
+    const url = args['url'];
+    const depth = args['depth'] || 'quick';
+    const limit = Math.min(Math.max(args['limit'] ?? 5, 1), 20);
+    // URL-based: map/discover pages on a domain
+    if (url && !query) {
+        const { mapDomain } = await import('../../core/map.js');
+        const results = await Promise.race([
+            mapDomain(url, { maxUrls: limit * 100 }),
+            timeout(600000, 'Map domain'),
+        ]);
+        return textResult(safeStringify(results));
+    }
+    if (!query)
+        throw new Error('Either query or url is required');
+    // Question-mode: if the query looks like a natural language question and depth
+    // isn't forced to 'deep', use the LLM-free BM25 Q&A path (search → fetch → BM25).
+    const isQuestion = /\?$/.test(query.trim()) ||
+        /^(what|how|when|where|why|who|which|can|does|is|are|do|did|will|would|could|should)\b/i.test(query.trim());
+    if (isQuestion && depth !== 'deep') {
+        const numSources = Math.min(limit, 5);
+        const { provider, apiKey } = getBestSearchProvider();
+        let searchResults;
+        try {
+            searchResults = (await Promise.race([
+                provider.searchWeb(query, { count: numSources, apiKey }),
+                timeout(30000, 'Ask search'),
+            ]));
+        }
+        catch {
+            searchResults = [];
+        }
+        if (searchResults.length === 0) {
+            return textResult(safeStringify({
+                question: query,
+                answer: null,
+                confidence: 0,
+                sources: [],
+                method: 'bm25',
+            }));
+        }
+        const fetched = await Promise.allSettled(searchResults.slice(0, numSources).map((r) => peel(r.url, { budget: 3000, format: 'markdown', timeout: 12000 }).then((result) => ({
+            result,
+            searchResult: r,
+        }))));
+        const answers = fetched
+            .filter((f) => f.status === 'fulfilled')
+            .map((f) => {
+            const { result, searchResult } = f.value;
+            const qa = quickAnswer({
+                question: query,
+                content: result.content || '',
+                url: result.url || searchResult.url,
+                maxPassages: 2,
+            });
+            return {
+                answer: qa.answer,
+                confidence: qa.confidence,
+                source: {
+                    url: result.url || searchResult.url,
+                    title: result.title || searchResult.title,
+                    snippet: searchResult.snippet,
+                },
+            };
+        })
+            .sort((a, b) => b.confidence - a.confidence);
+        const best = answers[0];
+        return textResult(safeStringify({
+            question: query,
+            answer: best?.answer || null,
+            confidence: best?.confidence || 0,
+            sources: answers.map((a) => ({ ...a.source, confidence: a.confidence })),
+            method: 'bm25',
+        }));
+    }
+    // Deep research mode
+    if (depth === 'deep') {
+        const { provider, apiKey } = getBestSearchProvider();
+        const searchResults = await Promise.race([
+            provider.searchWeb(query, { count: limit, apiKey }),
+            timeout(30000, 'Search'),
+        ]);
+        const results = Array.isArray(searchResults)
+            ? searchResults
+            : searchResults.results ?? [];
+        const topN = results.slice(0, limit);
+        if (topN.length === 0) {
+            return textResult(safeStringify({ query, sources: [], content: '', totalTokens: 0 }));
+        }
+        const urls = topN.map((r) => r.url).filter(Boolean);
+        const pages = await Promise.race([
+            peelBatch(urls, { concurrency: 5, format: 'markdown' }),
+            timeout(120000, 'Batch fetch'),
+        ]);
+        const sources = [];
+        const contentParts = [];
+        let totalTokens = 0;
+        for (let i = 0; i < pages.length; i++) {
+            const page = pages[i];
+            const sr = topN[i];
+            const pageUrl = urls[i];
+            const title = page.title || sr.title || pageUrl;
+            const relevanceScore = Math.round((1 - i / Math.max(pages.length, 1)) * 100) / 100;
+            sources.push({ url: pageUrl, title, relevanceScore });
+            if (page.content) {
+                contentParts.push(`## Source ${i + 1}: ${title}\n**URL:** ${pageUrl}\n\n${page.content}\n\n---\n`);
+                totalTokens += page.tokens || 0;
+            }
+        }
+        return textResult(safeStringify({
+            query,
+            sources,
+            content: contentParts.join('\n'),
+            totalTokens,
+        }));
+    }
+    // Quick search (default)
+    const validProviders = ['duckduckgo', 'brave', 'stealth', 'google'];
+    const providerId = (args['provider'] &&
+        validProviders.includes(args['provider']))
+        ? args['provider']
+        : 'duckduckgo';
+    const searchProvider = getSearchProvider(providerId);
+    const results = await Promise.race([
+        searchProvider.searchWeb(query, { count: limit }),
+        timeout(30000, 'Search'),
+    ]);
+    return textResult(safeStringify(results));
+};

package/dist/mcp/handlers/index.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Handler registry — maps tool names → handler functions.
+ * Imported by both the standalone MCP server and the HTTP MCP route.
+ * Single source of truth for all 24+ tool implementations.
+ */
+import type { McpHandler } from './types.js';
+export declare const handlers: Map<string, McpHandler>;
+/** Look up a handler by tool name. Returns undefined if unknown. */
+export declare function getHandler(toolName: string): McpHandler | undefined;
+/** All registered tool names (public + legacy). */
+export declare function getAllToolNames(): string[];
+export type { McpHandler, McpContext, McpToolResult } from './types.js';
+export { textResult, errorResult, safeStringify } from './types.js';

package/dist/mcp/handlers/index.js

@@ -0,0 +1,61 @@
+/**
+ * Handler registry — maps tool names → handler functions.
+ * Imported by both the standalone MCP server and the HTTP MCP route.
+ * Single source of truth for all 24+ tool implementations.
+ */
+import { handleMeta } from './meta.js';
+import { handleRead } from './read.js';
+import { handleSee } from './see.js';
+import { handleFind } from './find.js';
+import { handleExtract } from './extract.js';
+import { handleMonitor } from './monitor.js';
+import { handleAct } from './act.js';
+import { handleFetch } from './fetch.js';
+import * as legacy from './legacy.js';
+// ── Consolidated public tools (the 7 we expose) ────────────────────────────
+export const handlers = new Map([
+    ['webpeel', handleMeta],
+    ['webpeel_read', handleRead],
+    ['webpeel_see', handleSee],
+    ['webpeel_find', handleFind],
+    ['webpeel_extract', handleExtract],
+    ['webpeel_monitor', handleMonitor],
+    ['webpeel_act', handleAct],
+]);
+// ── Legacy tool names → route to appropriate handler ───────────────────────
+const legacyRoutes = {
+    webpeel_fetch: handleFetch,
+    webpeel_youtube: legacy.handleYoutube,
+    webpeel_screenshot: legacy.handleScreenshot,
+    webpeel_search: legacy.handleSearch,
+    webpeel_research: legacy.handleResearch,
+    webpeel_crawl: legacy.handleCrawl,
+    webpeel_map: legacy.handleMap,
+    webpeel_batch: legacy.handleBatch,
+    webpeel_deep_fetch: legacy.handleDeepFetch,
+    webpeel_summarize: legacy.handleSummarize,
+    webpeel_answer: legacy.handleAnswer,
+    webpeel_quick_answer: legacy.handleQuickAnswer,
+    webpeel_brand: legacy.handleBrand,
+    webpeel_change_track: legacy.handleChangeTrack,
+    webpeel_watch: legacy.handleWatch,
+    webpeel_hotels: legacy.handleHotels,
+    webpeel_design_analysis: legacy.handleDesignAnalysis,
+    webpeel_design_compare: legacy.handleDesignCompare,
+    webpeel_auto_extract: legacy.handleAutoExtract,
+    webpeel_agent: legacy.handleAgent,
+    // backward-compat alias (no webpeel_ prefix)
+    agent: legacy.handleAgent,
+};
+for (const [name, handler] of Object.entries(legacyRoutes)) {
+    handlers.set(name, handler);
+}
+/** Look up a handler by tool name. Returns undefined if unknown. */
+export function getHandler(toolName) {
+    return handlers.get(toolName);
+}
+/** All registered tool names (public + legacy). */
+export function getAllToolNames() {
+    return [...handlers.keys()];
+}
+export { textResult, errorResult, safeStringify } from './types.js';

package/dist/mcp/handlers/legacy.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Legacy tool handlers — backward-compatible implementations for all
+ * pre-consolidated tool names. Each handler delegates to the appropriate
+ * core module or consolidated handler.
+ */
+import { type McpHandler } from './types.js';
+export declare const handleYoutube: McpHandler;
+export declare const handleScreenshot: McpHandler;
+export declare const handleSearch: McpHandler;
+export declare const handleResearch: McpHandler;
+export declare const handleCrawl: McpHandler;
+export declare const handleMap: McpHandler;
+export declare const handleBatch: McpHandler;
+export declare const handleDeepFetch: McpHandler;
+export declare const handleSummarize: McpHandler;
+export declare const handleAnswer: McpHandler;
+export declare const handleQuickAnswer: McpHandler;
+export declare const handleBrand: McpHandler;
+export declare const handleChangeTrack: McpHandler;
+export declare const handleWatch: McpHandler;
+export declare const handleHotels: McpHandler;
+export declare const handleDesignAnalysis: McpHandler;
+export declare const handleDesignCompare: McpHandler;
+export declare const handleAutoExtract: McpHandler;
+export declare const handleAgent: McpHandler;