webpeel 0.20.2 → 0.20.3

package/dist/server/routes/activity.js

@@ -0,0 +1,74 @@
+/**
+ * Activity endpoint - provides recent API request history
+ */
+import { Router } from 'express';
+import { PostgresAuthStore } from '../pg-auth-store.js';
+export function createActivityRouter(authStore) {
+    const router = Router();
+    router.get('/v1/activity', async (req, res) => {
+        try {
+            // Require authentication (API key or JWT session token)
+            const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
+            if (!userId) {
+                res.status(401).json({ success: false, error: { type: 'unauthorized', message: 'Authentication required', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/errors#unauthorized' }, requestId: req.requestId });
+                return;
+            }
+            // Only works with PostgreSQL backend
+            if (!(authStore instanceof PostgresAuthStore)) {
+                res.status(501).json({
+                    success: false,
+                    error: {
+                        type: 'not_implemented',
+                        message: 'Activity endpoint requires PostgreSQL backend',
+                        docs: 'https://webpeel.dev/docs/errors#not_implemented',
+                    },
+                    requestId: req.requestId,
+                });
+                return;
+            }
+            // Access pool via any cast (pool is private but we need direct DB access)
+            const pgStore = authStore;
+            const limit = Math.min(parseInt(req.query.limit) || 50, 100);
+            // Get recent requests from usage_logs
+            const activityQuery = `
+        SELECT 
+          id,
+          url,
+          method,
+          status_code,
+          processing_time_ms,
+          tokens_used,
+          created_at
+        FROM usage_logs
+        WHERE user_id = $1
+        ORDER BY created_at DESC
+        LIMIT $2
+      `;
+            const result = await pgStore.pool.query(activityQuery, [userId, limit]);
+            // Transform to frontend format
+            const requests = result.rows.map((row) => ({
+                id: row.id,
+                url: row.url || 'N/A',
+                status: (row.status_code >= 200 && row.status_code < 300) ? 'success' : 'error',
+                responseTime: row.processing_time_ms || 0,
+                mode: row.method || 'basic',
+                timestamp: row.created_at,
+                tokensUsed: row.tokens_used || null,
+            }));
+            res.json({ requests });
+        }
+        catch (error) {
+            console.error('Activity error:', error);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Failed to retrieve activity',
+                    docs: 'https://webpeel.dev/docs/errors#internal_error',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/answer.d.ts

@@ -0,0 +1,5 @@
+/**
+ * /v1/answer — search + fetch + LLM-generated answer with citations (BYOK)
+ */
+import { Router } from 'express';
+export declare function createAnswerRouter(): Router;

package/dist/server/routes/answer.js

@@ -0,0 +1,125 @@
+/**
+ * /v1/answer — search + fetch + LLM-generated answer with citations (BYOK)
+ */
+import { Router } from 'express';
+import { answerQuestion, } from '../../core/answer.js';
+const VALID_LLM_PROVIDERS = ['openai', 'anthropic', 'google'];
+const VALID_SEARCH_PROVIDERS = ['duckduckgo', 'brave'];
+export function createAnswerRouter() {
+    const router = Router();
+    router.post('/v1/answer', async (req, res) => {
+        // Deprecation notice — prefer /v1/fetch?question=... which is LLM-free
+        res.setHeader('X-Deprecated', 'true');
+        res.setHeader('X-Deprecated-Use', '/v1/fetch?question=...');
+        // AUTH: require authentication (global middleware sets req.auth)
+        const ansAuthId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!ansAuthId) {
+            res.status(401).json({ success: false, error: { type: 'authentication_required', message: 'API key required. Get one at https://app.webpeel.dev/keys', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/errors#authentication_required' }, requestId: req.requestId });
+            return;
+        }
+        try {
+            const { question, searchProvider, searchApiKey, llmProvider, llmApiKey, llmModel, maxSources, stream, } = req.body;
+            // --- Validation -----------------------------------------------------------
+            if (!question || typeof question !== 'string' || question.trim().length === 0) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing or invalid "question" parameter', hint: 'Include a "question" string in the request body', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+                return;
+            }
+            if (question.length > 2000) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: '"question" too long (max 2000 characters)', hint: 'Keep the question under 2000 characters', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+                return;
+            }
+            if (!llmProvider || !VALID_LLM_PROVIDERS.includes(llmProvider)) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: `"llmProvider" is required and must be one of: ${VALID_LLM_PROVIDERS.join(', ')}`, hint: `Supported providers: ${VALID_LLM_PROVIDERS.join(', ')}`, docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+                return;
+            }
+            if (!llmApiKey || typeof llmApiKey !== 'string' || llmApiKey.trim().length === 0) {
+                res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing or invalid "llmApiKey" (BYOK required)', hint: 'Provide your own LLM API key in the "llmApiKey" field', docs: 'https://webpeel.dev/docs/errors#invalid_request' }, requestId: req.requestId });
+                return;
+            }
+            const resolvedSearchProvider = searchProvider && VALID_SEARCH_PROVIDERS.includes(searchProvider)
+                ? searchProvider
+                : 'duckduckgo';
+            // Accept search API key from body or header
+            const resolvedSearchApiKey = searchApiKey || req.headers['x-search-api-key'] || undefined;
+            const resolvedMaxSources = typeof maxSources === 'number'
+                ? Math.min(Math.max(maxSources, 1), 10)
+                : 5;
+            const shouldStream = stream === true;
+            // --- Streaming response (SSE) -------------------------------------------
+            if (shouldStream) {
+                res.setHeader('Content-Type', 'text/event-stream');
+                res.setHeader('Cache-Control', 'no-cache');
+                res.setHeader('Connection', 'keep-alive');
+                res.setHeader('X-Accel-Buffering', 'no'); // nginx
+                res.flushHeaders();
+                const answerReq = {
+                    question: question.trim(),
+                    searchProvider: resolvedSearchProvider,
+                    searchApiKey: resolvedSearchApiKey,
+                    llmProvider: llmProvider,
+                    llmApiKey: llmApiKey.trim(),
+                    llmModel,
+                    maxSources: resolvedMaxSources,
+                    stream: true,
+                    onChunk: (text) => {
+                        const payload = JSON.stringify({ type: 'chunk', text });
+                        res.write(`data: ${payload}\n\n`);
+                    },
+                };
+                try {
+                    const result = await answerQuestion(answerReq);
+                    const donePayload = JSON.stringify({
+                        type: 'done',
+                        citations: result.citations,
+                        searchProvider: result.searchProvider,
+                        llmProvider: result.llmProvider,
+                        llmModel: result.llmModel,
+                        tokensUsed: result.tokensUsed,
+                    });
+                    res.write(`data: ${donePayload}\n\n`);
+                }
+                catch (err) {
+                    const errMsg = err instanceof Error ? err.message : 'Unknown error';
+                    const errPayload = JSON.stringify({ type: 'error', message: errMsg });
+                    res.write(`data: ${errPayload}\n\n`);
+                }
+                res.end();
+                return;
+            }
+            // --- Non-streaming response ---------------------------------------------
+            const answerReq = {
+                question: question.trim(),
+                searchProvider: resolvedSearchProvider,
+                searchApiKey: resolvedSearchApiKey,
+                llmProvider: llmProvider,
+                llmApiKey: llmApiKey.trim(),
+                llmModel,
+                maxSources: resolvedMaxSources,
+                stream: false,
+            };
+            const result = await answerQuestion(answerReq);
+            res.json({
+                answer: result.answer,
+                citations: result.citations,
+                searchProvider: result.searchProvider,
+                llmProvider: result.llmProvider,
+                llmModel: result.llmModel,
+                tokensUsed: result.tokensUsed,
+            });
+        }
+        catch (error) {
+            const err = error;
+            console.error('Answer error:', err);
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'answer_failed',
+                    message: 'Failed to generate answer. Please try again.',
+                    docs: 'https://webpeel.dev/docs/errors#answer_failed',
+                },
+                requestId: req.requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/ask.d.ts

@@ -0,0 +1,28 @@
+/**
+ * GET /v1/ask?q=<question>&sources=3
+ * POST /v1/ask { "question": "...", "sources": 3 }
+ *
+ * LLM-free web Q&A: search → fetch top pages → BM25 → best answer
+ *
+ * Returns:
+ * {
+ *   question: string,
+ *   answer: string,           // best passage from top sources
+ *   confidence: number,       // 0-1
+ *   sources: [{url, title, snippet, confidence}],
+ *   method: "bm25",
+ *   elapsed: number           // ms
+ * }
+ *
+ * No LLM key required — 100% deterministic BM25 ranking.
+ * Competitors: Tavily charges $50/mo and requires an API key.
+ * We do this with zero LLM cost, included in every plan.
+ *
+ * Performance targets:
+ * - Source pages fetched in parallel with 5s timeout (no browser escalation)
+ * - Early termination when high-confidence answer found (>=0.85)
+ * - 10s hard timeout on the entire flow
+ * - 5-minute in-memory cache for repeated questions
+ */
+import { Router } from 'express';
+export declare function createAskRouter(): Router;

package/dist/server/routes/ask.js

@@ -0,0 +1,229 @@
+/**
+ * GET /v1/ask?q=<question>&sources=3
+ * POST /v1/ask { "question": "...", "sources": 3 }
+ *
+ * LLM-free web Q&A: search → fetch top pages → BM25 → best answer
+ *
+ * Returns:
+ * {
+ *   question: string,
+ *   answer: string,           // best passage from top sources
+ *   confidence: number,       // 0-1
+ *   sources: [{url, title, snippet, confidence}],
+ *   method: "bm25",
+ *   elapsed: number           // ms
+ * }
+ *
+ * No LLM key required — 100% deterministic BM25 ranking.
+ * Competitors: Tavily charges $50/mo and requires an API key.
+ * We do this with zero LLM cost, included in every plan.
+ *
+ * Performance targets:
+ * - Source pages fetched in parallel with 5s timeout (no browser escalation)
+ * - Early termination when high-confidence answer found (>=0.85)
+ * - 10s hard timeout on the entire flow
+ * - 5-minute in-memory cache for repeated questions
+ */
+import { Router } from 'express';
+import { peel } from '../../index.js';
+import { quickAnswer } from '../../core/quick-answer.js';
+import { getBestSearchProvider } from '../../core/search-provider.js';
+const resultCache = new Map();
+const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
+function getCacheKey(question, numSources) {
+    return `${question.trim().toLowerCase()}|${numSources}`;
+}
+function getFromCache(key) {
+    const entry = resultCache.get(key);
+    if (!entry)
+        return null;
+    if (Date.now() > entry.expiresAt) {
+        resultCache.delete(key);
+        return null;
+    }
+    return entry.result;
+}
+function setInCache(key, result) {
+    // Evict stale entries periodically (simple GC — keep max 500 entries)
+    if (resultCache.size >= 500) {
+        const now = Date.now();
+        for (const [k, v] of resultCache) {
+            if (v.expiresAt < now)
+                resultCache.delete(k);
+        }
+    }
+    resultCache.set(key, { result, expiresAt: Date.now() + CACHE_TTL_MS });
+}
+// ---------------------------------------------------------------------------
+// Route factory
+// ---------------------------------------------------------------------------
+export function createAskRouter() {
+    const router = Router();
+    async function handleAsk(question, numSources, req, res) {
+        const startMs = Date.now();
+        const elapsed = () => Date.now() - startMs;
+        if (!question?.trim()) {
+            res.status(400).json({ success: false, error: { type: 'missing_question', message: 'Provide q= or question= parameter', hint: 'GET /v1/ask?q=your+question or POST {"question": "your question"}', docs: 'https://webpeel.dev/docs/errors#missing_question' }, requestId: req.requestId });
+            return;
+        }
+        // Auth check — global middleware sets req.auth
+        const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!authId) {
+            res.status(401).json({ success: false, error: { type: 'authentication_required', message: 'API key required. Get one at https://app.webpeel.dev/keys', hint: 'Get a free API key at https://app.webpeel.dev/keys', docs: 'https://webpeel.dev/docs/errors#authentication_required' }, requestId: req.requestId });
+            return;
+        }
+        const clampedSources = Math.min(Math.max(numSources, 1), 5);
+        // Cache check — return cached result immediately for repeated questions
+        const cacheKey = getCacheKey(question, clampedSources);
+        const cached = getFromCache(cacheKey);
+        if (cached) {
+            if (process.env.DEBUG)
+                console.debug('[ask] cache hit in', elapsed(), 'ms');
+            res.json({ ...cached, elapsed: elapsed() });
+            return;
+        }
+        // -----------------------------------------------------------------------
+        // Total flow timeout — 10s hard cap.
+        // -----------------------------------------------------------------------
+        const TOTAL_TIMEOUT_MS = 10000;
+        let timedOut = false;
+        const totalTimer = setTimeout(() => { timedOut = true; }, TOTAL_TIMEOUT_MS);
+        try {
+            // Step 1: Search
+            const searchStart = Date.now();
+            const { provider, apiKey } = getBestSearchProvider();
+            let searchResults;
+            try {
+                searchResults = await provider.searchWeb(question.trim(), {
+                    count: clampedSources,
+                    apiKey,
+                });
+            }
+            catch {
+                searchResults = [];
+            }
+            if (process.env.DEBUG)
+                console.debug(`[ask] search ${Date.now() - searchStart}ms, ${searchResults.length} results`);
+            if (!searchResults.length) {
+                clearTimeout(totalTimer);
+                res.json({
+                    question,
+                    answer: null,
+                    confidence: 0,
+                    sources: [],
+                    method: 'bm25',
+                    elapsed: elapsed(),
+                });
+                return;
+            }
+            // -----------------------------------------------------------------------
+            // Step 2: Fetch top sources in parallel
+            // - noEscalate: true → skip browser escalation (simple HTTP only)
+            // - render: false    → don't start headless browser
+            // - timeout: 5000    → 5s per source max
+            // - budget: 3000     → keep content manageable
+            // -----------------------------------------------------------------------
+            const PER_SOURCE_TIMEOUT_MS = 5000;
+            const fetchStart = Date.now();
+            const sourceUrls = searchResults.slice(0, clampedSources);
+            const fetchPromises = sourceUrls.map((r) => Promise.race([
+                peel(r.url, {
+                    render: false,
+                    noEscalate: true,
+                    format: 'markdown',
+                    timeout: PER_SOURCE_TIMEOUT_MS,
+                    budget: 3000,
+                }).then((result) => ({ result, searchResult: r })),
+                new Promise((_, reject) => setTimeout(() => reject(new Error('per-source timeout')), PER_SOURCE_TIMEOUT_MS)),
+            ]));
+            const fetched = await Promise.allSettled(fetchPromises);
+            if (process.env.DEBUG) {
+                const ok = fetched.filter(f => f.status === 'fulfilled').length;
+                console.debug(`[ask] fetch ${Date.now() - fetchStart}ms, ${ok}/${sourceUrls.length} ok`);
+            }
+            // -----------------------------------------------------------------------
+            // Step 3: Score with quickAnswer, sort by confidence
+            // Early termination: if any source yields >=0.85 confidence, use it now
+            // -----------------------------------------------------------------------
+            const HIGH_CONFIDENCE_THRESHOLD = 0.85;
+            const answers = [];
+            for (const f of fetched) {
+                if (timedOut)
+                    break;
+                if (f.status !== 'fulfilled')
+                    continue;
+                const { result, searchResult } = f.value;
+                const qa = quickAnswer({
+                    question,
+                    content: result.content,
+                    url: result.url,
+                    maxPassages: 2,
+                });
+                answers.push({
+                    answer: qa.answer,
+                    confidence: qa.confidence,
+                    source: {
+                        url: result.url,
+                        title: result.title || searchResult.title,
+                        snippet: searchResult.snippet,
+                    },
+                });
+                // Early termination on high confidence
+                if (qa.confidence >= HIGH_CONFIDENCE_THRESHOLD) {
+                    if (process.env.DEBUG)
+                        console.debug(`[ask] early exit confidence=${qa.confidence.toFixed(2)} at ${elapsed()}ms`);
+                    break;
+                }
+            }
+            answers.sort((a, b) => b.confidence - a.confidence);
+            const best = answers[0];
+            clearTimeout(totalTimer);
+            const response = {
+                question,
+                answer: best?.answer || null,
+                confidence: best?.confidence || 0,
+                sources: answers.map((a) => ({
+                    ...a.source,
+                    confidence: a.confidence,
+                })),
+                method: 'bm25',
+                elapsed: elapsed(),
+            };
+            if (timedOut) {
+                response.warning = 'Partial result — 10s timeout reached';
+            }
+            // Cache successful results (only when we have an answer)
+            if (best?.answer && !timedOut) {
+                setInCache(cacheKey, response);
+            }
+            if (process.env.DEBUG)
+                console.debug(`[ask] done ${elapsed()}ms confidence=${best?.confidence?.toFixed(2) ?? 0}`);
+            res.json(response);
+        }
+        catch (err) {
+            clearTimeout(totalTimer);
+            if (process.env.DEBUG)
+                console.debug('[ask] error:', err);
+            res.json({
+                question,
+                answer: null,
+                confidence: 0,
+                sources: [],
+                method: 'bm25',
+                elapsed: elapsed(),
+                ...(timedOut ? { warning: 'Request timed out after 10s' } : {}),
+            });
+        }
+    }
+    router.get('/v1/ask', async (req, res) => {
+        const question = req.query.q || req.query.question || '';
+        const sources = Math.min(parseInt(req.query.sources || '3', 10) || 3, 5);
+        await handleAsk(question, sources, req, res);
+    });
+    router.post('/v1/ask', async (req, res) => {
+        const question = req.body?.question || req.body?.q || '';
+        const sources = Math.min(parseInt(req.body?.sources ?? 3, 10) || 3, 5);
+        await handleAsk(question, sources, req, res);
+    });
+    return router;
+}

package/dist/server/routes/batch.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Batch scrape API - process multiple URLs concurrently
+ */
+import { Router } from 'express';
+import type { IJobQueue } from '../job-queue.js';
+export declare function createBatchRouter(jobQueue: IJobQueue): Router;