webpeel 0.21.89 → 0.22.0

package/dist/server/routes/search.js

@@ -9,6 +9,9 @@ import { peel } from '../../index.js';
 import { simpleFetch } from '../../core/fetcher.js';
 import { searchCache } from '../../core/fetch-cache.js';
 import { getSearchProvider, getBestSearchProvider, } from '../../core/search-provider.js';
+import { BaiduSearchProvider, YandexSearchProvider, NaverSearchProvider, YahooJapanSearchProvider } from '../../core/search-engines.js';
+import { crossVerifySearch } from '../../core/cross-verify.js';
+import { searchShopping, searchNews as searchNewsVertical, searchImages as searchImagesVertical, searchVideos, } from '../../core/vertical-search.js';
 import { getSourceCredibility } from '../../core/source-credibility.js';
 import { checkAndSendDualAlert } from '../email-service.js';
 import { localSearch } from '../../core/local-search.js';
@@ -34,10 +37,10 @@ export function createSearchRouter(authStore) {
             // scrapeResults=true: fetches full page content for each result (like Firecrawl's scrape_options).
             // Adds `content` field to each result. Significantly increases response time and credits used.
             // Documented in OpenAPI spec under /v1/search parameters.
-            const { q, count, scrapeResults, enrich, sources, categories, tbs, country, location, local, language } = req.query;
+            const { q, count, scrapeResults, enrich, sources, categories, tbs, country, location, local, language, structured } = req.query;
             // --- Search provider (new: BYOK Brave support) ---
             const providerParam = (req.query.provider || '').toLowerCase() || 'auto';
-            const validProviders = ['duckduckgo', 'brave', 'stealth', 'google'];
+            const validProviders = ['duckduckgo', 'brave', 'stealth', 'google', 'baidu', 'yandex', 'naver', 'yahoo_japan'];
             const providerId = validProviders.includes(providerParam)
                 ? providerParam
                 : providerParam === 'auto' ? 'auto' : 'duckduckgo';
@@ -119,7 +122,8 @@ export function createSearchRouter(authStore) {
             }
             // Build cache key (include all parameters)
             const enrichCount = enrich ? Math.min(Math.max(parseInt(enrich, 10) || 0, 0), 5) : 0;
-            const cacheKey = `search:${providerId}:${q}:${resultCount}:${sourcesStr}:${shouldScrape}:${enrichCount}:${categoriesStr}:${tbsStr}:${countryStr}:${locationStr}`;
+            const isStructured = structured === 'true' || structured === '1';
+            const cacheKey = `search:${providerId}:${q}:${resultCount}:${sourcesStr}:${shouldScrape}:${enrichCount}:${categoriesStr}:${tbsStr}:${countryStr}:${locationStr}:${isStructured}`;
             const sharedCacheKey = searchCache.getKey(cacheKey, {});
             // Check cache (local LRU first, then shared singleton)
             const cached = cache.get(cacheKey);
@@ -160,6 +164,22 @@ export function createSearchRouter(authStore) {
                     searchProvider = best.provider;
                     effectiveApiKey = searchApiKey || best.apiKey;
                 }
+                else if (providerId === 'baidu') {
+                    searchProvider = new BaiduSearchProvider();
+                    effectiveApiKey = undefined;
+                }
+                else if (providerId === 'yandex') {
+                    searchProvider = new YandexSearchProvider();
+                    effectiveApiKey = undefined;
+                }
+                else if (providerId === 'naver') {
+                    searchProvider = new NaverSearchProvider();
+                    effectiveApiKey = undefined;
+                }
+                else if (providerId === 'yahoo_japan') {
+                    searchProvider = new YahooJapanSearchProvider();
+                    effectiveApiKey = undefined;
+                }
                 else {
                     searchProvider = getSearchProvider(providerId);
                     effectiveApiKey = searchApiKey || undefined;
@@ -170,12 +190,14 @@ export function createSearchRouter(authStore) {
                     tbs: tbsStr || undefined,
                     country: countryStr || undefined,
                     location: locationStr || undefined,
+                    structured: isStructured,
                 });
                 // Map to SearchResult (with optional content field)
                 let results = providerResults.map(r => ({
                     title: r.title,
                     url: r.url,
                     snippet: r.snippet,
+                    ...(r.serp ? { serp: r.serp } : {}),
                 }));
                 // Apply category filtering if specified
                 if (categoriesStr) {
@@ -458,5 +480,179 @@ export function createSearchRouter(authStore) {
             });
         }
     });
+    // ── GET /v1/search/shopping ──────────────────────────────────────────────
+    router.get('/v1/search/shopping', async (req, res) => {
+        const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!authId) {
+            res.status(401).json({ success: false, error: { type: 'authentication_required', message: 'API key required.' } });
+            return;
+        }
+        const { q, count, country, language } = req.query;
+        if (!q || typeof q !== 'string') {
+            res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing required "q" parameter.' } });
+            return;
+        }
+        const resultCount = count ? Math.min(Math.max(parseInt(count, 10) || 10, 1), 40) : 10;
+        const startTime = Date.now();
+        try {
+            const results = await searchShopping({
+                query: q,
+                count: resultCount,
+                country: country,
+                language: language,
+            });
+            const elapsed = Date.now() - startTime;
+            const pgStore = authStore;
+            if (req.auth?.keyInfo?.key && typeof pgStore.trackUsage === 'function') {
+                await pgStore.trackUsage(req.auth.keyInfo.key, 'search').catch(() => { });
+            }
+            res.setHeader('X-Credits-Used', '1');
+            res.setHeader('X-Processing-Time', elapsed.toString());
+            res.json({ success: true, data: { results, query: q, count: results.length, elapsed } });
+        }
+        catch (err) {
+            console.error('[search/shopping] error:', err);
+            res.status(500).json({ success: false, error: { type: 'search_failed', message: 'Shopping search failed.' } });
+        }
+    });
+    // ── GET /v1/search/news ──────────────────────────────────────────────────
+    router.get('/v1/search/news', async (req, res) => {
+        const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!authId) {
+            res.status(401).json({ success: false, error: { type: 'authentication_required', message: 'API key required.' } });
+            return;
+        }
+        const { q, count, language, freshness } = req.query;
+        if (!q || typeof q !== 'string') {
+            res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing required "q" parameter.' } });
+            return;
+        }
+        const resultCount = count ? Math.min(Math.max(parseInt(count, 10) || 10, 1), 40) : 10;
+        const startTime = Date.now();
+        try {
+            const results = await searchNewsVertical({
+                query: q,
+                count: resultCount,
+                language: language,
+                freshness: freshness,
+            });
+            const elapsed = Date.now() - startTime;
+            const pgStore = authStore;
+            if (req.auth?.keyInfo?.key && typeof pgStore.trackUsage === 'function') {
+                await pgStore.trackUsage(req.auth.keyInfo.key, 'search').catch(() => { });
+            }
+            res.setHeader('X-Credits-Used', '1');
+            res.setHeader('X-Processing-Time', elapsed.toString());
+            res.json({ success: true, data: { results, query: q, count: results.length, elapsed } });
+        }
+        catch (err) {
+            console.error('[search/news] error:', err);
+            res.status(500).json({ success: false, error: { type: 'search_failed', message: 'News search failed.' } });
+        }
+    });
+    // ── GET /v1/search/images ────────────────────────────────────────────────
+    router.get('/v1/search/images', async (req, res) => {
+        const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!authId) {
+            res.status(401).json({ success: false, error: { type: 'authentication_required', message: 'API key required.' } });
+            return;
+        }
+        const { q, count, country, language } = req.query;
+        if (!q || typeof q !== 'string') {
+            res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing required "q" parameter.' } });
+            return;
+        }
+        const resultCount = count ? Math.min(Math.max(parseInt(count, 10) || 20, 1), 50) : 20;
+        const startTime = Date.now();
+        try {
+            const results = await searchImagesVertical({
+                query: q,
+                count: resultCount,
+                country: country,
+                language: language,
+            });
+            const elapsed = Date.now() - startTime;
+            const pgStore = authStore;
+            if (req.auth?.keyInfo?.key && typeof pgStore.trackUsage === 'function') {
+                await pgStore.trackUsage(req.auth.keyInfo.key, 'search').catch(() => { });
+            }
+            res.setHeader('X-Credits-Used', '1');
+            res.setHeader('X-Processing-Time', elapsed.toString());
+            res.json({ success: true, data: { results, query: q, count: results.length, elapsed } });
+        }
+        catch (err) {
+            console.error('[search/images] error:', err);
+            res.status(500).json({ success: false, error: { type: 'search_failed', message: 'Image search failed.' } });
+        }
+    });
+    // ── GET /v1/search/verify ────────────────────────────────────────────────
+    // Cross-source verification: searches multiple engines and computes consensus
+    // GET /v1/search/verify?q=...&engines=google,duckduckgo,baidu&count=10
+    router.get('/v1/search/verify', async (req, res) => {
+        const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!authId) {
+            res.status(401).json({ success: false, error: { type: 'authentication_required', message: 'API key required.' } });
+            return;
+        }
+        const { q, engines, count } = req.query;
+        if (!q || typeof q !== 'string') {
+            res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing required "q" parameter.' } });
+            return;
+        }
+        const engineList = engines
+            ? engines.split(',').map(e => e.trim()).filter(Boolean)
+            : undefined;
+        const resultCount = count ? Math.min(Math.max(parseInt(count, 10) || 10, 1), 20) : 10;
+        const startTime = Date.now();
+        try {
+            const result = await crossVerifySearch(q, { engines: engineList, count: resultCount });
+            const elapsed = Date.now() - startTime;
+            const pgStore = authStore;
+            if (req.auth?.keyInfo?.key && typeof pgStore.trackUsage === 'function') {
+                await pgStore.trackUsage(req.auth.keyInfo.key, 'search').catch(() => { });
+            }
+            res.setHeader('X-Credits-Used', '1');
+            res.setHeader('X-Processing-Time', elapsed.toString());
+            res.json({ success: true, data: result });
+        }
+        catch (err) {
+            console.error('[search/verify] error:', err);
+            res.status(500).json({ success: false, error: { type: 'search_failed', message: 'Cross-verify search failed.' } });
+        }
+    });
+    // ── GET /v1/search/videos ────────────────────────────────────────────────
+    router.get('/v1/search/videos', async (req, res) => {
+        const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!authId) {
+            res.status(401).json({ success: false, error: { type: 'authentication_required', message: 'API key required.' } });
+            return;
+        }
+        const { q, count, language } = req.query;
+        if (!q || typeof q !== 'string') {
+            res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Missing required "q" parameter.' } });
+            return;
+        }
+        const resultCount = count ? Math.min(Math.max(parseInt(count, 10) || 10, 1), 20) : 10;
+        const startTime = Date.now();
+        try {
+            const results = await searchVideos({
+                query: q,
+                count: resultCount,
+                language: language,
+            });
+            const elapsed = Date.now() - startTime;
+            const pgStore = authStore;
+            if (req.auth?.keyInfo?.key && typeof pgStore.trackUsage === 'function') {
+                await pgStore.trackUsage(req.auth.keyInfo.key, 'search').catch(() => { });
+            }
+            res.setHeader('X-Credits-Used', '1');
+            res.setHeader('X-Processing-Time', elapsed.toString());
+            res.json({ success: true, data: { results, query: q, count: results.length, elapsed } });
+        }
+        catch (err) {
+            console.error('[search/videos] error:', err);
+            res.status(500).json({ success: false, error: { type: 'search_failed', message: 'Video search failed.' } });
+        }
+    });
     return router;
 }

package/dist/server/routes/smart-search/handlers/cars.d.ts

@@ -0,0 +1,2 @@
+import type { SearchIntent, SmartSearchResult } from '../types.js';
+export declare function handleCarSearch(intent: SearchIntent): Promise<SmartSearchResult>;

package/dist/server/routes/smart-search/handlers/cars.js

@@ -0,0 +1,99 @@
+import { peel } from '../../../../index.js';
+import { getBestSearchProvider } from '../../../../core/search-provider.js';
+import { addAffiliateTag, parsePrice } from '../utils.js';
+import { callLLMQuick, sanitizeSearchQuery, PROMPT_INJECTION_DEFENSE } from '../llm.js';
+export async function handleCarSearch(intent) {
+    const t0 = Date.now();
+    // Build a clean keyword: strip buying signals, price amounts, and common noise words
+    // NOTE: keep "car"/"cars" — they're needed for Cars.com search!
+    const keyword = intent.query
+        .replace(/\b(buy|cheap|cheapest|under|budget|price|used|new|for sale|listing|deal|best|good|find|search|looking for|want|need|in|near|around)\b/gi, '')
+        .replace(/[$]\d[\d,]*/g, '') // strip $30000, $30,000 etc.
+        .replace(/\b\d{4,}\b/g, '') // strip standalone 4+ digit numbers (prices, not model years)
+        // Remove location words that were already extracted to zip
+        .replace(/\b(long island|nassau|suffolk|manhattan|brooklyn|queens|bronx|nyc|new york|los angeles|chicago|houston|miami|boston|seattle|san francisco|washington dc)\b/gi, '')
+        .replace(/\s+/g, ' ')
+        .trim();
+    const params = new URLSearchParams({
+        keyword,
+        sort: 'list_price',
+        stock_type: 'all',
+        zip: intent.params.zip || '10001',
+        maximum_distance: '50',
+    });
+    if (intent.params.maxPrice)
+        params.set('list_price_max', intent.params.maxPrice);
+    const carSearchUrl = `https://www.cars.com/shopping/results/?${params.toString()}`;
+    // Search MULTIPLE car sites in parallel — first one with real listings wins
+    const { provider } = getBestSearchProvider();
+    const [carsComSettled, carGurusSettled, autotraderSettled, redditSettled] = await Promise.allSettled([
+        peel(carSearchUrl, { timeout: 15000 }),
+        provider.searchWeb(`${keyword} ${intent.params.maxPrice ? 'under $' + intent.params.maxPrice : ''} site:cargurus.com price listing`, { count: 3 }),
+        provider.searchWeb(`${keyword} ${intent.params.maxPrice ? 'under $' + intent.params.maxPrice : ''} site:autotrader.com price listing`, { count: 3 }),
+        provider.searchWeb(`${keyword} reddit review reliable problems`, { count: 3 }),
+    ]);
+    // Cars.com peel gives structured listings (best quality)
+    const carsComResult = carsComSettled.status === 'fulfilled' ? carsComSettled.value : null;
+    let carListings = carsComResult?.domainData?.structured?.listings || [];
+    // If Cars.com peel failed, combine results from CarGurus + Autotrader
+    if (carListings.length === 0) {
+        const carGurusResults = carGurusSettled.status === 'fulfilled' ? carGurusSettled.value : [];
+        const autotraderResults = autotraderSettled.status === 'fulfilled' ? autotraderSettled.value : [];
+        const allSearchResults = [...carGurusResults, ...autotraderResults];
+        carListings = allSearchResults
+            .filter(r => r.url && r.title)
+            .map(r => {
+            const textToSearch = `${r.title || ''} ${r.snippet || ''}`;
+            const price = parsePrice(textToSearch);
+            const isGenericPage = /\b(for sale near|cars for sale|search|browse|find)\b/i.test(r.title || '') && !price;
+            if (isGenericPage)
+                return null;
+            const yearMatch = (r.title || '').match(/\b(20\d{2}|19\d{2})\b/);
+            return {
+                title: r.title?.replace(/\s*[-|–—].*$/, '').trim() || 'Car Listing',
+                price,
+                year: yearMatch ? yearMatch[1] : '',
+                url: addAffiliateTag(r.url),
+                snippet: r.snippet || '',
+                source: (() => { try {
+                    return new URL(r.url).hostname.replace('www.', '');
+                }
+                catch {
+                    return '';
+                } })(),
+            };
+        })
+            .filter(Boolean)
+            .slice(0, 10);
+    }
+    const redditResults = redditSettled.status === 'fulfilled' ? redditSettled.value : [];
+    // AI synthesis: summarize top listings + Reddit input
+    let answer;
+    if (process.env.OLLAMA_URL) {
+        const listingSummary = carListings.slice(0, 5).map((l) => `${l.title || l.name || 'Car'}: ${l.price || 'price N/A'}, ${l.mileage || ''} miles`).join(', ');
+        const redditSnippets = redditResults.slice(0, 2).map(r => r.snippet || '').join(' ');
+        const aiPrompt = `${PROMPT_INJECTION_DEFENSE}You are a car buying advisor. The user searched: "${sanitizeSearchQuery(intent.query)}". Here are the top listings: ${listingSummary || 'no listings found'}. Reddit says: ${redditSnippets || 'no community input'}. Give a 2-3 sentence recommendation about the best value. Mention specific prices and models. Cite sources inline as [1], [2], etc. if available. Max 200 words.`;
+        const aiText = await callLLMQuick(aiPrompt, { maxTokens: 250, timeoutMs: 5000, temperature: 0.4 });
+        if (aiText && aiText.length > 20)
+            answer = aiText;
+    }
+    const content = carListings.length > 0
+        ? `# 🚗 Cars — ${intent.query}\n\n${carListings.map((l, i) => `${i + 1}. **${l.title || l.name}** — ${l.price || 'see price'}${l.mileage ? ` · ${String(l.mileage).replace(/\s*mi$/i, '')} mi` : ''}\n   ${l.snippet || ''}`).join('\n\n')}`
+        : (carsComResult?.content || `# 🚗 Cars — ${intent.query}\n\nNo listings found. Try a different search.`);
+    return {
+        type: 'cars',
+        source: 'Cars.com + CarGurus + Autotrader + Reddit',
+        sourceUrl: carSearchUrl,
+        content,
+        title: carsComResult?.title || `Cars — ${intent.query}`,
+        domainData: carsComResult?.domainData,
+        structured: carsComResult?.domainData?.structured || (carListings.length > 0 ? { listings: carListings } : undefined),
+        tokens: carsComResult?.tokens || content.split(/\s+/).length,
+        fetchTimeMs: Date.now() - t0,
+        ...(answer !== undefined ? { answer } : {}),
+        sources: [
+            { type: 'cars', url: carSearchUrl, count: carListings.length },
+            { type: 'reddit', threads: redditResults.map(r => ({ title: r.title, url: r.url, snippet: r.snippet })) },
+        ],
+    };
+}

package/dist/server/routes/smart-search/handlers/flights.d.ts

@@ -0,0 +1,2 @@
+import type { SearchIntent, SmartSearchResult } from '../types.js';
+export declare function handleFlightSearch(intent: SearchIntent): Promise<SmartSearchResult>;

package/dist/server/routes/smart-search/handlers/flights.js

@@ -0,0 +1,69 @@
+import { getBestSearchProvider } from '../../../../core/search-provider.js';
+import { addAffiliateTag } from '../utils.js';
+import { callLLMQuick, sanitizeSearchQuery, PROMPT_INJECTION_DEFENSE } from '../llm.js';
+export async function handleFlightSearch(intent) {
+    const t0 = Date.now();
+    const gfUrl = `https://www.google.com/travel/flights?q=Flights+${encodeURIComponent(intent.query)}+one+way`;
+    // Search for actual flight prices + Reddit tips in parallel
+    const { provider: searchProvider } = getBestSearchProvider();
+    const [kayakSettled, skyscannerSettled, momondoSettled, googleSettled, redditSettled] = await Promise.allSettled([
+        searchProvider.searchWeb(`${intent.query} cheapest price site:kayak.com`, { count: 2 }),
+        searchProvider.searchWeb(`${intent.query} cheapest flights site:skyscanner.com`, { count: 3 }),
+        searchProvider.searchWeb(`${intent.query} cheap flights site:momondo.com OR site:cheapflights.com`, { count: 3 }),
+        searchProvider.searchWeb(`${intent.query} flights site:google.com/travel`, { count: 2 }),
+        searchProvider.searchWeb(`${intent.query} flights reddit tips cheap`, { count: 3 }),
+    ]);
+    const flightResults = [
+        ...(kayakSettled.status === 'fulfilled' ? kayakSettled.value : []),
+        ...(skyscannerSettled.status === 'fulfilled' ? skyscannerSettled.value : []),
+        ...(momondoSettled.status === 'fulfilled' ? momondoSettled.value : []),
+        ...(googleSettled.status === 'fulfilled' ? googleSettled.value : []),
+    ];
+    const redditResults = redditSettled.status === 'fulfilled' ? redditSettled.value : [];
+    // Build content from search results + static booking links as fallback
+    const searchSection = flightResults.length > 0
+        ? `## 🔍 Flight Results\n\n${flightResults.slice(0, 6).map((r, i) => `${i + 1}. **[${r.title}](${r.url})**\n   ${r.snippet || ''}`).join('\n\n')}\n\n`
+        : '';
+    const content = `# ✈️ Flights — ${intent.query}
+
+${searchSection}## 📌 Book Directly
+
+1. **[Google Flights](${gfUrl})**  
+   Direct link to Google Flights search
+
+2. **[Kayak](https://www.kayak.com/flights?a=help)**  
+   Compare prices across all airlines
+
+3. **[Expedia](https://www.expedia.com/Flights)**  
+   Flights, hotels, bundles
+
+4. **[Skyscanner](https://www.skyscanner.com/)**  
+   Popular international flight search
+
+5. **[Momondo](https://www.momondo.com/)**  
+   Meta-search with lowest prices
+
+---
+`;
+    // AI synthesis from search results + Reddit tips
+    let answer;
+    if (process.env.OLLAMA_URL) {
+        const flightInfo = flightResults.slice(0, 5).map(r => `${r.title}: ${r.snippet || ''}`).join('\n');
+        const redditSnippets = redditResults.slice(0, 2).map(r => `${r.title}: ${r.snippet || ''}`).join('\n');
+        const aiPrompt = `${PROMPT_INJECTION_DEFENSE}You are a flight booking advisor. ONLY use information from the sources below. Do NOT make up prices, airlines, or routes not mentioned. User searched: "${sanitizeSearchQuery(intent.query)}". Web results: ${flightInfo || 'no results found'}. Reddit tips: ${redditSnippets || 'none'}. Give a 2-3 sentence tip about cheapest flights for this route based ONLY on the sources. Mention actual prices found and booking sites. Max 200 words. Cite sources inline as [1], [2], [3].`;
+        const aiText = await callLLMQuick(aiPrompt, { maxTokens: 250, timeoutMs: 5000, temperature: 0.4 });
+        if (aiText && aiText.length > 20)
+            answer = aiText;
+    }
+    return {
+        type: 'flights',
+        source: 'Flight Search',
+        sourceUrl: gfUrl,
+        content,
+        title: `Flights — ${intent.query}`,
+        structured: { listings: flightResults.slice(0, 6).map(r => ({ title: r.title, url: addAffiliateTag(r.url), snippet: r.snippet })) },
+        tokens: content.split(' ').length,
+        fetchTimeMs: Date.now() - t0,
+        ...(answer !== undefined ? { answer } : {}),
+    };
+}

package/dist/server/routes/smart-search/handlers/general.d.ts

@@ -0,0 +1,2 @@
+import type { SmartSearchResult } from '../types.js';
+export declare function handleGeneralSearch(query: string): Promise<SmartSearchResult>;