webpeel 0.21.89 → 0.22.0

package/dist/core/search-engines.js

@@ -0,0 +1,182 @@
+/**
+ * Additional search engine providers: Baidu, Yandex
+ * HTTP-only scraping with cheerio — no browser, no API key required.
+ */
+import { load } from 'cheerio';
+import { simpleFetch } from './fetcher.js';
+// ── Baidu Search ──────────────────────────────────────────────────────────
+export class BaiduSearchProvider {
+    id = 'baidu';
+    requiresApiKey = false;
+    async searchWeb(query, options) {
+        const { count = 10 } = options;
+        // Baidu search URL
+        const params = new URLSearchParams({
+            wd: query,
+            rn: String(Math.min(count, 50)),
+            ie: 'utf-8',
+        });
+        const url = `https://www.baidu.com/s?${params}`;
+        try {
+            const response = await simpleFetch(url, 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 15000);
+            if (!response.html)
+                return [];
+            const $ = load(response.html);
+            const results = [];
+            const seen = new Set();
+            // Baidu result selectors: .result or .c-container
+            $('.result, .c-container').each((_, elem) => {
+                const el = $(elem);
+                const linkEl = el.find('h3 a, .t a').first();
+                const title = linkEl.text().trim();
+                // Baidu uses redirect URLs — get the data-url or mu attribute for real URL
+                const href = el.attr('mu') || linkEl.attr('href') || '';
+                const snippet = el.find('.c-abstract, .c-span-last, .content-right_8Zs40').first().text().trim();
+                if (title && href && !seen.has(href)) {
+                    seen.add(href);
+                    results.push({ title, url: href, snippet });
+                }
+            });
+            return results.slice(0, count);
+        }
+        catch {
+            return [];
+        }
+    }
+}
+// ── Naver Search (Korea) ──────────────────────────────────────────────────
+export class NaverSearchProvider {
+    id = 'naver';
+    requiresApiKey = false;
+    async searchWeb(query, options) {
+        const { count = 10 } = options;
+        const params = new URLSearchParams({
+            query: query,
+            where: 'web',
+        });
+        const url = `https://search.naver.com/search.naver?${params}`;
+        try {
+            // Naver is heavily JS-rendered — use peel with render for full content
+            const { peel } = await import('../index.js');
+            const result = await peel(url, { render: true, format: 'html', wait: 2000, timeout: 15000 });
+            const html = result.content || '';
+            if (!html)
+                return [];
+            const $ = load(html);
+            const results = [];
+            const seen = new Set();
+            // Naver uses .type-web class for web results, titles in *-title-text classes
+            $('.type-web').each((_, elem) => {
+                const el = $(elem);
+                const parent = el.closest('li, section, [class*=item]').length ? el.closest('li, section, [class*=item]') : el.parent();
+                const title = parent.find('[class*="title-text"], [class*="Title"]').first().text().trim();
+                const href = parent.find('a[href^="http"]').first().attr('href') || '';
+                const snippet = parent.find('[class*="text-type-body"], [class*="desc"]').first().text().trim();
+                if (title && href && !href.includes('naver.com/search') && !seen.has(href)) {
+                    seen.add(href);
+                    results.push({ title, url: href, snippet });
+                }
+            });
+            // Fallback: try generic link extraction if .type-web yielded nothing
+            if (results.length === 0) {
+                $('a[href^="http"]').each((_, elem) => {
+                    const el = $(elem);
+                    const href = el.attr('href') || '';
+                    const title = el.text().trim();
+                    if (title.length > 5 && title.length < 200 && href && !href.includes('naver.com') && !seen.has(href)) {
+                        seen.add(href);
+                        results.push({ title, url: href, snippet: '' });
+                    }
+                });
+            }
+            return results.slice(0, count);
+        }
+        catch {
+            return [];
+        }
+    }
+}
+// ── Yahoo Japan Search ────────────────────────────────────────────────────
+export class YahooJapanSearchProvider {
+    id = 'yahoo_japan';
+    requiresApiKey = false;
+    async searchWeb(query, options) {
+        const { count = 10 } = options;
+        const params = new URLSearchParams({
+            p: query,
+            n: String(Math.min(count, 50)),
+            ei: 'UTF-8',
+        });
+        const url = `https://search.yahoo.co.jp/search?${params}`;
+        try {
+            const response = await simpleFetch(url, 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36', 15000);
+            if (!response.html)
+                return [];
+            const $ = load(response.html);
+            const results = [];
+            // Yahoo Japan result selectors (2026 layout uses sw-Card components)
+            const seen = new Set();
+            $('.sw-Card__title, .algo, .dd').each((_, elem) => {
+                const el = $(elem);
+                // Walk up to the card container to find the link and snippet
+                const card = el.closest('[class*="sw-Card"], .algo, .dd, li').length
+                    ? el.closest('[class*="sw-Card"], .algo, .dd, li')
+                    : el.parent();
+                const title = el.find('.sw-Card__titleMain, h3, a').first().text().trim() || el.text().trim();
+                const href = card.find('a[href^="http"]').first().attr('href') || '';
+                const snippet = card.find('.sw-Card__description, .sw-Card__floatText, p').first().text().trim();
+                // Filter Yahoo internal links
+                if (title && title.length > 3 && href && !href.includes('yahoo.co.jp/search') && !href.includes('cache.yahoofs') && !seen.has(href)) {
+                    seen.add(href);
+                    results.push({ title, url: href, snippet });
+                }
+            });
+            return results.slice(0, count);
+        }
+        catch {
+            return [];
+        }
+    }
+}
+// ── Yandex Search ──────────────────────────────────────────────────────────
+export class YandexSearchProvider {
+    id = 'yandex';
+    requiresApiKey = false;
+    async searchWeb(query, options) {
+        const { count = 10 } = options;
+        const params = new URLSearchParams({
+            text: query,
+            numdoc: String(Math.min(count, 50)),
+            lr: '84', // Default to US region; can be overridden
+        });
+        // Use Yandex HTML search
+        const url = `https://yandex.com/search/?${params}`;
+        try {
+            const response = await simpleFetch(url, 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 15000);
+            if (!response.html)
+                return [];
+            const $ = load(response.html);
+            const results = [];
+            const seen = new Set();
+            // Yandex result selectors
+            $('.serp-item, .organic').each((_, elem) => {
+                const el = $(elem);
+                const linkEl = el.find('.organic__url, .link, a[href]').first();
+                const title = el.find('.organic__title, .OrganicTitle-LinkText, h2').first().text().trim();
+                const href = linkEl.attr('href') || '';
+                const snippet = el.find('.organic__text, .OrganicText, .text-container').first().text().trim();
+                // Filter internal Yandex links
+                if (title && href && !href.includes('yandex.') && !seen.has(href)) {
+                    seen.add(href);
+                    // Normalize URL (Yandex sometimes uses relative paths)
+                    const fullUrl = href.startsWith('http') ? href : `https://${href}`;
+                    results.push({ title, url: fullUrl, snippet });
+                }
+            });
+            return results.slice(0, count);
+        }
+        catch {
+            return [];
+        }
+    }
+}

package/dist/core/search-provider.d.ts

@@ -12,7 +12,7 @@
  * In production with no API keys configured, getBestSearchProvider() returns
  * StealthSearchProvider since DDG HTTP is often blocked on datacenter IPs.
  */
-export type SearchProviderId = 'duckduckgo' | 'brave' | 'stealth' | 'google';
+export type SearchProviderId = 'duckduckgo' | 'brave' | 'stealth' | 'google' | 'baidu' | 'yandex' | 'naver' | 'yahoo_japan';
 export interface WebSearchResult {
     title: string;
     url: string;
@@ -21,6 +21,8 @@ export interface WebSearchResult {
     relevanceScore?: number;
     /** Thumbnail/image URL from SearXNG results (img_src or thumbnail field). */
     imageUrl?: string;
+    /** Structured SERP data when structured=true is passed in WebSearchOptions. */
+    serp?: import('./google-serp-parser.js').GoogleSerpResult;
 }
 export interface WebSearchOptions {
     /** Number of results (1-10) */
@@ -42,6 +44,8 @@ export interface WebSearchOptions {
     locale?: string;
     /** Optional AbortSignal */
     signal?: AbortSignal;
+    /** Return structured SERP data (knowledge panel, PAA, featured snippets, etc.) */
+    structured?: boolean;
 }
 export interface SearchProvider {
     readonly id: SearchProviderId;

package/dist/core/search-provider.js

@@ -1400,8 +1400,18 @@ export class GoogleSearchProvider {
                     const snippet = cleanText(snippetText, { maxLen: 500, stripEllipsisPadding: true });
                     results.push({ title, url: validated, snippet });
                 });
-                if (results.length > 0)
-                    return results.slice(0, count);
+                if (results.length > 0) {
+                    const sliced = results.slice(0, count);
+                    // Attach structured SERP data to the first result when structured=true
+                    if (options?.structured) {
+                        const { parseGoogleSerp } = await import('./google-serp-parser.js');
+                        const serp = parseGoogleSerp(html);
+                        if (sliced.length > 0) {
+                            sliced[0] = { ...sliced[0], serp };
+                        }
+                    }
+                    return sliced;
+                }
             }
         }
         catch (e) {
@@ -1536,6 +1546,9 @@ export function getSearchProvider(id) {
         return new StealthSearchProvider();
     if (id === 'google')
         return new GoogleSearchProvider();
+    // 'baidu' and 'yandex' are handled by BaiduSearchProvider / YandexSearchProvider
+    // from './search-engines.js'. They cannot be imported here (circular dependency).
+    // Use search-engines.ts directly for these providers.
     // Exhaustive fallback (should be unreachable due to typing)
     return new DuckDuckGoProvider();
 }

package/dist/core/vertical-search.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Vertical search — specialized endpoints for shopping, news, images, videos.
+ * Uses Google vertical search pages + cheerio parsing.
+ */
+export interface ShoppingResult {
+    title: string;
+    price?: string;
+    currency?: string;
+    store: string;
+    url: string;
+    imageUrl?: string;
+    rating?: number;
+    reviewCount?: number;
+    condition?: string;
+}
+export interface NewsResult {
+    title: string;
+    url: string;
+    source: string;
+    date?: string;
+    snippet?: string;
+    imageUrl?: string;
+    category?: string;
+}
+export interface ImageResult {
+    title: string;
+    url: string;
+    imageUrl: string;
+    width?: number;
+    height?: number;
+    source?: string;
+}
+export interface VideoResult {
+    title: string;
+    url: string;
+    platform: string;
+    duration?: string;
+    date?: string;
+    thumbnailUrl?: string;
+    channel?: string;
+    views?: string;
+}
+export interface VerticalSearchOptions {
+    query: string;
+    count?: number;
+    country?: string;
+    language?: string;
+    freshness?: string;
+}
+export declare function searchShopping(opts: VerticalSearchOptions): Promise<ShoppingResult[]>;
+export declare function searchNews(opts: VerticalSearchOptions): Promise<NewsResult[]>;
+export declare function searchImages(opts: VerticalSearchOptions): Promise<ImageResult[]>;
+export declare function searchVideos(opts: VerticalSearchOptions): Promise<VideoResult[]>;

package/dist/core/vertical-search.js

@@ -0,0 +1,231 @@
+/**
+ * Vertical search — specialized endpoints for shopping, news, images, videos.
+ * Uses Google vertical search pages + cheerio parsing.
+ */
+import { load } from 'cheerio';
+// ── Shopping Search ────────────────────────────────────────────────────────
+export async function searchShopping(opts) {
+    const { query, count = 10, country, language } = opts;
+    // Strategy: Use Google Shopping via peel() with render
+    const { peel } = await import('../index.js');
+    const params = new URLSearchParams({
+        q: query,
+        tbm: 'shop', // Google Shopping mode
+        num: String(Math.min(count * 2, 40)),
+    });
+    if (country)
+        params.set('gl', country.toLowerCase());
+    if (language)
+        params.set('hl', language);
+    const url = `https://www.google.com/search?${params}`;
+    try {
+        const result = await peel(url, {
+            render: true,
+            stealth: true,
+            format: 'html',
+            wait: 3000,
+            timeout: 15000,
+        });
+        const html = result.content || '';
+        if (!html || html.length < 500)
+            return [];
+        const $ = load(html);
+        const items = [];
+        // Google Shopping result selectors
+        $('.sh-dgr__content, .sh-dlr__list-result, .mnr-c .pla-unit, [data-docid], .KZmu8e').each((_, elem) => {
+            const el = $(elem);
+            const title = el.find('.tAxDx, .pymv4e, h3, .Xjkr3b').first().text().trim();
+            const price = el.find('.a8Pemb, .e10twf, .HRLxBb, .kHxwFf').first().text().trim();
+            const store = el.find('.aULzUe, .LbUacb, .dD8iuc, .IuHnof').first().text().trim();
+            const link = el.find('a[href]').first().attr('href') || '';
+            const img = el.find('img').first().attr('src') || '';
+            const ratingText = el.find('.Rsc7Yb, .yi40Hd').first().text().trim();
+            const reviewText = el.find('.QhqGkb, .RDApEe').first().text().trim();
+            if (title && (price || store)) {
+                items.push({
+                    title,
+                    price: price || undefined,
+                    store: store || 'Unknown',
+                    url: link.startsWith('http')
+                        ? link
+                        : link.startsWith('/')
+                            ? `https://www.google.com${link}`
+                            : link,
+                    imageUrl: img.startsWith('http') ? img : undefined,
+                    rating: parseFloat(ratingText) || undefined,
+                    reviewCount: parseInt(reviewText.replace(/[^0-9]/g, '')) || undefined,
+                });
+            }
+        });
+        return items.slice(0, count);
+    }
+    catch {
+        return [];
+    }
+}
+// ── News Search ────────────────────────────────────────────────────────────
+export async function searchNews(opts) {
+    const { query, count = 10, language, freshness } = opts;
+    const { peel } = await import('../index.js');
+    const params = new URLSearchParams({
+        q: query,
+        tbm: 'nws', // Google News mode
+        num: String(Math.min(count * 2, 40)),
+    });
+    if (language)
+        params.set('hl', language);
+    if (freshness === 'day')
+        params.set('tbs', 'qdr:d');
+    else if (freshness === 'week')
+        params.set('tbs', 'qdr:w');
+    else if (freshness === 'month')
+        params.set('tbs', 'qdr:m');
+    const url = `https://www.google.com/search?${params}`;
+    try {
+        const result = await peel(url, {
+            render: true,
+            stealth: true,
+            format: 'html',
+            wait: 3000,
+            timeout: 15000,
+        });
+        const html = result.content || '';
+        if (!html || html.length < 500)
+            return [];
+        const $ = load(html);
+        const items = [];
+        // Google News result selectors
+        $('.WlydOe, .JJZKK, .SoaBEf, .dbsr, [jscontroller="d0DtYd"]').each((_, elem) => {
+            const el = $(elem);
+            const title = el.find('[role="heading"], .mCBkyc, .nDgy9d, .JheGif').first().text().trim();
+            const link = el.find('a[href^="http"]').first().attr('href') || '';
+            const source = el.find('.NUnG9d, .CEMjEf, .XTjFC, .wEwyrc').first().text().trim();
+            const date = el.find('.OSrXXb, .WG9SHc, .f').first().text().trim();
+            const snippet = el.find('.GI74Re, .Y3v8qd, .VwiC3b').first().text().trim();
+            const img = el.find('img[src^="http"]').first().attr('src') || '';
+            if (title && link) {
+                items.push({
+                    title,
+                    url: link,
+                    source: source || 'Unknown',
+                    date: date || undefined,
+                    snippet: snippet || undefined,
+                    imageUrl: img || undefined,
+                });
+            }
+        });
+        return items.slice(0, count);
+    }
+    catch {
+        return [];
+    }
+}
+// ── Image Search ────────────────────────────────────────────────────────────
+export async function searchImages(opts) {
+    const { query, count = 20 } = opts;
+    // Use Bing Images (more scrape-friendly than Google Images)
+    const { peel } = await import('../index.js');
+    const params = new URLSearchParams({ q: query, form: 'HDRSC2', first: '1' });
+    const url = `https://www.bing.com/images/search?${params}`;
+    try {
+        const result = await peel(url, { render: true, wait: 2000, timeout: 15000 });
+        const html = result.content || '';
+        if (!html || html.length < 500)
+            return [];
+        const $ = load(html);
+        const items = [];
+        // Bing Images selectors
+        $('.iusc, .imgpt, [data-idx]').each((_, elem) => {
+            const el = $(elem);
+            // Bing stores image data in a JSON attribute 'm'
+            const mData = el.attr('m');
+            if (mData) {
+                try {
+                    const m = JSON.parse(mData);
+                    items.push({
+                        title: m.t || el.find('img').attr('alt') || '',
+                        url: m.purl || '',
+                        imageUrl: m.murl || m.turl || '',
+                        width: m.w || undefined,
+                        height: m.h || undefined,
+                        source: m.desc || undefined,
+                    });
+                }
+                catch {
+                    /* skip malformed JSON */
+                }
+            }
+            else {
+                // Fallback: direct img extraction
+                const img = el.find('img');
+                const imgSrc = img.attr('src') || img.attr('data-src') || '';
+                const title = img.attr('alt') || '';
+                if (imgSrc && imgSrc.startsWith('http')) {
+                    items.push({
+                        title,
+                        url: el.find('a[href]').first().attr('href') || '',
+                        imageUrl: imgSrc,
+                    });
+                }
+            }
+        });
+        return items.slice(0, count);
+    }
+    catch {
+        return [];
+    }
+}
+// ── Video Search ────────────────────────────────────────────────────────────
+export async function searchVideos(opts) {
+    const { query, count = 10 } = opts;
+    const { peel } = await import('../index.js');
+    const params = new URLSearchParams({
+        q: query,
+        tbm: 'vid', // Google Videos mode
+        num: String(Math.min(count * 2, 20)),
+    });
+    const url = `https://www.google.com/search?${params}`;
+    try {
+        const result = await peel(url, {
+            render: true,
+            stealth: true,
+            format: 'html',
+            wait: 3000,
+            timeout: 15000,
+        });
+        const html = result.content || '';
+        if (!html || html.length < 500)
+            return [];
+        const $ = load(html);
+        const items = [];
+        // Google Video result selectors
+        $('[data-surl], .dXiKIc, .g, .RzdJxc').each((_, elem) => {
+            const el = $(elem);
+            const title = el.find('h3, .fc9yUc, [aria-label]').first().text().trim();
+            const link = el.find('a[href^="http"]').first().attr('href') || '';
+            const duration = el.find('.J1mWY, .FGpTBd, .vdur').first().text().trim();
+            const date = el.find('.OSrXXb, .f').first().text().trim();
+            const channel = el.find('.pcJO7e, .GlPvmc').first().text().trim();
+            const thumb = el.find('img[src^="http"]').first().attr('src') || '';
+            if (title && link && !link.includes('google.com/search')) {
+                items.push({
+                    title,
+                    url: link,
+                    platform: link.includes('youtube')
+                        ? 'YouTube'
+                        : link.includes('vimeo')
+                            ? 'Vimeo'
+                            : 'Web',
+                    duration: duration || undefined,
+                    date: date || undefined,
+                    channel: channel || undefined,
+                    thumbnailUrl: thumb || undefined,
+                });
+            }
+        });
+        return items.slice(0, count);
+    }
+    catch {
+        return [];
+    }
+}

package/dist/index.d.ts

@@ -18,7 +18,10 @@ export { extractInlineJson, type InlineExtractOptions, type InlineExtractResult
 export { runAgent, type AgentOptions, type AgentResult, type AgentProgress, type AgentStreamEvent, type AgentDepth, type AgentTopic } from './core/agent.js';
 export { summarizeContent, type SummarizeOptions } from './core/summarize.js';
 export { getSearchProvider, DuckDuckGoProvider, BraveSearchProvider, providerStats, type SearchProvider, type SearchProviderId, type WebSearchResult, type WebSearchOptions, } from './core/search-provider.js';
+export { BaiduSearchProvider, YandexSearchProvider, NaverSearchProvider, YahooJapanSearchProvider } from './core/search-engines.js';
+export { crossVerifySearch, type CrossVerifyResult } from './core/cross-verify.js';
 export { answerQuestion, type AnswerRequest, type AnswerResponse, type AnswerCitation, type LLMProviderId, type TokensUsed, } from './core/answer.js';
+export { parseGoogleSerp, type GoogleSerpResult } from './core/google-serp-parser.js';
 export { searchJobs, type JobCard, type JobDetail, type JobSearchOptions, type JobSearchResult } from './core/jobs.js';
 export { RateGovernor, formatDuration, type RateConfig, type RateState, type CanApplyResult, } from './core/rate-governor.js';
 export { ApplicationTracker, type ApplicationRecord, type ApplicationFilter, type ApplicationStats, type ApplicationStatus, } from './core/application-tracker.js';
@@ -52,6 +55,8 @@ export { localSearch, type LocalSearchOptions, type LocalSearchResult, type Loca
 export { getBusinessIntel, type BusinessIntel } from './core/business-intel.js';
 export { CircuitBreaker, browserCircuitBreaker, type CircuitState } from './core/circuit-breaker.js';
 export { checkMemoryPressure } from './core/browser-pool.js';
+export { searchShopping, searchNews, searchImages, searchVideos } from './core/vertical-search.js';
+export type { ShoppingResult, NewsResult, ImageResult, VideoResult, VerticalSearchOptions } from './core/vertical-search.js';
 /**
  * Fetch and extract content from a URL
  *

package/dist/index.js

@@ -21,7 +21,10 @@ export { extractInlineJson } from './core/extract-inline.js';
 export { runAgent } from './core/agent.js';
 export { summarizeContent } from './core/summarize.js';
 export { getSearchProvider, DuckDuckGoProvider, BraveSearchProvider, providerStats, } from './core/search-provider.js';
+export { BaiduSearchProvider, YandexSearchProvider, NaverSearchProvider, YahooJapanSearchProvider } from './core/search-engines.js';
+export { crossVerifySearch } from './core/cross-verify.js';
 export { answerQuestion, } from './core/answer.js';
+export { parseGoogleSerp } from './core/google-serp-parser.js';
 export { searchJobs } from './core/jobs.js';
 export { RateGovernor, formatDuration, } from './core/rate-governor.js';
 export { ApplicationTracker, } from './core/application-tracker.js';
@@ -60,6 +63,7 @@ export { localSearch } from './core/local-search.js';
 export { getBusinessIntel } from './core/business-intel.js';
 export { CircuitBreaker, browserCircuitBreaker } from './core/circuit-breaker.js';
 export { checkMemoryPressure } from './core/browser-pool.js';
+export { searchShopping, searchNews, searchImages, searchVideos } from './core/vertical-search.js';
 /**
  * Fetch and extract content from a URL
  *

package/dist/server/app.js

@@ -19,7 +19,7 @@ import { createRateLimitMiddleware, RateLimiter } from './middleware/rate-limit.
 import { createHealthRouter } from './routes/health.js';
 import { createFetchRouter } from './routes/fetch.js';
 import { createSearchRouter } from './routes/search.js';
-import { createSmartSearchRouter } from './routes/smart-search.js';
+import { createSmartSearchRouter } from './routes/smart-search/index.js';
 import { createUserRouter } from './routes/users.js';
 import { createStripeRouter, createBillingPortalRouter } from './routes/stripe.js';
 import { createOAuthRouter } from './routes/oauth.js';