webpeel 0.16.0 → 0.17.0

package/dist/core/search-fallback.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Search-as-proxy fallback for blocked pages.
+ * When a protected site blocks our fetch, we search for the URL
+ * and extract info from search engine snippets.
+ *
+ * Uses the getBestSearchProvider() chain which handles:
+ *   Google CSE API → Brave API → Google stealth → DDG
+ * This avoids direct HTML scraping which is blocked by CAPTCHAs on datacenter IPs.
+ */
+export interface SearchFallbackResult {
+    title: string;
+    snippet: string;
+    cachedContent: string;
+    source: 'duckduckgo' | 'google' | 'none';
+    /** Extracted product data if this looks like a product page */
+    productData?: {
+        price?: string;
+        rating?: string;
+        brand?: string;
+        availability?: string;
+    };
+}
+/**
+ * Search for a URL using the best available search provider and extract the snippet.
+ * Returns the title, snippet, and any extracted product data.
+ */
+export declare function searchFallback(url: string): Promise<SearchFallbackResult>;
+//# sourceMappingURL=search-fallback.d.ts.map

package/dist/core/search-fallback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-fallback.d.ts","sourceRoot":"","sources":["../../src/core/search-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,YAAY,GAAG,QAAQ,GAAG,MAAM,CAAC;IACzC,+DAA+D;IAC/D,WAAW,CAAC,EAAE;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAmID;;;GAGG;AACH,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA4D/E"}

package/dist/core/search-fallback.js

@@ -0,0 +1,185 @@
+/**
+ * Search-as-proxy fallback for blocked pages.
+ * When a protected site blocks our fetch, we search for the URL
+ * and extract info from search engine snippets.
+ *
+ * Uses the getBestSearchProvider() chain which handles:
+ *   Google CSE API → Brave API → Google stealth → DDG
+ * This avoids direct HTML scraping which is blocked by CAPTCHAs on datacenter IPs.
+ */
+import { getBestSearchProvider } from './search-provider.js';
+/**
+ * Detect if a URL is likely a product page.
+ */
+function isProductUrl(url) {
+    try {
+        const u = new URL(url);
+        const host = u.hostname;
+        const path = u.pathname.toLowerCase();
+        const productHosts = ['bestbuy.com', 'walmart.com', 'amazon.com', 'target.com', 'newegg.com', 'ebay.com'];
+        const isProductHost = productHosts.some(h => host.includes(h));
+        const hasProductPath = path.includes('/dp/') ||
+            path.includes('/ip/') ||
+            path.includes('/site/') ||
+            path.includes('/p/') ||
+            path.includes('/product') ||
+            /\/\d{5,}/.test(path);
+        return isProductHost || hasProductPath;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Build an optimized search query for the given URL.
+ * For product pages, adds price/availability keywords for richer snippets.
+ */
+function buildSearchQuery(url) {
+    try {
+        const urlObj = new URL(url);
+        // Extract meaningful path terms (skip numeric IDs, short tokens)
+        const pathTerms = urlObj.pathname
+            .split(/[-/_]/)
+            .filter(t => t.length > 2 && !/^\d+$/.test(t) && !/^[a-z]{1,2}$/.test(t))
+            .slice(0, 6)
+            .join(' ');
+        const isProduct = isProductUrl(url);
+        if (isProduct) {
+            // For product pages, search with price intent for richer snippets
+            return `"${urlObj.hostname}" ${pathTerms} price`.trim();
+        }
+        return `site:${urlObj.hostname} ${pathTerms}`.trim();
+    }
+    catch {
+        return url;
+    }
+}
+/**
+ * Extract product-specific data from a search snippet.
+ * Google/Brave often include price, rating, availability in rich snippets.
+ */
+function extractProductData(title, snippet) {
+    const text = `${title} ${snippet}`;
+    const data = {
+        price: '',
+        rating: '',
+        brand: '',
+        availability: '',
+    };
+    // Price: $XX.XX or $X,XXX.XX
+    const priceMatch = text.match(/\$[\d,]+(?:\.\d{2})?/);
+    if (priceMatch)
+        data.price = priceMatch[0];
+    // Rating: "4.5 out of 5", "4.5/5", "4.5 stars"
+    const ratingMatch = text.match(/(\d(?:\.\d)?)\s*(?:out of 5|\/5|★|\bstars?\b)/i);
+    if (ratingMatch)
+        data.rating = `${ratingMatch[1]}/5`;
+    // Availability: in stock, out of stock, available
+    const availMatch = text.match(/\b(in stock|out of stock|available|sold out|ships in|pre-order)\b/i);
+    if (availMatch)
+        data.availability = availMatch[0];
+    // Brand: check for "by BrandName" or "Brand:" patterns
+    const brandMatch = text.match(/(?:by|brand:|sold by)\s+([A-Z][a-zA-Z\s&]{2,20}?)(?:\s*[|·•,]|\s+(?:at|on|for|price))/i);
+    if (brandMatch)
+        data.brand = brandMatch[1].trim();
+    const hasData = data.price || data.rating || data.availability;
+    if (!hasData)
+        return undefined;
+    return {
+        price: data.price || undefined,
+        rating: data.rating || undefined,
+        brand: data.brand || undefined,
+        availability: data.availability || undefined,
+    };
+}
+/**
+ * Build a rich markdown content block from search results.
+ * Formats differently for product pages vs general pages.
+ */
+function buildCachedContent(url, title, snippet, productData) {
+    const lines = [];
+    lines.push(`# ${title}`);
+    lines.push('');
+    // For product pages, add structured product info
+    if (productData) {
+        if (productData.price)
+            lines.push(`**Price:** ${productData.price}`);
+        if (productData.brand)
+            lines.push(`**Brand:** ${productData.brand}`);
+        if (productData.rating)
+            lines.push(`**Rating:** ${productData.rating}`);
+        if (productData.availability)
+            lines.push(`**Availability:** ${productData.availability}`);
+        lines.push('');
+    }
+    if (snippet) {
+        lines.push(snippet);
+        lines.push('');
+    }
+    lines.push(`---`);
+    lines.push(`*Source: Search engine snippet for ${url}*`);
+    lines.push(`*⚠️ Limited content — original page blocked direct access. For full data, configure GOOGLE_SEARCH_KEY or BRAVE_SEARCH_KEY.*`);
+    return lines.join('\n');
+}
+/**
+ * Search for a URL using the best available search provider and extract the snippet.
+ * Returns the title, snippet, and any extracted product data.
+ */
+export async function searchFallback(url) {
+    const emptyResult = {
+        title: '',
+        snippet: '',
+        cachedContent: '',
+        source: 'none',
+    };
+    try {
+        // Validate URL early — invalid URLs return empty result immediately
+        try {
+            new URL(url);
+        }
+        catch {
+            return emptyResult;
+        }
+        const searchQuery = buildSearchQuery(url);
+        const { provider, apiKey } = getBestSearchProvider();
+        const results = await provider.searchWeb(searchQuery, {
+            count: 3,
+            apiKey,
+        });
+        if (!results || results.length === 0) {
+            return emptyResult;
+        }
+        const topResult = results[0];
+        const title = topResult.title?.trim() || '';
+        const snippet = topResult.snippet?.trim() || '';
+        if (!title && !snippet) {
+            return emptyResult;
+        }
+        const productData = isProductUrl(url)
+            ? extractProductData(title, snippet)
+            : undefined;
+        const cachedContent = buildCachedContent(url, title, snippet, productData);
+        // Map provider ID to our source type
+        const sourceMap = {
+            duckduckgo: 'duckduckgo',
+            brave: 'google',
+            stealth: 'duckduckgo',
+            google: 'google',
+        };
+        const source = sourceMap[provider.id] ?? 'google';
+        return {
+            title,
+            snippet,
+            cachedContent,
+            source,
+            productData,
+        };
+    }
+    catch (e) {
+        if (process.env.DEBUG) {
+            console.debug('[webpeel]', 'Search fallback failed:', e instanceof Error ? e.message : e);
+        }
+        return emptyResult;
+    }
+}
+//# sourceMappingURL=search-fallback.js.map

package/dist/core/search-fallback.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-fallback.js","sourceRoot":"","sources":["../../src/core/search-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAgB7D;;GAEG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAEtC,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QAC1G,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,cAAc,GAClB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACrB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACrB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACvB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YACpB,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YACzB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAExB,OAAO,aAAa,IAAI,cAAc,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5B,iEAAiE;QACjE,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ;aAC9B,KAAK,CAAC,OAAO,CAAC;aACd,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACxE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;aACX,IAAI,CAAC,GAAG,CAAC,CAAC;QAEb,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAEpC,IAAI,SAAS,EAAE,CAAC;YACd,kEAAkE;YAClE,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC1D,CAAC;QAED,OAAO,QAAQ,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,KAAa,EAAE,OAAe;IACxD,MAAM,IAAI,GAAG,GAAG,KAAK,IAAI,OAAO,EAAE,CAAC;IACnC,MAAM,IAAI,GAA+D;QACvE,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,KAAK,EAAE,EAAE;QACT,YAAY,EAAE,EAAE;KACjB,CAAC;IAEF,6BAA6B;IAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACtD,IAAI,UAAU;QAAE,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAE3C,+CAA+C;IAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACjF,IAAI,WAAW;QAAE,IAAI,CAAC,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;IAErD,kDAAkD;IAClD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACpG,IAAI,UAAU;QAAE,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAElD,uDAAuD;IACvD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,wFAAwF,CAAC,CAAC;IACxH,IAAI,UAAU;QAAE,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAElD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,CAAC;IAC/D,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAE/B,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS;QAC9B,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,SAAS;QAChC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS;QAC9B,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,SAAS;KAC7C,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CACzB,GAAW,EACX,KAAa,EACb,OAAe,EACf,WAAiD;IAEjD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,iDAAiD;IACjD,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,WAAW,CAAC,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,cAAc,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC;QACrE,IAAI,WAAW,CAAC,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,cAAc,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC;QACrE,IAAI,WAAW,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,eAAe,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACxE,IAAI,WAAW,CAAC,YAAY;YAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,WAAW,CAAC,YAAY,EAAE,CAAC,CAAC;QAC1F,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,sCAAsC,GAAG,GAAG,CAAC,CAAC;IACzD,KAAK,CAAC,IAAI,CAAC,6HAA6H,CAAC,CAAC;IAE1I,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAW;IAC9C,MAAM,WAAW,GAAyB;QACxC,KAAK,EAAE,EAAE;QACT,OAAO,EAAE,EAAE;QACX,aAAa,EAAE,EAAE;QACjB,MAAM,EAAE,MAAM;KACf,CAAC;IAEF,IAAI,CAAC;QACH,oEAAoE;QACpE,IAAI,CAAC;YAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,WAAW,CAAC;QAAC,CAAC;QAEnD,MAAM,WAAW,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,qBAAqB,EAAE,CAAC;QAErD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,WAAW,EAAE;YACpD,KAAK,EAAE,CAAC;YACR,MAAM;SACP,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO,WAAW,CAAC;QACrB,CAAC;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAEhD,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,WAAW,CAAC;QACrB,CAAC;QAED,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC;YACnC,CAAC,CAAC,kBAAkB,CAAC,KAAK,EAAE,OAAO,CAAC;YACpC,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,aAAa,GAAG,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;QAE3E,qCAAqC;QACrC,MAAM,SAAS,GAA4C;YACzD,UAAU,EAAE,YAAY;YACxB,KAAK,EAAE,QAAQ;YACf,OAAO,EAAE,YAAY;YACrB,MAAM,EAAE,QAAQ;SACjB,CAAC;QACF,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC;QAElD,OAAO;YACL,KAAK;YACL,OAAO;YACP,aAAa;YACb,MAAM;YACN,WAAW;SACZ,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,yBAAyB,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5F,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;AACH,CAAC"}

package/dist/core/search-provider.d.ts

@@ -12,7 +12,7 @@
  * In production with no API keys configured, getBestSearchProvider() returns
  * StealthSearchProvider since DDG HTTP is often blocked on datacenter IPs.
  */
-export type SearchProviderId = 'duckduckgo' | 'brave' | 'stealth';
+export type SearchProviderId = 'duckduckgo' | 'brave' | 'stealth' | 'google';
 export interface WebSearchResult {
     title: string;
     url: string;
@@ -86,11 +86,54 @@ export declare class BraveSearchProvider implements SearchProvider {
     readonly requiresApiKey = true;
     searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
 }
+/**
+ * GoogleSearchProvider — Google Search via stealth browser or Custom Search JSON API
+ *
+ * Two modes:
+ *   1. Custom Search JSON API (BYOK): set GOOGLE_SEARCH_KEY + GOOGLE_SEARCH_CX env vars.
+ *      Reliable, structured, 100 free queries/day. Works from any IP.
+ *   2. Stealth browser scraping (no API key): uses playwright-extra stealth plugin to
+ *      scrape google.com/search directly. Works from datacenter IPs where DDG/Bing/Ecosia
+ *      are blocked. Gracefully returns [] if Playwright is unavailable.
+ *
+ * Docs: https://developers.google.com/custom-search/v1/overview
+ */
+export declare class GoogleSearchProvider implements SearchProvider {
+    readonly id: SearchProviderId;
+    /**
+     * requiresApiKey is false: works without API keys via stealth browser fallback.
+     */
+    readonly requiresApiKey = false;
+    /**
+     * Map standard freshness values to Google's dateRestrict format.
+     * Google dateRestrict: d[n]=past n days, w[n]=past n weeks,
+     *                      m[n]=past n months, y[n]=past n years.
+     */
+    private mapFreshnessToDateRestrict;
+    /** Validate URL; returns null if invalid/non-http */
+    private validateUrl;
+    /**
+     * Stealth browser scrape of google.com/search.
+     * Used when no Custom Search API key is configured.
+     * Strategy A: peel() with stealth rendering (consistent with StealthSearchProvider).
+     * Strategy B: direct playwright-extra launch (if peel returns no results).
+     */
+    private scrapeGoogleStealth;
+    /** Parse Google search result HTML using cheerio. No DOM lib types required. */
+    private _parseGoogleHtml;
+    searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+}
 export declare function getSearchProvider(id: SearchProviderId | undefined): SearchProvider;
 /**
- * Get the best available search provider based on configured API keys.
- * In production with no API keys configured, returns StealthSearchProvider
- * since DDG HTTP is often blocked on datacenter/server IPs.
+ * Get the best available search provider based on configured API keys and
+ * available runtime dependencies.
+ *
+ * Priority:
+ *   1. Google Custom Search JSON API (if GOOGLE_SEARCH_KEY + GOOGLE_SEARCH_CX set)
+ *   2. Brave Search (if BRAVE_SEARCH_KEY is set)
+ *   3. Google stealth browser scraping (works from datacenter IPs; no API key needed)
+ *      — only when playwright-extra is available in node_modules
+ *   4. DuckDuckGo with full fallback chain (DDG HTTP → DDG Lite → stealth multi-engine)
  */
 export declare function getBestSearchProvider(): {
     provider: SearchProvider;

package/dist/core/search-provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search-provider.d.ts","sourceRoot":"","sources":["../../src/core/search-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,MAAM,MAAM,gBAAgB,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,CAAC;AAElE,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2BAA2B;IAC3B,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,EAAE,gBAAgB,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IAEjC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACjF;AAwED;;;;;;;;GAQG;AACH,qBAAa,qBAAsB,YAAW,cAAc;IAC1D,QAAQ,CAAC,EAAE,EAAE,gBAAgB,CAAa;IAC1C,QAAQ,CAAC,cAAc,SAAS;IAEhC,qEAAqE;IACrE,OAAO,CAAC,WAAW;IAUnB;;;OAGG;YACW,SAAS;IAuDvB;;;OAGG;YACW,UAAU;IAqDxB;;;OAGG;YACW,YAAY;IA2DpB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CA8BtF;AAED,qBAAa,kBAAmB,YAAW,cAAc;IACvD,QAAQ,CAAC,EAAE,EAAE,gBAAgB,CAAgB;IAC7C,QAAQ,CAAC,cAAc,SAAS;IAEhC,OAAO,CAAC,kBAAkB;IA8C1B,OAAO,CAAC,cAAc;YAoBR,UAAU;IAuFxB;;;OAGG;YACW,UAAU;IAiElB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAyCtF;AAED,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,EAAE,EAAE,gBAAgB,CAAW;IACxC,QAAQ,CAAC,cAAc,QAAQ;IAEzB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CA8DtF;AAED,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,gBAAgB,GAAG,SAAS,GAAG,cAAc,CAOlF;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,IAAI;IAAE,QAAQ,EAAE,cAAc,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAWrF"}
+{"version":3,"file":"search-provider.d.ts","sourceRoot":"","sources":["../../src/core/search-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,MAAM,MAAM,gBAAgB,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE7E,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2BAA2B;IAC3B,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,EAAE,gBAAgB,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IAEjC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACjF;AAwED;;;;;;;;GAQG;AACH,qBAAa,qBAAsB,YAAW,cAAc;IAC1D,QAAQ,CAAC,EAAE,EAAE,gBAAgB,CAAa;IAC1C,QAAQ,CAAC,cAAc,SAAS;IAEhC,qEAAqE;IACrE,OAAO,CAAC,WAAW;IAUnB;;;OAGG;YACW,SAAS;IAuDvB;;;OAGG;YACW,UAAU;IAqDxB;;;OAGG;YACW,YAAY;IA2DpB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CA8BtF;AAED,qBAAa,kBAAmB,YAAW,cAAc;IACvD,QAAQ,CAAC,EAAE,EAAE,gBAAgB,CAAgB;IAC7C,QAAQ,CAAC,cAAc,SAAS;IAEhC,OAAO,CAAC,kBAAkB;IA8C1B,OAAO,CAAC,cAAc;YAoBR,UAAU;IAuFxB;;;OAGG;YACW,UAAU;IAiElB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAyCtF;AAED,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,EAAE,EAAE,gBAAgB,CAAW;IACxC,QAAQ,CAAC,cAAc,QAAQ;IAEzB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CA8DtF;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,YAAW,cAAc;IACzD,QAAQ,CAAC,EAAE,EAAE,gBAAgB,CAAY;IACzC;;OAEG;IACH,QAAQ,CAAC,cAAc,SAAS;IAEhC;;;;OAIG;IACH,OAAO,CAAC,0BAA0B;IAWlC,qDAAqD;IACrD,OAAO,CAAC,WAAW;IAUnB;;;;;OAKG;YACW,mBAAmB;IA4HjC,gFAAgF;IAChF,OAAO,CAAC,gBAAgB;IA4ClB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAuCtF;AAED,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,gBAAgB,GAAG,SAAS,GAAG,cAAc,CAQlF;AAkBD;;;;;;;;;;GAUG;AACH,wBAAgB,qBAAqB,IAAI;IAAE,QAAQ,EAAE,cAAc,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAuBrF"}

package/dist/core/search-provider.js

@@ -14,6 +14,8 @@
  */
 import { fetch as undiciFetch } from 'undici';
 import { load } from 'cheerio';
+import { existsSync } from 'fs';
+import { resolve as pathResolve } from 'path';
 function decodeHtmlEntities(input) {
     // Cheerio usually decodes entities when using `.text()`, but keep this as a
     // safety net since DuckDuckGo snippets sometimes leak encoded entities.
@@ -596,6 +598,243 @@ export class BraveSearchProvider {
         return results;
     }
 }
+/**
+ * GoogleSearchProvider — Google Search via stealth browser or Custom Search JSON API
+ *
+ * Two modes:
+ *   1. Custom Search JSON API (BYOK): set GOOGLE_SEARCH_KEY + GOOGLE_SEARCH_CX env vars.
+ *      Reliable, structured, 100 free queries/day. Works from any IP.
+ *   2. Stealth browser scraping (no API key): uses playwright-extra stealth plugin to
+ *      scrape google.com/search directly. Works from datacenter IPs where DDG/Bing/Ecosia
+ *      are blocked. Gracefully returns [] if Playwright is unavailable.
+ *
+ * Docs: https://developers.google.com/custom-search/v1/overview
+ */
+export class GoogleSearchProvider {
+    id = 'google';
+    /**
+     * requiresApiKey is false: works without API keys via stealth browser fallback.
+     */
+    requiresApiKey = false;
+    /**
+     * Map standard freshness values to Google's dateRestrict format.
+     * Google dateRestrict: d[n]=past n days, w[n]=past n weeks,
+     *                      m[n]=past n months, y[n]=past n years.
+     */
+    mapFreshnessToDateRestrict(tbs) {
+        if (!tbs)
+            return undefined;
+        const map = {
+            pd: 'd1',
+            pw: 'w1',
+            pm: 'm1',
+            py: 'y1',
+        };
+        return map[tbs];
+    }
+    /** Validate URL; returns null if invalid/non-http */
+    validateUrl(rawUrl) {
+        try {
+            const parsed = new URL(rawUrl);
+            if (!['http:', 'https:'].includes(parsed.protocol))
+                return null;
+            return parsed.href;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Stealth browser scrape of google.com/search.
+     * Used when no Custom Search API key is configured.
+     * Strategy A: peel() with stealth rendering (consistent with StealthSearchProvider).
+     * Strategy B: direct playwright-extra launch (if peel returns no results).
+     */
+    async scrapeGoogleStealth(query, count) {
+        // Strategy A: peel() + cheerio parse
+        try {
+            const { peel } = await import('../index.js');
+            const params = new URLSearchParams({
+                q: query,
+                num: String(Math.min(count * 2, 20)),
+                hl: 'en',
+                gl: 'us',
+            });
+            const url = `https://www.google.com/search?${params.toString()}`;
+            const result = await Promise.race([
+                peel(url, { render: true, stealth: true, format: 'html', wait: 3000 }),
+                new Promise((_, reject) => setTimeout(() => reject(new Error('Google stealth peel timeout')), 20_000)),
+            ]);
+            const html = result.content || '';
+            if (html) {
+                const $ = load(html);
+                const results = [];
+                const seen = new Set();
+                // Multiple selector patterns for resilience across Google HTML variants
+                const resultBlocks = $('#search .g, #rso .g, [data-hveid] .g');
+                resultBlocks.each((_i, elem) => {
+                    if (results.length >= count)
+                        return;
+                    const $r = $(elem);
+                    const $a = $r.find('a[href^="http"]').first();
+                    const $h3 = $r.find('h3').first();
+                    if (!$a.length || !$h3.length)
+                        return;
+                    const href = $a.attr('href') || '';
+                    if (href.includes('google.com/') ||
+                        href.includes('accounts.google') ||
+                        href.includes('/aclk') ||
+                        href.startsWith('#'))
+                        return;
+                    const validated = this.validateUrl(href);
+                    if (!validated)
+                        return;
+                    const key = normalizeUrlForDedupe(validated);
+                    if (seen.has(key))
+                        return;
+                    seen.add(key);
+                    const title = cleanText($h3.text(), { maxLen: 200 });
+                    if (!title)
+                        return;
+                    const snippetText = $r.find('[data-sncf]').first().text() ||
+                        $r.find('.VwiC3b').first().text() ||
+                        $r.find('[style*="-webkit-line-clamp"]').first().text() ||
+                        $r.find('.st').first().text() ||
+                        '';
+                    const snippet = cleanText(snippetText, { maxLen: 500, stripEllipsisPadding: true });
+                    results.push({ title, url: validated, snippet });
+                });
+                if (results.length > 0)
+                    return results.slice(0, count);
+            }
+        }
+        catch (e) {
+            if (process.env.DEBUG) {
+                console.debug('[webpeel] Google stealth (peel) error:', e.message);
+            }
+        }
+        // Strategy B: direct playwright-extra + stealth plugin
+        let browser;
+        let context;
+        let page;
+        try {
+            const pwExtra = await import('playwright-extra');
+            const StealthPlugin = (await import('puppeteer-extra-plugin-stealth')).default;
+            const stealthChromium = pwExtra.chromium;
+            stealthChromium.use(StealthPlugin());
+            const params = new URLSearchParams({
+                q: query,
+                num: String(Math.min(count * 2, 20)),
+                hl: 'en',
+                gl: 'us',
+            });
+            const url = `https://www.google.com/search?${params.toString()}`;
+            browser = await stealthChromium.launch({
+                headless: true,
+                args: [
+                    '--disable-blink-features=AutomationControlled',
+                    '--disable-dev-shm-usage',
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-gpu',
+                ],
+            });
+            context = await browser.newContext({
+                userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                viewport: { width: 1280, height: 720 },
+                locale: 'en-US',
+            });
+            page = await context.newPage();
+            await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 });
+            // Use page.content() + cheerio to avoid needing DOM lib types in tsconfig
+            const html = await page.content();
+            return this._parseGoogleHtml(html, count);
+        }
+        catch (e) {
+            if (process.env.DEBUG) {
+                console.debug('[webpeel] Google stealth (playwright) error:', e.message);
+            }
+            return [];
+        }
+        finally {
+            await page?.close().catch(() => { });
+            await context?.close().catch(() => { });
+            await browser?.close().catch(() => { });
+        }
+    }
+    /** Parse Google search result HTML using cheerio. No DOM lib types required. */
+    _parseGoogleHtml(html, count) {
+        const $ = load(html);
+        const results = [];
+        const seen = new Set();
+        const resultBlocks = $('#search .g, #rso .g, [data-hveid] .g');
+        resultBlocks.each((_i, elem) => {
+            if (results.length >= count)
+                return;
+            const $r = $(elem);
+            const $a = $r.find('a[href^="http"]').first();
+            const $h3 = $r.find('h3').first();
+            if (!$a.length || !$h3.length)
+                return;
+            const href = $a.attr('href') || '';
+            if (href.includes('google.com/') ||
+                href.includes('accounts.google') ||
+                href.includes('/aclk') ||
+                href.startsWith('#'))
+                return;
+            const validated = this.validateUrl(href);
+            if (!validated)
+                return;
+            const key = normalizeUrlForDedupe(validated);
+            if (seen.has(key))
+                return;
+            seen.add(key);
+            const title = cleanText($h3.text(), { maxLen: 200 });
+            if (!title)
+                return;
+            const snippetText = $r.find('[data-sncf]').first().text() ||
+                $r.find('.VwiC3b').first().text() ||
+                $r.find('[style*="-webkit-line-clamp"]').first().text() ||
+                $r.find('.st').first().text() ||
+                '';
+            const snippet = cleanText(snippetText, { maxLen: 500, stripEllipsisPadding: true });
+            results.push({ title, url: validated, snippet });
+        });
+        return results.slice(0, count);
+    }
+    async searchWeb(query, options) {
+        const { count, apiKey: optApiKey, tbs } = options;
+        const apiKey = optApiKey || process.env.GOOGLE_SEARCH_KEY || process.env.GOOGLE_API_KEY;
+        const cx = process.env.GOOGLE_SEARCH_CX;
+        // No API key — fall back to stealth browser scraping
+        if (!apiKey || !cx) {
+            return this.scrapeGoogleStealth(query, count);
+        }
+        // Custom Search JSON API path
+        const params = new URLSearchParams({
+            key: apiKey,
+            cx: cx,
+            q: query,
+            num: String(Math.min(count, 10)), // Google CSE max is 10 per request
+        });
+        const dateRestrict = this.mapFreshnessToDateRestrict(tbs);
+        if (dateRestrict)
+            params.set('dateRestrict', dateRestrict);
+        const response = await fetch(`https://www.googleapis.com/customsearch/v1?${params}`, {
+            signal: AbortSignal.timeout(10000),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            throw new Error(`Google search failed (${response.status}): ${text.substring(0, 200)}`);
+        }
+        const data = await response.json();
+        return (data.items || []).map((item) => ({
+            url: item.link,
+            title: item.title,
+            snippet: item.snippet || '',
+        }));
+    }
+}
 export function getSearchProvider(id) {
     if (!id || id === 'duckduckgo')
         return new DuckDuckGoProvider();
@@ -603,23 +842,55 @@ export function getSearchProvider(id) {
         return new BraveSearchProvider();
     if (id === 'stealth')
         return new StealthSearchProvider();
+    if (id === 'google')
+        return new GoogleSearchProvider();
     // Exhaustive fallback (should be unreachable due to typing)
     return new DuckDuckGoProvider();
 }
 /**
- * Get the best available search provider based on configured API keys.
- * In production with no API keys configured, returns StealthSearchProvider
- * since DDG HTTP is often blocked on datacenter/server IPs.
+ * Check whether playwright-extra is available synchronously.
+ * Uses fs.existsSync on node_modules to avoid making getBestSearchProvider async.
+ */
+function isPlaywrightExtraAvailable() {
+    try {
+        const cwd = process.cwd();
+        return (existsSync(pathResolve(cwd, 'node_modules', 'playwright-extra')) ||
+            existsSync(pathResolve(cwd, '..', 'node_modules', 'playwright-extra')));
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Get the best available search provider based on configured API keys and
+ * available runtime dependencies.
+ *
+ * Priority:
+ *   1. Google Custom Search JSON API (if GOOGLE_SEARCH_KEY + GOOGLE_SEARCH_CX set)
+ *   2. Brave Search (if BRAVE_SEARCH_KEY is set)
+ *   3. Google stealth browser scraping (works from datacenter IPs; no API key needed)
+ *      — only when playwright-extra is available in node_modules
+ *   4. DuckDuckGo with full fallback chain (DDG HTTP → DDG Lite → stealth multi-engine)
  */
 export function getBestSearchProvider() {
-    // Check for Brave
+    // 1. Google Custom Search JSON API (BYOK) — works from any IP
+    const googleKey = process.env.GOOGLE_SEARCH_KEY || process.env.GOOGLE_API_KEY;
+    const googleCx = process.env.GOOGLE_SEARCH_CX;
+    if (googleKey && googleCx) {
+        return { provider: new GoogleSearchProvider(), apiKey: googleKey };
+    }
+    // 2. Brave Search (BYOK)
     const braveKey = process.env.BRAVE_SEARCH_KEY || process.env.BRAVE_API_KEY;
     if (braveKey) {
         return { provider: new BraveSearchProvider(), apiKey: braveKey };
     }
-    // Always use DuckDuckGoProvider — it has the full fallback chain:
-    // DDG HTTP → DDG Lite → Brave (if key) → stealth multi-engine.
-    // This ensures search works even if Playwright isn't available (graceful degradation).
+    // 3. Google stealth browser — works from datacenter IPs where DDG/Bing/Ecosia fail.
+    // GoogleSearchProvider.searchWeb() falls back to stealth scraping when no API key is set.
+    if (isPlaywrightExtraAvailable()) {
+        return { provider: new GoogleSearchProvider() };
+    }
+    // 4. DuckDuckGo with full internal fallback chain
+    // (DDG HTTP → DDG Lite → stealth multi-engine)
     return { provider: new DuckDuckGoProvider() };
 }
 //# sourceMappingURL=search-provider.js.map