webpeel 0.7.0 → 0.7.1

package/dist/core/search-provider.js

@@ -0,0 +1,281 @@
+/**
+ * Search provider abstraction
+ *
+ * WebPeel supports multiple web search backends. DuckDuckGo is the default
+ * (no API key required). Brave Search is supported via BYOK.
+ */
+import { fetch as undiciFetch } from 'undici';
+import { load } from 'cheerio';
+function decodeHtmlEntities(input) {
+    // Cheerio usually decodes entities when using `.text()`, but keep this as a
+    // safety net since DuckDuckGo snippets sometimes leak encoded entities.
+    return input
+        .replace(/ /gi, ' ')
+        .replace(/&/gi, '&')
+        .replace(/&lt;/gi, '<')
+        .replace(/&gt;/gi, '>')
+        .replace(/&quot;/gi, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&#x([0-9a-f]+);/gi, (_m, hex) => {
+        const cp = Number.parseInt(String(hex), 16);
+        if (!Number.isFinite(cp) || cp < 0 || cp > 0x10ffff)
+            return _m;
+        try {
+            return String.fromCodePoint(cp);
+        }
+        catch {
+            return _m;
+        }
+    })
+        .replace(/&#(\d+);/g, (_m, num) => {
+        const cp = Number.parseInt(String(num), 10);
+        if (!Number.isFinite(cp) || cp < 0 || cp > 0x10ffff)
+            return _m;
+        try {
+            return String.fromCodePoint(cp);
+        }
+        catch {
+            return _m;
+        }
+    });
+}
+function cleanText(input, opts) {
+    let s = decodeHtmlEntities(input);
+    s = s.replace(/\s+/g, ' ').trim();
+    if (opts.stripEllipsisPadding) {
+        // Remove leading/trailing "..." or Unicode ellipsis padding.
+        s = s
+            .replace(/^(?:\.{3,}|…)+\s*/g, '')
+            .replace(/\s*(?:\.{3,}|…)+$/g, '')
+            .trim();
+    }
+    if (s.length > opts.maxLen)
+        s = s.slice(0, opts.maxLen);
+    return s;
+}
+function normalizeUrlForDedupe(rawUrl) {
+    try {
+        const u = new URL(rawUrl);
+        const host = u.hostname.toLowerCase().replace(/^www\./, '');
+        let path = u.pathname || '/';
+        path = path.replace(/\/+$/g, '');
+        return `${host}${path}`;
+    }
+    catch {
+        return rawUrl
+            .trim()
+            .toLowerCase()
+            .replace(/^https?:\/\//, '')
+            .replace(/^www\./, '')
+            .replace(/[?#].*$/, '')
+            .replace(/\/+$/g, '');
+    }
+}
+export class DuckDuckGoProvider {
+    id = 'duckduckgo';
+    requiresApiKey = false;
+    buildQueryAttempts(originalQuery) {
+        const q = originalQuery.trim();
+        if (!q)
+            return [];
+        const attempts = [];
+        // Required retry strategy order:
+        // 1) original query
+        // 2) quoted query
+        // 3) query site:*
+        attempts.push(q);
+        if (!/^".*"$/.test(q))
+            attempts.push(`"${q}"`);
+        attempts.push(`${q} site:*`);
+        // Single-word queries are disproportionately likely to return 0 results on
+        // the DDG HTML endpoint (e.g. "openai" vs "open ai"). When the first three
+        // attempts fail, try a few light-touch strategies that tend to coax the
+        // parser into returning web results.
+        const isSingleWord = !/\s/.test(q);
+        const looksLikeUrlOrDomain = /[./]/.test(q) || /^https?:/i.test(q);
+        if (isSingleWord && !looksLikeUrlOrDomain) {
+            // Try splitting a common suffix (e.g. openai -> open ai)
+            if (/^[a-z]{5,}ai$/i.test(q)) {
+                attempts.push(`${q.slice(0, -2)} ai`);
+            }
+            // Common suffixes that often return at least the official domain
+            attempts.push(`${q}.com`);
+            attempts.push(`site:${q}.com`);
+            attempts.push(`${q} website`);
+        }
+        // De-dupe attempts (case-insensitive)
+        const seen = new Set();
+        return attempts
+            .map((s) => s.trim())
+            .filter((s) => s.length > 0)
+            .filter((s) => {
+            const key = s.toLowerCase();
+            if (seen.has(key))
+                return false;
+            seen.add(key);
+            return true;
+        });
+    }
+    buildSearchUrl(query, options) {
+        const { tbs, country, location } = options;
+        const params = new URLSearchParams();
+        params.set('q', query);
+        // DuckDuckGo HTML endpoint supports some filtering
+        if (tbs) {
+            // DDG uses `df` for time filtering on html endpoint
+            params.set('df', tbs);
+        }
+        if (country || location) {
+            const region = (country || location || '').toLowerCase();
+            if (region)
+                params.set('kl', region);
+        }
+        return `https://html.duckduckgo.com/html/?${params.toString()}`;
+    }
+    async searchOnce(query, options) {
+        const { count, signal } = options;
+        const searchUrl = this.buildSearchUrl(query, options);
+        const response = await undiciFetch(searchUrl, {
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                'Accept-Language': 'en-US,en;q=0.9',
+            },
+            signal,
+        });
+        if (!response.ok) {
+            throw new Error(`Search failed: HTTP ${response.status}`);
+        }
+        const html = await response.text();
+        const $ = load(html);
+        const results = [];
+        const seen = new Set();
+        $('.result').each((_i, elem) => {
+            if (results.length >= count)
+                return;
+            const $result = $(elem);
+            // Be resilient to markup variations: title can be in .result__title or
+            // directly on the anchor.
+            const titleRaw = $result.find('.result__title').text() || $result.find('.result__a').text();
+            const rawUrl = $result.find('.result__a').attr('href') || '';
+            const snippetRaw = $result.find('.result__snippet').text();
+            let title = cleanText(titleRaw, { maxLen: 200 });
+            let snippet = cleanText(snippetRaw, { maxLen: 500, stripEllipsisPadding: true });
+            if (!title || !rawUrl)
+                return;
+            // Extract actual URL from DuckDuckGo redirect
+            let url = rawUrl;
+            try {
+                const ddgUrl = new URL(rawUrl, 'https://duckduckgo.com');
+                const uddg = ddgUrl.searchParams.get('uddg');
+                if (uddg)
+                    url = decodeURIComponent(uddg);
+            }
+            catch {
+                // Use raw URL if parsing fails
+            }
+            // SECURITY: Validate and sanitize results — only allow HTTP/HTTPS URLs
+            try {
+                let parsed;
+                try {
+                    parsed = new URL(url);
+                }
+                catch {
+                    // Handle protocol-relative or relative URLs (rare but possible)
+                    parsed = new URL(url, 'https://duckduckgo.com');
+                }
+                if (!['http:', 'https:'].includes(parsed.protocol)) {
+                    return;
+                }
+                url = parsed.href;
+            }
+            catch {
+                return;
+            }
+            // Deduplicate by normalized URL (strip query params, www, trailing slash)
+            const dedupeKey = normalizeUrlForDedupe(url);
+            if (seen.has(dedupeKey))
+                return;
+            seen.add(dedupeKey);
+            results.push({ title, url, snippet });
+        });
+        return results;
+    }
+    async searchWeb(query, options) {
+        const attempts = this.buildQueryAttempts(query);
+        // Retry only when DDG returns 0 results.
+        for (const q of attempts) {
+            const results = await this.searchOnce(q, options);
+            if (results.length > 0)
+                return results;
+        }
+        return [];
+    }
+}
+export class BraveSearchProvider {
+    id = 'brave';
+    requiresApiKey = true;
+    async searchWeb(query, options) {
+        const { count, apiKey, signal } = options;
+        if (!apiKey || apiKey.trim().length === 0) {
+            throw new Error('Brave Search requires an API key');
+        }
+        const url = new URL('https://api.search.brave.com/res/v1/web/search');
+        url.searchParams.set('q', query);
+        url.searchParams.set('count', String(Math.min(Math.max(count, 1), 10)));
+        const response = await undiciFetch(url.toString(), {
+            headers: {
+                'Accept': 'application/json',
+                'X-Subscription-Token': apiKey,
+            },
+            signal,
+        });
+        if (!response.ok) {
+            const text = await response.text().catch(() => '');
+            throw new Error(`Brave Search failed: HTTP ${response.status}${text ? ` - ${text}` : ''}`);
+        }
+        const data = await response.json();
+        const resultsArray = data?.web?.results;
+        if (!Array.isArray(resultsArray)) {
+            return [];
+        }
+        const results = [];
+        for (const r of resultsArray) {
+            if (results.length >= count)
+                break;
+            const title = typeof r?.title === 'string' ? r.title.trim() : '';
+            const rawUrl = typeof r?.url === 'string' ? r.url.trim() : '';
+            const snippet = typeof r?.description === 'string'
+                ? r.description.trim()
+                : typeof r?.snippet === 'string'
+                    ? r.snippet.trim()
+                    : '';
+            if (!title || !rawUrl)
+                continue;
+            // SECURITY: Validate URL protocol
+            try {
+                const parsed = new URL(rawUrl);
+                if (!['http:', 'https:'].includes(parsed.protocol))
+                    continue;
+            }
+            catch {
+                continue;
+            }
+            results.push({
+                title: title.slice(0, 200),
+                url: rawUrl,
+                snippet: snippet.slice(0, 500),
+            });
+        }
+        return results;
+    }
+}
+export function getSearchProvider(id) {
+    if (!id || id === 'duckduckgo')
+        return new DuckDuckGoProvider();
+    if (id === 'brave')
+        return new BraveSearchProvider();
+    // Exhaustive fallback (should be unreachable due to typing)
+    return new DuckDuckGoProvider();
+}
+//# sourceMappingURL=search-provider.js.map

package/dist/core/search-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-provider.js","sourceRoot":"","sources":["../../src/core/search-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAgC/B,SAAS,kBAAkB,CAAC,KAAa;IACvC,4EAA4E;IAC5E,wEAAwE;IACxE,OAAO,KAAK;SACT,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,mBAAmB,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,QAAQ;YAAE,OAAO,EAAE,CAAC;QAC/D,IAAI,CAAC;YACH,OAAO,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC,CAAC;SACD,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE;QAChC,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,QAAQ;YAAE,OAAO,EAAE,CAAC;QAC/D,IAAI,CAAC;YACH,OAAO,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,SAAS,CAChB,KAAa,EACb,IAGC;IAED,IAAI,CAAC,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAElC,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC9B,6DAA6D;QAC7D,CAAC,GAAG,CAAC;aACF,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC;aACjC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC;aACjC,IAAI,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;QAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACxD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAc;IAC3C,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAC5D,IAAI,IAAI,GAAG,CAAC,CAAC,QAAQ,IAAI,GAAG,CAAC;QAC7B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACjC,OAAO,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM;aACV,IAAI,EAAE;aACN,WAAW,EAAE;aACb,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;aAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;aACrB,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;aACtB,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,MAAM,OAAO,kBAAkB;IACpB,EAAE,GAAqB,YAAY,CAAC;IACpC,cAAc,GAAG,KAAK,CAAC;IAExB,kBAAkB,CAAC,aAAqB;QAC9C,MAAM,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAElB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,iCAAiC;QACjC,oBAAoB;QACpB,kBAAkB;QAClB,kBAAkB;QAClB,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAE7B,2EAA2E;QAC3E,2EAA2E;QAC3E,wEAAwE;QACxE,qCAAqC;QACrC,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,oBAAoB,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEnE,IAAI,YAAY,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC1C,yDAAyD;YACzD,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACxC,CAAC;YAED,iEAAiE;YACjE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC/B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAChC,CAAC;QAED,sCAAsC;QACtC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,OAAO,QAAQ;aACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;aAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACZ,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,cAAc,CAAC,KAAa,EAAE,OAAyB;QAC7D,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAEvB,mDAAmD;QACnD,IAAI,GAAG,EAAE,CAAC;YACR,oDAAoD;YACpD,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,CAAC,OAAO,IAAI,QAAQ,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YACzD,IAAI,MAAM;gBAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,qCAAqC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAa,EAAE,OAAyB;QAC/D,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAElC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAEtD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE;YAC5C,OAAO,EAAE;gBACP,YAAY,EAAE,oEAAoE;gBAClF,QAAQ,EAAE,iEAAiE;gBAC3E,iBAAiB,EAAE,gBAAgB;aACpC;YACD,MAAM;SACP,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QAErB,MAAM,OAAO,GAAsB,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE;YAC7B,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;gBAAE,OAAO;YAEpC,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YAExB,uEAAuE;YACvE,0BAA0B;YAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5F,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC7D,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,EAAE,CAAC;YAE3D,IAAI,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YACjD,IAAI,OAAO,GAAG,SAAS,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAC;YAEjF,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM;gBAAE,OAAO;YAE9B,8CAA8C;YAC9C,IAAI,GAAG,GAAG,MAAM,CAAC;YACjB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;gBACzD,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7C,IAAI,IAAI;oBAAE,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;YAAC,MAAM,CAAC;gBACP,+BAA+B;YACjC,CAAC;YAED,uEAAuE;YACvE,IAAI,CAAC;gBACH,IAAI,MAAW,CAAC;gBAChB,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC;gBAAC,MAAM,CAAC;oBACP,gEAAgE;oBAChE,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,wBAAwB,CAAC,CAAC;gBAClD,CAAC;gBACD,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACnD,OAAO;gBACT,CAAC;gBACD,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC;YACpB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;YACT,CAAC;YAED,0EAA0E;YAC1E,MAAM,SAAS,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,OAAO;YAChC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAEpB,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,OAAyB;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAEhD,yCAAyC;QACzC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAClD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,OAAO,CAAC;QACzC,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;CACF;AAED,MAAM,OAAO,mBAAmB;IACrB,EAAE,GAAqB,OAAO,CAAC;IAC/B,cAAc,GAAG,IAAI,CAAC;IAE/B,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,OAAyB;QACtD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAE1C,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,gDAAgD,CAAC,CAAC;QACtE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAExE,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YACjD,OAAO,EAAE;gBACP,QAAQ,EAAE,kBAAkB;gBAC5B,sBAAsB,EAAE,MAAM;aAC/B;YACD,MAAM;SACP,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,6BAA6B,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7F,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAS,CAAC;QAC1C,MAAM,YAAY,GAAU,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;QAE/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;YACjC,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;YAC7B,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;gBAAE,MAAM;YACnC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,MAAM,MAAM,GAAG,OAAO,CAAC,EAAE,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,MAAM,OAAO,GAAG,OAAO,CAAC,EAAE,WAAW,KAAK,QAAQ;gBAChD,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE;gBACtB,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,QAAQ;oBAC9B,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE;oBAClB,CAAC,CAAC,EAAE,CAAC;YAET,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM;gBAAE,SAAS;YAEhC,kCAAkC;YAClC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC/B,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;oBAAE,SAAS;YAC/D,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,OAAO,CAAC,IAAI,CAAC;gBACX,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBAC1B,GAAG,EAAE,MAAM;gBACX,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,MAAM,UAAU,iBAAiB,CAAC,EAAgC;IAChE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,YAAY;QAAE,OAAO,IAAI,kBAAkB,EAAE,CAAC;IAChE,IAAI,EAAE,KAAK,OAAO;QAAE,OAAO,IAAI,mBAAmB,EAAE,CAAC;IAErD,4DAA4D;IAC5D,OAAO,IAAI,kBAAkB,EAAE,CAAC;AAClC,CAAC"}

package/dist/core/strategies.d.ts

@@ -2,6 +2,7 @@
  * Smart escalation strategy: try simple fetch first, escalate to browser if needed
  */
 import { type FetchResult } from './fetcher.js';
+export declare function clearDomainIntel(): void;
 export interface StrategyOptions {
     /** Force browser mode (skip simple fetch) */
     forceBrowser?: boolean;
@@ -33,6 +34,10 @@ export interface StrategyOptions {
     }>;
     /** Keep browser page open for reuse (caller must close) */
     keepPageOpen?: boolean;
+    /** Disable response cache for this request */
+    noCache?: boolean;
+    /** Time to wait before launching browser in parallel with simple fetch */
+    raceTimeoutMs?: number;
     /** Location/language for geo-targeted scraping */
     location?: {
         country?: string;
@@ -40,19 +45,11 @@ export interface StrategyOptions {
     };
 }
 export interface StrategyResult extends FetchResult {
-    /** Which strategy succeeded: 'simple' | 'browser' | 'stealth' */
-    method: 'simple' | 'browser' | 'stealth';
+    /** Which strategy succeeded: 'simple' | 'browser' | 'stealth' | 'cached' */
+    method: 'simple' | 'browser' | 'stealth' | 'cached';
 }
 /**
  * Smart fetch with automatic escalation
- *
- * Strategy:
- * 1. Try simple HTTP fetch first (fast, ~200ms)
- * 2. If blocked (403, 503, Cloudflare, empty body) → try browser
- * 3. If browser gets blocked (403, CAPTCHA) → try stealth mode
- * 4. If stealth mode is explicitly requested → skip to stealth
- *
- * Returns the result along with which method worked
  */
 export declare function smartFetch(url: string, options?: StrategyOptions): Promise<StrategyResult>;
 //# sourceMappingURL=strategies.d.ts.map

package/dist/core/strategies.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAyC,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AAGvF,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,gDAAgD;IAChD,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,iBAAiB,GAAG,YAAY,CAAC;QACtH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;QAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,2DAA2D;IAC3D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,kDAAkD;IAClD,QAAQ,CAAC,EAAE;QACT,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,iEAAiE;IACjE,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;CAC1C;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC,CA6GpG"}
+{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAyC,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AAoIvF,wBAAgB,gBAAgB,IAAI,IAAI,CAGvC;AA6ED,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,gDAAgD;IAChD,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,iBAAiB,GAAG,YAAY,CAAC;QACtH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,EAAE,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;QAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,2DAA2D;IAC3D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,QAAQ,CAAC,EAAE;QACT,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,4EAA4E;IAC5E,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,CAAC;CACrD;AAuGD;;GAEG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC,CAqNpG"}