recker 1.0.79 → 1.0.80-next.70ea84d

package/dist/search/google.js

@@ -2,9 +2,8 @@ import { createClient } from '../core/client.js';
 import { ValidationError } from '../core/errors.js';
 import { HttpRequest } from '../core/request.js';
 import { ScrapeDocument } from '../scrape/document.js';
-import { detectBlock } from '../utils/block-detector.js';
+import { detectBlock, detectCaptcha, } from '../utils/block-detector.js';
 import { getRandomUserAgent } from '../utils/user-agent.js';
-const GOOGLE_SEARCH_BASE_URL = 'https://www.google.com/search';
 const GOOGLE_SEARCH_ORIGIN = 'https://www.google.com';
 const GOOGLE_RESULT_SNIPPET_SELECTOR_ORDER = [
     '[data-sncf="1"]',
@@ -23,6 +22,18 @@ const GOOGLE_RESULT_LINK_SELECTORS = [
     'a[href^="http://www.google.com/url?"]',
 ];
 const GOOGLE_RESULT_CONTAINER_SELECTORS = '[data-hveid], [data-ved], div[class*="g"], div[class*="MjjY"], div[class*="tF2Cxc"], [class*="xpd"]';
+const GOOGLE_BROWSER_PROFILE_GS_LCRP_VALUES = [
+    'EgZjaHJvbWUyBggAEEUYOdIBCDIwNzlqMGoxqAIAsAIA',
+    'EgZjaHJvbWUyBggAEEUYOdIBCDIwMjlqMGoxqAIAsAIA',
+    'EgZjaHJvbWUyBggAEEUYOdIBCDIwMzNqMGoxqAIAsAIA',
+];
+const GOOGLE_BROWSER_PROFILE_AQS_VALUES = [
+    'EgNQQxIEGAM%3D',
+    'EgZjaHJvbWUyBggAEEUYOdIB',
+    'EgYIARABGBQyGAM%3D',
+];
+const GOOGLE_SEARCH_PARAM_IE_VALUES = ['UTF-8', 'utf-8', 'UTF-8', 'utf-8'];
+const GOOGLE_BROWSER_SEI_LENGTH = 16;
 const GOOGLE_AD_CONTAINER_CLASS_HINTS = [
     'ad',
     'ads',
@@ -47,6 +58,47 @@ const GOOGLE_AD_TEXT_HINTS = [
     'anunciante',
 ];
 const COUNTRY_CODE_PATTERN = /^[a-z]{2}$/;
+const GOOGLE_SEARCH_HOST_OVERRIDES = {
+    br: 'google.com.br',
+    de: 'google.de',
+    fr: 'google.fr',
+    es: 'google.es',
+    it: 'google.it',
+    in: 'google.co.in',
+    uk: 'google.co.uk',
+    gb: 'google.co.uk',
+    us: 'google.com',
+    au: 'google.com.au',
+    ca: 'google.ca',
+    mx: 'google.com.mx',
+    ar: 'google.com.ar',
+};
+const GOOGLE_DEFAULT_LANGUAGE_BY_COUNTRY = {
+    br: 'pt-BR',
+    pt: 'pt-PT',
+    pt_br: 'pt-BR',
+    de: 'de',
+    fr: 'fr',
+    es: 'es',
+    it: 'it',
+    in: 'en-IN',
+    mx: 'es',
+    ar: 'es',
+    au: 'en',
+    ca: 'en',
+    gb: 'en',
+    uk: 'en',
+};
+const SERP_BLOCK_CONFIDENCE = 0.6;
+const URL_DOMAIN_MARKET_MAP = {
+    'co.uk': 'uk',
+    'com.br': 'br',
+    'co.br': 'br',
+    'com.au': 'au',
+    'co.in': 'in',
+    'com.ar': 'ar',
+    'com.mx': 'mx',
+};
 const COUNTRY_ALIASES = {
     us: 'us',
     usa: 'us',
@@ -85,6 +137,39 @@ const COUNTRY_ALIASES_NORMALIZED = Object.entries(COUNTRY_ALIASES).reduce((acc,
     acc[key.toLowerCase().replace(/[^a-z]/g, '_')] = value;
     return acc;
 }, {});
+const SERP_SEARCH_SOURCES = ['google'];
+function normalizeNumberTiming(value) {
+    if (typeof value !== 'number')
+        return undefined;
+    if (!Number.isFinite(value))
+        return undefined;
+    if (value <= 0)
+        return undefined;
+    return Math.round(value);
+}
+function toSearchTimings(timings) {
+    if (!timings || typeof timings !== 'object')
+        return undefined;
+    const parsed = timings;
+    const normalized = {
+        dns: normalizeNumberTiming(parsed.dns),
+        tcp: normalizeNumberTiming(parsed.tcp),
+        tls: normalizeNumberTiming(parsed.tls),
+        ttfb: normalizeNumberTiming(parsed.firstByte),
+        content: normalizeNumberTiming(parsed.content),
+        total: normalizeNumberTiming(parsed.total),
+    };
+    return Object.values(normalized).some(value => value !== undefined) ? normalized : undefined;
+}
+function detectSearchCaptcha(status, headers, body) {
+    const result = detectCaptcha({ status, headers }, body);
+    return {
+        detected: result.detected,
+        confidence: result.confidence,
+        provider: result.provider,
+        description: result.description,
+    };
+}
 function cleanText(value) {
     return value.replace(/\s+/g, ' ').trim();
 }
@@ -99,6 +184,41 @@ function toParamValue(value) {
     const trimmed = String(value).trim();
     return trimmed.length > 0 ? trimmed : undefined;
 }
+function pickRandomProfileValue(values) {
+    if (values.length === 0)
+        return '';
+    const index = Math.floor(Math.random() * values.length);
+    return values[index];
+}
+function generateSeiHint() {
+    const alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
+    const chars = [];
+    for (let i = 0; i < GOOGLE_BROWSER_SEI_LENGTH; i++) {
+        chars.push(alphabet[Math.floor(Math.random() * alphabet.length)] ?? 'a');
+    }
+    return chars.join('');
+}
+function normalizeHumanSearchProfile(value) {
+    if (typeof value !== 'string') {
+        return 'off';
+    }
+    const normalized = value.trim().toLowerCase();
+    if (!normalized || normalized === 'off' || normalized === 'false' || normalized === '0') {
+        return 'off';
+    }
+    if (normalized === 'chrome' || normalized === 'browser' || normalized === 'human') {
+        return 'chrome';
+    }
+    return 'off';
+}
+function normalizeSearchSource(value) {
+    if (typeof value !== 'string')
+        return 'google';
+    const normalized = value.trim().toLowerCase();
+    return SERP_SEARCH_SOURCES.includes(normalized)
+        ? normalized
+        : 'google';
+}
 function pick(...values) {
     for (const value of values) {
         if (value !== undefined) {
@@ -138,6 +258,84 @@ function resolveCountryCode(country, legacyGl) {
     }
     return legacyGl ? legacyGl.trim().toLowerCase() : '';
 }
+function parseCountryFromHostname(hostname) {
+    const normalized = hostname
+        .toLowerCase()
+        .replace(/^www\./, '')
+        .trim();
+    const parts = normalized.split('.');
+    if (parts.length < 2)
+        return undefined;
+    const suffix2 = parts.slice(-2).join('.');
+    const mapped2 = URL_DOMAIN_MARKET_MAP[suffix2];
+    if (mapped2)
+        return mapped2;
+    const last = parts[parts.length - 1];
+    if (COUNTRY_CODE_PATTERN.test(last))
+        return last;
+    return undefined;
+}
+function resolveGoogleSearchOrigin(gl) {
+    if (!gl)
+        return GOOGLE_SEARCH_ORIGIN;
+    const normalized = gl.toLowerCase();
+    const host = GOOGLE_SEARCH_HOST_OVERRIDES[normalized] ?? `google.${normalized}`;
+    return `https://www.${host}`;
+}
+function normalizeSearchOrigin(origin) {
+    try {
+        const parsed = new URL(origin);
+        return `${parsed.protocol}//${parsed.host}`;
+    }
+    catch {
+        if (origin.startsWith('http://') || origin.startsWith('https://'))
+            return origin;
+        return `https://www.google.com`;
+    }
+}
+function pickGoogleSearchOrigins(primary, country) {
+    const unique = [];
+    const push = (value) => {
+        if (!value)
+            return;
+        if (!unique.includes(value))
+            unique.push(value);
+    };
+    const normalizedPrimary = normalizeSearchOrigin(primary);
+    const canonical = 'https://www.google.com';
+    push(normalizedPrimary);
+    push(canonical);
+    const countryHost = country ? GOOGLE_SEARCH_HOST_OVERRIDES[country.toLowerCase()] : '';
+    if (countryHost) {
+        push(`https://www.${countryHost}`);
+    }
+    if (countryHost && `https://www.${countryHost}` !== canonical) {
+        push('https://www.google.com');
+    }
+    if (normalizedPrimary.startsWith('https://')) {
+        push(normalizedPrimary.replace(/^https:/, 'http:'));
+    }
+    return unique;
+}
+export function inferSearchLocaleFromUrl(targetUrl) {
+    try {
+        const parsed = new URL(targetUrl);
+        const country = parseCountryFromHostname(parsed.hostname);
+        if (!country)
+            return undefined;
+        const resolved = normalizeCountryCode(country);
+        if (!resolved)
+            return undefined;
+        return {
+            country: resolved,
+            gl: resolved,
+            hl: GOOGLE_DEFAULT_LANGUAGE_BY_COUNTRY[resolved],
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
 function normalizeOptions(query, options = {}) {
     const normalizedQuery = cleanText(query);
     if (!normalizedQuery) {
@@ -147,6 +345,9 @@ function normalizeOptions(query, options = {}) {
         });
     }
     const resolvedTransport = (options.transport ?? 'auto');
+    const gl = resolveCountryCode(options.country, options.gl);
+    const searchOrigin = resolveGoogleSearchOrigin(gl);
+    const source = normalizeSearchSource(options.source);
     return {
         ...options,
         as_q: pick(options.as_q, options.asQ) ?? '',
@@ -159,16 +360,29 @@ function normalizeOptions(query, options = {}) {
         as_rights: pick(options.as_rights, options.asRights) ?? '',
         as_nlo: toParamValue(pick(options.as_nlo, options.asNlo)) ?? '',
         as_nhi: toParamValue(pick(options.as_nhi, options.asNhi)) ?? '',
-        gl: resolveCountryCode(options.country, options.gl),
+        gl,
+        humanProfile: normalizeHumanSearchProfile(options.humanProfile),
+        source,
         transport: resolvedTransport,
         includeRawHtml: options.includeRawHtml ?? false,
+        searchOrigin,
+        searchBaseUrl: `${searchOrigin}/search`,
     };
 }
 function buildSearchUrl(query, options) {
     const params = new URLSearchParams();
     params.set('q', query);
-    params.set('ie', 'UTF-8');
-    params.set('oe', 'UTF-8');
+    params.set('oq', query);
+    params.set('sourceid', 'chrome');
+    params.set('ie', pickRandomProfileValue(GOOGLE_SEARCH_PARAM_IE_VALUES));
+    params.set('oe', pickRandomProfileValue(GOOGLE_SEARCH_PARAM_IE_VALUES));
+    if (options.humanProfile === 'chrome') {
+        const qs = query.trim().toLowerCase();
+        params.set('aqs', pickRandomProfileValue(GOOGLE_BROWSER_PROFILE_AQS_VALUES));
+        params.set('gs_lcrp', pickRandomProfileValue(GOOGLE_BROWSER_PROFILE_GS_LCRP_VALUES));
+        params.set('sei', generateSeiHint());
+        params.set('client', 'chrome');
+    }
     if (options.as_q)
         params.set('as_q', options.as_q);
     if (options.asEpq)
@@ -221,17 +435,26 @@ function buildSearchUrl(query, options) {
             }
         }
     }
-    return `${GOOGLE_SEARCH_BASE_URL}?${params.toString()}`;
+    return `${options.searchBaseUrl}?${params.toString()}`;
 }
-function normalizeRequestHeaders(inputHeaders, userAgent) {
+function normalizeRequestHeaders(inputHeaders, userAgent, referer, locale) {
+    const normalizedLocale = (locale || 'en-US').replace('_', '-');
+    const acceptLanguage = `${normalizedLocale},en-US;q=0.8,en;q=0.6`;
     const headers = new Headers({
-        accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-        'accept-language': 'en-US,en;q=0.9',
+        accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+        'accept-language': acceptLanguage,
         'cache-control': 'max-age=0',
-        'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="132", "Google Chrome";v="132"',
         'sec-ch-ua-platform': '"Windows"',
+        'sec-ch-ua-mobile': '?0',
+        'sec-fetch-dest': 'document',
+        'sec-fetch-mode': 'navigate',
+        'sec-fetch-site': 'same-origin',
+        'sec-fetch-user': '?1',
+        'upgrade-insecure-requests': '1',
         'user-agent': userAgent,
-        referer: GOOGLE_SEARCH_ORIGIN,
+        referer,
+        cookie: 'SOCS=CAI',
     });
     if (inputHeaders) {
         const incoming = new Headers(inputHeaders);
@@ -243,6 +466,24 @@ function normalizeRequestHeaders(inputHeaders, userAgent) {
     });
     return merged;
 }
+function isUsefulSearchResponse(parsed, fetchResult) {
+    if (parsed.results.length > 0)
+        return true;
+    if (fetchResult.block?.blocked && fetchResult.block.confidence >= SERP_BLOCK_CONFIDENCE)
+        return false;
+    if (fetchResult.captcha?.detected && fetchResult.captcha.confidence >= SERP_BLOCK_CONFIDENCE)
+        return false;
+    return false;
+}
+function isHardSearchBlock(parsed, fetchResult) {
+    if (fetchResult.captcha?.detected && fetchResult.captcha.confidence >= SERP_BLOCK_CONFIDENCE)
+        return true;
+    if (fetchResult.block?.blocked && fetchResult.block.confidence >= SERP_BLOCK_CONFIDENCE)
+        return true;
+    if (parsed.results.length === 0 && fetchResult.block?.reason === 'captcha')
+        return true;
+    return false;
+}
 async function hasImpersonateBinary() {
     try {
         const { hasImpersonate } = await import('../utils/binary-manager.js');
@@ -254,6 +495,7 @@ async function hasImpersonateBinary() {
 }
 async function fetchWithCurl(url, headers, timeout) {
     const { CurlTransport } = await import('../transport/curl.js');
+    const requestStart = performance.now();
     const transport = new CurlTransport();
     const request = new HttpRequest(url, {
         method: 'GET',
@@ -261,22 +503,32 @@ async function fetchWithCurl(url, headers, timeout) {
         timeout,
     });
     const response = await transport.dispatch(request);
+    const requestEnd = performance.now();
     const html = await response.text();
-    return { html, status: response.status };
+    const parsedTimings = toSearchTimings(response.timings);
+    const timings = parsedTimings ?? {
+        total: Math.round(requestEnd - requestStart),
+    };
+    return {
+        html,
+        status: response.status,
+        responseHeaders: response.headers,
+        timings,
+    };
 }
 async function fetchSearchResults(url, options) {
-    const headers = normalizeRequestHeaders(options.headers, options.userAgent ?? getRandomUserAgent('desktop.chrome'));
+    const headers = normalizeRequestHeaders(options.headers, options.userAgent ?? getRandomUserAgent('desktop.chrome'), options.searchOrigin, options.hl);
     const requestTimeout = options.timeout;
-    const impersonateAvailable = options.transport !== 'undici' && (await hasImpersonateBinary());
-    if (options.transport === 'curl' && !impersonateAvailable) {
-        throw new ValidationError('Transport "curl" requires curl-impersonate; install it with `rek setup`', {
-            field: 'transport',
-            value: options.transport,
-        });
-    }
+    const impersonateAvailable = await hasImpersonateBinary();
     if (options.transport === 'curl') {
+        if (!impersonateAvailable) {
+            throw new ValidationError('Transport "curl" requires curl-impersonate; install it with `rek setup`', {
+                field: 'transport',
+                value: options.transport,
+            });
+        }
         const directResponse = await fetchWithCurl(url, headers, requestTimeout);
-        const directBlock = detectBlock({ status: directResponse.status, headers: new Headers() }, directResponse.html);
+        const directBlock = detectBlock({ status: directResponse.status, headers: directResponse.responseHeaders }, directResponse.html);
         return {
             html: directResponse.html,
             status: directResponse.status,
@@ -284,13 +536,19 @@ async function fetchSearchResults(url, options) {
             fallbackUsed: false,
             impersonateAvailable,
             block: directBlock,
+            captcha: detectSearchCaptcha(directResponse.status, directResponse.responseHeaders, directResponse.html),
+            timings: directResponse.timings,
         };
     }
     const client = createClient({ timeout: requestTimeout });
     const performUndiciRequest = async () => {
-        const response = await client.get(url, { headers });
+        const response = await client.get(url, {
+            headers,
+            throwHttpErrors: false,
+        });
         const html = await response.text();
         const block = detectBlock({ status: response.status, headers: response.headers }, html);
+        const captcha = detectSearchCaptcha(response.status, response.headers, html);
         return {
             html,
             status: response.status,
@@ -298,18 +556,39 @@ async function fetchSearchResults(url, options) {
             fallbackUsed: false,
             impersonateAvailable,
             block,
+            captcha,
+            timings: toSearchTimings(response.timings),
         };
     };
-    if (options.transport === 'undici') {
-        return performUndiciRequest();
-    }
     if (!impersonateAvailable) {
         return performUndiciRequest();
     }
+    const shouldFallbackToUndici = (block, captcha) => {
+        if (block.blocked && block.confidence >= SERP_BLOCK_CONFIDENCE)
+            return true;
+        if (captcha.detected && captcha.confidence >= SERP_BLOCK_CONFIDENCE)
+            return true;
+        if (block.reason === 'captcha' || block.reason === 'rate-limit')
+            return true;
+        return false;
+    };
+    const getSearchResponseRisk = (response) => ((response.block?.confidence ?? 0) + (response.captcha?.confidence ?? 0));
+    const shouldKeepPrimaryImpersonateResponse = (primary, fallback) => (getSearchResponseRisk(primary) <= getSearchResponseRisk(fallback));
     try {
         const primaryImpersonateResponse = await fetchWithCurl(url, headers, requestTimeout);
-        const primaryImpersonateBlock = detectBlock({ status: primaryImpersonateResponse.status, headers: new Headers() }, primaryImpersonateResponse.html);
-        if (!primaryImpersonateBlock.blocked || primaryImpersonateBlock.confidence <= 0.7) {
+        const primaryImpersonateBlock = detectBlock({ status: primaryImpersonateResponse.status, headers: primaryImpersonateResponse.responseHeaders }, primaryImpersonateResponse.html);
+        const primaryImpersonateCaptcha = detectSearchCaptcha(primaryImpersonateResponse.status, primaryImpersonateResponse.responseHeaders, primaryImpersonateResponse.html);
+        const primaryResult = {
+            html: primaryImpersonateResponse.html,
+            status: primaryImpersonateResponse.status,
+            transport: 'curl',
+            fallbackUsed: false,
+            impersonateAvailable,
+            block: primaryImpersonateBlock,
+            captcha: primaryImpersonateCaptcha,
+            timings: primaryImpersonateResponse.timings,
+        };
+        if (!shouldFallbackToUndici(primaryImpersonateBlock, primaryImpersonateCaptcha)) {
             return {
                 html: primaryImpersonateResponse.html,
                 status: primaryImpersonateResponse.status,
@@ -317,8 +596,20 @@ async function fetchSearchResults(url, options) {
                 fallbackUsed: false,
                 impersonateAvailable,
                 block: primaryImpersonateBlock,
+                captcha: primaryImpersonateCaptcha,
+                timings: primaryImpersonateResponse.timings,
+            };
+        }
+        const fallback = await performUndiciRequest();
+        if (!shouldKeepPrimaryImpersonateResponse(primaryResult, fallback)) {
+            return {
+                ...fallback,
+                transport: fallback.transport,
+                fallbackUsed: true,
+                impersonateAvailable,
             };
         }
+        return primaryResult;
     }
     catch {
     }
@@ -330,19 +621,36 @@ async function fetchSearchResults(url, options) {
         impersonateAvailable,
     };
 }
-function resolveSearchResultUrl(rawHref) {
+function isGoogleOwnHost(hostname) {
+    const lower = hostname.toLowerCase();
+    return (lower === 'www.google.com' ||
+        lower === 'google.com' ||
+        lower.endsWith('.google.com') ||
+        lower.endsWith('.googleapis.com') ||
+        lower.endsWith('.googleusercontent.com') ||
+        lower.endsWith('.googlesyndication.com') ||
+        lower.endsWith('.doubleclick.net') ||
+        lower.endsWith('.gstatic.com') ||
+        lower.startsWith('www.google.') ||
+        /^([a-z0-9-]+\.)*google(\.[a-z]{2,})+$/.test(lower));
+}
+function resolveSearchResultUrl(rawHref, searchOrigin) {
     try {
         const normalized = rawHref.startsWith('//') ? `https:${rawHref}` : rawHref;
-        const parsed = new URL(normalized, GOOGLE_SEARCH_ORIGIN);
+        const parsed = new URL(normalized, searchOrigin);
         const rawResultUrl = parsed.searchParams.get('q') ?? parsed.searchParams.get('url');
-        if (!rawResultUrl)
-            return null;
-        const candidate = new URL(decodeURIComponent(rawResultUrl), GOOGLE_SEARCH_ORIGIN);
-        if (!candidate.protocol.startsWith('http'))
-            return null;
-        if (candidate.hostname === 'www.google.com' && candidate.pathname === '/search')
-            return null;
-        return candidate.toString();
+        if (rawResultUrl) {
+            const candidate = new URL(decodeURIComponent(rawResultUrl), searchOrigin);
+            if (!candidate.protocol.startsWith('http'))
+                return null;
+            if (candidate.pathname === '/search' && candidate.hostname.includes('google.'))
+                return null;
+            return candidate.toString();
+        }
+        if (parsed.protocol.startsWith('http') && !isGoogleOwnHost(parsed.hostname)) {
+            return parsed.toString();
+        }
+        return null;
     }
     catch {
         return null;
@@ -490,8 +798,8 @@ function parseResultStats(text) {
     const parsed = Number.parseInt(match[1], 10);
     return Number.isFinite(parsed) ? parsed : undefined;
 }
-function parseSearchPage(html, options) {
-    const doc = ScrapeDocument.createSync(html, { baseUrl: GOOGLE_SEARCH_ORIGIN });
+function parseSearchPage(html, options, searchOrigin) {
+    const doc = ScrapeDocument.createSync(html, { baseUrl: searchOrigin });
     const results = [];
     const seen = new Set();
     const maxResults = options.maxResults ? Number(options.maxResults) : undefined;
@@ -501,7 +809,7 @@ function parseSearchPage(html, options) {
         const rawHref = anchor.attr('href');
         if (!rawHref)
             continue;
-        const resultUrl = resolveSearchResultUrl(rawHref);
+        const resultUrl = resolveSearchResultUrl(rawHref, searchOrigin);
         if (!resultUrl || seen.has(resultUrl))
             continue;
         const titleText = (() => {
@@ -551,6 +859,49 @@ function parseSearchPage(html, options) {
             break;
         }
     }
+    for (const anchor of doc.selectAll('a[href^="https://"], a[href^="http://"]')) {
+        if (typeof maxResults === 'number' && Number.isFinite(maxResults) && results.length >= maxResults)
+            break;
+        const rawHref = anchor.attr('href');
+        if (!rawHref)
+            continue;
+        const resultUrl = resolveSearchResultUrl(rawHref, searchOrigin);
+        if (!resultUrl || seen.has(resultUrl))
+            continue;
+        const titleText = cleanText(anchor.find('h3').text());
+        if (!titleText)
+            continue;
+        const resultContainer = anchor.parents(GOOGLE_RESULT_CONTAINER_SELECTORS).first();
+        const placement = detectResultPlacement(anchor, resultContainer);
+        const snippet = (() => {
+            for (const selector of GOOGLE_RESULT_SNIPPET_SELECTOR_ORDER) {
+                const snippetNode = resultContainer.find(selector).first();
+                const snippetText = cleanText(snippetNode.text());
+                if (looksLikeSnippet(snippetText, titleText))
+                    return snippetText;
+            }
+            const fallbackElements = resultContainer.find('span,div').toArray();
+            for (const fallbackEl of fallbackElements) {
+                const fallbackText = cleanText(fallbackEl.text());
+                if (looksLikeSnippet(fallbackText, titleText))
+                    return fallbackText;
+            }
+            const rootFallback = cleanText(resultContainer.text());
+            if (looksLikeSnippet(rootFallback, titleText))
+                return rootFallback.slice(0, 240);
+            return undefined;
+        })();
+        results.push({
+            rank: results.length + 1,
+            title: titleText,
+            url: resultUrl,
+            snippet,
+            displayedUrl: extractDisplayedUrl(resultUrl, anchor.text()),
+            placement: placement.placement,
+            placementHint: placement.placementHint,
+        });
+        seen.add(resultUrl);
+    }
     const nextPageRaw = (() => {
         const candidate = doc.selectFirst('a#pnnext, a[aria-label="Next"], a[id="pnnext"]').first();
         if (candidate && candidate.length) {
@@ -558,7 +909,7 @@ function parseSearchPage(html, options) {
             if (!href)
                 return undefined;
             try {
-                return new URL(href, GOOGLE_SEARCH_ORIGIN).toString();
+                return new URL(href, searchOrigin).toString();
             }
             catch {
                 return undefined;
@@ -591,12 +942,41 @@ function parseSearchPage(html, options) {
 }
 export async function searchGoogleAdvanced(query, options = {}) {
     const normalized = normalizeOptions(query, options);
-    const searchUrl = buildSearchUrl(query, normalized);
-    const fetchResult = await fetchSearchResults(searchUrl, normalized);
-    const parsed = parseSearchPage(fetchResult.html, normalized);
+    const originCandidates = pickGoogleSearchOrigins(normalized.searchOrigin, normalized.gl);
+    let fetchResult;
+    let parsed;
+    let selectedSearchUrl = '';
+    for (let index = 0; index < originCandidates.length; index += 1) {
+        const origin = originCandidates[index];
+        const isFinalCandidate = index === originCandidates.length - 1;
+        const candidateOptions = {
+            ...normalized,
+            searchOrigin: origin,
+            searchBaseUrl: `${origin}/search`,
+        };
+        const candidateUrl = buildSearchUrl(query, candidateOptions);
+        const candidateResult = await fetchSearchResults(candidateUrl, candidateOptions);
+        const candidateParsed = parseSearchPage(candidateResult.html, candidateOptions, candidateOptions.searchOrigin);
+        fetchResult = candidateResult;
+        parsed = candidateParsed;
+        selectedSearchUrl = candidateUrl;
+        if (isUsefulSearchResponse(candidateParsed, candidateResult) || isFinalCandidate) {
+            break;
+        }
+        if (isHardSearchBlock(candidateParsed, candidateResult)) {
+            break;
+        }
+        if (candidateParsed.results.length === 0) {
+            continue;
+        }
+    }
+    if (!fetchResult || !parsed) {
+        throw new ValidationError('Failed to execute Google search attempts');
+    }
     const response = {
         query: normalized.query,
-        searchUrl,
+        source: normalized.source,
+        searchUrl: selectedSearchUrl,
         results: parsed.results,
         transport: {
             requested: normalized.transport,
@@ -605,7 +985,9 @@ export async function searchGoogleAdvanced(query, options = {}) {
             impersonateAvailable: fetchResult.impersonateAvailable,
         },
         status: fetchResult.status,
+        captcha: fetchResult.captcha,
         block: fetchResult.block,
+        timings: fetchResult.timings,
         nextPageUrl: parsed.nextPageUrl,
         nextPageStart: parsed.nextPageStart,
         resultStats: parsed.resultStats,

package/dist/search/index.d.ts

@@ -1,3 +1,3 @@
 export { searchGoogleAdvanced } from './google.js';
-export type { GoogleSearchAdvancedOptions, GoogleSearchResult, GoogleSearchResultPlacement, GoogleSearchResponse, SearchTransportDetails, } from './google.js';
+export type { GoogleSearchAdvancedOptions, GoogleSearchResult, GoogleSearchResultPlacement, GoogleSearchResponse, SerpSearchSource, SearchTransportDetails, } from './google.js';
 export type { SearchTransport } from './google.js';

package/dist/seo/analyzer.d.ts

@@ -13,6 +13,7 @@ export declare class SeoAnalyzer {
     analyze(): SeoReport;
     private getMainBody;
     private detectPageType;
+    private extractContentSectionSignals;
     private getVisibleText;
     private buildRuleContext;
     private analyzeUrlQuality;