webcrawl 1.4.2 → 1.4.3

package/dist/core/local-engine.js

@@ -9,7 +9,7 @@
  * Every community improvement to extractors, parsers, or targets in `src/` immediately
  * powers both this local engine (`npx webcrawl`) and the hosted `webcrawl.app` API.
  */
-import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, isChallengeMarkdown, mapWebsiteUrls, setCachedPage, sha256, } from "./engine.js";
+import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, isChallengeMarkdown, isLocalChromeAvailable, mapWebsiteUrls, renderViaLocalChrome, setCachedPage, sha256, } from "./engine.js";
 import { isDomainRenderNeeded, markDomainRenderNeeded, } from "./engine/session.js";
 import { isRenderProviderEnabled, renderViaProvider } from "./engine/render.js";
 import { fetchResource, isAntiBotChallenge, lastFetchTier, needsBrowserEscalation, } from "./engine/fetcher.js";
@@ -20,6 +20,7 @@ import { parseYouTubeCached, youtubeVideoId } from "./parse/youtube.js";
 import { rankDocuments } from "./ranker.js";
 import { getSearchCategoriesCatalog, searchWeb } from "./search.js";
 import { inspectTarget, listTargets, resolveTarget } from "./targets.js";
+import { hasCustomExtractRequest } from "./extract/structured.js";
 /** Lowercased hostname of a URL (memo key for the Tier R render memo). */
 function safeDomainOf(url) {
     try {
@@ -34,9 +35,10 @@ function safeDomainOf(url) {
  * `success: false` with no markdown, so callers never mistake the block page
  * for content. Exported for unit tests.
  */
-export function blockedScrapeResult(url, statusCode, tier, latencyMs) {
-    const error = `Blocked by the site's bot check (HTTP ${statusCode}). No content was returned. ` +
-        "Retry with `--mode cloud` (hosted stealth browser), or self-host a render provider via WEBCRAWL_RENDER_URL.";
+export function blockedScrapeResult(url, statusCode, tier, latencyMs, reason) {
+    const error = reason ??
+        (`Blocked by the site's bot check (HTTP ${statusCode}). No content was returned. ` +
+            "Retry with `--mode cloud` (hosted stealth browser), or self-host a render provider via WEBCRAWL_RENDER_URL.");
     return {
         success: false,
         mode: "local",
@@ -51,6 +53,86 @@ export function blockedScrapeResult(url, statusCode, tier, latencyMs) {
         evidence: { tier, blocked: true, statusCode, latencyMs, limitations: [error] },
     };
 }
+/** Shells shorter than this are unrendered JS / empty pages and must not be cached as content. */
+export const MIN_CACHEABLE_MARKDOWN = 200;
+/** Attach source HTML when the caller asked for the `html` format. */
+export function withOptionalHtml(result, html, formats) {
+    if (!formats?.includes("html") || !html)
+        return result;
+    const data = result.data && typeof result.data === "object" && !Array.isArray(result.data)
+        ? result.data
+        : {};
+    return { ...result, html, data: { ...data, html } };
+}
+/** Structured extract payload: custom prompt/schema → only those fields, no `target` sidecar. */
+export function extractOutputData(extracted, opts) {
+    if (hasCustomExtractRequest(opts.prompt, opts.schema))
+        return extracted;
+    return {
+        ...extracted,
+        ...(opts.target ? { target: opts.target } : {}),
+    };
+}
+function nativeScrapeTier() {
+    const t = lastFetchTier();
+    return t === "impersonate" || t === "proxy-impersonate" ? t : "impit-tls";
+}
+function emptyPageError(statusCode) {
+    return (`Page returned no readable content (HTTP ${statusCode}). ` +
+        "The site may be blocking automated clients or serving a client-rendered shell. " +
+        "Retry with `--mode cloud` (hosted stealth browser), or self-host a render provider via WEBCRAWL_RENDER_URL.");
+}
+/**
+ * JS / challenge shell → render provider, then local Chrome dump-dom, then stealth.
+ * Returns null when no renderer produced real content.
+ */
+async function escalateJsRender(url) {
+    if (isRenderProviderEnabled()) {
+        const rendered = await renderViaProvider(url);
+        if (rendered && rendered.markdown.trim().length >= 40) {
+            return {
+                title: rendered.title,
+                markdown: rendered.markdown,
+                html: "",
+                links: [],
+                tier: "render",
+            };
+        }
+    }
+    if (isLocalChromeAvailable()) {
+        const html = await renderViaLocalChrome(url);
+        if (html && html.length > 100) {
+            const parsed = fastHtmlToMarkdown(html, url);
+            if (parsed.markdown.trim().length > 40 &&
+                !isAntiBotChallenge(html, 200) &&
+                !isChallengeMarkdown(parsed.title, parsed.markdown)) {
+                return { ...parsed, html, tier: "browser" };
+            }
+        }
+    }
+    try {
+        const { stealthAvailable } = await import("./engine/stealth.js");
+        const { escalateToSandboxOrBrowser, isChallengeResult } = await import("./sandbox.js");
+        if (await stealthAvailable()) {
+            const esc = await escalateToSandboxOrBrowser(url, { browser: true, timeoutMs: 25_000 });
+            if (esc &&
+                esc.markdown.trim().length > 40 &&
+                !isChallengeResult(esc.html, esc.markdown, esc.statusCode)) {
+                return {
+                    title: esc.title,
+                    markdown: esc.markdown,
+                    html: esc.html,
+                    links: esc.links,
+                    tier: esc.tier,
+                };
+            }
+        }
+    }
+    catch {
+        /* stealth optional */
+    }
+    return null;
+}
 /**
  * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
  * Matching is a case-insensitive substring of the URL. Exported for unit tests.
@@ -90,7 +172,7 @@ export function createLocalEngineClient() {
             const resolved = resolveTarget({ query, category });
             return {
                 ...res,
-                success: true,
+                success: res.success !== false,
                 mode: "local",
                 target: {
                     id: resolved.plugin.id,
@@ -161,8 +243,11 @@ export function createLocalEngineClient() {
             if (!p.noCache) {
                 const cached = await getCachedPageAsync(targetUrl, "none");
                 // Entries written before blocked pages stopped being cached may hold a
-                // bot-check page: treat those as a miss and fetch again.
-                if (cached && !isChallengeMarkdown(cached.title, cached.markdown)) {
+                // bot-check page: treat those as a miss and fetch again. Thin shells
+                // (header-only JS pages) must also miss so local Chrome can hydrate them.
+                if (cached &&
+                    !isChallengeMarkdown(cached.title, cached.markdown) &&
+                    cached.markdown.trim().length >= MIN_CACHEABLE_MARKDOWN) {
                     const targetInspection = await inspectTarget({
                         url: targetUrl,
                         markdown: cached.markdown,
@@ -235,76 +320,64 @@ export function createLocalEngineClient() {
                 // ladder — the memo may be stale or the provider flaky.
             }
             const fetched = await fetchWithTlsImpersonation(targetUrl);
-            const { title, markdown, links } = fastHtmlToMarkdown(fetched.html, targetUrl);
-            const contentHash = sha256(markdown || fetched.html);
-            // A bot-check / 403 / 429 / 503 response is not the page: never cache it,
-            // or every retry for the next 24h would get the block page back.
-            const blocked = isAntiBotChallenge(fetched.html, fetched.status);
-            if (!blocked) {
-                setCachedPage(targetUrl, "none", {
-                    title,
-                    markdown,
-                    links,
-                    statusCode: fetched.status,
-                    contentHash,
-                });
-            }
-            const targetInspection = await inspectTarget({
-                url: targetUrl,
-                html: fetched.html,
-                markdown,
-            });
-            //    Tier R: the native ladder passed the bot check but the page is a
-            //    JS shell / challenge page the ladder cannot render. Memoize the
-            //    domain and either render via the provider (when configured) or
-            //    return the shell with an honest limitation note.
-            if (needsBrowserEscalation(fetched.status, fetched.html, markdown)) {
+            const parsed = fastHtmlToMarkdown(fetched.html, targetUrl);
+            let title = parsed.title;
+            let markdown = parsed.markdown;
+            let links = parsed.links;
+            let html = fetched.html;
+            const blocked = isAntiBotChallenge(html, fetched.status);
+            const needsRender = needsBrowserEscalation(fetched.status, html, markdown);
+            const nativeTier = nativeScrapeTier();
+            //    JS shell / challenge: memoize the domain and escalate to a renderer
+            //    (hosted provider, local Chrome dump-dom, then stealth). Never cache
+            //    the unrendered shell.
+            if (needsRender) {
                 await markDomainRenderNeeded(renderDomain);
-                if (isRenderProviderEnabled()) {
-                    const rendered = await renderViaProvider(targetUrl);
-                    if (rendered) {
-                        const renderHash = sha256(rendered.markdown);
+                // 403/429/503 is already a refusal — don't spend a Chrome launch on the block page.
+                const rendered = blocked && fetched.status >= 400 ? null : await escalateJsRender(targetUrl);
+                if (rendered) {
+                    const renderHash = sha256(rendered.markdown);
+                    if (rendered.markdown.trim().length >= MIN_CACHEABLE_MARKDOWN) {
                         setCachedPage(targetUrl, "none", {
                             title: rendered.title,
                             markdown: rendered.markdown,
-                            links: [],
+                            links: rendered.links,
                             statusCode: 200,
                             contentHash: renderHash,
                         });
-                        return {
-                            success: true,
-                            mode: "local",
+                    }
+                    return withOptionalHtml({
+                        success: true,
+                        mode: "local",
+                        url: targetUrl,
+                        title: rendered.title || targetUrl,
+                        markdown: rendered.markdown,
+                        links: rendered.links,
+                        data: {
                             url: targetUrl,
                             title: rendered.title || targetUrl,
                             markdown: rendered.markdown,
-                            links: [],
-                            data: {
-                                url: targetUrl,
-                                title: rendered.title || targetUrl,
-                                markdown: rendered.markdown,
-                                links: [],
-                            },
-                            evidence: {
-                                targetId: "render-provider",
-                                tier: "render",
-                                contentHash: renderHash,
-                                latencyMs: Date.now() - start,
-                                sourceUrl: rendered.sourceUrl,
-                                matchedSignals: ["render:provider"],
-                                limitations: rendered.warnings,
-                            },
-                        };
-                    }
+                            links: rendered.links,
+                        },
+                        evidence: {
+                            targetId: rendered.tier === "render" ? "render-provider" : "web-spa",
+                            tier: rendered.tier,
+                            contentHash: renderHash,
+                            latencyMs: Date.now() - start,
+                            sourceUrl: targetUrl,
+                            matchedSignals: rendered.tier === "browser" ? ["render:local-chrome"] : [`render:${rendered.tier}`],
+                        },
+                    }, rendered.html, p.formats);
                 }
-                const nativeTier = lastFetchTier() === "impersonate"
-                    ? "impersonate"
-                    : lastFetchTier() === "proxy-impersonate"
-                        ? "proxy-impersonate"
-                        : "impit-tls";
-                if (blocked) {
-                    return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start);
+                if (blocked || !markdown.trim()) {
+                    return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start, blocked ? undefined : emptyPageError(fetched.status));
                 }
-                return {
+                const targetInspection = await inspectTarget({
+                    url: targetUrl,
+                    html,
+                    markdown,
+                });
+                return withOptionalHtml({
                     success: true,
                     mode: "local",
                     url: targetUrl,
@@ -321,20 +394,38 @@ export function createLocalEngineClient() {
                     evidence: {
                         ...targetInspection.evidence,
                         tier: nativeTier,
-                        contentHash,
+                        contentHash: sha256(markdown || html),
                         latencyMs: Date.now() - start,
                         limitations: [
                             ...(Array.isArray(targetInspection.evidence?.limitations)
                                 ? targetInspection.evidence.limitations
                                 : []),
-                            "Page needs JS rendering; the local native tiers cannot render it. " +
-                                "Run `webcrawl login` and retry with `--mode cloud` (rendering runs in the hosted tier), " +
+                            "Page needs JS rendering; local Chrome was not available or produced no content. " +
+                                "Install Chrome (see `webcrawl doctor`), run `webcrawl login` and retry with `--mode cloud`, " +
                                 "or self-host a render provider via WEBCRAWL_RENDER_URL.",
                         ],
                     },
-                };
+                }, html, p.formats);
             }
-            return {
+            if (blocked || !markdown.trim()) {
+                return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start, blocked ? undefined : emptyPageError(fetched.status));
+            }
+            const contentHash = sha256(markdown || html);
+            if (markdown.trim().length >= MIN_CACHEABLE_MARKDOWN) {
+                setCachedPage(targetUrl, "none", {
+                    title,
+                    markdown,
+                    links,
+                    statusCode: fetched.status,
+                    contentHash,
+                });
+            }
+            const targetInspection = await inspectTarget({
+                url: targetUrl,
+                html,
+                markdown,
+            });
+            return withOptionalHtml({
                 success: true,
                 mode: "local",
                 url: targetUrl,
@@ -350,15 +441,11 @@ export function createLocalEngineClient() {
                 },
                 evidence: {
                     ...targetInspection.evidence,
-                    tier: lastFetchTier() === "impersonate"
-                        ? "impersonate"
-                        : lastFetchTier() === "proxy-impersonate"
-                            ? "proxy-impersonate"
-                            : "impit-tls",
+                    tier: nativeTier,
                     contentHash,
                     latencyMs: Date.now() - start,
                 },
-            };
+            }, html, p.formats);
         },
         async parse(p) {
             const url = typeof p.url === "string" ? p.url : undefined;
@@ -464,63 +551,51 @@ export function createLocalEngineClient() {
         },
         async extract(p) {
             const targetUrl = /^https?:\/\//i.test(p.url) ? p.url : `https://${p.url}`;
-            const fetched = await fetchWithTlsImpersonation(targetUrl);
-            const fastCheck = fastHtmlToMarkdown(fetched.html, targetUrl);
-            //    Tier R: shell page the native ladder cannot render — retry
-            //    through scrape() so the render path + cache apply, then extract
-            //    from the rendered Markdown. Without a provider, surface the
-            //    login/cloud limitation instead of silently returning {}.
-            if (needsBrowserEscalation(fetched.status, fetched.html, fastCheck.markdown)) {
-                if (isRenderProviderEnabled()) {
-                    const scraped = (await this.scrape({ url: targetUrl, noCache: true }));
-                    const renderedMarkdown = (scraped.data && scraped.data.markdown) || scraped.markdown || "";
-                    if (renderedMarkdown.trim().length > 40) {
-                        const targetInspection = await inspectTarget({
-                            url: targetUrl,
-                            markdown: renderedMarkdown,
-                        });
-                        const siteData = await extractWebsiteData(targetUrl);
-                        return {
-                            success: true,
-                            mode: "local",
-                            url: targetUrl,
-                            data: {
-                                ...siteData,
-                                ...(targetInspection.structured ? { target: targetInspection.structured } : {}),
-                            },
-                            evidence: targetInspection.evidence,
-                        };
-                    }
-                }
+            // Scrape first (local Chrome / render provider) so JS pages yield rendered HTML, not the shell.
+            const scraped = (await this.scrape({
+                url: targetUrl,
+                formats: ["markdown", "html"],
+                noCache: true,
+            }));
+            const markdown = (scraped.data && scraped.data.markdown) || scraped.markdown || "";
+            const html = (scraped.data && scraped.data.html) || scraped.html || "";
+            if (scraped.blocked || scraped.success === false) {
                 return {
-                    success: true,
+                    success: false,
+                    blocked: true,
                     mode: "local",
                     url: targetUrl,
                     data: {},
+                    error: scraped.error || emptyPageError(200),
                     evidence: {
                         tier: lastFetchTier(),
                         sourceUrl: targetUrl,
-                        limitations: [
-                            "Page needs JS rendering; the local native tiers cannot render it. " +
-                                "Run `webcrawl login` and retry with `--mode cloud` (rendering runs in the hosted tier), " +
-                                "or self-host a render provider via WEBCRAWL_RENDER_URL.",
-                        ],
+                        limitations: [String(scraped.error || emptyPageError(200))],
                     },
                 };
             }
             const targetInspection = await inspectTarget({
                 url: targetUrl,
-                html: fetched.html,
+                html,
+                markdown,
+            });
+            const siteData = await extractWebsiteData(targetUrl, {
+                prompt: p.prompt,
+                schema: p.schema,
+                markdown,
+                html,
+                bypassCache: true,
             });
-            const siteData = await extractWebsiteData(targetUrl);
+            const { cached: _c, cacheLayer: _l, ...extracted } = siteData;
             return {
                 success: true,
                 mode: "local",
                 url: targetUrl,
-                data: {
-                    ...siteData,
-                    ...(targetInspection.structured ? { target: targetInspection.structured } : {}),
-                },
+                data: extractOutputData(extracted, {
+                    prompt: p.prompt,
+                    schema: p.schema,
+                    target: targetInspection.structured,
+                }),
                 evidence: targetInspection.evidence,
             };
         },

package/dist/core/ranker/lexical.js

@@ -6,6 +6,7 @@ const ENGINE_WEIGHTS = {
     jina: 1.3,
     "brave-native": 1.25,
     "ddg-lite": 1.15,
+    "bing-rss": 1.12,
     "google-news-rss": 1.05,
     wikipedia: 1.1,
     "hn-algolia": 1.0,

package/dist/core/ranker/types.d.ts

@@ -1,4 +1,4 @@
-export type SearchEngineSource = "bing-shopping" | "google-shopping" | "brave-native" | "ddg-lite" | "google-news-rss" | "wikipedia" | "hn-algolia" | "arxiv" | "nvd" | "osv" | "serper" | "brave-api" | "jina";
+export type SearchEngineSource = "bing-shopping" | "google-shopping" | "brave-native" | "ddg-lite" | "bing-rss" | "google-news-rss" | "wikipedia" | "hn-algolia" | "arxiv" | "nvd" | "osv" | "serper" | "brave-api" | "jina";
 export type DecisionModelId = "baseten/mammoth-1" | "baseten/elephant-1" | "convaiinnovations/laya" | "auto" | "none";
 export interface RawCandidateDocument {
     title: string;

package/dist/core/search/catalog.d.ts

@@ -107,6 +107,8 @@ export interface SearchResponse {
     /** Structured CVE records from NVD and OSV: CVSS score and severity, CWE, affected products and versions, fixed versions, CISA KEV flag, dates and references (populated when category === "security") */
     cves?: CveRecord[];
     provider: "native-federated" | "brave" | "serper" | "jina" | "ddg-edge";
+    /** Set when a keyless search returns no hits (engines blocked or empty). */
+    error?: string;
     /** Set when the primary engines came back thin and a paid fallback engine filled in. */
     fallback?: "jina";
     cached?: boolean;

package/dist/core/search/pipeline.js

@@ -18,7 +18,7 @@ function isHotelOta(url) {
     return hostMatchesAny(url, HOTEL_OTA_HOSTS);
 }
 import { hostMatchesAny, normalizeHostList, resolveCategory, resolveUserLocation, SEARCH_CATEGORY_CATALOG, } from "./catalog.js";
-import { fetchHedgedNativeFederation, fetchJinaSearch, fetchNvdByIds, } from "./providers.js";
+import { EMPTY_SEARCH_ERROR, fetchHedgedNativeFederation, fetchJinaSearch, fetchNvdByIds, } from "./providers.js";
 import { runInBackground, upsertCves } from "../extract/cve-cache.js";
 const searchCache = new Map();
 const SEARCH_CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours warm edge cache
@@ -653,23 +653,26 @@ export async function searchWeb(req) {
     // An explicitly chosen provider is not a fallback.
     const usedJinaFallback = (!req.provider || req.provider === "auto") &&
         finalItems.some((item) => item.sources?.includes("jina"));
-    const provider = ranked.rankingMetrics.enginesContributing.length > 1
+    const engines = ranked.rankingMetrics.enginesContributing;
+    const provider = engines.length > 1
         ? "native-federated"
-        : ranked.rankingMetrics.enginesContributing.includes("serper")
+        : engines.includes("serper")
             ? "serper"
-            : ranked.rankingMetrics.enginesContributing.includes("jina")
+            : engines.includes("jina")
                 ? "jina"
-                : ranked.rankingMetrics.enginesContributing.includes("brave-api") ||
-                    ranked.rankingMetrics.enginesContributing.includes("brave-native")
+                : engines.includes("brave-api") || engines.includes("brave-native")
                     ? "brave"
-                    : "ddg-edge";
-    if (finalItems.length > 0 ||
+                    : engines.includes("ddg-lite")
+                        ? "ddg-edge"
+                        : "native-federated";
+    const hasHits = finalItems.length > 0 ||
         (products && products.length > 0) ||
         (dedupedListings && dedupedListings.length > 0) ||
         (dedupedFlights && dedupedFlights.length > 0) ||
         (dedupedHotels && dedupedHotels.length > 0) ||
         (dedupedBusinesses && dedupedBusinesses.length > 0) ||
-        (dedupedCves && dedupedCves.length > 0)) {
+        (dedupedCves && dedupedCves.length > 0);
+    if (hasHits) {
         searchCache.set(cacheKey, {
             data: finalItems,
             ...(products ? { products } : {}),
@@ -684,6 +687,18 @@ export async function searchWeb(req) {
             expiresAt: Date.now() + SEARCH_CACHE_TTL_MS,
         });
     }
+    if (!hasHits) {
+        return {
+            success: false,
+            data: [],
+            provider,
+            error: EMPTY_SEARCH_ERROR,
+            cached: false,
+            latencyMs: Math.round(performance.now() - start),
+            ...(hasGeo ? { userLocation } : {}),
+            rankingMetrics: ranked.rankingMetrics,
+        };
+    }
     return {
         success: true,
         data: finalItems,

package/dist/core/search/providers.d.ts

@@ -16,6 +16,16 @@ export declare function isEntityQuery(q: string): boolean;
 export declare function fetchHedgedNativeFederation(rawQuery: string, localizedQuery: string, perEngineLimit: number, req: SearchRequest, userLocation: UserLocationContext, quorumTimeoutMs: number, targetSites: string[], onOffer: (offer: ProductOffer) => void, onCve?: (cve: CveRecord) => void): Promise<RawCandidateDocument[]>;
 /** Below this many unique URLs from the free engines, the Jina fallback runs. */
 export declare const JINA_FALLBACK_MIN_URLS = 3;
+/** True when a scraped SERP is a bot-check / captcha rather than results. */
+export declare function isSerpChallengeHtml(html: string, status?: number): boolean;
+export declare const EMPTY_SEARCH_ERROR: string;
+/** Parse Bing's `format=rss` web-search feed into candidate docs. */
+export declare function parseBingWebRss(xml: string, limit: number): RawCandidateDocument[];
+/**
+ * Last-resort keyless engines used when DDG/Brave HTML SERPs come back empty.
+ * Wikipedia OpenSearch, HN Algolia, and Bing RSS do not require a JS challenge.
+ */
+export declare function fetchKeylessSearchFallbacks(query: string, limit: number): Promise<RawCandidateDocument[]>;
 export declare function fetchJinaSearch(query: string, limit: number, req?: {
     country?: string;
     lang?: string;

package/dist/core/search/providers.js

@@ -219,10 +219,105 @@ export async function fetchHedgedNativeFederation(rawQuery, localizedQuery, perE
         }
         collected.push(...jina);
     }
+    // HTML SERPs (DDG, Brave) are frequently challenge/429 pages. When they
+    // yield nothing, fall back to keyless APIs that still work without a key.
+    if (collected.length === 0 && (!req.provider || req.provider === "auto")) {
+        const extras = await fetchKeylessSearchFallbacks(localizedQuery, perEngineLimit);
+        collected.push(...extras);
+    }
     return collected;
 }
 /** Below this many unique URLs from the free engines, the Jina fallback runs. */
 export const JINA_FALLBACK_MIN_URLS = 3;
+/** True when a scraped SERP is a bot-check / captcha rather than results. */
+export function isSerpChallengeHtml(html, status = 200) {
+    if (status === 403 || status === 429 || status === 503)
+        return true;
+    const lower = String(html ?? "").slice(0, 8000).toLowerCase();
+    return (lower.includes("bots use duckduckgo") ||
+        lower.includes("anomaly-modal") ||
+        lower.includes("please complete the following challenge") ||
+        lower.includes("schedule a captcha") ||
+        lower.includes("flagged as being suspicious") ||
+        lower.includes("enable javascript to function"));
+}
+export const EMPTY_SEARCH_ERROR = "Keyless search engines returned no results (DuckDuckGo and Brave often serve a bot challenge). " +
+    "Tried Wikipedia, Hacker News, and Bing RSS as well. " +
+    "Set BRAVE_SEARCH_API_KEY, SERPER_API_KEY, or JINA_API_KEY, or retry later.";
+/** Parse Bing's `format=rss` web-search feed into candidate docs. */
+export function parseBingWebRss(xml, limit) {
+    const items = [];
+    const itemRe = /<item>([\s\S]*?)<\/item>/gi;
+    let match;
+    while ((match = itemRe.exec(xml)) !== null && items.length < limit) {
+        const block = match[1];
+        const title = decodeHtmlEntities((block.match(/<title>([\s\S]*?)<\/title>/i)?.[1] || "")
+            .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+            .trim());
+        const link = (block.match(/<link>([\s\S]*?)<\/link>/i)?.[1] || "")
+            .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+            .trim();
+        const description = decodeHtmlEntities((block.match(/<description>([\s\S]*?)<\/description>/i)?.[1] || "")
+            .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+            .replace(/<[^>]+>/g, " ")
+            .replace(/\s+/g, " ")
+            .trim());
+        if (!title || !link.startsWith("http"))
+            continue;
+        items.push({
+            title,
+            url: link,
+            description: description || title,
+            engine: "bing-rss",
+            engineRank: items.length + 1,
+        });
+    }
+    return items;
+}
+async function fetchBingWebRss(query, limit) {
+    try {
+        const rssUrl = `https://www.bing.com/search?format=rss&q=${encodeURIComponent(query)}`;
+        const res = await fetch(rssUrl, {
+            headers: {
+                Accept: "application/rss+xml, application/xml, text/xml;q=0.9, */*;q=0.8",
+                "User-Agent": "Mozilla/5.0 (compatible; WebCrawl/1.4; +https://webcrawl.app)",
+            },
+            signal: AbortSignal.timeout(4000),
+        });
+        if (!res.ok)
+            return [];
+        const xml = await res.text();
+        if (!xml.includes("<item>"))
+            return [];
+        return parseBingWebRss(xml, limit);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Last-resort keyless engines used when DDG/Brave HTML SERPs come back empty.
+ * Wikipedia OpenSearch, HN Algolia, and Bing RSS do not require a JS challenge.
+ */
+export async function fetchKeylessSearchFallbacks(query, limit) {
+    const per = Math.max(limit, 5);
+    const [hn, wiki, bing] = await Promise.all([
+        fetchHackerNewsAlgolia(query, per),
+        fetchWikipediaOpenSearch(query, per),
+        fetchBingWebRss(query, per),
+    ]);
+    const seen = new Set();
+    const out = [];
+    for (const item of [...hn, ...wiki, ...bing]) {
+        if (!item.url || seen.has(item.url))
+            continue;
+        seen.add(item.url);
+        out.push(item);
+        if (out.length >= Math.max(limit + 4, 8))
+            break;
+    }
+    return out;
+}
 export async function fetchJinaSearch(query, limit, req) {
     const key = process.env.JINA_API_KEY;
     if (!key) {
@@ -435,7 +530,7 @@ export async function fetchBraveNativeSerp(query, limit, userLocation) {
             : "";
         const url = `https://search.brave.com/search?q=${encodeURIComponent(query)}&source=web${countryParam}`;
         const res = await fetchWithTlsImpersonation(url, 2200);
-        if (res.status !== 200 || !res.html) {
+        if (res.status !== 200 || !res.html || isSerpChallengeHtml(res.html, res.status)) {
             return [];
         }
         const html = res.html;
@@ -545,7 +640,7 @@ async function fetchDuckDuckGoLiteOnly(query, limit, userLocation) {
         const kl = ddgRegion(countryCode, undefined);
         const klParam = kl ? `&kl=${encodeURIComponent(kl)}` : "";
         const liteRes = await fetchWithTlsImpersonation(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}${klParam}`, 2200);
-        if (liteRes && liteRes.html) {
+        if (liteRes && liteRes.html && !isSerpChallengeHtml(liteRes.html, liteRes.status)) {
             const html = liteRes.html;
             const linkMatches = Array.from(html.matchAll(/<a\s+[^>]*href=["']([^"']+)["'][^>]*class=["']result-link["'][^>]*>([\s\S]*?)<\/a>|<a\s+[^>]*class=["']result-link["'][^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi));
             const snippetMatches = Array.from(html.matchAll(/<td\s+[^>]*class=["']result-snippet["'][^>]*>([\s\S]*?)<\/td>/gi));