webcrawl 1.4.2 → 1.4.4

package/dist/core/local-engine.js

@@ -9,7 +9,7 @@
  * Every community improvement to extractors, parsers, or targets in `src/` immediately
  * powers both this local engine (`npx webcrawl`) and the hosted `webcrawl.app` API.
  */
-import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, isChallengeMarkdown, mapWebsiteUrls, setCachedPage, sha256, } from "./engine.js";
+import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, isChallengeMarkdown, isLocalChromeAvailable, mapWebsiteUrls, renderViaLocalChromeDetailed, setCachedPage, sha256, } from "./engine.js";
 import { isDomainRenderNeeded, markDomainRenderNeeded, } from "./engine/session.js";
 import { isRenderProviderEnabled, renderViaProvider } from "./engine/render.js";
 import { fetchResource, isAntiBotChallenge, lastFetchTier, needsBrowserEscalation, } from "./engine/fetcher.js";
@@ -20,6 +20,7 @@ import { parseYouTubeCached, youtubeVideoId } from "./parse/youtube.js";
 import { rankDocuments } from "./ranker.js";
 import { getSearchCategoriesCatalog, searchWeb } from "./search.js";
 import { inspectTarget, listTargets, resolveTarget } from "./targets.js";
+import { hasCustomExtractRequest, isEmptyExtractResult, } from "./extract/structured.js";
 /** Lowercased hostname of a URL (memo key for the Tier R render memo). */
 function safeDomainOf(url) {
     try {
@@ -34,9 +35,10 @@ function safeDomainOf(url) {
  * `success: false` with no markdown, so callers never mistake the block page
  * for content. Exported for unit tests.
  */
-export function blockedScrapeResult(url, statusCode, tier, latencyMs) {
-    const error = `Blocked by the site's bot check (HTTP ${statusCode}). No content was returned. ` +
-        "Retry with `--mode cloud` (hosted stealth browser), or self-host a render provider via WEBCRAWL_RENDER_URL.";
+export function blockedScrapeResult(url, statusCode, tier, latencyMs, reason) {
+    const error = reason ??
+        (`Blocked by the site's bot check (HTTP ${statusCode}). No content was returned. ` +
+            "Retry with `--mode cloud` (hosted stealth browser), or self-host a render provider via WEBCRAWL_RENDER_URL.");
     return {
         success: false,
         mode: "local",
@@ -51,6 +53,139 @@ export function blockedScrapeResult(url, statusCode, tier, latencyMs) {
         evidence: { tier, blocked: true, statusCode, latencyMs, limitations: [error] },
     };
 }
+/**
+ * JS page whose renderer failed. `success: false` so callers never treat the
+ * unrendered nav shell as content (v1.4.3 smoke test: 114 chars of nav with
+ * success:true after Chrome hung on --dump-dom).
+ */
+export function jsRenderFailedResult(url, statusCode, tier, latencyMs, reason) {
+    return {
+        success: false,
+        mode: "local",
+        url,
+        blocked: false,
+        needsJs: true,
+        statusCode,
+        error: reason,
+        title: url,
+        markdown: "",
+        links: [],
+        data: { url, title: url, markdown: "", links: [] },
+        evidence: {
+            tier,
+            blocked: false,
+            needsJs: true,
+            statusCode,
+            latencyMs,
+            limitations: [reason],
+        },
+    };
+}
+/** Shells shorter than this are unrendered JS / empty pages and must not be cached as content. */
+export const MIN_CACHEABLE_MARKDOWN = 200;
+/** Attach source HTML when the caller asked for the `html` format. */
+export function withOptionalHtml(result, html, formats) {
+    if (!formats?.includes("html") || !html)
+        return result;
+    const data = result.data && typeof result.data === "object" && !Array.isArray(result.data)
+        ? result.data
+        : {};
+    return { ...result, html, data: { ...data, html } };
+}
+/** Structured extract payload: custom prompt/schema → only those fields, no `target` sidecar. */
+export function extractOutputData(extracted, opts) {
+    if (hasCustomExtractRequest(opts.prompt, opts.schema))
+        return extracted;
+    return {
+        ...extracted,
+        ...(opts.target ? { target: opts.target } : {}),
+    };
+}
+function nativeScrapeTier() {
+    const t = lastFetchTier();
+    return t === "impersonate" || t === "proxy-impersonate" ? t : "impit-tls";
+}
+function emptyPageError(statusCode) {
+    return (`Page returned no readable content (HTTP ${statusCode}). ` +
+        "The site may be blocking automated clients or serving a client-rendered shell. " +
+        "Retry with `--mode cloud` (hosted stealth browser), or self-host a render provider via WEBCRAWL_RENDER_URL.");
+}
+function defaultJsRenderError() {
+    return ("Page needs JavaScript rendering; local Chrome was not available or produced no content. " +
+        "Install Chrome (see `webcrawl doctor`), point WEBCRAWL_CHROME_PATH at chrome-headless-shell " +
+        "(`npx playwright install chromium-headless-shell`), run `webcrawl login` and retry with `--mode cloud`, " +
+        "or self-host a render provider via WEBCRAWL_RENDER_URL.");
+}
+/**
+ * JS / challenge shell → render provider, then local Chrome (CDP, then dump-dom),
+ * then stealth. Returns null when no renderer produced real content.
+ */
+async function escalateJsRender(url) {
+    const errors = [];
+    if (isRenderProviderEnabled()) {
+        const rendered = await renderViaProvider(url);
+        if (rendered && rendered.markdown.trim().length >= 40) {
+            return {
+                page: {
+                    title: rendered.title,
+                    markdown: rendered.markdown,
+                    html: "",
+                    links: [],
+                    tier: "render",
+                },
+            };
+        }
+        errors.push("render provider produced no content");
+    }
+    if (isLocalChromeAvailable()) {
+        const chrome = await renderViaLocalChromeDetailed(url);
+        if (chrome.html && chrome.html.length > 100) {
+            const parsed = fastHtmlToMarkdown(chrome.html, url);
+            if (parsed.markdown.trim().length > 40 &&
+                !isAntiBotChallenge(chrome.html, 200) &&
+                !isChallengeMarkdown(parsed.title, parsed.markdown)) {
+                return { page: { ...parsed, html: chrome.html, tier: "browser" } };
+            }
+            errors.push(chrome.method
+                ? `${chrome.method} rendered HTML but markdown was still a shell`
+                : "local Chrome rendered HTML but markdown was still a shell");
+        }
+        else if (chrome.error) {
+            errors.push(chrome.error);
+        }
+        else {
+            errors.push("local Chrome produced no content");
+        }
+    }
+    else {
+        errors.push("no local Chrome/Chromium binary");
+    }
+    try {
+        const { stealthAvailable } = await import("./engine/stealth.js");
+        const { escalateToSandboxOrBrowser, isChallengeResult } = await import("./sandbox.js");
+        if (await stealthAvailable()) {
+            const esc = await escalateToSandboxOrBrowser(url, { browser: true, timeoutMs: 25_000 });
+            if (esc &&
+                esc.markdown.trim().length > 40 &&
+                !isChallengeResult(esc.html, esc.markdown, esc.statusCode)) {
+                return {
+                    page: {
+                        title: esc.title,
+                        markdown: esc.markdown,
+                        html: esc.html,
+                        links: esc.links,
+                        tier: esc.tier,
+                    },
+                };
+            }
+            errors.push("stealth browser produced no content");
+        }
+    }
+    catch {
+        /* stealth optional */
+    }
+    return { page: null, error: errors.filter(Boolean).join("; ") || defaultJsRenderError() };
+}
 /**
  * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
  * Matching is a case-insensitive substring of the URL. Exported for unit tests.
@@ -90,7 +225,7 @@ export function createLocalEngineClient() {
             const resolved = resolveTarget({ query, category });
             return {
                 ...res,
-                success: true,
+                success: res.success !== false,
                 mode: "local",
                 target: {
                     id: resolved.plugin.id,
@@ -161,8 +296,11 @@ export function createLocalEngineClient() {
             if (!p.noCache) {
                 const cached = await getCachedPageAsync(targetUrl, "none");
                 // Entries written before blocked pages stopped being cached may hold a
-                // bot-check page: treat those as a miss and fetch again.
-                if (cached && !isChallengeMarkdown(cached.title, cached.markdown)) {
+                // bot-check page: treat those as a miss and fetch again. Thin shells
+                // (header-only JS pages) must also miss so local Chrome can hydrate them.
+                if (cached &&
+                    !isChallengeMarkdown(cached.title, cached.markdown) &&
+                    cached.markdown.trim().length >= MIN_CACHEABLE_MARKDOWN) {
                     const targetInspection = await inspectTarget({
                         url: targetUrl,
                         markdown: cached.markdown,
@@ -235,106 +373,82 @@ export function createLocalEngineClient() {
                 // ladder — the memo may be stale or the provider flaky.
             }
             const fetched = await fetchWithTlsImpersonation(targetUrl);
-            const { title, markdown, links } = fastHtmlToMarkdown(fetched.html, targetUrl);
-            const contentHash = sha256(markdown || fetched.html);
-            // A bot-check / 403 / 429 / 503 response is not the page: never cache it,
-            // or every retry for the next 24h would get the block page back.
-            const blocked = isAntiBotChallenge(fetched.html, fetched.status);
-            if (!blocked) {
-                setCachedPage(targetUrl, "none", {
-                    title,
-                    markdown,
-                    links,
-                    statusCode: fetched.status,
-                    contentHash,
-                });
-            }
-            const targetInspection = await inspectTarget({
-                url: targetUrl,
-                html: fetched.html,
-                markdown,
-            });
-            //    Tier R: the native ladder passed the bot check but the page is a
-            //    JS shell / challenge page the ladder cannot render. Memoize the
-            //    domain and either render via the provider (when configured) or
-            //    return the shell with an honest limitation note.
-            if (needsBrowserEscalation(fetched.status, fetched.html, markdown)) {
+            const parsed = fastHtmlToMarkdown(fetched.html, targetUrl);
+            let title = parsed.title;
+            let markdown = parsed.markdown;
+            let links = parsed.links;
+            let html = fetched.html;
+            const blocked = isAntiBotChallenge(html, fetched.status);
+            const needsRender = needsBrowserEscalation(fetched.status, html, markdown);
+            const nativeTier = nativeScrapeTier();
+            //    JS shell / challenge: memoize the domain and escalate to a renderer
+            //    (hosted provider, local Chrome CDP, then stealth). Never cache
+            //    the unrendered shell, and never report the shell as success:true.
+            if (needsRender) {
                 await markDomainRenderNeeded(renderDomain);
-                if (isRenderProviderEnabled()) {
-                    const rendered = await renderViaProvider(targetUrl);
-                    if (rendered) {
-                        const renderHash = sha256(rendered.markdown);
+                // 403/429/503 is already a refusal — don't spend a Chrome launch on the block page.
+                const escalated = blocked && fetched.status >= 400
+                    ? { page: null, error: undefined }
+                    : await escalateJsRender(targetUrl);
+                const rendered = escalated.page;
+                if (rendered) {
+                    const renderHash = sha256(rendered.markdown);
+                    if (rendered.markdown.trim().length >= MIN_CACHEABLE_MARKDOWN) {
                         setCachedPage(targetUrl, "none", {
                             title: rendered.title,
                             markdown: rendered.markdown,
-                            links: [],
+                            links: rendered.links,
                             statusCode: 200,
                             contentHash: renderHash,
                         });
-                        return {
-                            success: true,
-                            mode: "local",
+                    }
+                    return withOptionalHtml({
+                        success: true,
+                        mode: "local",
+                        url: targetUrl,
+                        title: rendered.title || targetUrl,
+                        markdown: rendered.markdown,
+                        links: rendered.links,
+                        data: {
                             url: targetUrl,
                             title: rendered.title || targetUrl,
                             markdown: rendered.markdown,
-                            links: [],
-                            data: {
-                                url: targetUrl,
-                                title: rendered.title || targetUrl,
-                                markdown: rendered.markdown,
-                                links: [],
-                            },
-                            evidence: {
-                                targetId: "render-provider",
-                                tier: "render",
-                                contentHash: renderHash,
-                                latencyMs: Date.now() - start,
-                                sourceUrl: rendered.sourceUrl,
-                                matchedSignals: ["render:provider"],
-                                limitations: rendered.warnings,
-                            },
-                        };
-                    }
+                            links: rendered.links,
+                        },
+                        evidence: {
+                            targetId: rendered.tier === "render" ? "render-provider" : "web-spa",
+                            tier: rendered.tier,
+                            contentHash: renderHash,
+                            latencyMs: Date.now() - start,
+                            sourceUrl: targetUrl,
+                            matchedSignals: rendered.tier === "browser" ? ["render:local-chrome"] : [`render:${rendered.tier}`],
+                        },
+                    }, rendered.html, p.formats);
                 }
-                const nativeTier = lastFetchTier() === "impersonate"
-                    ? "impersonate"
-                    : lastFetchTier() === "proxy-impersonate"
-                        ? "proxy-impersonate"
-                        : "impit-tls";
                 if (blocked) {
                     return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start);
                 }
-                return {
-                    success: true,
-                    mode: "local",
-                    url: targetUrl,
-                    title: title || targetInspection.title || targetUrl,
-                    markdown: targetInspection.markdown || markdown,
+                return jsRenderFailedResult(targetUrl, fetched.status, nativeTier, Date.now() - start, escalated.error || (!markdown.trim() ? emptyPageError(fetched.status) : defaultJsRenderError()));
+            }
+            if (blocked || !markdown.trim()) {
+                return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start, blocked ? undefined : emptyPageError(fetched.status));
+            }
+            const contentHash = sha256(markdown || html);
+            if (markdown.trim().length >= MIN_CACHEABLE_MARKDOWN) {
+                setCachedPage(targetUrl, "none", {
+                    title,
+                    markdown,
                     links,
-                    data: {
-                        url: targetUrl,
-                        title: title || targetInspection.title || targetUrl,
-                        markdown: targetInspection.markdown || markdown,
-                        links,
-                        structured: targetInspection.structured,
-                    },
-                    evidence: {
-                        ...targetInspection.evidence,
-                        tier: nativeTier,
-                        contentHash,
-                        latencyMs: Date.now() - start,
-                        limitations: [
-                            ...(Array.isArray(targetInspection.evidence?.limitations)
-                                ? targetInspection.evidence.limitations
-                                : []),
-                            "Page needs JS rendering; the local native tiers cannot render it. " +
-                                "Run `webcrawl login` and retry with `--mode cloud` (rendering runs in the hosted tier), " +
-                                "or self-host a render provider via WEBCRAWL_RENDER_URL.",
-                        ],
-                    },
-                };
+                    statusCode: fetched.status,
+                    contentHash,
+                });
             }
-            return {
+            const targetInspection = await inspectTarget({
+                url: targetUrl,
+                html,
+                markdown,
+            });
+            return withOptionalHtml({
                 success: true,
                 mode: "local",
                 url: targetUrl,
@@ -350,15 +464,11 @@ export function createLocalEngineClient() {
                 },
                 evidence: {
                     ...targetInspection.evidence,
-                    tier: lastFetchTier() === "impersonate"
-                        ? "impersonate"
-                        : lastFetchTier() === "proxy-impersonate"
-                            ? "proxy-impersonate"
-                            : "impit-tls",
+                    tier: nativeTier,
                     contentHash,
                     latencyMs: Date.now() - start,
                 },
-            };
+            }, html, p.formats);
         },
         async parse(p) {
             const url = typeof p.url === "string" ? p.url : undefined;
@@ -464,63 +574,87 @@ export function createLocalEngineClient() {
         },
         async extract(p) {
             const targetUrl = /^https?:\/\//i.test(p.url) ? p.url : `https://${p.url}`;
-            const fetched = await fetchWithTlsImpersonation(targetUrl);
-            const fastCheck = fastHtmlToMarkdown(fetched.html, targetUrl);
-            //    Tier R: shell page the native ladder cannot render — retry
-            //    through scrape() so the render path + cache apply, then extract
-            //    from the rendered Markdown. Without a provider, surface the
-            //    login/cloud limitation instead of silently returning {}.
-            if (needsBrowserEscalation(fetched.status, fetched.html, fastCheck.markdown)) {
-                if (isRenderProviderEnabled()) {
-                    const scraped = (await this.scrape({ url: targetUrl, noCache: true }));
-                    const renderedMarkdown = (scraped.data && scraped.data.markdown) || scraped.markdown || "";
-                    if (renderedMarkdown.trim().length > 40) {
-                        const targetInspection = await inspectTarget({
-                            url: targetUrl,
-                            markdown: renderedMarkdown,
-                        });
-                        const siteData = await extractWebsiteData(targetUrl);
-                        return {
-                            success: true,
-                            mode: "local",
-                            url: targetUrl,
-                            data: {
-                                ...siteData,
-                                ...(targetInspection.structured ? { target: targetInspection.structured } : {}),
-                            },
-                            evidence: targetInspection.evidence,
-                        };
+            // Scrape first (local Chrome / render provider) so JS pages yield rendered HTML, not the shell.
+            const scraped = (await this.scrape({
+                url: targetUrl,
+                formats: ["markdown", "html"],
+                noCache: true,
+            }));
+            let markdown = (scraped.data && scraped.data.markdown) || scraped.markdown || "";
+            let html = (scraped.data && scraped.data.html) || scraped.html || "";
+            if (scraped.blocked || scraped.success === false) {
+                // Bot-blocked pages stay failures. A JS shell whose Chrome render failed
+                // can still yield structured data from JSON islands once script tags are
+                // stripped for card matching — never from `document.write` templates.
+                if (scraped.needsJs && scraped.blocked !== true) {
+                    try {
+                        const fetched = await fetchWithTlsImpersonation(targetUrl);
+                        html = fetched.html || html;
+                        const parsedShell = fastHtmlToMarkdown(html, targetUrl);
+                        markdown = parsedShell.markdown || markdown;
                     }
+                    catch {
+                        /* keep whatever scrape returned */
+                    }
+                }
+                else {
+                    return {
+                        success: false,
+                        blocked: true,
+                        mode: "local",
+                        url: targetUrl,
+                        data: {},
+                        error: scraped.error || emptyPageError(200),
+                        evidence: {
+                            tier: lastFetchTier(),
+                            sourceUrl: targetUrl,
+                            limitations: [String(scraped.error || emptyPageError(200))],
+                        },
+                    };
                 }
+            }
+            const targetInspection = await inspectTarget({
+                url: targetUrl,
+                html,
+                markdown,
+            });
+            const siteData = await extractWebsiteData(targetUrl, {
+                prompt: p.prompt,
+                schema: p.schema,
+                markdown,
+                html,
+                bypassCache: true,
+            });
+            const { cached: _c, cacheLayer: _l, ...extracted } = siteData;
+            const data = extractOutputData(extracted, {
+                prompt: p.prompt,
+                schema: p.schema,
+                target: targetInspection.structured,
+            });
+            if (hasCustomExtractRequest(p.prompt, p.schema) && isEmptyExtractResult(data)) {
+                const error = "Extraction produced no data matching the prompt/schema. " +
+                    "If this is a JS-rendered page, install Chrome (see `webcrawl doctor`) and retry; " +
+                    "empty objects are not reported as success.";
                 return {
-                    success: true,
+                    success: false,
                     mode: "local",
                     url: targetUrl,
                     data: {},
+                    error,
                     evidence: {
-                        tier: lastFetchTier(),
-                        sourceUrl: targetUrl,
+                        ...targetInspection.evidence,
                         limitations: [
-                            "Page needs JS rendering; the local native tiers cannot render it. " +
-                                "Run `webcrawl login` and retry with `--mode cloud` (rendering runs in the hosted tier), " +
-                                "or self-host a render provider via WEBCRAWL_RENDER_URL.",
+                            ...(targetInspection.evidence?.limitations ?? []),
+                            error,
                         ],
                     },
                 };
             }
-            const targetInspection = await inspectTarget({
-                url: targetUrl,
-                html: fetched.html,
-            });
-            const siteData = await extractWebsiteData(targetUrl);
             return {
                 success: true,
                 mode: "local",
                 url: targetUrl,
-                data: {
-                    ...siteData,
-                    ...(targetInspection.structured ? { target: targetInspection.structured } : {}),
-                },
+                data,
                 evidence: targetInspection.evidence,
             };
         },

package/dist/core/ranker/lexical.js

@@ -6,6 +6,7 @@ const ENGINE_WEIGHTS = {
     jina: 1.3,
     "brave-native": 1.25,
     "ddg-lite": 1.15,
+    "bing-rss": 1.12,
     "google-news-rss": 1.05,
     wikipedia: 1.1,
     "hn-algolia": 1.0,

package/dist/core/ranker/types.d.ts

@@ -1,4 +1,4 @@
-export type SearchEngineSource = "bing-shopping" | "google-shopping" | "brave-native" | "ddg-lite" | "google-news-rss" | "wikipedia" | "hn-algolia" | "arxiv" | "nvd" | "osv" | "serper" | "brave-api" | "jina";
+export type SearchEngineSource = "bing-shopping" | "google-shopping" | "brave-native" | "ddg-lite" | "bing-rss" | "google-news-rss" | "wikipedia" | "hn-algolia" | "arxiv" | "nvd" | "osv" | "serper" | "brave-api" | "jina";
 export type DecisionModelId = "baseten/mammoth-1" | "baseten/elephant-1" | "convaiinnovations/laya" | "auto" | "none";
 export interface RawCandidateDocument {
     title: string;

package/dist/core/search/catalog.d.ts

@@ -52,7 +52,7 @@ export interface SearchRequest {
     bypassCache?: boolean;
     /**
      * Pick one search provider and skip the default multi-engine pipeline.
-     * `auto` (default) uses the free engines and falls back to Jina when results are thin.
+     * `auto` (default) uses the free engines and falls back to a keyed provider when results are thin.
      */
     provider?: SearchProviderChoice;
     rankOptions?: {
@@ -107,6 +107,8 @@ export interface SearchResponse {
     /** Structured CVE records from NVD and OSV: CVSS score and severity, CWE, affected products and versions, fixed versions, CISA KEV flag, dates and references (populated when category === "security") */
     cves?: CveRecord[];
     provider: "native-federated" | "brave" | "serper" | "jina" | "ddg-edge";
+    /** Set when a keyless search returns no hits (engines blocked or empty). */
+    error?: string;
     /** Set when the primary engines came back thin and a paid fallback engine filled in. */
     fallback?: "jina";
     cached?: boolean;

package/dist/core/search/catalog.js

@@ -11,7 +11,7 @@ export const SEARCH_PROVIDER_ENV_KEYS = {
 export const SEARCH_CATEGORY_CATALOG = {
     general: {
         description: "Broad web search across general-purpose engines.",
-        engines: ["brave-native", "ddg-lite", "wikipedia"],
+        engines: ["brave-native", "ddg-lite", "wikipedia", "hn-algolia", "bing-rss"],
         localizeBy: "none",
         preferLocalDomains: false,
         subcategories: {

package/dist/core/search/pipeline.js

@@ -18,7 +18,7 @@ function isHotelOta(url) {
     return hostMatchesAny(url, HOTEL_OTA_HOSTS);
 }
 import { hostMatchesAny, normalizeHostList, resolveCategory, resolveUserLocation, SEARCH_CATEGORY_CATALOG, } from "./catalog.js";
-import { fetchHedgedNativeFederation, fetchJinaSearch, fetchNvdByIds, } from "./providers.js";
+import { EMPTY_SEARCH_ERROR, fetchHedgedNativeFederation, fetchJinaSearch, fetchNvdByIds, } from "./providers.js";
 import { runInBackground, upsertCves } from "../extract/cve-cache.js";
 const searchCache = new Map();
 const SEARCH_CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours warm edge cache
@@ -115,7 +115,9 @@ export async function searchWeb(req) {
         }
     }
     const perEngineLimit = Math.min(Math.max(limit + 4, 8), 15);
-    const quorumTimeoutMs = req.rankOptions?.quorumTimeoutMs ?? 1500;
+    // Must outlast DDG/Brave impersonation (2200ms) so a slow SERP is not cut off
+    // after Wikipedia's 2 hits — the v1.4.3 keyless-search failure mode.
+    const quorumTimeoutMs = req.rankOptions?.quorumTimeoutMs ?? 2800;
     // Stage 1: Hedged Multi-Engine Native Federation
     const collectedOffers = [];
     const offerByUrl = new Map();
@@ -257,7 +259,7 @@ export async function searchWeb(req) {
         filterContext,
     };
     let ranked = await rankDocuments(localizedSearchQuery, rankCandidates, rankOpts);
-    // Thin result set after ranking and filtering: ask Jina once, then re-rank the combined pool.
+    // Thin result set after ranking and filtering: ask the keyed search provider once, then re-rank.
     const minResults = Math.min(3, limit);
     if ((!req.provider || req.provider === "auto") &&
         ranked.results.length < minResults &&
@@ -649,27 +651,30 @@ export async function searchWeb(req) {
             ...(matchedCves?.length ? { cves: matchedCves } : {}),
         };
     });
-    // Charge the fallback surcharge only when a Jina result actually reached the caller.
+    // Charge the fallback surcharge only when a keyed-provider result actually reached the caller.
     // An explicitly chosen provider is not a fallback.
     const usedJinaFallback = (!req.provider || req.provider === "auto") &&
         finalItems.some((item) => item.sources?.includes("jina"));
-    const provider = ranked.rankingMetrics.enginesContributing.length > 1
+    const engines = ranked.rankingMetrics.enginesContributing;
+    const provider = engines.length > 1
         ? "native-federated"
-        : ranked.rankingMetrics.enginesContributing.includes("serper")
+        : engines.includes("serper")
             ? "serper"
-            : ranked.rankingMetrics.enginesContributing.includes("jina")
+            : engines.includes("jina")
                 ? "jina"
-                : ranked.rankingMetrics.enginesContributing.includes("brave-api") ||
-                    ranked.rankingMetrics.enginesContributing.includes("brave-native")
+                : engines.includes("brave-api") || engines.includes("brave-native")
                     ? "brave"
-                    : "ddg-edge";
-    if (finalItems.length > 0 ||
+                    : engines.includes("ddg-lite")
+                        ? "ddg-edge"
+                        : "native-federated";
+    const hasHits = finalItems.length > 0 ||
         (products && products.length > 0) ||
         (dedupedListings && dedupedListings.length > 0) ||
         (dedupedFlights && dedupedFlights.length > 0) ||
         (dedupedHotels && dedupedHotels.length > 0) ||
         (dedupedBusinesses && dedupedBusinesses.length > 0) ||
-        (dedupedCves && dedupedCves.length > 0)) {
+        (dedupedCves && dedupedCves.length > 0);
+    if (hasHits) {
         searchCache.set(cacheKey, {
             data: finalItems,
             ...(products ? { products } : {}),
@@ -684,6 +689,18 @@ export async function searchWeb(req) {
             expiresAt: Date.now() + SEARCH_CACHE_TTL_MS,
         });
     }
+    if (!hasHits) {
+        return {
+            success: false,
+            data: [],
+            provider,
+            error: EMPTY_SEARCH_ERROR,
+            cached: false,
+            latencyMs: Math.round(performance.now() - start),
+            ...(hasGeo ? { userLocation } : {}),
+            rankingMetrics: ranked.rankingMetrics,
+        };
+    }
     return {
         success: true,
         data: finalItems,