webcrawl 1.4.1 → 1.4.3

package/dist/core/local-engine.d.ts

@@ -32,6 +32,22 @@ export interface LocalEngineBackend {
     targets(): Promise<Record<string, unknown>>;
     usage(): Promise<Record<string, unknown>>;
 }
+/**
+ * Scrape result for a page the site refused (bot check, 403/429/503).
+ * `success: false` with no markdown, so callers never mistake the block page
+ * for content. Exported for unit tests.
+ */
+export declare function blockedScrapeResult(url: string, statusCode: number, tier: string, latencyMs: number, reason?: string): Record<string, unknown>;
+/** Shells shorter than this are unrendered JS / empty pages and must not be cached as content. */
+export declare const MIN_CACHEABLE_MARKDOWN = 200;
+/** Attach source HTML when the caller asked for the `html` format. */
+export declare function withOptionalHtml(result: Record<string, unknown>, html: string | undefined, formats?: Array<"markdown" | "links" | "html">): Record<string, unknown>;
+/** Structured extract payload: custom prompt/schema → only those fields, no `target` sidecar. */
+export declare function extractOutputData(extracted: Record<string, unknown>, opts: {
+    prompt?: string;
+    schema?: Record<string, unknown>;
+    target?: unknown;
+}): Record<string, unknown>;
 /**
  * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
  * Matching is a case-insensitive substring of the URL. Exported for unit tests.

package/dist/core/local-engine.js

@@ -9,10 +9,10 @@
  * Every community improvement to extractors, parsers, or targets in `src/` immediately
  * powers both this local engine (`npx webcrawl`) and the hosted `webcrawl.app` API.
  */
-import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, mapWebsiteUrls, setCachedPage, sha256, } from "./engine.js";
+import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, isChallengeMarkdown, isLocalChromeAvailable, mapWebsiteUrls, renderViaLocalChrome, setCachedPage, sha256, } from "./engine.js";
 import { isDomainRenderNeeded, markDomainRenderNeeded, } from "./engine/session.js";
 import { isRenderProviderEnabled, renderViaProvider } from "./engine/render.js";
-import { fetchResource, lastFetchTier, needsBrowserEscalation } from "./engine/fetcher.js";
+import { fetchResource, isAntiBotChallenge, lastFetchTier, needsBrowserEscalation, } from "./engine/fetcher.js";
 import { parseFile, publicParseMeta } from "./parse/index.js";
 import { isBinaryContentType, looksLikeBinaryUrl } from "./parse/fetch.js";
 import { maxDownloadBytes } from "./parse/limits.js";
@@ -20,6 +20,7 @@ import { parseYouTubeCached, youtubeVideoId } from "./parse/youtube.js";
 import { rankDocuments } from "./ranker.js";
 import { getSearchCategoriesCatalog, searchWeb } from "./search.js";
 import { inspectTarget, listTargets, resolveTarget } from "./targets.js";
+import { hasCustomExtractRequest } from "./extract/structured.js";
 /** Lowercased hostname of a URL (memo key for the Tier R render memo). */
 function safeDomainOf(url) {
     try {
@@ -29,6 +30,109 @@ function safeDomainOf(url) {
         return "";
     }
 }
+/**
+ * Scrape result for a page the site refused (bot check, 403/429/503).
+ * `success: false` with no markdown, so callers never mistake the block page
+ * for content. Exported for unit tests.
+ */
+export function blockedScrapeResult(url, statusCode, tier, latencyMs, reason) {
+    const error = reason ??
+        (`Blocked by the site's bot check (HTTP ${statusCode}). No content was returned. ` +
+            "Retry with `--mode cloud` (hosted stealth browser), or self-host a render provider via WEBCRAWL_RENDER_URL.");
+    return {
+        success: false,
+        mode: "local",
+        url,
+        blocked: true,
+        statusCode,
+        error,
+        title: url,
+        markdown: "",
+        links: [],
+        data: { url, title: url, markdown: "", links: [] },
+        evidence: { tier, blocked: true, statusCode, latencyMs, limitations: [error] },
+    };
+}
+/** Shells shorter than this are unrendered JS / empty pages and must not be cached as content. */
+export const MIN_CACHEABLE_MARKDOWN = 200;
+/** Attach source HTML when the caller asked for the `html` format. */
+export function withOptionalHtml(result, html, formats) {
+    if (!formats?.includes("html") || !html)
+        return result;
+    const data = result.data && typeof result.data === "object" && !Array.isArray(result.data)
+        ? result.data
+        : {};
+    return { ...result, html, data: { ...data, html } };
+}
+/** Structured extract payload: custom prompt/schema → only those fields, no `target` sidecar. */
+export function extractOutputData(extracted, opts) {
+    if (hasCustomExtractRequest(opts.prompt, opts.schema))
+        return extracted;
+    return {
+        ...extracted,
+        ...(opts.target ? { target: opts.target } : {}),
+    };
+}
+function nativeScrapeTier() {
+    const t = lastFetchTier();
+    return t === "impersonate" || t === "proxy-impersonate" ? t : "impit-tls";
+}
+function emptyPageError(statusCode) {
+    return (`Page returned no readable content (HTTP ${statusCode}). ` +
+        "The site may be blocking automated clients or serving a client-rendered shell. " +
+        "Retry with `--mode cloud` (hosted stealth browser), or self-host a render provider via WEBCRAWL_RENDER_URL.");
+}
+/**
+ * JS / challenge shell → render provider, then local Chrome dump-dom, then stealth.
+ * Returns null when no renderer produced real content.
+ */
+async function escalateJsRender(url) {
+    if (isRenderProviderEnabled()) {
+        const rendered = await renderViaProvider(url);
+        if (rendered && rendered.markdown.trim().length >= 40) {
+            return {
+                title: rendered.title,
+                markdown: rendered.markdown,
+                html: "",
+                links: [],
+                tier: "render",
+            };
+        }
+    }
+    if (isLocalChromeAvailable()) {
+        const html = await renderViaLocalChrome(url);
+        if (html && html.length > 100) {
+            const parsed = fastHtmlToMarkdown(html, url);
+            if (parsed.markdown.trim().length > 40 &&
+                !isAntiBotChallenge(html, 200) &&
+                !isChallengeMarkdown(parsed.title, parsed.markdown)) {
+                return { ...parsed, html, tier: "browser" };
+            }
+        }
+    }
+    try {
+        const { stealthAvailable } = await import("./engine/stealth.js");
+        const { escalateToSandboxOrBrowser, isChallengeResult } = await import("./sandbox.js");
+        if (await stealthAvailable()) {
+            const esc = await escalateToSandboxOrBrowser(url, { browser: true, timeoutMs: 25_000 });
+            if (esc &&
+                esc.markdown.trim().length > 40 &&
+                !isChallengeResult(esc.html, esc.markdown, esc.statusCode)) {
+                return {
+                    title: esc.title,
+                    markdown: esc.markdown,
+                    html: esc.html,
+                    links: esc.links,
+                    tier: esc.tier,
+                };
+            }
+        }
+    }
+    catch {
+        /* stealth optional */
+    }
+    return null;
+}
 /**
  * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
  * Matching is a case-insensitive substring of the URL. Exported for unit tests.
@@ -68,7 +172,7 @@ export function createLocalEngineClient() {
             const resolved = resolveTarget({ query, category });
             return {
                 ...res,
-                success: true,
+                success: res.success !== false,
                 mode: "local",
                 target: {
                     id: resolved.plugin.id,
@@ -138,7 +242,12 @@ export function createLocalEngineClient() {
             // 3. Check local cache first
             if (!p.noCache) {
                 const cached = await getCachedPageAsync(targetUrl, "none");
-                if (cached) {
+                // Entries written before blocked pages stopped being cached may hold a
+                // bot-check page: treat those as a miss and fetch again. Thin shells
+                // (header-only JS pages) must also miss so local Chrome can hydrate them.
+                if (cached &&
+                    !isChallengeMarkdown(cached.title, cached.markdown) &&
+                    cached.markdown.trim().length >= MIN_CACHEABLE_MARKDOWN) {
                     const targetInspection = await inspectTarget({
                         url: targetUrl,
                         markdown: cached.markdown,
@@ -211,68 +320,64 @@ export function createLocalEngineClient() {
                 // ladder — the memo may be stale or the provider flaky.
             }
             const fetched = await fetchWithTlsImpersonation(targetUrl);
-            const { title, markdown, links } = fastHtmlToMarkdown(fetched.html, targetUrl);
-            const contentHash = sha256(markdown || fetched.html);
-            setCachedPage(targetUrl, "none", {
-                title,
-                markdown,
-                links,
-                statusCode: fetched.status,
-                contentHash,
-            });
-            const targetInspection = await inspectTarget({
-                url: targetUrl,
-                html: fetched.html,
-                markdown,
-            });
-            //    Tier R: the native ladder passed the bot check but the page is a
-            //    JS shell / challenge page the ladder cannot render. Memoize the
-            //    domain and either render via the provider (when configured) or
-            //    return the shell with an honest limitation note.
-            if (needsBrowserEscalation(fetched.status, fetched.html, markdown)) {
+            const parsed = fastHtmlToMarkdown(fetched.html, targetUrl);
+            let title = parsed.title;
+            let markdown = parsed.markdown;
+            let links = parsed.links;
+            let html = fetched.html;
+            const blocked = isAntiBotChallenge(html, fetched.status);
+            const needsRender = needsBrowserEscalation(fetched.status, html, markdown);
+            const nativeTier = nativeScrapeTier();
+            //    JS shell / challenge: memoize the domain and escalate to a renderer
+            //    (hosted provider, local Chrome dump-dom, then stealth). Never cache
+            //    the unrendered shell.
+            if (needsRender) {
                 await markDomainRenderNeeded(renderDomain);
-                if (isRenderProviderEnabled()) {
-                    const rendered = await renderViaProvider(targetUrl);
-                    if (rendered) {
-                        const renderHash = sha256(rendered.markdown);
+                // 403/429/503 is already a refusal — don't spend a Chrome launch on the block page.
+                const rendered = blocked && fetched.status >= 400 ? null : await escalateJsRender(targetUrl);
+                if (rendered) {
+                    const renderHash = sha256(rendered.markdown);
+                    if (rendered.markdown.trim().length >= MIN_CACHEABLE_MARKDOWN) {
                         setCachedPage(targetUrl, "none", {
                             title: rendered.title,
                             markdown: rendered.markdown,
-                            links: [],
+                            links: rendered.links,
                             statusCode: 200,
                             contentHash: renderHash,
                         });
-                        return {
-                            success: true,
-                            mode: "local",
+                    }
+                    return withOptionalHtml({
+                        success: true,
+                        mode: "local",
+                        url: targetUrl,
+                        title: rendered.title || targetUrl,
+                        markdown: rendered.markdown,
+                        links: rendered.links,
+                        data: {
                             url: targetUrl,
                             title: rendered.title || targetUrl,
                             markdown: rendered.markdown,
-                            links: [],
-                            data: {
-                                url: targetUrl,
-                                title: rendered.title || targetUrl,
-                                markdown: rendered.markdown,
-                                links: [],
-                            },
-                            evidence: {
-                                targetId: "render-provider",
-                                tier: "render",
-                                contentHash: renderHash,
-                                latencyMs: Date.now() - start,
-                                sourceUrl: rendered.sourceUrl,
-                                matchedSignals: ["render:provider"],
-                                limitations: rendered.warnings,
-                            },
-                        };
-                    }
+                            links: rendered.links,
+                        },
+                        evidence: {
+                            targetId: rendered.tier === "render" ? "render-provider" : "web-spa",
+                            tier: rendered.tier,
+                            contentHash: renderHash,
+                            latencyMs: Date.now() - start,
+                            sourceUrl: targetUrl,
+                            matchedSignals: rendered.tier === "browser" ? ["render:local-chrome"] : [`render:${rendered.tier}`],
+                        },
+                    }, rendered.html, p.formats);
                 }
-                const nativeTier = lastFetchTier() === "impersonate"
-                    ? "impersonate"
-                    : lastFetchTier() === "proxy-impersonate"
-                        ? "proxy-impersonate"
-                        : "impit-tls";
-                return {
+                if (blocked || !markdown.trim()) {
+                    return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start, blocked ? undefined : emptyPageError(fetched.status));
+                }
+                const targetInspection = await inspectTarget({
+                    url: targetUrl,
+                    html,
+                    markdown,
+                });
+                return withOptionalHtml({
                     success: true,
                     mode: "local",
                     url: targetUrl,
@@ -289,20 +394,38 @@ export function createLocalEngineClient() {
                     evidence: {
                         ...targetInspection.evidence,
                         tier: nativeTier,
-                        contentHash,
+                        contentHash: sha256(markdown || html),
                         latencyMs: Date.now() - start,
                         limitations: [
                             ...(Array.isArray(targetInspection.evidence?.limitations)
                                 ? targetInspection.evidence.limitations
                                 : []),
-                            "Page needs JS rendering; the local native tiers cannot render it. " +
-                                "Run `webcrawl login` and retry with `--mode cloud` (rendering runs in the hosted tier), " +
+                            "Page needs JS rendering; local Chrome was not available or produced no content. " +
+                                "Install Chrome (see `webcrawl doctor`), run `webcrawl login` and retry with `--mode cloud`, " +
                                 "or self-host a render provider via WEBCRAWL_RENDER_URL.",
                         ],
                     },
-                };
+                }, html, p.formats);
             }
-            return {
+            if (blocked || !markdown.trim()) {
+                return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start, blocked ? undefined : emptyPageError(fetched.status));
+            }
+            const contentHash = sha256(markdown || html);
+            if (markdown.trim().length >= MIN_CACHEABLE_MARKDOWN) {
+                setCachedPage(targetUrl, "none", {
+                    title,
+                    markdown,
+                    links,
+                    statusCode: fetched.status,
+                    contentHash,
+                });
+            }
+            const targetInspection = await inspectTarget({
+                url: targetUrl,
+                html,
+                markdown,
+            });
+            return withOptionalHtml({
                 success: true,
                 mode: "local",
                 url: targetUrl,
@@ -318,15 +441,11 @@ export function createLocalEngineClient() {
                 },
                 evidence: {
                     ...targetInspection.evidence,
-                    tier: lastFetchTier() === "impersonate"
-                        ? "impersonate"
-                        : lastFetchTier() === "proxy-impersonate"
-                            ? "proxy-impersonate"
-                            : "impit-tls",
+                    tier: nativeTier,
                     contentHash,
                     latencyMs: Date.now() - start,
                 },
-            };
+            }, html, p.formats);
         },
         async parse(p) {
             const url = typeof p.url === "string" ? p.url : undefined;
@@ -432,63 +551,51 @@ export function createLocalEngineClient() {
         },
         async extract(p) {
             const targetUrl = /^https?:\/\//i.test(p.url) ? p.url : `https://${p.url}`;
-            const fetched = await fetchWithTlsImpersonation(targetUrl);
-            const fastCheck = fastHtmlToMarkdown(fetched.html, targetUrl);
-            //    Tier R: shell page the native ladder cannot render — retry
-            //    through scrape() so the render path + cache apply, then extract
-            //    from the rendered Markdown. Without a provider, surface the
-            //    login/cloud limitation instead of silently returning {}.
-            if (needsBrowserEscalation(fetched.status, fetched.html, fastCheck.markdown)) {
-                if (isRenderProviderEnabled()) {
-                    const scraped = (await this.scrape({ url: targetUrl, noCache: true }));
-                    const renderedMarkdown = (scraped.data && scraped.data.markdown) || scraped.markdown || "";
-                    if (renderedMarkdown.trim().length > 40) {
-                        const targetInspection = await inspectTarget({
-                            url: targetUrl,
-                            markdown: renderedMarkdown,
-                        });
-                        const siteData = await extractWebsiteData(targetUrl);
-                        return {
-                            success: true,
-                            mode: "local",
-                            url: targetUrl,
-                            data: {
-                                ...siteData,
-                                ...(targetInspection.structured ? { target: targetInspection.structured } : {}),
-                            },
-                            evidence: targetInspection.evidence,
-                        };
-                    }
-                }
+            // Scrape first (local Chrome / render provider) so JS pages yield rendered HTML, not the shell.
+            const scraped = (await this.scrape({
+                url: targetUrl,
+                formats: ["markdown", "html"],
+                noCache: true,
+            }));
+            const markdown = (scraped.data && scraped.data.markdown) || scraped.markdown || "";
+            const html = (scraped.data && scraped.data.html) || scraped.html || "";
+            if (scraped.blocked || scraped.success === false) {
                 return {
-                    success: true,
+                    success: false,
+                    blocked: true,
                     mode: "local",
                     url: targetUrl,
                     data: {},
+                    error: scraped.error || emptyPageError(200),
                     evidence: {
                         tier: lastFetchTier(),
                         sourceUrl: targetUrl,
-                        limitations: [
-                            "Page needs JS rendering; the local native tiers cannot render it. " +
-                                "Run `webcrawl login` and retry with `--mode cloud` (rendering runs in the hosted tier), " +
-                                "or self-host a render provider via WEBCRAWL_RENDER_URL.",
-                        ],
+                        limitations: [String(scraped.error || emptyPageError(200))],
                     },
                 };
             }
             const targetInspection = await inspectTarget({
                 url: targetUrl,
-                html: fetched.html,
+                html,
+                markdown,
+            });
+            const siteData = await extractWebsiteData(targetUrl, {
+                prompt: p.prompt,
+                schema: p.schema,
+                markdown,
+                html,
+                bypassCache: true,
             });
-            const siteData = await extractWebsiteData(targetUrl);
+            const { cached: _c, cacheLayer: _l, ...extracted } = siteData;
             return {
                 success: true,
                 mode: "local",
                 url: targetUrl,
-                data: {
-                    ...siteData,
-                    ...(targetInspection.structured ? { target: targetInspection.structured } : {}),
-                },
+                data: extractOutputData(extracted, {
+                    prompt: p.prompt,
+                    schema: p.schema,
+                    target: targetInspection.structured,
+                }),
                 evidence: targetInspection.evidence,
             };
         },

package/dist/core/ranker/lexical.js

@@ -6,6 +6,7 @@ const ENGINE_WEIGHTS = {
     jina: 1.3,
     "brave-native": 1.25,
     "ddg-lite": 1.15,
+    "bing-rss": 1.12,
     "google-news-rss": 1.05,
     wikipedia: 1.1,
     "hn-algolia": 1.0,

package/dist/core/ranker/types.d.ts

@@ -1,4 +1,4 @@
-export type SearchEngineSource = "bing-shopping" | "google-shopping" | "brave-native" | "ddg-lite" | "google-news-rss" | "wikipedia" | "hn-algolia" | "arxiv" | "nvd" | "osv" | "serper" | "brave-api" | "jina";
+export type SearchEngineSource = "bing-shopping" | "google-shopping" | "brave-native" | "ddg-lite" | "bing-rss" | "google-news-rss" | "wikipedia" | "hn-algolia" | "arxiv" | "nvd" | "osv" | "serper" | "brave-api" | "jina";
 export type DecisionModelId = "baseten/mammoth-1" | "baseten/elephant-1" | "convaiinnovations/laya" | "auto" | "none";
 export interface RawCandidateDocument {
     title: string;

package/dist/core/search/catalog.d.ts

@@ -107,6 +107,8 @@ export interface SearchResponse {
     /** Structured CVE records from NVD and OSV: CVSS score and severity, CWE, affected products and versions, fixed versions, CISA KEV flag, dates and references (populated when category === "security") */
     cves?: CveRecord[];
     provider: "native-federated" | "brave" | "serper" | "jina" | "ddg-edge";
+    /** Set when a keyless search returns no hits (engines blocked or empty). */
+    error?: string;
     /** Set when the primary engines came back thin and a paid fallback engine filled in. */
     fallback?: "jina";
     cached?: boolean;

package/dist/core/search/pipeline.js

@@ -18,7 +18,7 @@ function isHotelOta(url) {
     return hostMatchesAny(url, HOTEL_OTA_HOSTS);
 }
 import { hostMatchesAny, normalizeHostList, resolveCategory, resolveUserLocation, SEARCH_CATEGORY_CATALOG, } from "./catalog.js";
-import { fetchHedgedNativeFederation, fetchJinaSearch, fetchNvdByIds, } from "./providers.js";
+import { EMPTY_SEARCH_ERROR, fetchHedgedNativeFederation, fetchJinaSearch, fetchNvdByIds, } from "./providers.js";
 import { runInBackground, upsertCves } from "../extract/cve-cache.js";
 const searchCache = new Map();
 const SEARCH_CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours warm edge cache
@@ -653,23 +653,26 @@ export async function searchWeb(req) {
     // An explicitly chosen provider is not a fallback.
     const usedJinaFallback = (!req.provider || req.provider === "auto") &&
         finalItems.some((item) => item.sources?.includes("jina"));
-    const provider = ranked.rankingMetrics.enginesContributing.length > 1
+    const engines = ranked.rankingMetrics.enginesContributing;
+    const provider = engines.length > 1
         ? "native-federated"
-        : ranked.rankingMetrics.enginesContributing.includes("serper")
+        : engines.includes("serper")
             ? "serper"
-            : ranked.rankingMetrics.enginesContributing.includes("jina")
+            : engines.includes("jina")
                 ? "jina"
-                : ranked.rankingMetrics.enginesContributing.includes("brave-api") ||
-                    ranked.rankingMetrics.enginesContributing.includes("brave-native")
+                : engines.includes("brave-api") || engines.includes("brave-native")
                     ? "brave"
-                    : "ddg-edge";
-    if (finalItems.length > 0 ||
+                    : engines.includes("ddg-lite")
+                        ? "ddg-edge"
+                        : "native-federated";
+    const hasHits = finalItems.length > 0 ||
         (products && products.length > 0) ||
         (dedupedListings && dedupedListings.length > 0) ||
         (dedupedFlights && dedupedFlights.length > 0) ||
         (dedupedHotels && dedupedHotels.length > 0) ||
         (dedupedBusinesses && dedupedBusinesses.length > 0) ||
-        (dedupedCves && dedupedCves.length > 0)) {
+        (dedupedCves && dedupedCves.length > 0);
+    if (hasHits) {
         searchCache.set(cacheKey, {
             data: finalItems,
             ...(products ? { products } : {}),
@@ -684,6 +687,18 @@ export async function searchWeb(req) {
             expiresAt: Date.now() + SEARCH_CACHE_TTL_MS,
         });
     }
+    if (!hasHits) {
+        return {
+            success: false,
+            data: [],
+            provider,
+            error: EMPTY_SEARCH_ERROR,
+            cached: false,
+            latencyMs: Math.round(performance.now() - start),
+            ...(hasGeo ? { userLocation } : {}),
+            rankingMetrics: ranked.rankingMetrics,
+        };
+    }
     return {
         success: true,
         data: finalItems,

package/dist/core/search/providers.d.ts

@@ -16,6 +16,16 @@ export declare function isEntityQuery(q: string): boolean;
 export declare function fetchHedgedNativeFederation(rawQuery: string, localizedQuery: string, perEngineLimit: number, req: SearchRequest, userLocation: UserLocationContext, quorumTimeoutMs: number, targetSites: string[], onOffer: (offer: ProductOffer) => void, onCve?: (cve: CveRecord) => void): Promise<RawCandidateDocument[]>;
 /** Below this many unique URLs from the free engines, the Jina fallback runs. */
 export declare const JINA_FALLBACK_MIN_URLS = 3;
+/** True when a scraped SERP is a bot-check / captcha rather than results. */
+export declare function isSerpChallengeHtml(html: string, status?: number): boolean;
+export declare const EMPTY_SEARCH_ERROR: string;
+/** Parse Bing's `format=rss` web-search feed into candidate docs. */
+export declare function parseBingWebRss(xml: string, limit: number): RawCandidateDocument[];
+/**
+ * Last-resort keyless engines used when DDG/Brave HTML SERPs come back empty.
+ * Wikipedia OpenSearch, HN Algolia, and Bing RSS do not require a JS challenge.
+ */
+export declare function fetchKeylessSearchFallbacks(query: string, limit: number): Promise<RawCandidateDocument[]>;
 export declare function fetchJinaSearch(query: string, limit: number, req?: {
     country?: string;
     lang?: string;