webcrawl 1.3.3 → 1.3.5

package/dist/cli.d.ts

@@ -15,5 +15,5 @@ export interface Deps {
     /** Working directory override for `webcrawl setup` tests. */
     cwd?: string;
 }
-export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in (--email <addr> [--code <6-digit>], --web, or --key <key>)\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in or create an account in your browser via OAuth PKCE redirect\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
+export declare const HELP = "Webcrawl Anything: search, scrape, parse and crawl any target on the internet\n\nUsage: webcrawl <command> [options]\n\nCommands\n  search <query>      Search the web or verticals. -c category, --sub, -n limit, --site <domain>\n  scrape <url>        Inspect a URL (website, SPA, PDF, Office, YouTube, GitHub, ArXiv, llms.txt)\n  map <url>           List a site's URLs. --filter <text>, -n limit\n  crawl <url>         Read several pages. -n max pages (default 5, max 25)\n  extract <url>       Structured JSON. --prompt <text> or --schema <file.json>\n  rerank <query>      Rank documents. --doc <text> (repeatable), --file <path>, or lines on stdin. -n top\n  targets             List all supported crawl & search targets and their requirements\n  categories          List search categories and vertical filters\n  setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)\n  doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration\n  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>\n  whoami              Show execution mode (local vs cloud), account email, key source, and credit balance\n\nOptions\n  --mode <m>          Execution mode: auto (default: local without key, cloud with key), local, or cloud\n  --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)\n  --code <6-digit>    6-digit email verification code from `webcrawl login --email <addr>`\n  --name <name>       Optional display name when creating an account with `--email`\n  --web               Sign in via browser OAuth (default for login; no need to pass it)\n  --json              Print the raw JSON response\n  --api-key <key>     Override WEBCRAWL_API_KEY\n  --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)\n  --country, --city   Where to search from (default: this machine's timezone)\n  --no-cache          Skip cached pages (scrape)\n  --agent <name>      Target agent for setup: claude, cursor, codex, gemini, windsurf, or all (repeatable)\n  --schema-profile    MCP schema profile for setup: compact (default) or full\n  -h, --help\n\nConfig order: flags, then WEBCRAWL_API_KEY / WEBCRAWL_URL / WEBCRAWL_MODE, then ~/.config/webcrawl/config.json";
 export declare function main(argv: string[], io: Io, deps?: Deps): Promise<number>;

package/dist/cli.js

@@ -22,7 +22,7 @@ Commands
   categories          List search categories and vertical filters
   setup               Register Webcrawl MCP server and Agent Skill (Claude, Cursor, Codex, Gemini, Windsurf)
   doctor              Check local crawl engine, parsers, stealth browser, and cloud configuration
-  login / signup      Create an account or sign in (--email <addr> [--code <6-digit>], --web, or --key <key>)
+  login / signup      Create an account or sign in in your browser (default), or --email <addr> [--code <6-digit>], or --key <key>
   whoami              Show execution mode (local vs cloud), account email, key source, and credit balance
 
 Options
@@ -30,7 +30,7 @@ Options
   --email <addr>      Email address for CLI account creation / OTP sign-in (assigns 1,000 free credits)
   --code <6-digit>    6-digit email verification code from \`webcrawl login --email <addr>\`
   --name <name>       Optional display name when creating an account with \`--email\`
-  --web               Sign in or create an account in your browser via OAuth PKCE redirect
+  --web               Sign in via browser OAuth (default for login; no need to pass it)
   --json              Print the raw JSON response
   --api-key <key>     Override WEBCRAWL_API_KEY
   --url <base>        Override WEBCRAWL_URL (default https://webcrawl.app)
@@ -302,7 +302,7 @@ export async function main(argv, io, deps = {}) {
             "     webcrawl login --email <you@domain.com>",
             "     webcrawl login --email <you@domain.com> --code <6-digit-code>",
             "  2. Browser OAuth redirect:",
-            "     webcrawl login --web",
+            "     webcrawl login",
             "  3. Or run in zero-key local mode:",
             `     webcrawl --mode local ${cmd} ${rest.join(" ")}`,
         ].join("\n");
@@ -314,7 +314,7 @@ export async function main(argv, io, deps = {}) {
                 loginCommands: {
                     sendOtp: "webcrawl login --email <you@domain.com>",
                     verifyOtp: "webcrawl login --email <you@domain.com> --code <6-digit-code>",
-                    browserRedirect: "webcrawl login --web",
+                    browserRedirect: "webcrawl login",
                     signupUrl: `${cfg.baseUrl}/?signup=cli`,
                 },
             }, null, 2)}\n`);
@@ -447,8 +447,8 @@ export async function main(argv, io, deps = {}) {
                     fetch: deps.fetch,
                     userAgent: "webcrawl-cli/1.2",
                 });
-                // 1. Browser OAuth PKCE redirect (`webcrawl login --web`)
-                if (v.web) {
+                // 1. Browser OAuth PKCE redirect (default for `webcrawl login`; `--web` kept for compatibility)
+                if (v.web || (!v.email && !v.key)) {
                     const webRes = await runWebOAuthLogin(cfg.baseUrl, deps.fetch ?? globalThis.fetch, io);
                     const existing = readConfigFile(file);
                     writeConfigFile({ ...existing, apiKey: webRes.apiKey, ...(v.url ? { baseUrl: v.url } : {}) }, file);
@@ -524,9 +524,9 @@ export async function main(argv, io, deps = {}) {
                     return 0;
                 }
                 // 3. Direct API key (`webcrawl login --key wc_live_...` or interactive paste)
-                const key = (v.key ?? (await readKeyInteractively(io))).trim();
+                const key = v.key?.trim() ?? (await readKeyInteractively(io)).trim();
                 if (!key) {
-                    throw new UsageError("No key provided. Pass --key <wc_live_...>, --email <you@domain.com>, or --web.");
+                    throw new UsageError("No key provided. Pass --key <wc_live_...> or --email <you@domain.com>, or run `webcrawl login` to sign in via your browser.");
                 }
                 const existing = readConfigFile(file);
                 writeConfigFile({ ...existing, apiKey: key, ...(v.url ? { baseUrl: v.url } : {}) }, file);
@@ -538,7 +538,7 @@ export async function main(argv, io, deps = {}) {
                 const lines = [
                     `endpoint: ${cfg.baseUrl} (${cfg.source.baseUrl})`,
                     `mode:     ${cfg.effectiveMode} (${cfg.source.mode})`,
-                    `key:      ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` (`--email <you@domain.com>` or `--web`) or set WEBCRAWL_API_KEY (local mode works with zero key)."}`,
+                    `key:      ${masked}${cfg.apiKey ? ` (${cfg.source.apiKey})` : ". Run `webcrawl login` to sign in via your browser (or `--email <you@domain.com>`), or set WEBCRAWL_API_KEY (local mode works with zero key)."}`,
                 ];
                 if (cfg.apiKey) {
                     const u = await client.usage();

package/dist/core/engine/fetcher.js

@@ -465,5 +465,11 @@ export function needsBrowserEscalation(status, html, markdown) {
             lower.includes('id="__next"'))) {
         return true;
     }
+    // Big-HTML/no-content SPA shell: a very large page (JS bundles / inline
+    // state, e.g. StubHub category pages ~3MB) that yields almost no
+    // extractable text is serving its real content client-side.
+    if (html.length > 250_000 && markdown.length < 500) {
+        return true;
+    }
     return false;
 }

package/dist/core/engine/index.d.ts

@@ -2,4 +2,5 @@ export * from "./cache.js";
 export * from "./markdown.js";
 export * from "./session.js";
 export * from "./fetcher.js";
+export * from "./render.js";
 export * from "./crawler.js";

package/dist/core/engine/index.js

@@ -2,4 +2,5 @@ export * from "./cache.js";
 export * from "./markdown.js";
 export * from "./session.js";
 export * from "./fetcher.js";
+export * from "./render.js";
 export * from "./crawler.js";

package/dist/core/engine/render.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Generic render-provider tier (Tier R): a server-side JS-rendering provider
+ * for pages the native fetch ladder cannot render (JS SPAs, challenge shells).
+ *
+ * Same pattern as the residential-proxy tier: the provider URL comes from
+ * `WEBCRAWL_RENDER_URL` and is unset in the open-source repo — `null` disables
+ * the tier entirely, so the OSS build never requires a provider. The hosted
+ * webcrawl.app server sets the env (and the optional key), so cloud-mode calls
+ * render JS pages through the provider while local zero-key mode degrades to
+ * the `webcrawl login` prompt.
+ *
+ * The wire format parsed here is the de-facto standard "reader" text format
+ * (`Title:` / `URL Source:` / `Markdown Content:` headers), so any compatible
+ * provider works without code changes.
+ */
+/** Render-provider endpoint base URL, or null when the tier is disabled. */
+export declare function renderProviderUrl(): string | null;
+/** Optional bearer key for the render provider (hosted tier). */
+export declare function renderProviderKey(): string | null;
+/** Whether the render-provider tier is configured and usable at all. */
+export declare function isRenderProviderEnabled(): boolean;
+export interface RenderedPage {
+    title: string;
+    markdown: string;
+    sourceUrl: string;
+    warnings: string[];
+}
+/** Splits a reader-format response body into its header fields and markdown. */
+export declare function parseRenderedResponse(body: string): {
+    title: string;
+    sourceUrl: string;
+    markdown: string;
+};
+/** Test helper: clears the in-memory rate-limit window (external env untouched). */
+export declare function resetRenderStateForTests(): void;
+/**
+ * Renders `url` through the configured render provider. Returns null when the
+ * tier is disabled, rate-limited, times out, or the provider itself reports a
+ * failed render — the caller then keeps its native result.
+ */
+export declare function renderViaProvider(url: string): Promise<RenderedPage | null>;

package/dist/core/engine/render.js

@@ -0,0 +1,138 @@
+/**
+ * Generic render-provider tier (Tier R): a server-side JS-rendering provider
+ * for pages the native fetch ladder cannot render (JS SPAs, challenge shells).
+ *
+ * Same pattern as the residential-proxy tier: the provider URL comes from
+ * `WEBCRAWL_RENDER_URL` and is unset in the open-source repo — `null` disables
+ * the tier entirely, so the OSS build never requires a provider. The hosted
+ * webcrawl.app server sets the env (and the optional key), so cloud-mode calls
+ * render JS pages through the provider while local zero-key mode degrades to
+ * the `webcrawl login` prompt.
+ *
+ * The wire format parsed here is the de-facto standard "reader" text format
+ * (`Title:` / `URL Source:` / `Markdown Content:` headers), so any compatible
+ * provider works without code changes.
+ */
+/** Render-provider endpoint base URL, or null when the tier is disabled. */
+export function renderProviderUrl() {
+    const url = (process.env.WEBCRAWL_RENDER_URL ?? "").trim();
+    return url.length > 0 ? url : null;
+}
+/** Optional bearer key for the render provider (hosted tier). */
+export function renderProviderKey() {
+    const key = (process.env.WEBCRAWL_RENDER_KEY ?? "").trim();
+    return key.length > 0 ? key : null;
+}
+/** Whether the render-provider tier is configured and usable at all. */
+export function isRenderProviderEnabled() {
+    return renderProviderUrl() !== null;
+}
+/** Default provider rate limit (requests/min) when no key is configured. */
+const KEYLESS_RATE_LIMIT_PER_MIN = 15;
+/** Provider rate limit (requests/min) assumed with a key configured. */
+const KEYED_RATE_LIMIT_PER_MIN = 60;
+/** Hard timeout for a single render request. */
+const RENDER_TIMEOUT_MS = 45_000;
+/** Max bytes accepted from the provider (matches the fetch ladder cap). */
+const RENDER_MAX_BYTES = 2 * 1024 * 1024;
+const rateWindow = [];
+function withinRateLimit() {
+    const now = Date.now();
+    while (rateWindow.length > 0 && rateWindow[0] <= now - 60_000)
+        rateWindow.shift();
+    const limit = renderProviderKey() ? KEYED_RATE_LIMIT_PER_MIN : KEYLESS_RATE_LIMIT_PER_MIN;
+    if (rateWindow.length >= limit)
+        return false;
+    rateWindow.push(now);
+    return true;
+}
+/**
+ * Failure markers providers prepend/emit when the target page could not be
+ * rendered (challenge, CAPTCHA, upstream error). A matching response is not
+ * rendered content and must not be returned as such.
+ */
+const RENDER_FAILURE_MARKERS = [
+    "target url returned error",
+    "may be requiring captcha",
+    "maybe requiring captcha",
+    "please make sure you are authorized",
+];
+/** Splits a reader-format response body into its header fields and markdown. */
+export function parseRenderedResponse(body) {
+    const lines = body.split("\n");
+    let title = "";
+    let sourceUrl = "";
+    let markdownStart = lines.length;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        if (line.startsWith("Title:")) {
+            title = line.slice("Title:".length).trim();
+            continue;
+        }
+        if (line.startsWith("URL Source:")) {
+            sourceUrl = line.slice("URL Source:".length).trim();
+            continue;
+        }
+        if (/^Markdown Content:?\s*$/.test(line)) {
+            markdownStart = i + 1;
+            break;
+        }
+    }
+    let markdown = lines.slice(markdownStart).join("\n").trim();
+    if (markdownStart === lines.length && title) {
+        // No "Markdown Content:" separator: treat the whole body after the header
+        // block as markdown (some providers omit the separator).
+        markdown = lines
+            .filter((l) => !l.startsWith("Title:") && !l.startsWith("URL Source:"))
+            .join("\n")
+            .trim();
+    }
+    return { title, sourceUrl, markdown };
+}
+/** Test helper: clears the in-memory rate-limit window (external env untouched). */
+export function resetRenderStateForTests() {
+    rateWindow.length = 0;
+}
+/**
+ * Renders `url` through the configured render provider. Returns null when the
+ * tier is disabled, rate-limited, times out, or the provider itself reports a
+ * failed render — the caller then keeps its native result.
+ */
+export async function renderViaProvider(url) {
+    const base = renderProviderUrl();
+    if (!base)
+        return null;
+    if (!withinRateLimit())
+        return null;
+    const key = renderProviderKey();
+    const headers = {
+        Accept: "text/plain",
+        ...(key ? { Authorization: `Bearer ${key}` } : {}),
+    };
+    let res;
+    try {
+        res = await fetch(`${base}${url}`, {
+            headers,
+            signal: AbortSignal.timeout(RENDER_TIMEOUT_MS),
+            redirect: "follow",
+        });
+    }
+    catch {
+        return null; // network error / timeout / DNS — keep native result
+    }
+    if (!res.ok)
+        return null;
+    const body = (await res.text()).slice(0, RENDER_MAX_BYTES);
+    const lower = body.slice(0, 2000).toLowerCase();
+    if (RENDER_FAILURE_MARKERS.some((m) => lower.includes(m)))
+        return null;
+    const { title, sourceUrl, markdown } = parseRenderedResponse(body);
+    if (markdown.trim().length < 40)
+        return null; // nothing real came back
+    return {
+        title,
+        markdown,
+        sourceUrl: sourceUrl || url,
+        warnings: [],
+    };
+}

package/dist/core/engine/session.d.ts

@@ -38,6 +38,9 @@ export interface ExternalSessionStore {
     /** Optional persistence for the residential-proxy memo (Tier P1). */
     markDomainProxyNeeded?(domain: string): Promise<void>;
     isDomainProxyNeeded?(domain: string): Promise<boolean>;
+    /** Optional persistence for the render-provider memo (Tier R). */
+    markDomainRenderNeeded?(domain: string): Promise<void>;
+    isDomainRenderNeeded?(domain: string): Promise<boolean>;
 }
 /** How long a solved session is offered for warm reconnect / cookie replay. */
 export declare const WARM_TTL_MS: number;
@@ -49,6 +52,8 @@ export declare const WARM_CAP = 4;
 export declare const CHALLENGE_MEMO_TTL_MS: number;
 /** How long a domain stays on the residential-proxy memo (Tier P1). */
 export declare const PROXY_MEMO_TTL_MS: number;
+/** How long a domain stays on the render-provider memo (Tier R). */
+export declare const RENDER_MEMO_TTL_MS: number;
 export declare function registerExternalSessionStore(store: ExternalSessionStore): void;
 export interface CookieLike {
     name: string;
@@ -77,5 +82,12 @@ export declare function isDomainChallenged(domain: string): Promise<boolean>;
  */
 export declare function markDomainProxyNeeded(domain: string): Promise<void>;
 export declare function isDomainProxyNeeded(domain: string): Promise<boolean>;
+/**
+ * Marks a domain as needing the render-provider tier (Tier R): the native
+ * fetch ladder passes the bot check but the page's content only appears after
+ * JS renders, so future runs skip the ladder and go provider-first.
+ */
+export declare function markDomainRenderNeeded(domain: string): Promise<void>;
+export declare function isDomainRenderNeeded(domain: string): Promise<boolean>;
 /** Test helper: resets all in-memory state (external stores are untouched). */
 export declare function resetSessionStateForTests(): void;

package/dist/core/engine/session.js

@@ -29,6 +29,8 @@ export const WARM_CAP = 4;
 export const CHALLENGE_MEMO_TTL_MS = 30 * 60_000;
 /** How long a domain stays on the residential-proxy memo (Tier P1). */
 export const PROXY_MEMO_TTL_MS = 60 * 60_000;
+/** How long a domain stays on the render-provider memo (Tier R). */
+export const RENDER_MEMO_TTL_MS = 12 * 60 * 60_000;
 let external = null;
 export function registerExternalSessionStore(store) {
     external = store;
@@ -36,6 +38,7 @@ export function registerExternalSessionStore(store) {
 const warm = new Map();
 const challenged = new Map();
 const proxyNeeded = new Map();
+const renderNeeded = new Map();
 function reap(now = Date.now()) {
     for (const [d, s] of warm)
         if (s.expiresAt <= now)
@@ -46,6 +49,9 @@ function reap(now = Date.now()) {
     for (const [d, t] of proxyNeeded)
         if (t <= now)
             proxyNeeded.delete(d);
+    for (const [d, t] of renderNeeded)
+        if (t <= now)
+            renderNeeded.delete(d);
 }
 /** Serializes captured browser cookies into a `Cookie:` header value (drops expired). */
 export function cookieHeaderFrom(cookies) {
@@ -195,9 +201,50 @@ export async function isDomainProxyNeeded(domain) {
     }
     return false;
 }
+/**
+ * Marks a domain as needing the render-provider tier (Tier R): the native
+ * fetch ladder passes the bot check but the page's content only appears after
+ * JS renders, so future runs skip the ladder and go provider-first.
+ */
+export async function markDomainRenderNeeded(domain) {
+    if (!domain)
+        return;
+    renderNeeded.set(domain, Date.now() + RENDER_MEMO_TTL_MS);
+    if (external?.markDomainRenderNeeded) {
+        try {
+            await external.markDomainRenderNeeded(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+export async function isDomainRenderNeeded(domain) {
+    if (!domain)
+        return false;
+    reap();
+    const until = renderNeeded.get(domain);
+    if (until && until > Date.now())
+        return true;
+    if (until)
+        renderNeeded.delete(domain);
+    if (external?.isDomainRenderNeeded) {
+        try {
+            if (await external.isDomainRenderNeeded(domain)) {
+                renderNeeded.set(domain, Date.now() + RENDER_MEMO_TTL_MS);
+                return true;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return false;
+}
 /** Test helper: resets all in-memory state (external stores are untouched). */
 export function resetSessionStateForTests() {
     warm.clear();
     challenged.clear();
     proxyNeeded.clear();
+    renderNeeded.clear();
 }

package/dist/core/local-engine.js

@@ -10,7 +10,9 @@
  * powers both this local engine (`npx webcrawl`) and the hosted `webcrawl.app` API.
  */
 import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, mapWebsiteUrls, setCachedPage, sha256, } from "./engine.js";
-import { fetchResource, lastFetchTier } from "./engine/fetcher.js";
+import { isDomainRenderNeeded, markDomainRenderNeeded, } from "./engine/session.js";
+import { isRenderProviderEnabled, renderViaProvider } from "./engine/render.js";
+import { fetchResource, lastFetchTier, needsBrowserEscalation } from "./engine/fetcher.js";
 import { parseFile, publicParseMeta } from "./parse/index.js";
 import { isBinaryContentType, looksLikeBinaryUrl } from "./parse/fetch.js";
 import { maxDownloadBytes } from "./parse/limits.js";
@@ -18,6 +20,15 @@ import { parseYouTubeCached, youtubeVideoId } from "./parse/youtube.js";
 import { rankDocuments } from "./ranker.js";
 import { getSearchCategoriesCatalog, searchWeb } from "./search.js";
 import { inspectTarget, listTargets, resolveTarget } from "./targets.js";
+/** Lowercased hostname of a URL (memo key for the Tier R render memo). */
+function safeDomainOf(url) {
+    try {
+        return new URL(url).hostname.toLowerCase();
+    }
+    catch {
+        return "";
+    }
+}
 export function createLocalEngineClient() {
     return {
         baseUrl: "local://engine",
@@ -142,7 +153,49 @@ export function createLocalEngineClient() {
                     };
                 }
             }
-            // 4. Fetch with TLS impersonation (impit) + target-aware extraction
+            // 4. Fetch with TLS impersonation (impit) + target-aware extraction.
+            //    Tier R: memoized render-needed domains skip the native ladder
+            //    entirely when a render provider is configured — cache (step 3)
+            //    already absorbed repeats, so the provider is the next best hop.
+            const renderDomain = safeDomainOf(targetUrl);
+            if (isRenderProviderEnabled() && (await isDomainRenderNeeded(renderDomain))) {
+                const rendered = await renderViaProvider(targetUrl);
+                if (rendered) {
+                    const renderHash = sha256(rendered.markdown);
+                    setCachedPage(targetUrl, "none", {
+                        title: rendered.title,
+                        markdown: rendered.markdown,
+                        links: [],
+                        statusCode: 200,
+                        contentHash: renderHash,
+                    });
+                    return {
+                        success: true,
+                        mode: "local",
+                        url: targetUrl,
+                        title: rendered.title || targetUrl,
+                        markdown: rendered.markdown,
+                        links: [],
+                        data: {
+                            url: targetUrl,
+                            title: rendered.title || targetUrl,
+                            markdown: rendered.markdown,
+                            links: [],
+                        },
+                        evidence: {
+                            targetId: "render-provider",
+                            tier: "render",
+                            contentHash: renderHash,
+                            latencyMs: Date.now() - start,
+                            sourceUrl: rendered.sourceUrl,
+                            matchedSignals: ["render:provider"],
+                            limitations: rendered.warnings,
+                        },
+                    };
+                }
+                // Provider failed on a memo'd domain: fall through to the native
+                // ladder — the memo may be stale or the provider flaky.
+            }
             const fetched = await fetchWithTlsImpersonation(targetUrl);
             const { title, markdown, links } = fastHtmlToMarkdown(fetched.html, targetUrl);
             const contentHash = sha256(markdown || fetched.html);
@@ -158,6 +211,83 @@ export function createLocalEngineClient() {
                 html: fetched.html,
                 markdown,
             });
+            //    Tier R: the native ladder passed the bot check but the page is a
+            //    JS shell / challenge page the ladder cannot render. Memoize the
+            //    domain and either render via the provider (when configured) or
+            //    return the shell with an honest limitation note.
+            if (needsBrowserEscalation(fetched.status, fetched.html, markdown)) {
+                await markDomainRenderNeeded(renderDomain);
+                if (isRenderProviderEnabled()) {
+                    const rendered = await renderViaProvider(targetUrl);
+                    if (rendered) {
+                        const renderHash = sha256(rendered.markdown);
+                        setCachedPage(targetUrl, "none", {
+                            title: rendered.title,
+                            markdown: rendered.markdown,
+                            links: [],
+                            statusCode: 200,
+                            contentHash: renderHash,
+                        });
+                        return {
+                            success: true,
+                            mode: "local",
+                            url: targetUrl,
+                            title: rendered.title || targetUrl,
+                            markdown: rendered.markdown,
+                            links: [],
+                            data: {
+                                url: targetUrl,
+                                title: rendered.title || targetUrl,
+                                markdown: rendered.markdown,
+                                links: [],
+                            },
+                            evidence: {
+                                targetId: "render-provider",
+                                tier: "render",
+                                contentHash: renderHash,
+                                latencyMs: Date.now() - start,
+                                sourceUrl: rendered.sourceUrl,
+                                matchedSignals: ["render:provider"],
+                                limitations: rendered.warnings,
+                            },
+                        };
+                    }
+                }
+                const nativeTier = lastFetchTier() === "impersonate"
+                    ? "impersonate"
+                    : lastFetchTier() === "proxy-impersonate"
+                        ? "proxy-impersonate"
+                        : "impit-tls";
+                return {
+                    success: true,
+                    mode: "local",
+                    url: targetUrl,
+                    title: title || targetInspection.title || targetUrl,
+                    markdown: targetInspection.markdown || markdown,
+                    links,
+                    data: {
+                        url: targetUrl,
+                        title: title || targetInspection.title || targetUrl,
+                        markdown: targetInspection.markdown || markdown,
+                        links,
+                        structured: targetInspection.structured,
+                    },
+                    evidence: {
+                        ...targetInspection.evidence,
+                        tier: nativeTier,
+                        contentHash,
+                        latencyMs: Date.now() - start,
+                        limitations: [
+                            ...(Array.isArray(targetInspection.evidence?.limitations)
+                                ? targetInspection.evidence.limitations
+                                : []),
+                            "Page needs JS rendering; the local native tiers cannot render it. " +
+                                "Run `webcrawl login` and retry with `--mode cloud` (rendering runs in the hosted tier), " +
+                                "or self-host a render provider via WEBCRAWL_RENDER_URL.",
+                        ],
+                    },
+                };
+            }
             return {
                 success: true,
                 mode: "local",
@@ -287,6 +417,49 @@ export function createLocalEngineClient() {
         async extract(p) {
             const targetUrl = /^https?:\/\//i.test(p.url) ? p.url : `https://${p.url}`;
             const fetched = await fetchWithTlsImpersonation(targetUrl);
+            const fastCheck = fastHtmlToMarkdown(fetched.html, targetUrl);
+            //    Tier R: shell page the native ladder cannot render — retry
+            //    through scrape() so the render path + cache apply, then extract
+            //    from the rendered Markdown. Without a provider, surface the
+            //    login/cloud limitation instead of silently returning {}.
+            if (needsBrowserEscalation(fetched.status, fetched.html, fastCheck.markdown)) {
+                if (isRenderProviderEnabled()) {
+                    const scraped = (await this.scrape({ url: targetUrl, noCache: true }));
+                    const renderedMarkdown = (scraped.data && scraped.data.markdown) || scraped.markdown || "";
+                    if (renderedMarkdown.trim().length > 40) {
+                        const targetInspection = await inspectTarget({
+                            url: targetUrl,
+                            markdown: renderedMarkdown,
+                        });
+                        const siteData = await extractWebsiteData(targetUrl);
+                        return {
+                            success: true,
+                            mode: "local",
+                            url: targetUrl,
+                            data: {
+                                ...siteData,
+                                ...(targetInspection.structured ? { target: targetInspection.structured } : {}),
+                            },
+                            evidence: targetInspection.evidence,
+                        };
+                    }
+                }
+                return {
+                    success: true,
+                    mode: "local",
+                    url: targetUrl,
+                    data: {},
+                    evidence: {
+                        tier: lastFetchTier(),
+                        sourceUrl: targetUrl,
+                        limitations: [
+                            "Page needs JS rendering; the local native tiers cannot render it. " +
+                                "Run `webcrawl login` and retry with `--mode cloud` (rendering runs in the hosted tier), " +
+                                "or self-host a render provider via WEBCRAWL_RENDER_URL.",
+                        ],
+                    },
+                };
+            }
             const targetInspection = await inspectTarget({
                 url: targetUrl,
                 html: fetched.html,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.3.3",
+  "version": "1.3.5",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.3.3",
-    "@webcrawl/curl-impersonate": "^1.3.3",
+    "@webcrawl/client": "^1.3.5",
+    "@webcrawl/curl-impersonate": "^1.3.5",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",