webcrawl 1.3.3 → 1.3.4

package/dist/core/engine/fetcher.js

@@ -465,5 +465,11 @@ export function needsBrowserEscalation(status, html, markdown) {
             lower.includes('id="__next"'))) {
         return true;
     }
+    // Big-HTML/no-content SPA shell: a very large page (JS bundles / inline
+    // state, e.g. StubHub category pages ~3MB) that yields almost no
+    // extractable text is serving its real content client-side.
+    if (html.length > 250_000 && markdown.length < 500) {
+        return true;
+    }
     return false;
 }

package/dist/core/engine/index.d.ts

@@ -2,4 +2,5 @@ export * from "./cache.js";
 export * from "./markdown.js";
 export * from "./session.js";
 export * from "./fetcher.js";
+export * from "./render.js";
 export * from "./crawler.js";

package/dist/core/engine/index.js

@@ -2,4 +2,5 @@ export * from "./cache.js";
 export * from "./markdown.js";
 export * from "./session.js";
 export * from "./fetcher.js";
+export * from "./render.js";
 export * from "./crawler.js";

package/dist/core/engine/render.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Generic render-provider tier (Tier R): a server-side JS-rendering provider
+ * for pages the native fetch ladder cannot render (JS SPAs, challenge shells).
+ *
+ * Same pattern as the residential-proxy tier: the provider URL comes from
+ * `WEBCRAWL_RENDER_URL` and is unset in the open-source repo — `null` disables
+ * the tier entirely, so the OSS build never requires a provider. The hosted
+ * webcrawl.app server sets the env (and the optional key), so cloud-mode calls
+ * render JS pages through the provider while local zero-key mode degrades to
+ * the `webcrawl login` prompt.
+ *
+ * The wire format parsed here is the de-facto standard "reader" text format
+ * (`Title:` / `URL Source:` / `Markdown Content:` headers), so any compatible
+ * provider works without code changes.
+ */
+/** Render-provider endpoint base URL, or null when the tier is disabled. */
+export declare function renderProviderUrl(): string | null;
+/** Optional bearer key for the render provider (hosted tier). */
+export declare function renderProviderKey(): string | null;
+/** Whether the render-provider tier is configured and usable at all. */
+export declare function isRenderProviderEnabled(): boolean;
+export interface RenderedPage {
+    title: string;
+    markdown: string;
+    sourceUrl: string;
+    warnings: string[];
+}
+/** Splits a reader-format response body into its header fields and markdown. */
+export declare function parseRenderedResponse(body: string): {
+    title: string;
+    sourceUrl: string;
+    markdown: string;
+};
+/** Test helper: clears the in-memory rate-limit window (external env untouched). */
+export declare function resetRenderStateForTests(): void;
+/**
+ * Renders `url` through the configured render provider. Returns null when the
+ * tier is disabled, rate-limited, times out, or the provider itself reports a
+ * failed render — the caller then keeps its native result.
+ */
+export declare function renderViaProvider(url: string): Promise<RenderedPage | null>;

package/dist/core/engine/render.js

@@ -0,0 +1,138 @@
+/**
+ * Generic render-provider tier (Tier R): a server-side JS-rendering provider
+ * for pages the native fetch ladder cannot render (JS SPAs, challenge shells).
+ *
+ * Same pattern as the residential-proxy tier: the provider URL comes from
+ * `WEBCRAWL_RENDER_URL` and is unset in the open-source repo — `null` disables
+ * the tier entirely, so the OSS build never requires a provider. The hosted
+ * webcrawl.app server sets the env (and the optional key), so cloud-mode calls
+ * render JS pages through the provider while local zero-key mode degrades to
+ * the `webcrawl login` prompt.
+ *
+ * The wire format parsed here is the de-facto standard "reader" text format
+ * (`Title:` / `URL Source:` / `Markdown Content:` headers), so any compatible
+ * provider works without code changes.
+ */
+/** Render-provider endpoint base URL, or null when the tier is disabled. */
+export function renderProviderUrl() {
+    const url = (process.env.WEBCRAWL_RENDER_URL ?? "").trim();
+    return url.length > 0 ? url : null;
+}
+/** Optional bearer key for the render provider (hosted tier). */
+export function renderProviderKey() {
+    const key = (process.env.WEBCRAWL_RENDER_KEY ?? "").trim();
+    return key.length > 0 ? key : null;
+}
+/** Whether the render-provider tier is configured and usable at all. */
+export function isRenderProviderEnabled() {
+    return renderProviderUrl() !== null;
+}
+/** Default provider rate limit (requests/min) when no key is configured. */
+const KEYLESS_RATE_LIMIT_PER_MIN = 15;
+/** Provider rate limit (requests/min) assumed with a key configured. */
+const KEYED_RATE_LIMIT_PER_MIN = 60;
+/** Hard timeout for a single render request. */
+const RENDER_TIMEOUT_MS = 45_000;
+/** Max bytes accepted from the provider (matches the fetch ladder cap). */
+const RENDER_MAX_BYTES = 2 * 1024 * 1024;
+const rateWindow = [];
+function withinRateLimit() {
+    const now = Date.now();
+    while (rateWindow.length > 0 && rateWindow[0] <= now - 60_000)
+        rateWindow.shift();
+    const limit = renderProviderKey() ? KEYED_RATE_LIMIT_PER_MIN : KEYLESS_RATE_LIMIT_PER_MIN;
+    if (rateWindow.length >= limit)
+        return false;
+    rateWindow.push(now);
+    return true;
+}
+/**
+ * Failure markers providers prepend/emit when the target page could not be
+ * rendered (challenge, CAPTCHA, upstream error). A matching response is not
+ * rendered content and must not be returned as such.
+ */
+const RENDER_FAILURE_MARKERS = [
+    "target url returned error",
+    "may be requiring captcha",
+    "maybe requiring captcha",
+    "please make sure you are authorized",
+];
+/** Splits a reader-format response body into its header fields and markdown. */
+export function parseRenderedResponse(body) {
+    const lines = body.split("\n");
+    let title = "";
+    let sourceUrl = "";
+    let markdownStart = lines.length;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        if (line.startsWith("Title:")) {
+            title = line.slice("Title:".length).trim();
+            continue;
+        }
+        if (line.startsWith("URL Source:")) {
+            sourceUrl = line.slice("URL Source:".length).trim();
+            continue;
+        }
+        if (/^Markdown Content:?\s*$/.test(line)) {
+            markdownStart = i + 1;
+            break;
+        }
+    }
+    let markdown = lines.slice(markdownStart).join("\n").trim();
+    if (markdownStart === lines.length && title) {
+        // No "Markdown Content:" separator: treat the whole body after the header
+        // block as markdown (some providers omit the separator).
+        markdown = lines
+            .filter((l) => !l.startsWith("Title:") && !l.startsWith("URL Source:"))
+            .join("\n")
+            .trim();
+    }
+    return { title, sourceUrl, markdown };
+}
+/** Test helper: clears the in-memory rate-limit window (external env untouched). */
+export function resetRenderStateForTests() {
+    rateWindow.length = 0;
+}
+/**
+ * Renders `url` through the configured render provider. Returns null when the
+ * tier is disabled, rate-limited, times out, or the provider itself reports a
+ * failed render — the caller then keeps its native result.
+ */
+export async function renderViaProvider(url) {
+    const base = renderProviderUrl();
+    if (!base)
+        return null;
+    if (!withinRateLimit())
+        return null;
+    const key = renderProviderKey();
+    const headers = {
+        Accept: "text/plain",
+        ...(key ? { Authorization: `Bearer ${key}` } : {}),
+    };
+    let res;
+    try {
+        res = await fetch(`${base}${url}`, {
+            headers,
+            signal: AbortSignal.timeout(RENDER_TIMEOUT_MS),
+            redirect: "follow",
+        });
+    }
+    catch {
+        return null; // network error / timeout / DNS — keep native result
+    }
+    if (!res.ok)
+        return null;
+    const body = (await res.text()).slice(0, RENDER_MAX_BYTES);
+    const lower = body.slice(0, 2000).toLowerCase();
+    if (RENDER_FAILURE_MARKERS.some((m) => lower.includes(m)))
+        return null;
+    const { title, sourceUrl, markdown } = parseRenderedResponse(body);
+    if (markdown.trim().length < 40)
+        return null; // nothing real came back
+    return {
+        title,
+        markdown,
+        sourceUrl: sourceUrl || url,
+        warnings: [],
+    };
+}

package/dist/core/engine/session.d.ts

@@ -38,6 +38,9 @@ export interface ExternalSessionStore {
     /** Optional persistence for the residential-proxy memo (Tier P1). */
     markDomainProxyNeeded?(domain: string): Promise<void>;
     isDomainProxyNeeded?(domain: string): Promise<boolean>;
+    /** Optional persistence for the render-provider memo (Tier R). */
+    markDomainRenderNeeded?(domain: string): Promise<void>;
+    isDomainRenderNeeded?(domain: string): Promise<boolean>;
 }
 /** How long a solved session is offered for warm reconnect / cookie replay. */
 export declare const WARM_TTL_MS: number;
@@ -49,6 +52,8 @@ export declare const WARM_CAP = 4;
 export declare const CHALLENGE_MEMO_TTL_MS: number;
 /** How long a domain stays on the residential-proxy memo (Tier P1). */
 export declare const PROXY_MEMO_TTL_MS: number;
+/** How long a domain stays on the render-provider memo (Tier R). */
+export declare const RENDER_MEMO_TTL_MS: number;
 export declare function registerExternalSessionStore(store: ExternalSessionStore): void;
 export interface CookieLike {
     name: string;
@@ -77,5 +82,12 @@ export declare function isDomainChallenged(domain: string): Promise<boolean>;
  */
 export declare function markDomainProxyNeeded(domain: string): Promise<void>;
 export declare function isDomainProxyNeeded(domain: string): Promise<boolean>;
+/**
+ * Marks a domain as needing the render-provider tier (Tier R): the native
+ * fetch ladder passes the bot check but the page's content only appears after
+ * JS renders, so future runs skip the ladder and go provider-first.
+ */
+export declare function markDomainRenderNeeded(domain: string): Promise<void>;
+export declare function isDomainRenderNeeded(domain: string): Promise<boolean>;
 /** Test helper: resets all in-memory state (external stores are untouched). */
 export declare function resetSessionStateForTests(): void;

package/dist/core/engine/session.js

@@ -29,6 +29,8 @@ export const WARM_CAP = 4;
 export const CHALLENGE_MEMO_TTL_MS = 30 * 60_000;
 /** How long a domain stays on the residential-proxy memo (Tier P1). */
 export const PROXY_MEMO_TTL_MS = 60 * 60_000;
+/** How long a domain stays on the render-provider memo (Tier R). */
+export const RENDER_MEMO_TTL_MS = 12 * 60 * 60_000;
 let external = null;
 export function registerExternalSessionStore(store) {
     external = store;
@@ -36,6 +38,7 @@ export function registerExternalSessionStore(store) {
 const warm = new Map();
 const challenged = new Map();
 const proxyNeeded = new Map();
+const renderNeeded = new Map();
 function reap(now = Date.now()) {
     for (const [d, s] of warm)
         if (s.expiresAt <= now)
@@ -46,6 +49,9 @@ function reap(now = Date.now()) {
     for (const [d, t] of proxyNeeded)
         if (t <= now)
             proxyNeeded.delete(d);
+    for (const [d, t] of renderNeeded)
+        if (t <= now)
+            renderNeeded.delete(d);
 }
 /** Serializes captured browser cookies into a `Cookie:` header value (drops expired). */
 export function cookieHeaderFrom(cookies) {
@@ -195,9 +201,50 @@ export async function isDomainProxyNeeded(domain) {
     }
     return false;
 }
+/**
+ * Marks a domain as needing the render-provider tier (Tier R): the native
+ * fetch ladder passes the bot check but the page's content only appears after
+ * JS renders, so future runs skip the ladder and go provider-first.
+ */
+export async function markDomainRenderNeeded(domain) {
+    if (!domain)
+        return;
+    renderNeeded.set(domain, Date.now() + RENDER_MEMO_TTL_MS);
+    if (external?.markDomainRenderNeeded) {
+        try {
+            await external.markDomainRenderNeeded(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+export async function isDomainRenderNeeded(domain) {
+    if (!domain)
+        return false;
+    reap();
+    const until = renderNeeded.get(domain);
+    if (until && until > Date.now())
+        return true;
+    if (until)
+        renderNeeded.delete(domain);
+    if (external?.isDomainRenderNeeded) {
+        try {
+            if (await external.isDomainRenderNeeded(domain)) {
+                renderNeeded.set(domain, Date.now() + RENDER_MEMO_TTL_MS);
+                return true;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return false;
+}
 /** Test helper: resets all in-memory state (external stores are untouched). */
 export function resetSessionStateForTests() {
     warm.clear();
     challenged.clear();
     proxyNeeded.clear();
+    renderNeeded.clear();
 }

package/dist/core/local-engine.js

@@ -10,7 +10,9 @@
  * powers both this local engine (`npx webcrawl`) and the hosted `webcrawl.app` API.
  */
 import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, mapWebsiteUrls, setCachedPage, sha256, } from "./engine.js";
-import { fetchResource, lastFetchTier } from "./engine/fetcher.js";
+import { isDomainRenderNeeded, markDomainRenderNeeded, } from "./engine/session.js";
+import { isRenderProviderEnabled, renderViaProvider } from "./engine/render.js";
+import { fetchResource, lastFetchTier, needsBrowserEscalation } from "./engine/fetcher.js";
 import { parseFile, publicParseMeta } from "./parse/index.js";
 import { isBinaryContentType, looksLikeBinaryUrl } from "./parse/fetch.js";
 import { maxDownloadBytes } from "./parse/limits.js";
@@ -18,6 +20,15 @@ import { parseYouTubeCached, youtubeVideoId } from "./parse/youtube.js";
 import { rankDocuments } from "./ranker.js";
 import { getSearchCategoriesCatalog, searchWeb } from "./search.js";
 import { inspectTarget, listTargets, resolveTarget } from "./targets.js";
+/** Lowercased hostname of a URL (memo key for the Tier R render memo). */
+function safeDomainOf(url) {
+    try {
+        return new URL(url).hostname.toLowerCase();
+    }
+    catch {
+        return "";
+    }
+}
 export function createLocalEngineClient() {
     return {
         baseUrl: "local://engine",
@@ -142,7 +153,49 @@ export function createLocalEngineClient() {
                     };
                 }
             }
-            // 4. Fetch with TLS impersonation (impit) + target-aware extraction
+            // 4. Fetch with TLS impersonation (impit) + target-aware extraction.
+            //    Tier R: memoized render-needed domains skip the native ladder
+            //    entirely when a render provider is configured — cache (step 3)
+            //    already absorbed repeats, so the provider is the next best hop.
+            const renderDomain = safeDomainOf(targetUrl);
+            if (isRenderProviderEnabled() && (await isDomainRenderNeeded(renderDomain))) {
+                const rendered = await renderViaProvider(targetUrl);
+                if (rendered) {
+                    const renderHash = sha256(rendered.markdown);
+                    setCachedPage(targetUrl, "none", {
+                        title: rendered.title,
+                        markdown: rendered.markdown,
+                        links: [],
+                        statusCode: 200,
+                        contentHash: renderHash,
+                    });
+                    return {
+                        success: true,
+                        mode: "local",
+                        url: targetUrl,
+                        title: rendered.title || targetUrl,
+                        markdown: rendered.markdown,
+                        links: [],
+                        data: {
+                            url: targetUrl,
+                            title: rendered.title || targetUrl,
+                            markdown: rendered.markdown,
+                            links: [],
+                        },
+                        evidence: {
+                            targetId: "render-provider",
+                            tier: "render",
+                            contentHash: renderHash,
+                            latencyMs: Date.now() - start,
+                            sourceUrl: rendered.sourceUrl,
+                            matchedSignals: ["render:provider"],
+                            limitations: rendered.warnings,
+                        },
+                    };
+                }
+                // Provider failed on a memo'd domain: fall through to the native
+                // ladder — the memo may be stale or the provider flaky.
+            }
             const fetched = await fetchWithTlsImpersonation(targetUrl);
             const { title, markdown, links } = fastHtmlToMarkdown(fetched.html, targetUrl);
             const contentHash = sha256(markdown || fetched.html);
@@ -158,6 +211,83 @@ export function createLocalEngineClient() {
                 html: fetched.html,
                 markdown,
             });
+            //    Tier R: the native ladder passed the bot check but the page is a
+            //    JS shell / challenge page the ladder cannot render. Memoize the
+            //    domain and either render via the provider (when configured) or
+            //    return the shell with an honest limitation note.
+            if (needsBrowserEscalation(fetched.status, fetched.html, markdown)) {
+                await markDomainRenderNeeded(renderDomain);
+                if (isRenderProviderEnabled()) {
+                    const rendered = await renderViaProvider(targetUrl);
+                    if (rendered) {
+                        const renderHash = sha256(rendered.markdown);
+                        setCachedPage(targetUrl, "none", {
+                            title: rendered.title,
+                            markdown: rendered.markdown,
+                            links: [],
+                            statusCode: 200,
+                            contentHash: renderHash,
+                        });
+                        return {
+                            success: true,
+                            mode: "local",
+                            url: targetUrl,
+                            title: rendered.title || targetUrl,
+                            markdown: rendered.markdown,
+                            links: [],
+                            data: {
+                                url: targetUrl,
+                                title: rendered.title || targetUrl,
+                                markdown: rendered.markdown,
+                                links: [],
+                            },
+                            evidence: {
+                                targetId: "render-provider",
+                                tier: "render",
+                                contentHash: renderHash,
+                                latencyMs: Date.now() - start,
+                                sourceUrl: rendered.sourceUrl,
+                                matchedSignals: ["render:provider"],
+                                limitations: rendered.warnings,
+                            },
+                        };
+                    }
+                }
+                const nativeTier = lastFetchTier() === "impersonate"
+                    ? "impersonate"
+                    : lastFetchTier() === "proxy-impersonate"
+                        ? "proxy-impersonate"
+                        : "impit-tls";
+                return {
+                    success: true,
+                    mode: "local",
+                    url: targetUrl,
+                    title: title || targetInspection.title || targetUrl,
+                    markdown: targetInspection.markdown || markdown,
+                    links,
+                    data: {
+                        url: targetUrl,
+                        title: title || targetInspection.title || targetUrl,
+                        markdown: targetInspection.markdown || markdown,
+                        links,
+                        structured: targetInspection.structured,
+                    },
+                    evidence: {
+                        ...targetInspection.evidence,
+                        tier: nativeTier,
+                        contentHash,
+                        latencyMs: Date.now() - start,
+                        limitations: [
+                            ...(Array.isArray(targetInspection.evidence?.limitations)
+                                ? targetInspection.evidence.limitations
+                                : []),
+                            "Page needs JS rendering; the local native tiers cannot render it. " +
+                                "Run `webcrawl login` and retry with `--mode cloud` (rendering runs in the hosted tier), " +
+                                "or self-host a render provider via WEBCRAWL_RENDER_URL.",
+                        ],
+                    },
+                };
+            }
             return {
                 success: true,
                 mode: "local",
@@ -287,6 +417,49 @@ export function createLocalEngineClient() {
         async extract(p) {
             const targetUrl = /^https?:\/\//i.test(p.url) ? p.url : `https://${p.url}`;
             const fetched = await fetchWithTlsImpersonation(targetUrl);
+            const fastCheck = fastHtmlToMarkdown(fetched.html, targetUrl);
+            //    Tier R: shell page the native ladder cannot render — retry
+            //    through scrape() so the render path + cache apply, then extract
+            //    from the rendered Markdown. Without a provider, surface the
+            //    login/cloud limitation instead of silently returning {}.
+            if (needsBrowserEscalation(fetched.status, fetched.html, fastCheck.markdown)) {
+                if (isRenderProviderEnabled()) {
+                    const scraped = (await this.scrape({ url: targetUrl, noCache: true }));
+                    const renderedMarkdown = (scraped.data && scraped.data.markdown) || scraped.markdown || "";
+                    if (renderedMarkdown.trim().length > 40) {
+                        const targetInspection = await inspectTarget({
+                            url: targetUrl,
+                            markdown: renderedMarkdown,
+                        });
+                        const siteData = await extractWebsiteData(targetUrl);
+                        return {
+                            success: true,
+                            mode: "local",
+                            url: targetUrl,
+                            data: {
+                                ...siteData,
+                                ...(targetInspection.structured ? { target: targetInspection.structured } : {}),
+                            },
+                            evidence: targetInspection.evidence,
+                        };
+                    }
+                }
+                return {
+                    success: true,
+                    mode: "local",
+                    url: targetUrl,
+                    data: {},
+                    evidence: {
+                        tier: lastFetchTier(),
+                        sourceUrl: targetUrl,
+                        limitations: [
+                            "Page needs JS rendering; the local native tiers cannot render it. " +
+                                "Run `webcrawl login` and retry with `--mode cloud` (rendering runs in the hosted tier), " +
+                                "or self-host a render provider via WEBCRAWL_RENDER_URL.",
+                        ],
+                    },
+                };
+            }
             const targetInspection = await inspectTarget({
                 url: targetUrl,
                 html: fetched.html,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.3.3",
+  "version": "1.3.4",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.3.3",
-    "@webcrawl/curl-impersonate": "^1.3.3",
+    "@webcrawl/client": "^1.3.4",
+    "@webcrawl/curl-impersonate": "^1.3.4",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",