webcrawl 1.2.8 → 1.3.0

package/dist/core/engine/fetcher.d.ts

@@ -17,12 +17,18 @@ export interface ScrapeResult {
     links?: string[];
     statusCode: number;
     contentHash: string;
-    tier: "cache" | "static" | "sandbox" | "cloudflare" | "browser";
+    tier: "cache" | "static" | "sandbox" | "cloudflare" | "browser" | "impersonate";
     latencyMs: number;
     error?: string;
 }
 export declare function getBrowserHeaders(targetUrl: string, extraHeaders?: Record<string, string>): Record<string, string>;
 export declare function setDomainCookie(hostname: string, cookieHeader: string): void;
+/**
+ * User-Agent paired with a domain's replayed session cookies. Cloudflare's
+ * `cf_clearance` (and similar bound tokens) only validate when the exact UA
+ * that solved the challenge is reused, so replay sets both together.
+ */
+export declare function setDomainUserAgent(hostname: string, userAgent: string): void;
 /**
  * True when an HTML response (with optional status) is an AWS WAF challenge
  * interstitial: a marker match proves it regardless of status; otherwise a
@@ -31,8 +37,10 @@ export declare function setDomainCookie(hostname: string, cookieHeader: string):
  */
 export declare function isAwsWafChallenge(html: string, status?: number): boolean;
 export declare function isAntiBotChallenge(html: string, status: number): boolean;
+/** Transport tier that won the last fetch ladder run (for result evidence). */
+export declare function lastFetchTier(): "impit-tls" | "impersonate" | "fetch";
 /**
- * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)
+ * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)sonation)
  * so Cloudflare Bot Management and Vercel Security Checkpoint treat requests as real Chrome.
  * Falls back to standard `fetch` if `impit` is unavailable on the current platform.
  * Every redirect hop is checked against the SSRF guard.

package/dist/core/engine/fetcher.js

@@ -1,15 +1,16 @@
 import { followSafely, UnsafeUrlError } from "./safe-url.js";
 const domainCookies = new Map();
+const domainUserAgents = new Map();
 const BROWSER_USER_AGENTS = [
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
 ];
 export function getBrowserHeaders(targetUrl, extraHeaders) {
-    const ua = BROWSER_USER_AGENTS[Math.abs(hashCode(targetUrl)) % BROWSER_USER_AGENTS.length];
+    const parsed = new URL(targetUrl);
+    const ua = domainUserAgents.get(parsed.hostname) ?? BROWSER_USER_AGENTS[Math.abs(hashCode(targetUrl)) % BROWSER_USER_AGENTS.length];
     let hostCookie;
     try {
-        const parsed = new URL(targetUrl);
         hostCookie = domainCookies.get(parsed.hostname);
     }
     catch {
@@ -39,6 +40,16 @@ export function setDomainCookie(hostname, cookieHeader) {
         domainCookies.set(hostname, cookieHeader);
     }
 }
+/**
+ * User-Agent paired with a domain's replayed session cookies. Cloudflare's
+ * `cf_clearance` (and similar bound tokens) only validate when the exact UA
+ * that solved the challenge is reused, so replay sets both together.
+ */
+export function setDomainUserAgent(hostname, userAgent) {
+    if (hostname && userAgent) {
+        domainUserAgents.set(hostname, userAgent);
+    }
+}
 function hashCode(str) {
     let hash = 0;
     for (let i = 0; i < str.length; i++) {
@@ -98,6 +109,12 @@ export function isAntiBotChallenge(html, status) {
     return isAwsWafChallenge(html, status);
 }
 let sharedImpit = null;
+/** Tier that won the most recent rawGet ladder ("impit-tls" → "impersonate" → "fetch"). */
+let lastTier = "impit-tls";
+/** Transport tier that won the last fetch ladder run (for result evidence). */
+export function lastFetchTier() {
+    return lastTier;
+}
 /** One request, redirects NOT followed (callers validate each hop via followSafely). */
 async function rawGet(url, reqHeaders, timeoutMs, maxBytes) {
     const read = async (status, headers, getBuf) => {
@@ -121,26 +138,122 @@ async function rawGet(url, reqHeaders, timeoutMs, maxBytes) {
             timeout: timeoutMs,
             redirect: "manual",
         });
-        return await read(res.status, headersToObject(res.headers), () => res.arrayBuffer());
+        const hop = await read(res.status, headersToObject(res.headers), () => res.arrayBuffer());
+        // AWS-WAF-style blocks (403/429/challenge shells) on the impit tier
+        // (Chrome 136) retry once on the curl-impersonate tier (Chrome 150),
+        // which passes WAFs that block the older fingerprint.
+        if (isAntiBotChallenge(new TextDecoder().decode(hop.body).slice(0, 8000), hop.status)) {
+            const impersonateHop = await rawGetImpersonate(url, reqHeaders, timeoutMs, maxBytes);
+            if (impersonateHop) {
+                lastTier = "impersonate";
+                return impersonateHop;
+            }
+        }
+        lastTier = "impit-tls";
+        return hop;
     }
     catch (err) {
         if (err instanceof ResourceTooLargeError || err instanceof UnsafeUrlError)
             throw err;
+        // impit unavailable/failed: curl-impersonate tier, then plain fetch.
+        const impersonateHop = await rawGetImpersonate(url, reqHeaders, timeoutMs, maxBytes);
+        if (impersonateHop) {
+            lastTier = "impersonate";
+            return impersonateHop;
+        }
         const res = await fetch(url, {
             headers: reqHeaders,
             redirect: "manual",
             signal: AbortSignal.timeout(timeoutMs),
         });
+        lastTier = "fetch";
         return await read(res.status, headersToObject(res.headers), () => res.arrayBuffer());
     }
 }
 /**
- * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)
+ * One GET hop with the curl-impersonate tier (Chrome 150 TLS/HTTP2 fingerprint).
+ * Used by `rawGet` when the impit hop is blocked (403/429/challenge) — newer
+ * fingerprints pass AWS WAF domains (stubhub, expedia) that block Chrome 136.
+ * Falls back to plain `fetch` when the binary is unavailable.
+ */
+async function rawGetImpersonate(url, reqHeaders, timeoutMs, maxBytes) {
+    let impersonate;
+    try {
+        impersonate = await import("@webcrawl/curl-impersonate");
+    }
+    catch {
+        return null;
+    }
+    // The chrome150 profile's own identity headers MUST survive (profileHeaders
+    // merges extras last, so a stale UA would override the profile UA and the
+    // UA/TLS mismatch is exactly what DataDome/AWS WAF block). Keep cookies and
+    // referer, drop UA + sec-ch/client-hint identity headers.
+    const extra = {};
+    for (const [key, value] of Object.entries(reqHeaders)) {
+        const lower = key.toLowerCase();
+        if (lower === "user-agent" ||
+            lower.startsWith("sec-ch-ua") ||
+            lower.startsWith("sec-fetch-") ||
+            lower === "upgrade-insecure-requests" ||
+            lower === "priority") {
+            continue;
+        }
+        extra[key] = value;
+    }
+    try {
+        const res = await impersonate.impersonateFetch(url, {
+            headers: extra,
+            timeoutMs,
+            maxBytes,
+        });
+        const headers = res.headers;
+        const isRedirect = res.status >= 300 && res.status < 400 && !!headers["location"];
+        if (!isRedirect && maxBytes !== undefined)
+            checkDeclaredLength(headers, maxBytes);
+        return {
+            status: res.status,
+            headers,
+            body: isRedirect ? new ArrayBuffer(0) : res.body.buffer,
+        };
+    }
+    catch (err) {
+        if (err instanceof UnsafeUrlError)
+            throw err;
+        return null; // tier unavailable or fetch failed — caller falls back
+    }
+}
+/**
+ * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)sonation)
  * so Cloudflare Bot Management and Vercel Security Checkpoint treat requests as real Chrome.
  * Falls back to standard `fetch` if `impit` is unavailable on the current platform.
  * Every redirect hop is checked against the SSRF guard.
  */
 export async function fetchWithTlsImpersonation(url, timeoutMs = 12_000) {
+    // Session hydration: when a warm stealth session exists for this domain, its
+    // solved cookies (and the UA they were solved under — bound cookies require
+    // the pair) ride the request so the lightweight tier can skip the challenge.
+    let hostname = "";
+    try {
+        hostname = new URL(url).hostname;
+    }
+    catch {
+        /* invalid URL handled below */
+    }
+    if (hostname) {
+        try {
+            const { getWarmSession } = await import("./session.js");
+            const warm = await getWarmSession(hostname);
+            if (warm) {
+                if (warm.cookieHeader)
+                    setDomainCookie(hostname, warm.cookieHeader);
+                if (warm.userAgent)
+                    setDomainUserAgent(hostname, warm.userAgent);
+            }
+        }
+        catch {
+            /* session module unavailable — plain fetch */
+        }
+    }
     const { res, finalUrl } = await followSafely(url, (hop) => rawGet(hop, getBrowserHeaders(hop), timeoutMs));
     if (res.headers["set-cookie"]) {
         try {

package/dist/core/engine/index.d.ts

@@ -1,4 +1,5 @@
 export * from "./cache.js";
 export * from "./markdown.js";
+export * from "./session.js";
 export * from "./fetcher.js";
 export * from "./crawler.js";

package/dist/core/engine/index.js

@@ -1,4 +1,5 @@
 export * from "./cache.js";
 export * from "./markdown.js";
+export * from "./session.js";
 export * from "./fetcher.js";
 export * from "./crawler.js";

package/dist/core/engine/session.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Stealth session reuse: warm-sandbox registry, cookie-replay state and the
+ * JS-challenge memo.
+ *
+ * A solved bot check (AWS WAF, Cloudflare, Vercel checkpoint) leaves behind a
+ * valuable session: the sandbox's egress IP, its User-Agent and the cleared
+ * cookies. Instead of throwing that away with the sandbox, this module keeps:
+ *   - a warm registry (domain -> sandboxId + UA + cookies) so the next query
+ *     reconnects to the still-running sandbox (`Sandbox.get`) instead of
+ *     cold-booting and re-solving;
+ *   - the cookie header for replay on the lightweight TLS-impersonated tier
+ *     (works only where the provider does not bind tokens to the solver IP —
+ *     Cloudflare's `cf_clearance` does; AWS WAF's 300 s immunity may not);
+ *   - a memo of domains known to serve JS challenges, so the useless
+ *     curl-sandbox tier is skipped for them.
+ *
+ * Everything is in-memory by default; `registerExternalSessionStore` plugs in
+ * persistence (Postgres in the hosted app) so warm sessions survive across
+ * serverless instances. A registered store is always optional: every call
+ * degrades gracefully when it is unavailable.
+ */
+/** A solved stealth session kept for reuse. */
+export interface WarmSession {
+    domain: string;
+    sandboxId: string;
+    userAgent: string;
+    /** `Cookie:` header value for the solved session (UA-paired: replay both or neither). */
+    cookieHeader: string;
+    /** Epoch ms after which the entry is stale. */
+    expiresAt: number;
+}
+export interface ExternalSessionStore {
+    getWarmSession(domain: string): Promise<WarmSession | null>;
+    saveWarmSession(session: WarmSession): Promise<void>;
+    deleteWarmSession(domain: string): Promise<void>;
+    markDomainChallenged(domain: string): Promise<void>;
+    isDomainChallenged(domain: string): Promise<boolean>;
+}
+/** How long a solved session is offered for warm reconnect / cookie replay. */
+export declare const WARM_TTL_MS: number;
+/** VM `timeout` headroom set at Sandbox.create so a warm sandbox outlives its registry TTL. */
+export declare const WARM_SANDBOX_TIMEOUT_MS: number;
+/** Cap on concurrently warm sandboxes; LRU entries beyond the cap are evicted. */
+export declare const WARM_CAP = 4;
+/** How long a domain stays on the JS-challenge memo. */
+export declare const CHALLENGE_MEMO_TTL_MS: number;
+export declare function registerExternalSessionStore(store: ExternalSessionStore): void;
+export interface CookieLike {
+    name: string;
+    value: string;
+    domain?: string;
+    path?: string;
+    expires?: number;
+}
+/** Serializes captured browser cookies into a `Cookie:` header value (drops expired). */
+export declare function cookieHeaderFrom(cookies: CookieLike[]): string;
+/** Best-available warm session for a domain (in-memory first, then the external store). */
+export declare function getWarmSession(domain: string): Promise<WarmSession | null>;
+/**
+ * Registers a solved session as warm. Returns the sandboxIds evicted by the
+ * cap — the caller must stop those sandboxes so idle compute does not linger.
+ */
+export declare function saveWarmSession(session: WarmSession): Promise<string[]>;
+export declare function deleteWarmSession(domain: string): Promise<void>;
+/** Marks a domain as known to serve JS challenges (skips the curl tier next time). */
+export declare function markDomainChallenged(domain: string): Promise<void>;
+export declare function isDomainChallenged(domain: string): Promise<boolean>;
+/** Test helper: resets all in-memory state (external stores are untouched). */
+export declare function resetSessionStateForTests(): void;

package/dist/core/engine/session.js

@@ -0,0 +1,156 @@
+/**
+ * Stealth session reuse: warm-sandbox registry, cookie-replay state and the
+ * JS-challenge memo.
+ *
+ * A solved bot check (AWS WAF, Cloudflare, Vercel checkpoint) leaves behind a
+ * valuable session: the sandbox's egress IP, its User-Agent and the cleared
+ * cookies. Instead of throwing that away with the sandbox, this module keeps:
+ *   - a warm registry (domain -> sandboxId + UA + cookies) so the next query
+ *     reconnects to the still-running sandbox (`Sandbox.get`) instead of
+ *     cold-booting and re-solving;
+ *   - the cookie header for replay on the lightweight TLS-impersonated tier
+ *     (works only where the provider does not bind tokens to the solver IP —
+ *     Cloudflare's `cf_clearance` does; AWS WAF's 300 s immunity may not);
+ *   - a memo of domains known to serve JS challenges, so the useless
+ *     curl-sandbox tier is skipped for them.
+ *
+ * Everything is in-memory by default; `registerExternalSessionStore` plugs in
+ * persistence (Postgres in the hosted app) so warm sessions survive across
+ * serverless instances. A registered store is always optional: every call
+ * degrades gracefully when it is unavailable.
+ */
+/** How long a solved session is offered for warm reconnect / cookie replay. */
+export const WARM_TTL_MS = 8 * 60_000;
+/** VM `timeout` headroom set at Sandbox.create so a warm sandbox outlives its registry TTL. */
+export const WARM_SANDBOX_TIMEOUT_MS = 10 * 60_000;
+/** Cap on concurrently warm sandboxes; LRU entries beyond the cap are evicted. */
+export const WARM_CAP = 4;
+/** How long a domain stays on the JS-challenge memo. */
+export const CHALLENGE_MEMO_TTL_MS = 30 * 60_000;
+let external = null;
+export function registerExternalSessionStore(store) {
+    external = store;
+}
+const warm = new Map();
+const challenged = new Map();
+function reap(now = Date.now()) {
+    for (const [d, s] of warm)
+        if (s.expiresAt <= now)
+            warm.delete(d);
+    for (const [d, t] of challenged)
+        if (t <= now)
+            challenged.delete(d);
+}
+/** Serializes captured browser cookies into a `Cookie:` header value (drops expired). */
+export function cookieHeaderFrom(cookies) {
+    const now = Date.now() / 1000;
+    return cookies
+        .filter((c) => c.name && c.value && (!c.expires || c.expires > now))
+        .map((c) => `${c.name}=${c.value}`)
+        .join("; ");
+}
+/** Best-available warm session for a domain (in-memory first, then the external store). */
+export async function getWarmSession(domain) {
+    if (!domain)
+        return null;
+    reap();
+    const local = warm.get(domain);
+    if (local && local.expiresAt > Date.now())
+        return local;
+    if (local)
+        warm.delete(domain);
+    if (external) {
+        try {
+            const stored = await external.getWarmSession(domain);
+            if (stored && stored.expiresAt > Date.now()) {
+                warm.set(domain, stored);
+                return stored;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return null;
+}
+/**
+ * Registers a solved session as warm. Returns the sandboxIds evicted by the
+ * cap — the caller must stop those sandboxes so idle compute does not linger.
+ */
+export async function saveWarmSession(session) {
+    if (!session.domain)
+        return [];
+    reap();
+    warm.delete(session.domain);
+    warm.set(session.domain, { ...session });
+    const evicted = [];
+    while (warm.size > WARM_CAP) {
+        const [d, s] = warm.entries().next().value;
+        warm.delete(d);
+        if (s.sandboxId && s.sandboxId !== session.sandboxId)
+            evicted.push(s.sandboxId);
+    }
+    if (external) {
+        try {
+            await external.saveWarmSession(session);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return evicted;
+}
+export async function deleteWarmSession(domain) {
+    if (!domain)
+        return;
+    warm.delete(domain);
+    if (external) {
+        try {
+            await external.deleteWarmSession(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+/** Marks a domain as known to serve JS challenges (skips the curl tier next time). */
+export async function markDomainChallenged(domain) {
+    if (!domain)
+        return;
+    challenged.set(domain, Date.now() + CHALLENGE_MEMO_TTL_MS);
+    if (external) {
+        try {
+            await external.markDomainChallenged(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+export async function isDomainChallenged(domain) {
+    if (!domain)
+        return false;
+    reap();
+    const until = challenged.get(domain);
+    if (until && until > Date.now())
+        return true;
+    if (until)
+        challenged.delete(domain);
+    if (external) {
+        try {
+            if (await external.isDomainChallenged(domain)) {
+                challenged.set(domain, Date.now() + CHALLENGE_MEMO_TTL_MS);
+                return true;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return false;
+}
+/** Test helper: resets all in-memory state (external stores are untouched). */
+export function resetSessionStateForTests() {
+    warm.clear();
+    challenged.clear();
+}

package/dist/core/engine/stealth.d.ts

@@ -45,6 +45,8 @@ export declare function stealthFetch(entryUrl: string, items: StealthItem[], opt
     followSitemaps?: number;
     /** Allow the paid remote provider. Off for unbilled (anonymous) runs. Default true. */
     remote?: boolean;
+    /** Keep the solved session warm (sandbox left running + cookies saved) for repeat queries. */
+    keepWarm?: boolean;
 }): Promise<StealthResult>;
 /** Extracts `<loc>` entries from a sitemap or sitemap index. */
 export declare function parseSitemapLocs(xml: string): {

package/dist/core/engine/stealth.js

@@ -10,6 +10,8 @@
  * Never fabricates content: a page still showing a check comes back `blocked`.
  */
 import { isAntiBotChallenge } from "./fetcher.js";
+import { setDomainCookie, setDomainUserAgent } from "./fetcher.js";
+import { cookieHeaderFrom, getWarmSession, saveWarmSession, WARM_TTL_MS } from "./session.js";
 import { remoteStealthProvider } from "../interact/remote.js";
 /**
  * True when a real browser can run here: our own (sandbox in prod, local
@@ -37,19 +39,29 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
     }
     const { startBrowser, runBrowserStep, stopBrowser } = await import("../interact/driver.js");
     const provider = opts.remote === false ? null : remoteStealthProvider();
-    const tag = provider ? { provider } : {};
+    const domain = hostOf(entryUrl);
+    // A warm session for this domain means the challenge is already solved: short
+    // solve budget. A cold solve needs the full budget.
+    const hadWarm = provider ? null : await getWarmSession(domain).catch(() => null);
+    const tag = provider ? { provider } : { warm: Boolean(hadWarm) };
     let handle = null;
     // Filled in by `finally` once the provider reports the session cost.
     const out = {};
+    let capturedCookies = [];
+    let capturedUa = "";
     try {
         const budgetMs = opts.budgetMs ?? 90_000;
-        handle = await startBrowser({ timeoutMs: Math.min(5 * 60_000, budgetMs + 90_000), remote: provider });
+        handle = await startBrowser({
+            timeoutMs: Math.min(5 * 60_000, budgetMs + 90_000),
+            remote: provider,
+            domain: provider ? undefined : domain,
+        });
         const res = await runBrowserStep(handle, {
             type: "fetchMany",
             entry: entryUrl,
             items: items.slice(0, 40),
             budgetMs,
-            solveMs: opts.solveMs ?? 20_000,
+            solveMs: hadWarm ? Math.min(opts.solveMs ?? 20_000, 8_000) : (opts.solveMs ?? 20_000),
             followSitemaps: opts.followSitemaps ?? 0,
         });
         if (!res.ok) {
@@ -57,6 +69,13 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
         }
         else {
             const value = (res.data?.value ?? {});
+            capturedCookies = Array.isArray(value.cookies) ? value.cookies : [];
+            capturedUa = value.userAgent ?? "";
+            if (capturedUa)
+                setDomainUserAgent(domain, capturedUa);
+            if (capturedCookies.length > 0) {
+                setDomainCookie(domain, cookieHeaderFrom(capturedCookies));
+            }
             out.result = {
                 solved: Boolean(value.solved),
                 entry: value.entry ?? null,
@@ -70,9 +89,32 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
         out.result = { solved: false, entry: null, docs: [], durationMs: 0, error: err instanceof Error ? err.message : String(err), ...tag };
     }
     finally {
-        const cost = handle ? await stopBrowser(handle).catch(() => null) : null;
+        const solved = out.result?.solved === true;
+        const keepWarm = !provider && solved && (opts.keepWarm ?? true) && Boolean(handle && handle.kind === "sandbox" && capturedUa);
+        const cost = handle
+            ? await stopBrowser(handle, {
+                keepWarm,
+                domain,
+                userAgent: capturedUa,
+                cookieHeader: capturedCookies.length > 0 ? cookieHeaderFrom(capturedCookies) : "",
+            }).catch(() => null)
+            : null;
         if (out.result && cost)
             out.result.providerCostUsd = cost.costUsd;
+        if (!keepWarm && solved && capturedCookies.length > 0) {
+            // Even without a warm sandbox, remember the solved cookies for replay on
+            // the lightweight tier (UA-paired; rejection there just escalates).
+            await saveWarmSession({ domain, sandboxId: "", userAgent: capturedUa, cookieHeader: cookieHeaderFrom(capturedCookies), expiresAt: Date.now() + WARM_TTL_MS })
+                .catch(() => { });
+        }
+        console.info("[stealth] session result", {
+            domain,
+            solved,
+            warm: Boolean(hadWarm),
+            keepWarm,
+            cookieCount: capturedCookies.length,
+            durationMs: Date.now() - t0,
+        });
     }
     const result = out.result;
     result.durationMs = Date.now() - t0;
@@ -88,6 +130,15 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
     }
     return result;
 }
+/** Hostname of a URL, or "" when unparseable. */
+function hostOf(url) {
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return "";
+    }
+}
 /** Extracts `<loc>` entries from a sitemap or sitemap index. */
 export function parseSitemapLocs(xml) {
     const pages = [];

package/dist/core/interact/driver.d.ts

@@ -42,8 +42,21 @@ export declare function startBrowser(opts?: {
     timeoutMs?: number;
     userAgent?: string;
     remote?: RemoteProvider | null;
+    domain?: string;
 }): Promise<BrowserHandle>;
 export declare function runBrowserStep(handle: BrowserHandle, action: StepAction): Promise<StepResultLine>;
-/** Stops the browser. For a remote provider, returns what the session cost (null when unknown). */
-export declare function stopBrowser(handle: BrowserHandle): Promise<RemoteSessionCost | null>;
+/**
+ * Stops the browser. For a remote provider, returns what the session cost (null
+ * when unknown). With `keepWarm`, a sandbox browser is left running and
+ * registered (domain + sandboxId + UA) so the next call reconnects to it via
+ * `Sandbox.get` instead of cold-booting — the caller must pass `domain`.
+ * Evicted warm entries (over the cap) are stopped here so idle compute
+ * does not linger.
+ */
+export declare function stopBrowser(handle: BrowserHandle, opts?: {
+    keepWarm?: boolean;
+    domain?: string;
+    userAgent?: string;
+    cookieHeader?: string;
+}): Promise<RemoteSessionCost | null>;
 export { CDP_PORT };

package/dist/core/interact/driver.js

@@ -15,6 +15,7 @@ import path from "node:path";
 import { Sandbox } from "@vercel/sandbox";
 import { CDP_PORT, SETUP_SCRIPT, STEP_SCRIPT, STEP_SCRIPT_PATH, VM_ROOT, WAIT_FOR_CDP_SCRIPT, chromeArgs, parseStepOutput, } from "./vm.js";
 import { startRemoteSession, stopRemoteSession } from "./remote.js";
+import { WARM_SANDBOX_TIMEOUT_MS, WARM_TTL_MS } from "../engine/session.js";
 const snapshotId = () => process.env.WEBCRAWL_INTERACT_SNAPSHOT_ID || process.env.INTERACT_SNAPSHOT_ID || "";
 const localMode = () => process.env.INTERACT_LOCAL_PLAYWRIGHT === "true";
 /**
@@ -91,6 +92,44 @@ async function startSandboxBrowser(opts) {
         throw err;
     }
 }
+/** True when the sandbox's Chromium still answers on CDP (warm reconnect check). */
+async function sandboxChromeAlive(sandbox) {
+    const ping = await sh(sandbox, `curl -sf -m 3 http://127.0.0.1:${CDP_PORT}/json/version >/dev/null && echo alive`, false);
+    return ping.exitCode === 0 && ping.stdout.includes("alive");
+}
+/**
+ * Warm reconnect: fetch the sandbox this domain solved its challenge in, verify
+ * Chromium still lives, and reuse it (same IP, same UA, same cleared cookies).
+ * Returns null when no warm entry exists or the sandbox is gone (cold boot).
+ */
+async function startWarmSandboxBrowser(domain) {
+    if (!domain)
+        return null;
+    const { getWarmSession } = await import("../engine/session.js");
+    let warm;
+    try {
+        warm = await getWarmSession(domain);
+    }
+    catch {
+        return null;
+    }
+    if (!warm || !warm.sandboxId)
+        return null;
+    try {
+        const sandbox = await Sandbox.get({ sandboxId: warm.sandboxId });
+        if (!(await sandboxChromeAlive(sandbox))) {
+            await sandbox.stop().catch(() => { });
+            return null;
+        }
+        // Refresh the step runner in case this instance ships newer code.
+        await sandbox.writeFiles([{ path: STEP_SCRIPT_PATH, content: Buffer.from(STEP_SCRIPT) }]).catch(() => { });
+        console.info("[stealth] warm reconnect", { domain, sandboxId: warm.sandboxId });
+        return { kind: "sandbox", sandboxId: warm.sandboxId };
+    }
+    catch {
+        return null;
+    }
+}
 /* ------------------------------ local backend ----------------------------- */
 async function resolveLocalPlaywright() {
     const candidates = [
@@ -213,7 +252,18 @@ export async function startBrowser(opts = {}) {
         return startLocalBrowser(opts);
     if (!interactBrowserAvailable())
         throw new Error("browser_unavailable");
-    return startSandboxBrowser({ timeoutMs: opts.timeoutMs ?? 5 * 60_000, userAgent: opts.userAgent });
+    // Warm reconnect first: a sandbox that already solved this domain's bot check
+    // keeps its IP / UA / cleared cookies — reusing it skips boot + re-solve.
+    if (opts.domain) {
+        const warm = await startWarmSandboxBrowser(opts.domain).catch(() => null);
+        if (warm)
+            return warm;
+    }
+    return startSandboxBrowser({
+        // Long VM timeout headroom so the sandbox can stay warm after this call.
+        timeoutMs: Math.max(opts.timeoutMs ?? 5 * 60_000, WARM_SANDBOX_TIMEOUT_MS),
+        userAgent: opts.userAgent,
+    });
 }
 function runLocalStep(scriptPath, playwright, cdp, action, extraEnv = {}) {
     return new Promise((resolve) => {
@@ -243,8 +293,15 @@ export async function runBrowserStep(handle, action) {
     }
     return runSandboxStep(handle.sandboxId, action);
 }
-/** Stops the browser. For a remote provider, returns what the session cost (null when unknown). */
-export async function stopBrowser(handle) {
+/**
+ * Stops the browser. For a remote provider, returns what the session cost (null
+ * when unknown). With `keepWarm`, a sandbox browser is left running and
+ * registered (domain + sandboxId + UA) so the next call reconnects to it via
+ * `Sandbox.get` instead of cold-booting — the caller must pass `domain`.
+ * Evicted warm entries (over the cap) are stopped here so idle compute
+ * does not linger.
+ */
+export async function stopBrowser(handle, opts = {}) {
     if (handle.kind === "local") {
         try {
             process.kill(handle.pid);
@@ -258,6 +315,32 @@ export async function stopBrowser(handle) {
         const [cost] = await Promise.all([stopRemoteSession(handle), stopRunner(handle.runner)]);
         return cost;
     }
+    if (opts.keepWarm && opts.domain) {
+        const { saveWarmSession } = await import("../engine/session.js");
+        try {
+            const evicted = await saveWarmSession({
+                domain: opts.domain,
+                sandboxId: handle.sandboxId,
+                userAgent: opts.userAgent ?? "",
+                cookieHeader: opts.cookieHeader ?? "",
+                expiresAt: Date.now() + WARM_TTL_MS,
+            });
+            await Promise.all(evicted.map(async (id) => {
+                try {
+                    const sandbox = await Sandbox.get({ sandboxId: id });
+                    await sandbox.stop();
+                }
+                catch {
+                    /* already stopped */
+                }
+            }));
+            console.info("[stealth] kept warm", { domain: opts.domain, sandboxId: handle.sandboxId, evicted: evicted.length });
+            return null;
+        }
+        catch {
+            /* registry unavailable — fall through to full stop */
+        }
+    }
     await stopRunner(handle);
     return null;
 }

package/dist/core/interact/vm.js

@@ -261,7 +261,16 @@ try {
           docs.push({ url, ok: false, error: String(e && e.message || e).split("\n")[0].slice(0, 300) });
         }
       }
-      data = { value: { entry, docs, solved: entry ? !entry.blocked : true, userAgent: ua } };
+      // Session capture: cookies + UA ride the result so the solved session can
+      // be replayed on the lightweight tier or the sandbox kept warm + reconnected.
+      let cookies = [];
+      try {
+        cookies = (await ctx.cookies())
+          .filter((c) => !c.expires || c.expires > Date.now() / 1000)
+          .slice(0, 50)
+          .map((c) => ({ name: c.name, value: c.value, domain: c.domain, path: c.path, expires: c.expires }));
+      } catch {}
+      data = { value: { entry, docs, solved: entry ? !entry.blocked : true, userAgent: ua, cookies } };
       break;
     }
     default:

package/dist/core/local-engine.js

@@ -10,7 +10,7 @@
  * powers both this local engine (`npx webcrawl`) and the hosted `webcrawl.app` API.
  */
 import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, mapWebsiteUrls, setCachedPage, sha256, } from "./engine.js";
-import { fetchResource } from "./engine/fetcher.js";
+import { fetchResource, lastFetchTier } from "./engine/fetcher.js";
 import { parseFile, publicParseMeta } from "./parse/index.js";
 import { isBinaryContentType, looksLikeBinaryUrl } from "./parse/fetch.js";
 import { maxDownloadBytes } from "./parse/limits.js";
@@ -174,7 +174,7 @@ export function createLocalEngineClient() {
                 },
                 evidence: {
                     ...targetInspection.evidence,
-                    tier: "impit-tls",
+                    tier: lastFetchTier() === "impersonate" ? "impersonate" : "impit-tls",
                     contentHash,
                     latencyMs: Date.now() - start,
                 },

package/dist/core/sandbox.js

@@ -1,5 +1,6 @@
 import { fastHtmlToMarkdown, setDomainCookie } from "./engine.js";
 import { isAntiBotChallenge } from "./engine/fetcher.js";
+import { getWarmSession, isDomainChallenged, markDomainChallenged } from "./engine/session.js";
 /** Bot-check copy that survives HTML → markdown conversion (Cloudflare `/markdown`). */
 const CHALLENGE_MARKDOWN = [
     "just a moment",
@@ -79,74 +80,93 @@ export async function escalateToSandboxOrBrowser(url, options = {}) {
     }
     // Option B: Vercel Sandbox microVM execution (`@vercel/sandbox`)
     if (vercelSandboxAvailable()) {
+        // Challenge memo: curl can never solve a JS challenge, so for domains known
+        // to serve one (or with a warm solved session) skip this tier entirely and
+        // save its ~4-6 s sandbox boot.
+        let host = "";
         try {
-            const { Sandbox } = await import("@vercel/sandbox");
-            const sandbox = await Sandbox.create({
-                timeout: options.timeoutMs ?? 20_000,
-            });
+            host = new URL(url).hostname;
+        }
+        catch {
+            /* fall through to curl */
+        }
+        const skipCurlTier = host
+            ? (await isDomainChallenged(host).catch(() => false)) || Boolean(await getWarmSession(host).catch(() => null))
+            : false;
+        if (!skipCurlTier) {
             try {
-                // Execute curl with browser cipher list + cookie capture inside the clean Linux microVM egress IP
-                const proxyArg = process.env.PROXY_URL ? `-x "${process.env.PROXY_URL}"` : "";
-                const cmd = await sandbox.runCommand({
-                    cmd: "curl",
-                    args: [
-                        "-sSL",
-                        "--compressed",
-                        "--max-time",
-                        "12",
-                        "-D",
-                        "/tmp/headers.txt",
-                        ...(proxyArg ? ["-x", process.env.PROXY_URL] : []),
-                        "-H",
-                        "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
-                        "-H",
-                        "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                        "-H",
-                        "Accept-Language: en-US,en;q=0.9",
-                        url,
-                    ],
+                const { Sandbox } = await import("@vercel/sandbox");
+                const sandbox = await Sandbox.create({
+                    timeout: options.timeoutMs ?? 20_000,
                 });
-                const html = await cmd.stdout();
                 try {
-                    const headersCmd = await sandbox.runCommand({
-                        cmd: "cat",
-                        args: ["/tmp/headers.txt"],
+                    // Execute curl with browser cipher list + cookie capture inside the clean Linux microVM egress IP
+                    const proxyArg = process.env.PROXY_URL ? `-x "${process.env.PROXY_URL}"` : "";
+                    const cmd = await sandbox.runCommand({
+                        cmd: "curl",
+                        args: [
+                            "-sSL",
+                            "--compressed",
+                            "--max-time",
+                            "12",
+                            "-D",
+                            "/tmp/headers.txt",
+                            ...(proxyArg ? ["-x", process.env.PROXY_URL] : []),
+                            "-H",
+                            "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+                            "-H",
+                            "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                            "-H",
+                            "Accept-Language: en-US,en;q=0.9",
+                            url,
+                        ],
                     });
-                    const headersText = await headersCmd.stdout();
-                    const setCookies = Array.from(headersText.matchAll(/^set-cookie:\s*([^;\r\n]+)/gim)).map((m) => m[1]);
-                    if (setCookies.length > 0) {
-                        setDomainCookie(new URL(url).hostname, setCookies.join("; "));
+                    const html = await cmd.stdout();
+                    try {
+                        const headersCmd = await sandbox.runCommand({
+                            cmd: "cat",
+                            args: ["/tmp/headers.txt"],
+                        });
+                        const headersText = await headersCmd.stdout();
+                        const setCookies = Array.from(headersText.matchAll(/^set-cookie:\s*([^;\r\n]+)/gim)).map((m) => m[1]);
+                        if (setCookies.length > 0) {
+                            setDomainCookie(new URL(url).hostname, setCookies.join("; "));
+                        }
                     }
-                }
-                catch {
-                    // ignore cookie extraction failure
-                }
-                // A challenge page is not content: never return it as a successful escalation.
-                // An empty shell (e.g. an AWS WAF 202 interstitial) is not content either:
-                // parse first and fall through to the browser tier instead.
-                if (html && html.length > 100 && !isAntiBotChallenge(html, 200)) {
-                    const parsed = fastHtmlToMarkdown(html, url, {
-                        retainImages: options.retainImages,
-                    });
-                    if (parsed.markdown.trim().length > 0) {
-                        return {
-                            html,
-                            markdown: parsed.markdown,
-                            title: parsed.title,
-                            links: parsed.links,
-                            statusCode: 200,
-                            tier: "sandbox",
-                        };
+                    catch {
+                        // ignore cookie extraction failure
+                    }
+                    // A challenge page is not content: never return it as a successful escalation.
+                    // An empty shell (e.g. an AWS WAF 202 interstitial) is not content either:
+                    // parse first and fall through to the browser tier instead.
+                    if (html && html.length > 100 && !isAntiBotChallenge(html, 200)) {
+                        const parsed = fastHtmlToMarkdown(html, url, {
+                            retainImages: options.retainImages,
+                        });
+                        if (parsed.markdown.trim().length > 0) {
+                            return {
+                                html,
+                                markdown: parsed.markdown,
+                                title: parsed.title,
+                                links: parsed.links,
+                                statusCode: 200,
+                                tier: "sandbox",
+                            };
+                        }
                     }
+                    // The clean-VM curl could not produce content: this domain serves a JS
+                    // challenge (or similar), so memo it and skip this tier next time.
+                    if (host)
+                        await markDomainChallenged(host).catch(() => { });
+                }
+                finally {
+                    await sandbox.stop().catch(() => { });
                 }
             }
-            finally {
-                await sandbox.stop().catch(() => { });
+            catch {
+                // Sandbox unavailable or not configured in this environment
             }
         }
-        catch {
-            // Sandbox unavailable or not configured in this environment
-        }
     }
     // Option C: real browser that waits out the bot check (opt-in, slow).
     if (options.browser) {

package/dist/core/targets/types.d.ts

@@ -6,7 +6,7 @@
  * and returns structured results paired with verifiable Evidence & Provenance.
  */
 export type TargetCategory = "web" | "documents" | "media" | "commerce" | "travel" | "real-estate" | "local" | "security" | "developer" | "academic";
-export type TargetTier = "cache" | "static" | "impit-tls" | "cloudflare" | "stealth-browser" | "sandbox" | "native-parser" | "ocr" | "api";
+export type TargetTier = "cache" | "static" | "impit-tls" | "impersonate" | "cloudflare" | "stealth-browser" | "sandbox" | "native-parser" | "ocr" | "api";
 export interface TargetEvidence {
     /** Which target plugin handled the inspection */
     targetId: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.2.8",
+  "version": "1.3.0",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,7 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.2.8",
+    "@webcrawl/client": "^1.3.0",
+    "@webcrawl/curl-impersonate": "^1.3.0",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",