webcrawl 1.2.7 → 1.2.9

package/dist/core/engine/awswaf.d.ts

@@ -1,5 +1,27 @@
-/** True when an HTML response (with optional status) is an AWS WAF challenge interstitial. */
-export declare function isAwsWafChallenge(html: string, status?: number): boolean;
+/**
+ * AWS WAF challenge handling (local prototype).
+ *
+ * `webcrawl`'s bot-check detector (`isAntiBotChallenge`) only matches
+ * Cloudflare and Vercel Security Checkpoint signatures. AWS WAF serves
+ * its invisible JS challenge as **HTTP 202** with an empty `<title>` and
+ * none of those markers, so such pages come back as
+ * `Empty page (HTTP 202)` with `blocked: false` and never reach the
+ * stealth browser. This module adds the missing detection plus a local
+ * stealth-routing path so those pages can be solved and read anyway.
+ *
+ * Detection markers are layered (no single one is proof):
+ *   - status 202 (documented Challenge interstitial status)
+ *   - `awsWafCookieDomainList` / `AwsWafIntegration` (documented JS API)
+ *   - `aws-waf-token` cookie reference
+ *   - `<div id="challenge-container">` + `/__challenge_<id>/.../challenge.js`
+ *
+ * The detection itself now lives in the shared detector
+ * (`isAwsWafChallenge` in `./fetcher.js`, also folded into
+ * `isAntiBotChallenge`); this module re-exports it and adds the local
+ * stealth-routing path so challenge pages can be solved and read anyway.
+ */
+import { isAwsWafChallenge } from "./fetcher.js";
+export { isAwsWafChallenge };
 /** True when a fetch response is any known bot check (upstream detector + AWS WAF). */
 export declare function isBlockedLike(html: string, status?: number): boolean;
 export interface AwsWafPage {

package/dist/core/engine/awswaf.js

@@ -14,40 +14,15 @@
  *   - `awsWafCookieDomainList` / `AwsWafIntegration` (documented JS API)
  *   - `aws-waf-token` cookie reference
  *   - `<div id="challenge-container">` + `/__challenge_<id>/.../challenge.js`
+ *
+ * The detection itself now lives in the shared detector
+ * (`isAwsWafChallenge` in `./fetcher.js`, also folded into
+ * `isAntiBotChallenge`); this module re-exports it and adds the local
+ * stealth-routing path so challenge pages can be solved and read anyway.
  */
-import { fetchWithTlsImpersonation, fastHtmlToMarkdown, isAntiBotChallenge } from "../engine.js";
-/** Markers that only appear on an AWS WAF challenge interstitial. */
-const AWS_WAF_MARKERS = [
-    "awswafcookiedomainlist",
-    "awswafintegration",
-    "aws-waf-token",
-    "id=\"challenge-container\"",
-    "/__challenge_",
-];
-/** True when an HTML response (with optional status) is an AWS WAF challenge interstitial. */
-export function isAwsWafChallenge(html, status) {
-    const lower = String(html ?? "").slice(0, 8000).toLowerCase();
-    const hasMarker = AWS_WAF_MARKERS.some((m) => lower.includes(m));
-    if (hasMarker)
-        return true;
-    // 202 alone is the documented Challenge status, but APIs can legitimately
-    // return 202 - only treat it as a challenge when the body is empty/challenge-ish.
-    if (status === 202) {
-        // 202 is the documented Challenge interstitial status. The interstitial's
-        // entire payload is <script>/<noscript>, so visible text is empty.
-        if (!lower.includes("<html"))
-            return false;
-        const visibleText = lower
-            .slice(0, 8000)
-            .replace(/<script[\s\S]*?<\/script>/g, "")
-            .replace(/<noscript[\s\S]*?<\/noscript>/g, "")
-            .replace(/<style[\s\S]*?<\/style>/g, "")
-            .replace(/<[^>]+>/g, " ")
-            .trim();
-        return visibleText.length === 0;
-    }
-    return false;
-}
+import { fetchWithTlsImpersonation, isAntiBotChallenge, isAwsWafChallenge } from "./fetcher.js";
+import { fastHtmlToMarkdown } from "./markdown.js";
+export { isAwsWafChallenge };
 /** True when a fetch response is any known bot check (upstream detector + AWS WAF). */
 export function isBlockedLike(html, status) {
     return isAntiBotChallenge(html ?? "", status ?? 200) || isAwsWafChallenge(html ?? "", status);

package/dist/core/engine/fetcher.d.ts

@@ -23,6 +23,19 @@ export interface ScrapeResult {
 }
 export declare function getBrowserHeaders(targetUrl: string, extraHeaders?: Record<string, string>): Record<string, string>;
 export declare function setDomainCookie(hostname: string, cookieHeader: string): void;
+/**
+ * User-Agent paired with a domain's replayed session cookies. Cloudflare's
+ * `cf_clearance` (and similar bound tokens) only validate when the exact UA
+ * that solved the challenge is reused, so replay sets both together.
+ */
+export declare function setDomainUserAgent(hostname: string, userAgent: string): void;
+/**
+ * True when an HTML response (with optional status) is an AWS WAF challenge
+ * interstitial: a marker match proves it regardless of status; otherwise a
+ * documented HTTP 202 whose payload is a `<script>`-only HTML shell with zero
+ * visible text (APIs legitimately return 202 too, so status alone is not proof).
+ */
+export declare function isAwsWafChallenge(html: string, status?: number): boolean;
 export declare function isAntiBotChallenge(html: string, status: number): boolean;
 /**
  * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)

package/dist/core/engine/fetcher.js

@@ -1,15 +1,16 @@
 import { followSafely, UnsafeUrlError } from "./safe-url.js";
 const domainCookies = new Map();
+const domainUserAgents = new Map();
 const BROWSER_USER_AGENTS = [
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
 ];
 export function getBrowserHeaders(targetUrl, extraHeaders) {
-    const ua = BROWSER_USER_AGENTS[Math.abs(hashCode(targetUrl)) % BROWSER_USER_AGENTS.length];
+    const parsed = new URL(targetUrl);
+    const ua = domainUserAgents.get(parsed.hostname) ?? BROWSER_USER_AGENTS[Math.abs(hashCode(targetUrl)) % BROWSER_USER_AGENTS.length];
     let hostCookie;
     try {
-        const parsed = new URL(targetUrl);
         hostCookie = domainCookies.get(parsed.hostname);
     }
     catch {
@@ -39,6 +40,16 @@ export function setDomainCookie(hostname, cookieHeader) {
         domainCookies.set(hostname, cookieHeader);
     }
 }
+/**
+ * User-Agent paired with a domain's replayed session cookies. Cloudflare's
+ * `cf_clearance` (and similar bound tokens) only validate when the exact UA
+ * that solved the challenge is reused, so replay sets both together.
+ */
+export function setDomainUserAgent(hostname, userAgent) {
+    if (hostname && userAgent) {
+        domainUserAgents.set(hostname, userAgent);
+    }
+}
 function hashCode(str) {
     let hash = 0;
     for (let i = 0; i < str.length; i++) {
@@ -47,18 +58,55 @@ function hashCode(str) {
     }
     return hash;
 }
+/** Markers that only appear on an AWS WAF challenge interstitial. */
+const AWS_WAF_CHALLENGE_MARKERS = [
+    "awswafcookiedomainlist",
+    "awswafintegration",
+    "aws-waf-token",
+    'id="challenge-container"',
+    "/__challenge_",
+];
+/**
+ * True when an HTML response (with optional status) is an AWS WAF challenge
+ * interstitial: a marker match proves it regardless of status; otherwise a
+ * documented HTTP 202 whose payload is a `<script>`-only HTML shell with zero
+ * visible text (APIs legitimately return 202 too, so status alone is not proof).
+ */
+export function isAwsWafChallenge(html, status) {
+    const lower = String(html ?? "").slice(0, 8000).toLowerCase();
+    if (AWS_WAF_CHALLENGE_MARKERS.some((m) => lower.includes(m)))
+        return true;
+    if (status === 202) {
+        if (!lower.includes("<html"))
+            return false;
+        const visibleText = lower
+            .slice(0, 8000)
+            .replace(/<script[\s\S]*?<\/script>/g, "")
+            .replace(/<noscript[\s\S]*?<\/noscript>/g, "")
+            .replace(/<style[\s\S]*?<\/style>/g, "")
+            .replace(/<[^>]+>/g, " ")
+            .trim();
+        return visibleText.length === 0;
+    }
+    return false;
+}
 export function isAntiBotChallenge(html, status) {
     if (status === 403 || status === 429 || status === 503) {
         return true;
     }
     const lower = html.slice(0, 6000).toLowerCase();
-    return (lower.includes("cf-browser-verification") ||
+    if (lower.includes("cf-browser-verification") ||
         lower.includes("challenge-platform") ||
         lower.includes("just a moment...") ||
         lower.includes("vercel security checkpoint") ||
         lower.includes("_vercel/security") ||
         lower.includes("enable javascript and cookies to continue") ||
-        lower.includes("_cf_chl_opt"));
+        lower.includes("_cf_chl_opt")) {
+        return true;
+    }
+    // AWS WAF serves its invisible JS challenge as HTTP 202 with none of the
+    // Cloudflare/Vercel markers, so those pages used to pass as real content.
+    return isAwsWafChallenge(html, status);
 }
 let sharedImpit = null;
 /** One request, redirects NOT followed (callers validate each hop via followSafely). */
@@ -104,6 +152,31 @@ async function rawGet(url, reqHeaders, timeoutMs, maxBytes) {
  * Every redirect hop is checked against the SSRF guard.
  */
 export async function fetchWithTlsImpersonation(url, timeoutMs = 12_000) {
+    // Session hydration: when a warm stealth session exists for this domain, its
+    // solved cookies (and the UA they were solved under — bound cookies require
+    // the pair) ride the request so the lightweight tier can skip the challenge.
+    let hostname = "";
+    try {
+        hostname = new URL(url).hostname;
+    }
+    catch {
+        /* invalid URL handled below */
+    }
+    if (hostname) {
+        try {
+            const { getWarmSession } = await import("./session.js");
+            const warm = await getWarmSession(hostname);
+            if (warm) {
+                if (warm.cookieHeader)
+                    setDomainCookie(hostname, warm.cookieHeader);
+                if (warm.userAgent)
+                    setDomainUserAgent(hostname, warm.userAgent);
+            }
+        }
+        catch {
+            /* session module unavailable — plain fetch */
+        }
+    }
     const { res, finalUrl } = await followSafely(url, (hop) => rawGet(hop, getBrowserHeaders(hop), timeoutMs));
     if (res.headers["set-cookie"]) {
         try {

package/dist/core/engine/index.d.ts

@@ -1,4 +1,5 @@
 export * from "./cache.js";
 export * from "./markdown.js";
+export * from "./session.js";
 export * from "./fetcher.js";
 export * from "./crawler.js";

package/dist/core/engine/index.js

@@ -1,4 +1,5 @@
 export * from "./cache.js";
 export * from "./markdown.js";
+export * from "./session.js";
 export * from "./fetcher.js";
 export * from "./crawler.js";

package/dist/core/engine/session.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Stealth session reuse: warm-sandbox registry, cookie-replay state and the
+ * JS-challenge memo.
+ *
+ * A solved bot check (AWS WAF, Cloudflare, Vercel checkpoint) leaves behind a
+ * valuable session: the sandbox's egress IP, its User-Agent and the cleared
+ * cookies. Instead of throwing that away with the sandbox, this module keeps:
+ *   - a warm registry (domain -> sandboxId + UA + cookies) so the next query
+ *     reconnects to the still-running sandbox (`Sandbox.get`) instead of
+ *     cold-booting and re-solving;
+ *   - the cookie header for replay on the lightweight TLS-impersonated tier
+ *     (works only where the provider does not bind tokens to the solver IP —
+ *     Cloudflare's `cf_clearance` does; AWS WAF's 300 s immunity may not);
+ *   - a memo of domains known to serve JS challenges, so the useless
+ *     curl-sandbox tier is skipped for them.
+ *
+ * Everything is in-memory by default; `registerExternalSessionStore` plugs in
+ * persistence (Postgres in the hosted app) so warm sessions survive across
+ * serverless instances. A registered store is always optional: every call
+ * degrades gracefully when it is unavailable.
+ */
+/** A solved stealth session kept for reuse. */
+export interface WarmSession {
+    domain: string;
+    sandboxId: string;
+    userAgent: string;
+    /** `Cookie:` header value for the solved session (UA-paired: replay both or neither). */
+    cookieHeader: string;
+    /** Epoch ms after which the entry is stale. */
+    expiresAt: number;
+}
+export interface ExternalSessionStore {
+    getWarmSession(domain: string): Promise<WarmSession | null>;
+    saveWarmSession(session: WarmSession): Promise<void>;
+    deleteWarmSession(domain: string): Promise<void>;
+    markDomainChallenged(domain: string): Promise<void>;
+    isDomainChallenged(domain: string): Promise<boolean>;
+}
+/** How long a solved session is offered for warm reconnect / cookie replay. */
+export declare const WARM_TTL_MS: number;
+/** VM `timeout` headroom set at Sandbox.create so a warm sandbox outlives its registry TTL. */
+export declare const WARM_SANDBOX_TIMEOUT_MS: number;
+/** Cap on concurrently warm sandboxes; LRU entries beyond the cap are evicted. */
+export declare const WARM_CAP = 4;
+/** How long a domain stays on the JS-challenge memo. */
+export declare const CHALLENGE_MEMO_TTL_MS: number;
+export declare function registerExternalSessionStore(store: ExternalSessionStore): void;
+export interface CookieLike {
+    name: string;
+    value: string;
+    domain?: string;
+    path?: string;
+    expires?: number;
+}
+/** Serializes captured browser cookies into a `Cookie:` header value (drops expired). */
+export declare function cookieHeaderFrom(cookies: CookieLike[]): string;
+/** Best-available warm session for a domain (in-memory first, then the external store). */
+export declare function getWarmSession(domain: string): Promise<WarmSession | null>;
+/**
+ * Registers a solved session as warm. Returns the sandboxIds evicted by the
+ * cap — the caller must stop those sandboxes so idle compute does not linger.
+ */
+export declare function saveWarmSession(session: WarmSession): Promise<string[]>;
+export declare function deleteWarmSession(domain: string): Promise<void>;
+/** Marks a domain as known to serve JS challenges (skips the curl tier next time). */
+export declare function markDomainChallenged(domain: string): Promise<void>;
+export declare function isDomainChallenged(domain: string): Promise<boolean>;
+/** Test helper: resets all in-memory state (external stores are untouched). */
+export declare function resetSessionStateForTests(): void;

package/dist/core/engine/session.js

@@ -0,0 +1,156 @@
+/**
+ * Stealth session reuse: warm-sandbox registry, cookie-replay state and the
+ * JS-challenge memo.
+ *
+ * A solved bot check (AWS WAF, Cloudflare, Vercel checkpoint) leaves behind a
+ * valuable session: the sandbox's egress IP, its User-Agent and the cleared
+ * cookies. Instead of throwing that away with the sandbox, this module keeps:
+ *   - a warm registry (domain -> sandboxId + UA + cookies) so the next query
+ *     reconnects to the still-running sandbox (`Sandbox.get`) instead of
+ *     cold-booting and re-solving;
+ *   - the cookie header for replay on the lightweight TLS-impersonated tier
+ *     (works only where the provider does not bind tokens to the solver IP —
+ *     Cloudflare's `cf_clearance` does; AWS WAF's 300 s immunity may not);
+ *   - a memo of domains known to serve JS challenges, so the useless
+ *     curl-sandbox tier is skipped for them.
+ *
+ * Everything is in-memory by default; `registerExternalSessionStore` plugs in
+ * persistence (Postgres in the hosted app) so warm sessions survive across
+ * serverless instances. A registered store is always optional: every call
+ * degrades gracefully when it is unavailable.
+ */
+/** How long a solved session is offered for warm reconnect / cookie replay. */
+export const WARM_TTL_MS = 8 * 60_000;
+/** VM `timeout` headroom set at Sandbox.create so a warm sandbox outlives its registry TTL. */
+export const WARM_SANDBOX_TIMEOUT_MS = 10 * 60_000;
+/** Cap on concurrently warm sandboxes; LRU entries beyond the cap are evicted. */
+export const WARM_CAP = 4;
+/** How long a domain stays on the JS-challenge memo. */
+export const CHALLENGE_MEMO_TTL_MS = 30 * 60_000;
+let external = null;
+export function registerExternalSessionStore(store) {
+    external = store;
+}
+const warm = new Map();
+const challenged = new Map();
+function reap(now = Date.now()) {
+    for (const [d, s] of warm)
+        if (s.expiresAt <= now)
+            warm.delete(d);
+    for (const [d, t] of challenged)
+        if (t <= now)
+            challenged.delete(d);
+}
+/** Serializes captured browser cookies into a `Cookie:` header value (drops expired). */
+export function cookieHeaderFrom(cookies) {
+    const now = Date.now() / 1000;
+    return cookies
+        .filter((c) => c.name && c.value && (!c.expires || c.expires > now))
+        .map((c) => `${c.name}=${c.value}`)
+        .join("; ");
+}
+/** Best-available warm session for a domain (in-memory first, then the external store). */
+export async function getWarmSession(domain) {
+    if (!domain)
+        return null;
+    reap();
+    const local = warm.get(domain);
+    if (local && local.expiresAt > Date.now())
+        return local;
+    if (local)
+        warm.delete(domain);
+    if (external) {
+        try {
+            const stored = await external.getWarmSession(domain);
+            if (stored && stored.expiresAt > Date.now()) {
+                warm.set(domain, stored);
+                return stored;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return null;
+}
+/**
+ * Registers a solved session as warm. Returns the sandboxIds evicted by the
+ * cap — the caller must stop those sandboxes so idle compute does not linger.
+ */
+export async function saveWarmSession(session) {
+    if (!session.domain)
+        return [];
+    reap();
+    warm.delete(session.domain);
+    warm.set(session.domain, { ...session });
+    const evicted = [];
+    while (warm.size > WARM_CAP) {
+        const [d, s] = warm.entries().next().value;
+        warm.delete(d);
+        if (s.sandboxId && s.sandboxId !== session.sandboxId)
+            evicted.push(s.sandboxId);
+    }
+    if (external) {
+        try {
+            await external.saveWarmSession(session);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return evicted;
+}
+export async function deleteWarmSession(domain) {
+    if (!domain)
+        return;
+    warm.delete(domain);
+    if (external) {
+        try {
+            await external.deleteWarmSession(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+/** Marks a domain as known to serve JS challenges (skips the curl tier next time). */
+export async function markDomainChallenged(domain) {
+    if (!domain)
+        return;
+    challenged.set(domain, Date.now() + CHALLENGE_MEMO_TTL_MS);
+    if (external) {
+        try {
+            await external.markDomainChallenged(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+export async function isDomainChallenged(domain) {
+    if (!domain)
+        return false;
+    reap();
+    const until = challenged.get(domain);
+    if (until && until > Date.now())
+        return true;
+    if (until)
+        challenged.delete(domain);
+    if (external) {
+        try {
+            if (await external.isDomainChallenged(domain)) {
+                challenged.set(domain, Date.now() + CHALLENGE_MEMO_TTL_MS);
+                return true;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return false;
+}
+/** Test helper: resets all in-memory state (external stores are untouched). */
+export function resetSessionStateForTests() {
+    warm.clear();
+    challenged.clear();
+}

package/dist/core/engine/stealth.d.ts

@@ -45,6 +45,8 @@ export declare function stealthFetch(entryUrl: string, items: StealthItem[], opt
     followSitemaps?: number;
     /** Allow the paid remote provider. Off for unbilled (anonymous) runs. Default true. */
     remote?: boolean;
+    /** Keep the solved session warm (sandbox left running + cookies saved) for repeat queries. */
+    keepWarm?: boolean;
 }): Promise<StealthResult>;
 /** Extracts `<loc>` entries from a sitemap or sitemap index. */
 export declare function parseSitemapLocs(xml: string): {

package/dist/core/engine/stealth.js

@@ -10,6 +10,8 @@
  * Never fabricates content: a page still showing a check comes back `blocked`.
  */
 import { isAntiBotChallenge } from "./fetcher.js";
+import { setDomainCookie, setDomainUserAgent } from "./fetcher.js";
+import { cookieHeaderFrom, getWarmSession, saveWarmSession, WARM_TTL_MS } from "./session.js";
 import { remoteStealthProvider } from "../interact/remote.js";
 /**
  * True when a real browser can run here: our own (sandbox in prod, local
@@ -37,19 +39,29 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
     }
     const { startBrowser, runBrowserStep, stopBrowser } = await import("../interact/driver.js");
     const provider = opts.remote === false ? null : remoteStealthProvider();
-    const tag = provider ? { provider } : {};
+    const domain = hostOf(entryUrl);
+    // A warm session for this domain means the challenge is already solved: short
+    // solve budget. A cold solve needs the full budget.
+    const hadWarm = provider ? null : await getWarmSession(domain).catch(() => null);
+    const tag = provider ? { provider } : { warm: Boolean(hadWarm) };
     let handle = null;
     // Filled in by `finally` once the provider reports the session cost.
     const out = {};
+    let capturedCookies = [];
+    let capturedUa = "";
     try {
         const budgetMs = opts.budgetMs ?? 90_000;
-        handle = await startBrowser({ timeoutMs: Math.min(5 * 60_000, budgetMs + 90_000), remote: provider });
+        handle = await startBrowser({
+            timeoutMs: Math.min(5 * 60_000, budgetMs + 90_000),
+            remote: provider,
+            domain: provider ? undefined : domain,
+        });
         const res = await runBrowserStep(handle, {
             type: "fetchMany",
             entry: entryUrl,
             items: items.slice(0, 40),
             budgetMs,
-            solveMs: opts.solveMs ?? 20_000,
+            solveMs: hadWarm ? Math.min(opts.solveMs ?? 20_000, 8_000) : (opts.solveMs ?? 20_000),
             followSitemaps: opts.followSitemaps ?? 0,
         });
         if (!res.ok) {
@@ -57,6 +69,13 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
         }
         else {
             const value = (res.data?.value ?? {});
+            capturedCookies = Array.isArray(value.cookies) ? value.cookies : [];
+            capturedUa = value.userAgent ?? "";
+            if (capturedUa)
+                setDomainUserAgent(domain, capturedUa);
+            if (capturedCookies.length > 0) {
+                setDomainCookie(domain, cookieHeaderFrom(capturedCookies));
+            }
             out.result = {
                 solved: Boolean(value.solved),
                 entry: value.entry ?? null,
@@ -70,9 +89,32 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
         out.result = { solved: false, entry: null, docs: [], durationMs: 0, error: err instanceof Error ? err.message : String(err), ...tag };
     }
     finally {
-        const cost = handle ? await stopBrowser(handle).catch(() => null) : null;
+        const solved = out.result?.solved === true;
+        const keepWarm = !provider && solved && (opts.keepWarm ?? true) && Boolean(handle && handle.kind === "sandbox" && capturedUa);
+        const cost = handle
+            ? await stopBrowser(handle, {
+                keepWarm,
+                domain,
+                userAgent: capturedUa,
+                cookieHeader: capturedCookies.length > 0 ? cookieHeaderFrom(capturedCookies) : "",
+            }).catch(() => null)
+            : null;
         if (out.result && cost)
             out.result.providerCostUsd = cost.costUsd;
+        if (!keepWarm && solved && capturedCookies.length > 0) {
+            // Even without a warm sandbox, remember the solved cookies for replay on
+            // the lightweight tier (UA-paired; rejection there just escalates).
+            await saveWarmSession({ domain, sandboxId: "", userAgent: capturedUa, cookieHeader: cookieHeaderFrom(capturedCookies), expiresAt: Date.now() + WARM_TTL_MS })
+                .catch(() => { });
+        }
+        console.info("[stealth] session result", {
+            domain,
+            solved,
+            warm: Boolean(hadWarm),
+            keepWarm,
+            cookieCount: capturedCookies.length,
+            durationMs: Date.now() - t0,
+        });
     }
     const result = out.result;
     result.durationMs = Date.now() - t0;
@@ -88,6 +130,15 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
     }
     return result;
 }
+/** Hostname of a URL, or "" when unparseable. */
+function hostOf(url) {
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return "";
+    }
+}
 /** Extracts `<loc>` entries from a sitemap or sitemap index. */
 export function parseSitemapLocs(xml) {
     const pages = [];

package/dist/core/interact/driver.d.ts

@@ -42,8 +42,21 @@ export declare function startBrowser(opts?: {
     timeoutMs?: number;
     userAgent?: string;
     remote?: RemoteProvider | null;
+    domain?: string;
 }): Promise<BrowserHandle>;
 export declare function runBrowserStep(handle: BrowserHandle, action: StepAction): Promise<StepResultLine>;
-/** Stops the browser. For a remote provider, returns what the session cost (null when unknown). */
-export declare function stopBrowser(handle: BrowserHandle): Promise<RemoteSessionCost | null>;
+/**
+ * Stops the browser. For a remote provider, returns what the session cost (null
+ * when unknown). With `keepWarm`, a sandbox browser is left running and
+ * registered (domain + sandboxId + UA) so the next call reconnects to it via
+ * `Sandbox.get` instead of cold-booting — the caller must pass `domain`.
+ * Evicted warm entries (over the cap) are stopped here so idle compute
+ * does not linger.
+ */
+export declare function stopBrowser(handle: BrowserHandle, opts?: {
+    keepWarm?: boolean;
+    domain?: string;
+    userAgent?: string;
+    cookieHeader?: string;
+}): Promise<RemoteSessionCost | null>;
 export { CDP_PORT };

package/dist/core/interact/driver.js

@@ -15,6 +15,7 @@ import path from "node:path";
 import { Sandbox } from "@vercel/sandbox";
 import { CDP_PORT, SETUP_SCRIPT, STEP_SCRIPT, STEP_SCRIPT_PATH, VM_ROOT, WAIT_FOR_CDP_SCRIPT, chromeArgs, parseStepOutput, } from "./vm.js";
 import { startRemoteSession, stopRemoteSession } from "./remote.js";
+import { WARM_SANDBOX_TIMEOUT_MS, WARM_TTL_MS } from "../engine/session.js";
 const snapshotId = () => process.env.WEBCRAWL_INTERACT_SNAPSHOT_ID || process.env.INTERACT_SNAPSHOT_ID || "";
 const localMode = () => process.env.INTERACT_LOCAL_PLAYWRIGHT === "true";
 /**
@@ -91,6 +92,44 @@ async function startSandboxBrowser(opts) {
         throw err;
     }
 }
+/** True when the sandbox's Chromium still answers on CDP (warm reconnect check). */
+async function sandboxChromeAlive(sandbox) {
+    const ping = await sh(sandbox, `curl -sf -m 3 http://127.0.0.1:${CDP_PORT}/json/version >/dev/null && echo alive`, false);
+    return ping.exitCode === 0 && ping.stdout.includes("alive");
+}
+/**
+ * Warm reconnect: fetch the sandbox this domain solved its challenge in, verify
+ * Chromium still lives, and reuse it (same IP, same UA, same cleared cookies).
+ * Returns null when no warm entry exists or the sandbox is gone (cold boot).
+ */
+async function startWarmSandboxBrowser(domain) {
+    if (!domain)
+        return null;
+    const { getWarmSession } = await import("../engine/session.js");
+    let warm;
+    try {
+        warm = await getWarmSession(domain);
+    }
+    catch {
+        return null;
+    }
+    if (!warm || !warm.sandboxId)
+        return null;
+    try {
+        const sandbox = await Sandbox.get({ sandboxId: warm.sandboxId });
+        if (!(await sandboxChromeAlive(sandbox))) {
+            await sandbox.stop().catch(() => { });
+            return null;
+        }
+        // Refresh the step runner in case this instance ships newer code.
+        await sandbox.writeFiles([{ path: STEP_SCRIPT_PATH, content: Buffer.from(STEP_SCRIPT) }]).catch(() => { });
+        console.info("[stealth] warm reconnect", { domain, sandboxId: warm.sandboxId });
+        return { kind: "sandbox", sandboxId: warm.sandboxId };
+    }
+    catch {
+        return null;
+    }
+}
 /* ------------------------------ local backend ----------------------------- */
 async function resolveLocalPlaywright() {
     const candidates = [
@@ -213,7 +252,18 @@ export async function startBrowser(opts = {}) {
         return startLocalBrowser(opts);
     if (!interactBrowserAvailable())
         throw new Error("browser_unavailable");
-    return startSandboxBrowser({ timeoutMs: opts.timeoutMs ?? 5 * 60_000, userAgent: opts.userAgent });
+    // Warm reconnect first: a sandbox that already solved this domain's bot check
+    // keeps its IP / UA / cleared cookies — reusing it skips boot + re-solve.
+    if (opts.domain) {
+        const warm = await startWarmSandboxBrowser(opts.domain).catch(() => null);
+        if (warm)
+            return warm;
+    }
+    return startSandboxBrowser({
+        // Long VM timeout headroom so the sandbox can stay warm after this call.
+        timeoutMs: Math.max(opts.timeoutMs ?? 5 * 60_000, WARM_SANDBOX_TIMEOUT_MS),
+        userAgent: opts.userAgent,
+    });
 }
 function runLocalStep(scriptPath, playwright, cdp, action, extraEnv = {}) {
     return new Promise((resolve) => {
@@ -243,8 +293,15 @@ export async function runBrowserStep(handle, action) {
     }
     return runSandboxStep(handle.sandboxId, action);
 }
-/** Stops the browser. For a remote provider, returns what the session cost (null when unknown). */
-export async function stopBrowser(handle) {
+/**
+ * Stops the browser. For a remote provider, returns what the session cost (null
+ * when unknown). With `keepWarm`, a sandbox browser is left running and
+ * registered (domain + sandboxId + UA) so the next call reconnects to it via
+ * `Sandbox.get` instead of cold-booting — the caller must pass `domain`.
+ * Evicted warm entries (over the cap) are stopped here so idle compute
+ * does not linger.
+ */
+export async function stopBrowser(handle, opts = {}) {
     if (handle.kind === "local") {
         try {
             process.kill(handle.pid);
@@ -258,6 +315,32 @@ export async function stopBrowser(handle) {
         const [cost] = await Promise.all([stopRemoteSession(handle), stopRunner(handle.runner)]);
         return cost;
     }
+    if (opts.keepWarm && opts.domain) {
+        const { saveWarmSession } = await import("../engine/session.js");
+        try {
+            const evicted = await saveWarmSession({
+                domain: opts.domain,
+                sandboxId: handle.sandboxId,
+                userAgent: opts.userAgent ?? "",
+                cookieHeader: opts.cookieHeader ?? "",
+                expiresAt: Date.now() + WARM_TTL_MS,
+            });
+            await Promise.all(evicted.map(async (id) => {
+                try {
+                    const sandbox = await Sandbox.get({ sandboxId: id });
+                    await sandbox.stop();
+                }
+                catch {
+                    /* already stopped */
+                }
+            }));
+            console.info("[stealth] kept warm", { domain: opts.domain, sandboxId: handle.sandboxId, evicted: evicted.length });
+            return null;
+        }
+        catch {
+            /* registry unavailable — fall through to full stop */
+        }
+    }
     await stopRunner(handle);
     return null;
 }

package/dist/core/interact/vm.js

@@ -155,7 +155,7 @@ try {
       const PER_DOC = 1500000;
       const TOTAL = 10000000;
       const started = Date.now();
-      const markers = ["vercel security checkpoint", "just a moment...", "cf-browser-verification", "challenge-platform", "_cf_chl_opt", "enable javascript and cookies to continue", "_vercel/security", "attention required! | cloudflare"];
+      const markers = ["vercel security checkpoint", "just a moment...", "cf-browser-verification", "challenge-platform", "_cf_chl_opt", "enable javascript and cookies to continue", "_vercel/security", "attention required! | cloudflare", "awswafcookiedomainlist", "awswafintegration", "aws-waf-token", 'id="challenge-container"', "/__challenge_"];
       const looksBlocked = (s) => { const l = String(s || "").slice(0, 8000).toLowerCase(); return markers.some((m) => l.includes(m)); };
       let ua = "";
       try {
@@ -261,7 +261,16 @@ try {
           docs.push({ url, ok: false, error: String(e && e.message || e).split("\n")[0].slice(0, 300) });
         }
       }
-      data = { value: { entry, docs, solved: entry ? !entry.blocked : true, userAgent: ua } };
+      // Session capture: cookies + UA ride the result so the solved session can
+      // be replayed on the lightweight tier or the sandbox kept warm + reconnected.
+      let cookies = [];
+      try {
+        cookies = (await ctx.cookies())
+          .filter((c) => !c.expires || c.expires > Date.now() / 1000)
+          .slice(0, 50)
+          .map((c) => ({ name: c.name, value: c.value, domain: c.domain, path: c.path, expires: c.expires }));
+      } catch {}
+      data = { value: { entry, docs, solved: entry ? !entry.blocked : true, userAgent: ua, cookies } };
       break;
     }
     default:

package/dist/core/sandbox.js

@@ -1,5 +1,6 @@
 import { fastHtmlToMarkdown, setDomainCookie } from "./engine.js";
 import { isAntiBotChallenge } from "./engine/fetcher.js";
+import { getWarmSession, isDomainChallenged, markDomainChallenged } from "./engine/session.js";
 /** Bot-check copy that survives HTML → markdown conversion (Cloudflare `/markdown`). */
 const CHALLENGE_MARKDOWN = [
     "just a moment",
@@ -79,70 +80,93 @@ export async function escalateToSandboxOrBrowser(url, options = {}) {
     }
     // Option B: Vercel Sandbox microVM execution (`@vercel/sandbox`)
     if (vercelSandboxAvailable()) {
+        // Challenge memo: curl can never solve a JS challenge, so for domains known
+        // to serve one (or with a warm solved session) skip this tier entirely and
+        // save its ~4-6 s sandbox boot.
+        let host = "";
         try {
-            const { Sandbox } = await import("@vercel/sandbox");
-            const sandbox = await Sandbox.create({
-                timeout: options.timeoutMs ?? 20_000,
-            });
+            host = new URL(url).hostname;
+        }
+        catch {
+            /* fall through to curl */
+        }
+        const skipCurlTier = host
+            ? (await isDomainChallenged(host).catch(() => false)) || Boolean(await getWarmSession(host).catch(() => null))
+            : false;
+        if (!skipCurlTier) {
             try {
-                // Execute curl with browser cipher list + cookie capture inside the clean Linux microVM egress IP
-                const proxyArg = process.env.PROXY_URL ? `-x "${process.env.PROXY_URL}"` : "";
-                const cmd = await sandbox.runCommand({
-                    cmd: "curl",
-                    args: [
-                        "-sSL",
-                        "--compressed",
-                        "--max-time",
-                        "12",
-                        "-D",
-                        "/tmp/headers.txt",
-                        ...(proxyArg ? ["-x", process.env.PROXY_URL] : []),
-                        "-H",
-                        "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
-                        "-H",
-                        "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                        "-H",
-                        "Accept-Language: en-US,en;q=0.9",
-                        url,
-                    ],
+                const { Sandbox } = await import("@vercel/sandbox");
+                const sandbox = await Sandbox.create({
+                    timeout: options.timeoutMs ?? 20_000,
                 });
-                const html = await cmd.stdout();
                 try {
-                    const headersCmd = await sandbox.runCommand({
-                        cmd: "cat",
-                        args: ["/tmp/headers.txt"],
+                    // Execute curl with browser cipher list + cookie capture inside the clean Linux microVM egress IP
+                    const proxyArg = process.env.PROXY_URL ? `-x "${process.env.PROXY_URL}"` : "";
+                    const cmd = await sandbox.runCommand({
+                        cmd: "curl",
+                        args: [
+                            "-sSL",
+                            "--compressed",
+                            "--max-time",
+                            "12",
+                            "-D",
+                            "/tmp/headers.txt",
+                            ...(proxyArg ? ["-x", process.env.PROXY_URL] : []),
+                            "-H",
+                            "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+                            "-H",
+                            "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                            "-H",
+                            "Accept-Language: en-US,en;q=0.9",
+                            url,
+                        ],
                     });
-                    const headersText = await headersCmd.stdout();
-                    const setCookies = Array.from(headersText.matchAll(/^set-cookie:\s*([^;\r\n]+)/gim)).map((m) => m[1]);
-                    if (setCookies.length > 0) {
-                        setDomainCookie(new URL(url).hostname, setCookies.join("; "));
+                    const html = await cmd.stdout();
+                    try {
+                        const headersCmd = await sandbox.runCommand({
+                            cmd: "cat",
+                            args: ["/tmp/headers.txt"],
+                        });
+                        const headersText = await headersCmd.stdout();
+                        const setCookies = Array.from(headersText.matchAll(/^set-cookie:\s*([^;\r\n]+)/gim)).map((m) => m[1]);
+                        if (setCookies.length > 0) {
+                            setDomainCookie(new URL(url).hostname, setCookies.join("; "));
+                        }
                     }
+                    catch {
+                        // ignore cookie extraction failure
+                    }
+                    // A challenge page is not content: never return it as a successful escalation.
+                    // An empty shell (e.g. an AWS WAF 202 interstitial) is not content either:
+                    // parse first and fall through to the browser tier instead.
+                    if (html && html.length > 100 && !isAntiBotChallenge(html, 200)) {
+                        const parsed = fastHtmlToMarkdown(html, url, {
+                            retainImages: options.retainImages,
+                        });
+                        if (parsed.markdown.trim().length > 0) {
+                            return {
+                                html,
+                                markdown: parsed.markdown,
+                                title: parsed.title,
+                                links: parsed.links,
+                                statusCode: 200,
+                                tier: "sandbox",
+                            };
+                        }
+                    }
+                    // The clean-VM curl could not produce content: this domain serves a JS
+                    // challenge (or similar), so memo it and skip this tier next time.
+                    if (host)
+                        await markDomainChallenged(host).catch(() => { });
                 }
-                catch {
-                    // ignore cookie extraction failure
-                }
-                // A challenge page is not content: never return it as a successful escalation.
-                if (html && html.length > 100 && !isAntiBotChallenge(html, 200)) {
-                    const parsed = fastHtmlToMarkdown(html, url, {
-                        retainImages: options.retainImages,
-                    });
-                    return {
-                        html,
-                        markdown: parsed.markdown,
-                        title: parsed.title,
-                        links: parsed.links,
-                        statusCode: 200,
-                        tier: "sandbox",
-                    };
+                finally {
+                    await sandbox.stop().catch(() => { });
                 }
             }
-            finally {
-                await sandbox.stop().catch(() => { });
+            catch {
+                // Sandbox unavailable or not configured in this environment
             }
         }
-        catch {
-            // Sandbox unavailable or not configured in this environment
-        }
     }
     // Option C: real browser that waits out the bot check (opt-in, slow).
     if (options.browser) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.2.7",
+  "version": "1.2.9",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,7 +127,7 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.2.7",
+    "@webcrawl/client": "^1.2.9",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",