webcrawl 1.3.1 → 1.3.3

package/dist/core/engine/fetcher.d.ts

@@ -17,7 +17,7 @@ export interface ScrapeResult {
     links?: string[];
     statusCode: number;
     contentHash: string;
-    tier: "cache" | "static" | "sandbox" | "cloudflare" | "browser" | "impersonate";
+    tier: "cache" | "static" | "sandbox" | "cloudflare" | "browser" | "impersonate" | "proxy-impersonate";
     latencyMs: number;
     error?: string;
 }
@@ -38,7 +38,18 @@ export declare function setDomainUserAgent(hostname: string, userAgent: string):
 export declare function isAwsWafChallenge(html: string, status?: number): boolean;
 export declare function isAntiBotChallenge(html: string, status: number): boolean;
 /** Transport tier that won the last fetch ladder run (for result evidence). */
-export declare function lastFetchTier(): "impit-tls" | "impersonate" | "fetch";
+export declare function lastFetchTier(): "impit-tls" | "impersonate" | "proxy-impersonate" | "fetch";
+/** Bytes transferred through the residential proxy on the last ladder run (bandwidth accounting). */
+export declare function lastProxyBytesCount(): number;
+/**
+ * Residential proxy URL from env, or null when the tier is disabled.
+ * Generic: a full `WEBCRAWL_PROXY_URL` (`http://user:pass@host:port`) or the
+ * split `WEBCRAWL_PROXY_USER` + `WEBCRAWL_PROXY_PWD` pair, with the gateway
+ * host/port overridable via `WEBCRAWL_PROXY_HOST` / `WEBCRAWL_PROXY_PORT`.
+ * `DECODO_*` equivalents are honored as a legacy fallback so existing
+ * deployments keep working (their default gateway is `gate.decodo.com:7000`).
+ */
+export declare function residentialProxyUrl(): string | null;
 /**
  * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)sonation)
  * so Cloudflare Bot Management and Vercel Security Checkpoint treat requests as real Chrome.

package/dist/core/engine/fetcher.js

@@ -1,4 +1,5 @@
 import { followSafely, UnsafeUrlError } from "./safe-url.js";
+import { isDomainProxyNeeded, markDomainProxyNeeded, } from "./session.js";
 const domainCookies = new Map();
 const domainUserAgents = new Map();
 const BROWSER_USER_AGENTS = [
@@ -119,12 +120,39 @@ export function isAntiBotChallenge(html, status) {
     return isAwsWafChallenge(html, status);
 }
 let sharedImpit = null;
-/** Tier that won the most recent rawGet ladder ("impit-tls" → "impersonate" → "fetch"). */
+/** Tier that won the most recent rawGet ladder ("impit-tls" → "impersonate" → "proxy-impersonate" → "fetch"). */
 let lastTier = "impit-tls";
+/** Bytes pulled through the residential proxy on the most recent ladder run. */
+let lastProxyBytes = 0;
 /** Transport tier that won the last fetch ladder run (for result evidence). */
 export function lastFetchTier() {
     return lastTier;
 }
+/** Bytes transferred through the residential proxy on the last ladder run (bandwidth accounting). */
+export function lastProxyBytesCount() {
+    return lastProxyBytes;
+}
+/**
+ * Residential proxy URL from env, or null when the tier is disabled.
+ * Generic: a full `WEBCRAWL_PROXY_URL` (`http://user:pass@host:port`) or the
+ * split `WEBCRAWL_PROXY_USER` + `WEBCRAWL_PROXY_PWD` pair, with the gateway
+ * host/port overridable via `WEBCRAWL_PROXY_HOST` / `WEBCRAWL_PROXY_PORT`.
+ * `DECODO_*` equivalents are honored as a legacy fallback so existing
+ * deployments keep working (their default gateway is `gate.decodo.com:7000`).
+ */
+export function residentialProxyUrl() {
+    const direct = process.env.WEBCRAWL_PROXY_URL?.trim() || process.env.DECODO_URL?.trim();
+    if (direct)
+        return direct;
+    const user = process.env.WEBCRAWL_PROXY_USER?.trim() || process.env.DECODO_USER?.trim();
+    const pwd = process.env.WEBCRAWL_PROXY_PWD?.trim() || process.env.DECODO_PWD?.trim();
+    if (user && pwd) {
+        const host = process.env.WEBCRAWL_PROXY_HOST?.trim() || "gate.decodo.com";
+        const port = process.env.WEBCRAWL_PROXY_PORT?.trim() || "7000";
+        return `http://${encodeURIComponent(user)}:${encodeURIComponent(pwd)}@${host}:${port}`;
+    }
+    return null;
+}
 /** One request, redirects NOT followed (callers validate each hop via followSafely). */
 async function rawGet(url, reqHeaders, timeoutMs, maxBytes) {
     const read = async (status, headers, getBuf) => {
@@ -133,6 +161,25 @@ async function rawGet(url, reqHeaders, timeoutMs, maxBytes) {
             checkDeclaredLength(headers, maxBytes);
         return { status, headers, body: isRedirect ? new ArrayBuffer(0) : await getBuf() };
     };
+    lastProxyBytes = 0;
+    // Tier P1 fast path: domains memoized as IP-blocked (DataDome) skip the
+    // datacenter attempts entirely and go straight to the residential proxy —
+    // saves latency and never burns proxy bytes on a doomed direct hop first.
+    const proxyUrl = residentialProxyUrl();
+    let host = "";
+    try {
+        host = new URL(url).hostname;
+    }
+    catch {
+        /* let the direct hops surface the URL error */
+    }
+    if (proxyUrl && host && (await isDomainProxyNeeded(host).catch(() => false))) {
+        const proxyHop = await rawGetImpersonateProxy(url, reqHeaders, timeoutMs, maxBytes, proxyUrl);
+        if (proxyHop) {
+            lastTier = "proxy-impersonate";
+            return proxyHop;
+        }
+    }
     try {
         if (!sharedImpit) {
             const impitMod = await import("impit");
@@ -155,6 +202,18 @@ async function rawGet(url, reqHeaders, timeoutMs, maxBytes) {
         if (isAntiBotChallenge(new TextDecoder().decode(hop.body).slice(0, 8000), hop.status)) {
             const impersonateHop = await rawGetImpersonate(url, reqHeaders, timeoutMs, maxBytes);
             if (impersonateHop) {
+                if (isAntiBotChallenge(new TextDecoder().decode(impersonateHop.body).slice(0, 8000), impersonateHop.status)) {
+                    // Direct datacenter egress is IP-blocked (DataDome): last cheap try
+                    // is the residential proxy hop before browser escalation. Costs
+                    // ~100-300KB of proxy bandwidth instead of a 5-15MB page load.
+                    if (proxyUrl) {
+                        const proxyHop = await rawGetImpersonateProxy(url, reqHeaders, timeoutMs, maxBytes, proxyUrl);
+                        if (proxyHop) {
+                            lastTier = "proxy-impersonate";
+                            return proxyHop;
+                        }
+                    }
+                }
                 lastTier = "impersonate";
                 return impersonateHop;
             }
@@ -232,6 +291,71 @@ async function rawGetImpersonate(url, reqHeaders, timeoutMs, maxBytes) {
         return null; // tier unavailable or fetch failed — caller falls back
     }
 }
+/**
+ * Tier P1 — one GET hop with curl-impersonate through the residential proxy
+ * (Decodo or any `http://user:pass@host:port` HTTP proxy). The target sees a
+ * clean residential egress IP while the TLS/HTTP2 fingerprint stays a real
+ * Chrome's. Only reached for domains whose datacenter egress is proven
+ * IP-blocked, so the domain is memoized and later ladder runs skip the
+ * doomed direct attempts entirely. Gzipped HTML only (~100-300KB/request);
+ * a 2MB byte cap keeps a runaway response from draining proxy bandwidth.
+ */
+async function rawGetImpersonateProxy(url, reqHeaders, timeoutMs, maxBytes, proxyUrl) {
+    let host = "";
+    try {
+        host = new URL(url).hostname;
+    }
+    catch {
+        /* URL errors surface via the direct hops */
+    }
+    if (host)
+        await markDomainProxyNeeded(host).catch(() => { });
+    let impersonate;
+    try {
+        impersonate = await import("@webcrawl/curl-impersonate");
+    }
+    catch {
+        return null;
+    }
+    // Same identity-header hygiene as the direct impersonate hop: the profile's
+    // UA/sec-ch headers must survive or the UA/TLS mismatch is what DataDome
+    // blocks even from a clean IP.
+    const extra = {};
+    for (const [key, value] of Object.entries(reqHeaders)) {
+        const lower = key.toLowerCase();
+        if (lower === "user-agent" ||
+            lower.startsWith("sec-ch-ua") ||
+            lower.startsWith("sec-fetch-") ||
+            lower === "upgrade-insecure-requests" ||
+            lower === "priority") {
+            continue;
+        }
+        extra[key] = value;
+    }
+    try {
+        const res = await impersonate.impersonateFetch(url, {
+            headers: extra,
+            timeoutMs,
+            maxBytes: maxBytes ?? 2 * 1024 * 1024,
+            proxy: proxyUrl,
+        });
+        lastProxyBytes += res.bytes;
+        const headers = res.headers;
+        const isRedirect = res.status >= 300 && res.status < 400 && !!headers["location"];
+        if (!isRedirect && maxBytes !== undefined)
+            checkDeclaredLength(headers, maxBytes);
+        return {
+            status: res.status,
+            headers,
+            body: isRedirect ? new ArrayBuffer(0) : res.body.buffer,
+        };
+    }
+    catch (err) {
+        if (err instanceof UnsafeUrlError)
+            throw err;
+        return null; // proxy tier unavailable or fetch failed — caller escalates
+    }
+}
 /**
  * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)sonation)
  * so Cloudflare Bot Management and Vercel Security Checkpoint treat requests as real Chrome.

package/dist/core/engine/session.d.ts

@@ -35,6 +35,9 @@ export interface ExternalSessionStore {
     deleteWarmSession(domain: string): Promise<void>;
     markDomainChallenged(domain: string): Promise<void>;
     isDomainChallenged(domain: string): Promise<boolean>;
+    /** Optional persistence for the residential-proxy memo (Tier P1). */
+    markDomainProxyNeeded?(domain: string): Promise<void>;
+    isDomainProxyNeeded?(domain: string): Promise<boolean>;
 }
 /** How long a solved session is offered for warm reconnect / cookie replay. */
 export declare const WARM_TTL_MS: number;
@@ -44,6 +47,8 @@ export declare const WARM_SANDBOX_TIMEOUT_MS: number;
 export declare const WARM_CAP = 4;
 /** How long a domain stays on the JS-challenge memo. */
 export declare const CHALLENGE_MEMO_TTL_MS: number;
+/** How long a domain stays on the residential-proxy memo (Tier P1). */
+export declare const PROXY_MEMO_TTL_MS: number;
 export declare function registerExternalSessionStore(store: ExternalSessionStore): void;
 export interface CookieLike {
     name: string;
@@ -65,5 +70,12 @@ export declare function deleteWarmSession(domain: string): Promise<void>;
 /** Marks a domain as known to serve JS challenges (skips the curl tier next time). */
 export declare function markDomainChallenged(domain: string): Promise<void>;
 export declare function isDomainChallenged(domain: string): Promise<boolean>;
+/**
+ * Marks a domain as needing the residential-proxy tier (Tier P1): every
+ * datacenter egress attempt is blocked by IP reputation (DataDome et al), so
+ * future ladder runs skip straight to the proxy hop.
+ */
+export declare function markDomainProxyNeeded(domain: string): Promise<void>;
+export declare function isDomainProxyNeeded(domain: string): Promise<boolean>;
 /** Test helper: resets all in-memory state (external stores are untouched). */
 export declare function resetSessionStateForTests(): void;

package/dist/core/engine/session.js

@@ -27,12 +27,15 @@ export const WARM_SANDBOX_TIMEOUT_MS = 10 * 60_000;
 export const WARM_CAP = 4;
 /** How long a domain stays on the JS-challenge memo. */
 export const CHALLENGE_MEMO_TTL_MS = 30 * 60_000;
+/** How long a domain stays on the residential-proxy memo (Tier P1). */
+export const PROXY_MEMO_TTL_MS = 60 * 60_000;
 let external = null;
 export function registerExternalSessionStore(store) {
     external = store;
 }
 const warm = new Map();
 const challenged = new Map();
+const proxyNeeded = new Map();
 function reap(now = Date.now()) {
     for (const [d, s] of warm)
         if (s.expiresAt <= now)
@@ -40,6 +43,9 @@ function reap(now = Date.now()) {
     for (const [d, t] of challenged)
         if (t <= now)
             challenged.delete(d);
+    for (const [d, t] of proxyNeeded)
+        if (t <= now)
+            proxyNeeded.delete(d);
 }
 /** Serializes captured browser cookies into a `Cookie:` header value (drops expired). */
 export function cookieHeaderFrom(cookies) {
@@ -149,8 +155,49 @@ export async function isDomainChallenged(domain) {
     }
     return false;
 }
+/**
+ * Marks a domain as needing the residential-proxy tier (Tier P1): every
+ * datacenter egress attempt is blocked by IP reputation (DataDome et al), so
+ * future ladder runs skip straight to the proxy hop.
+ */
+export async function markDomainProxyNeeded(domain) {
+    if (!domain)
+        return;
+    proxyNeeded.set(domain, Date.now() + PROXY_MEMO_TTL_MS);
+    if (external?.markDomainProxyNeeded) {
+        try {
+            await external.markDomainProxyNeeded(domain);
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+}
+export async function isDomainProxyNeeded(domain) {
+    if (!domain)
+        return false;
+    reap();
+    const until = proxyNeeded.get(domain);
+    if (until && until > Date.now())
+        return true;
+    if (until)
+        proxyNeeded.delete(domain);
+    if (external?.isDomainProxyNeeded) {
+        try {
+            if (await external.isDomainProxyNeeded(domain)) {
+                proxyNeeded.set(domain, Date.now() + PROXY_MEMO_TTL_MS);
+                return true;
+            }
+        }
+        catch {
+            /* store unavailable */
+        }
+    }
+    return false;
+}
 /** Test helper: resets all in-memory state (external stores are untouched). */
 export function resetSessionStateForTests() {
     warm.clear();
     challenged.clear();
+    proxyNeeded.clear();
 }

package/dist/core/engine/stealth.d.ts

@@ -47,6 +47,8 @@ export declare function stealthFetch(entryUrl: string, items: StealthItem[], opt
     remote?: boolean;
     /** Keep the solved session warm (sandbox left running + cookies saved) for repeat queries. */
     keepWarm?: boolean;
+    /** Tier P2: egress the stealth browser through this residential proxy URL. */
+    proxy?: string;
 }): Promise<StealthResult>;
 /** Extracts `<loc>` entries from a sitemap or sitemap index. */
 export declare function parseSitemapLocs(xml: string): {

package/dist/core/engine/stealth.js

@@ -40,9 +40,23 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
     const { startBrowser, runBrowserStep, stopBrowser } = await import("../interact/driver.js");
     const provider = opts.remote === false ? null : remoteStealthProvider();
     const domain = hostOf(entryUrl);
+    const proxyUrl = opts.proxy ?? null;
+    let proxyUser = "";
+    let proxyPass = "";
+    if (proxyUrl) {
+        try {
+            const parsed = new URL(proxyUrl);
+            proxyUser = decodeURIComponent(parsed.username);
+            proxyPass = decodeURIComponent(parsed.password);
+        }
+        catch {
+            /* bad proxy URL: browser launch will surface the error */
+        }
+    }
     // A warm session for this domain means the challenge is already solved: short
-    // solve budget. A cold solve needs the full budget.
-    const hadWarm = provider ? null : await getWarmSession(domain).catch(() => null);
+    // solve budget. A cold solve needs the full budget. Proxy runs never reuse
+    // warm sessions (their cookies were earned from a datacenter egress IP).
+    const hadWarm = provider || proxyUrl ? null : await getWarmSession(domain).catch(() => null);
     const tag = provider ? { provider } : { warm: Boolean(hadWarm) };
     let handle = null;
     // Filled in by `finally` once the provider reports the session cost.
@@ -54,7 +68,8 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
         handle = await startBrowser({
             timeoutMs: Math.min(5 * 60_000, budgetMs + 90_000),
             remote: provider,
-            domain: provider ? undefined : domain,
+            domain: provider || proxyUrl ? undefined : domain,
+            proxy: proxyUrl ?? undefined,
         });
         const res = await runBrowserStep(handle, {
             type: "fetchMany",
@@ -63,6 +78,7 @@ export async function stealthFetch(entryUrl, items, opts = {}) {
             budgetMs,
             solveMs: hadWarm ? Math.min(opts.solveMs ?? 20_000, 8_000) : (opts.solveMs ?? 20_000),
             followSitemaps: opts.followSitemaps ?? 0,
+            ...(proxyUser ? { proxyUser, proxyPass, proxyByteCap: 6_000_000 } : {}),
         });
         if (!res.ok) {
             out.result = { solved: false, entry: null, docs: [], durationMs: 0, error: res.error || "Stealth browser step failed", ...tag };

package/dist/core/interact/driver.d.ts

@@ -38,11 +38,21 @@ export type StepAction = Record<string, unknown> & {
  */
 export declare function interactBrowserAvailable(): boolean;
 export declare function interactBackend(): "local" | "sandbox" | "none";
+/**
+ * Path to a real, user-installed Chrome when one exists, else null.
+ * Anti-bot vendors (DataDome, Akamai) fingerprint the automation build of
+ * Chromium and serve degraded pages to it, while the same engine driving a
+ * real installed Chrome gets full content — practitioner reports confirm
+ * this on DataDome-protected sites. Priority: explicit env override, then
+ * the standard per-OS install locations.
+ */
+export declare function installedChromePath(): string | null;
 export declare function startBrowser(opts?: {
     timeoutMs?: number;
     userAgent?: string;
     remote?: RemoteProvider | null;
     domain?: string;
+    proxy?: string;
 }): Promise<BrowserHandle>;
 export declare function runBrowserStep(handle: BrowserHandle, action: StepAction): Promise<StepResultLine>;
 /**

package/dist/core/interact/driver.js

@@ -9,7 +9,7 @@
  *            (Local World / `next dev`).
  */
 import { execFile, spawn } from "node:child_process";
-import { mkdtempSync, writeFileSync } from "node:fs";
+import { existsSync, mkdtempSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
 import { Sandbox } from "@vercel/sandbox";
@@ -81,7 +81,7 @@ async function startSandboxBrowser(opts) {
         const exe = (await sh(sandbox, `cat ${VM_ROOT}/chrome-path`)).stdout.trim().split("\n").pop();
         if (!exe)
             throw new Error("Browser binary not found in sandbox");
-        await sandbox.runCommand({ cmd: exe, args: chromeArgs({ userAgent: opts.userAgent }), detached: true, sudo: true });
+        await sandbox.runCommand({ cmd: exe, args: chromeArgs({ userAgent: opts.userAgent, proxy: opts.proxy }), detached: true, sudo: true });
         const ready = await sh(sandbox, WAIT_FOR_CDP_SCRIPT, false);
         if (ready.exitCode !== 0)
             throw new Error("Browser did not start");
@@ -131,6 +131,38 @@ async function startWarmSandboxBrowser(domain) {
     }
 }
 /* ------------------------------ local backend ----------------------------- */
+/**
+ * Path to a real, user-installed Chrome when one exists, else null.
+ * Anti-bot vendors (DataDome, Akamai) fingerprint the automation build of
+ * Chromium and serve degraded pages to it, while the same engine driving a
+ * real installed Chrome gets full content — practitioner reports confirm
+ * this on DataDome-protected sites. Priority: explicit env override, then
+ * the standard per-OS install locations.
+ */
+export function installedChromePath() {
+    if (process.env.INTERACT_CHROME_PATH)
+        return process.env.INTERACT_CHROME_PATH;
+    const candidates = [];
+    if (process.platform === "darwin") {
+        candidates.push("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", `${process.env.HOME || ""}/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`);
+    }
+    else if (process.platform === "win32") {
+        candidates.push(`${process.env["ProgramFiles"] || "C:\\Program Files"}\\Google\\Chrome\\Application\\chrome.exe`, `${process.env["ProgramFiles(x86)"] || "C:\\Program Files (x86)"}\\Google\\Chrome\\Application\\chrome.exe`);
+    }
+    else {
+        candidates.push("/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium", "/snap/bin/chromium");
+    }
+    for (const p of candidates) {
+        try {
+            if (existsSync(p))
+                return p;
+        }
+        catch {
+            /* not installed here */
+        }
+    }
+    return null;
+}
 async function resolveLocalPlaywright() {
     const candidates = [
         process.env.WC_PLAYWRIGHT,
@@ -140,7 +172,10 @@ async function resolveLocalPlaywright() {
     for (const p of candidates) {
         try {
             const mod = (await import(/* webpackIgnore: true */ p));
-            const chrome = process.env.INTERACT_CHROME_PATH || mod.chromium.executablePath();
+            // Prefer a real installed Chrome over playwright's bundled automation
+            // Chromium (see installedChromePath); fall back to the bundle when the
+            // host has no Chrome (CI, containers, minimal servers).
+            const chrome = installedChromePath() || mod.chromium.executablePath();
             return { playwright: p, chrome };
         }
         catch {
@@ -153,7 +188,7 @@ async function startLocalBrowser(opts) {
     const { playwright, chrome } = await resolveLocalPlaywright();
     const port = 9300 + Math.floor(Math.random() * 600);
     const dir = mkdtempSync(path.join(tmpdir(), "wc-interact-"));
-    const args = chromeArgs({ userAgent: opts.userAgent })
+    const args = chromeArgs({ userAgent: opts.userAgent, proxy: opts.proxy })
         .map((a) => (a.startsWith("--remote-debugging-port=") ? `--remote-debugging-port=${port}` : a))
         .map((a) => (a.startsWith("--user-data-dir=") ? `--user-data-dir=${dir}/profile` : a));
     args.unshift("--disable-crash-reporter", "--disable-breakpad");
@@ -254,7 +289,8 @@ export async function startBrowser(opts = {}) {
         throw new Error("browser_unavailable");
     // Warm reconnect first: a sandbox that already solved this domain's bot check
     // keeps its IP / UA / cleared cookies — reusing it skips boot + re-solve.
-    if (opts.domain) {
+    // Never warm-reconnect when proxying: the warm sandbox egresses direct.
+    if (opts.domain && !opts.proxy) {
         const warm = await startWarmSandboxBrowser(opts.domain).catch(() => null);
         if (warm)
             return warm;
@@ -263,6 +299,7 @@ export async function startBrowser(opts = {}) {
         // Long VM timeout headroom so the sandbox can stay warm after this call.
         timeoutMs: Math.max(opts.timeoutMs ?? 5 * 60_000, WARM_SANDBOX_TIMEOUT_MS),
         userAgent: opts.userAgent,
+        proxy: opts.proxy,
     });
 }
 function runLocalStep(scriptPath, playwright, cdp, action, extraEnv = {}) {

package/dist/core/interact/vm.d.ts

@@ -18,6 +18,7 @@ export declare function chromeArgs(opts?: {
     userAgent?: string;
     width?: number;
     height?: number;
+    proxy?: string;
 }): string[];
 /** Waits until the CDP endpoint answers. Exit 1 if it never does. */
 export declare const WAIT_FOR_CDP_SCRIPT = "for i in $(seq 1 50); do curl -sf http://127.0.0.1:9222/json/version >/dev/null && echo up && exit 0; sleep 0.2; done; echo \"browser did not start\" >&2; exit 1";

package/dist/core/interact/vm.js

@@ -58,6 +58,11 @@ export function chromeArgs(opts = {}) {
     ];
     if (opts.userAgent)
         args.push(`--user-agent=${opts.userAgent}`);
+    // Residential-proxy browser tier (P2): all browser traffic egresses via the
+    // proxy. Chromium cannot parse credentials in --proxy-server, so they are
+    // stripped here and re-supplied per-page via proxy auth in the step script.
+    if (opts.proxy)
+        args.push(`--proxy-server=${opts.proxy.replace(/\/\/[^@/]+@/, "//")}`);
     args.push("about:blank");
     return args;
 }
@@ -178,9 +183,37 @@ try {
         await cdp.send("Page.addScriptToEvaluateOnNewDocument", { source: "Object.defineProperty(Navigator.prototype,'webdriver',{get:()=>false});" });
       } catch {}
       // Remote browsers bill proxy traffic per GB: skip bytes markdown never uses.
-      if (action.lightweight) {
-        const skip = new Set(["image", "media", "font"]);
-        await page.route("**/*", (r) => (skip.has(r.request().resourceType()) ? r.abort() : r.continue())).catch(() => {});
+      // The proxy tier (P2) blocks harder — stylesheets and analytics beacons
+      // too — because residential-proxy bandwidth is billed per MB and markdown
+      // only needs the document, its scripts and the challenge sensor (the
+      // sensor host is never on the ad blocklist). A byte cap aborts the rest
+      // of the page once the billed total blows past the budget.
+      // Anti-bot sensor/challenge hosts are ALWAYS allowed, before any block
+      // rule: blocking them would make the DataDome / Cloudflare / AWS WAF
+      // challenge unsolvable — no cookie is minted, the interstitial loops
+      // forever, and the failed sensor POST is itself a bot signal. The
+      // sensor payload is a few KB of form-encoded text, so allowing it
+      // costs essentially nothing against the byte budget.
+      const allowRe = /(api-js\.datadome\.co|js\.datadome\.co|ct\.captcha-delivery\.com|geo\.captcha-delivery\.com|challenges\.cloudflare\.com|[a-z0-9-]+\.awswaf\.com)/i;
+      let proxyBilled = 0;
+      if (action.lightweight || action.proxyUser) {
+        const skip = new Set(action.proxyUser ? ["image", "media", "font", "stylesheet"] : ["image", "media", "font"]);
+        const adRe = /(google-analytics|googletagmanager|doubleclick|adservice\.google|facebook\.net|connect\.facebook|hotjar|segment\.io|clarity\.ms|bat\.bing|scorecardresearch|newrelic|optimizely|criteo|taboola|outbrain)/i;
+        const cap = action.proxyUser ? Math.max(2_000_000, Math.min(Number(action.proxyByteCap) || 6_000_000, 12_000_000)) : Infinity;
+        page.on("response", (r) => { try { proxyBilled += Number(r.headers()["content-length"] || 0) || 0; } catch {} });
+        await page.route("**/*", (r) => {
+          try {
+            const req = r.request();
+            if (allowRe.test(req.url())) return r.continue(); // never block a challenge sensor, even past the cap
+            if (proxyBilled > cap) return r.abort();
+            if (skip.has(req.resourceType())) return r.abort();
+            if (adRe.test(req.url())) return r.abort();
+            return r.continue();
+          } catch { return r.continue(); }
+        }).catch(() => {});
+        if (action.proxyUser) {
+          try { await page.authenticate({ username: String(action.proxyUser), password: String(action.proxyPass || "") }); } catch {}
+        }
       }
       let navStatus = null;
       page.on("response", (r) => { try { if (r.request().isNavigationRequest() && r.frame() === page.mainFrame()) navStatus = r.status(); } catch {} });

package/dist/core/local-engine.js

@@ -174,7 +174,11 @@ export function createLocalEngineClient() {
                 },
                 evidence: {
                     ...targetInspection.evidence,
-                    tier: lastFetchTier() === "impersonate" ? "impersonate" : "impit-tls",
+                    tier: lastFetchTier() === "impersonate"
+                        ? "impersonate"
+                        : lastFetchTier() === "proxy-impersonate"
+                            ? "proxy-impersonate"
+                            : "impit-tls",
                     contentHash,
                     latencyMs: Date.now() - start,
                 },

package/dist/core/sandbox.js

@@ -181,7 +181,23 @@ async function escalateViaStealthBrowser(url, options) {
         const { stealthAvailable, stealthFetch } = await import("./engine/stealth.js");
         if (!(await stealthAvailable()))
             return null;
-        const res = await stealthFetch(url, [], { budgetMs: options.browserBudgetMs ?? 30_000 });
+        // Tier P2: when the domain is memoized as needing a residential egress,
+        // the stealth browser itself egresses through the proxy (resource-blocked,
+        // byte-capped inside stealthFetch) instead of a doomed datacenter IP.
+        let proxy;
+        try {
+            const { residentialProxyUrl } = await import("./engine/fetcher.js");
+            const { isDomainProxyNeeded } = await import("./engine/session.js");
+            let host = "";
+            try {
+                host = new URL(url).hostname;
+            }
+            catch { /* bad url handled below */ }
+            if (host && (await isDomainProxyNeeded(host)))
+                proxy = residentialProxyUrl() ?? undefined;
+        }
+        catch { /* proxy memo unavailable — datacenter stealth as before */ }
+        const res = await stealthFetch(url, [], { budgetMs: options.browserBudgetMs ?? 30_000, ...(proxy ? { proxy } : {}) });
         const entry = res.entry;
         if (!entry?.body || entry.blocked || isAntiBotChallenge(entry.body, entry.status ?? 200)) {
             return null;

package/dist/core/targets/types.d.ts

@@ -6,7 +6,7 @@
  * and returns structured results paired with verifiable Evidence & Provenance.
  */
 export type TargetCategory = "web" | "documents" | "media" | "commerce" | "travel" | "real-estate" | "local" | "security" | "developer" | "academic";
-export type TargetTier = "cache" | "static" | "impit-tls" | "impersonate" | "cloudflare" | "stealth-browser" | "sandbox" | "native-parser" | "ocr" | "api";
+export type TargetTier = "cache" | "static" | "impit-tls" | "impersonate" | "proxy-impersonate" | "cloudflare" | "stealth-browser" | "sandbox" | "native-parser" | "ocr" | "api";
 export interface TargetEvidence {
     /** Which target plugin handled the inspection */
     targetId: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.3.1",
+  "version": "1.3.3",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.3.1",
-    "@webcrawl/curl-impersonate": "^1.3.1",
+    "@webcrawl/client": "^1.3.3",
+    "@webcrawl/curl-impersonate": "^1.3.3",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",