webcrawl 1.2.9 → 1.3.1

package/dist/core/engine/fetcher.d.ts

@@ -17,7 +17,7 @@ export interface ScrapeResult {
     links?: string[];
     statusCode: number;
     contentHash: string;
-    tier: "cache" | "static" | "sandbox" | "cloudflare" | "browser";
+    tier: "cache" | "static" | "sandbox" | "cloudflare" | "browser" | "impersonate";
     latencyMs: number;
     error?: string;
 }
@@ -37,8 +37,10 @@ export declare function setDomainUserAgent(hostname: string, userAgent: string):
  */
 export declare function isAwsWafChallenge(html: string, status?: number): boolean;
 export declare function isAntiBotChallenge(html: string, status: number): boolean;
+/** Transport tier that won the last fetch ladder run (for result evidence). */
+export declare function lastFetchTier(): "impit-tls" | "impersonate" | "fetch";
 /**
- * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)
+ * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)sonation)
  * so Cloudflare Bot Management and Vercel Security Checkpoint treat requests as real Chrome.
  * Falls back to standard `fetch` if `impit` is unavailable on the current platform.
  * Every redirect hop is checked against the SSRF guard.

package/dist/core/engine/fetcher.js

@@ -104,11 +104,27 @@ export function isAntiBotChallenge(html, status) {
         lower.includes("_cf_chl_opt")) {
         return true;
     }
+    // IP-reputation interstitials (stubhub, expedia) that pass the TLS gate
+    // but gate datacenter IPs behind a "enable JS / are you human" shell.
+    // These can arrive cached or served with status 200, so text markers are
+    // required on top of the 403/429 status check above.
+    if (lower.includes("enable javascript and disable any ad blocker") ||
+        lower.includes("show us your human side") ||
+        lower.includes("we can't tell if you're a human or a bot") ||
+        lower.includes("we can’t tell if you’re a human or a bot")) {
+        return true;
+    }
     // AWS WAF serves its invisible JS challenge as HTTP 202 with none of the
     // Cloudflare/Vercel markers, so those pages used to pass as real content.
     return isAwsWafChallenge(html, status);
 }
 let sharedImpit = null;
+/** Tier that won the most recent rawGet ladder ("impit-tls" → "impersonate" → "fetch"). */
+let lastTier = "impit-tls";
+/** Transport tier that won the last fetch ladder run (for result evidence). */
+export function lastFetchTier() {
+    return lastTier;
+}
 /** One request, redirects NOT followed (callers validate each hop via followSafely). */
 async function rawGet(url, reqHeaders, timeoutMs, maxBytes) {
     const read = async (status, headers, getBuf) => {
@@ -132,21 +148,92 @@ async function rawGet(url, reqHeaders, timeoutMs, maxBytes) {
             timeout: timeoutMs,
             redirect: "manual",
         });
-        return await read(res.status, headersToObject(res.headers), () => res.arrayBuffer());
+        const hop = await read(res.status, headersToObject(res.headers), () => res.arrayBuffer());
+        // AWS-WAF-style blocks (403/429/challenge shells) on the impit tier
+        // (Chrome 136) retry once on the curl-impersonate tier (Chrome 150),
+        // which passes WAFs that block the older fingerprint.
+        if (isAntiBotChallenge(new TextDecoder().decode(hop.body).slice(0, 8000), hop.status)) {
+            const impersonateHop = await rawGetImpersonate(url, reqHeaders, timeoutMs, maxBytes);
+            if (impersonateHop) {
+                lastTier = "impersonate";
+                return impersonateHop;
+            }
+        }
+        lastTier = "impit-tls";
+        return hop;
     }
     catch (err) {
         if (err instanceof ResourceTooLargeError || err instanceof UnsafeUrlError)
             throw err;
+        // impit unavailable/failed: curl-impersonate tier, then plain fetch.
+        const impersonateHop = await rawGetImpersonate(url, reqHeaders, timeoutMs, maxBytes);
+        if (impersonateHop) {
+            lastTier = "impersonate";
+            return impersonateHop;
+        }
         const res = await fetch(url, {
             headers: reqHeaders,
             redirect: "manual",
             signal: AbortSignal.timeout(timeoutMs),
         });
+        lastTier = "fetch";
         return await read(res.status, headersToObject(res.headers), () => res.arrayBuffer());
     }
 }
 /**
- * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)
+ * One GET hop with the curl-impersonate tier (Chrome 150 TLS/HTTP2 fingerprint).
+ * Used by `rawGet` when the impit hop is blocked (403/429/challenge) — newer
+ * fingerprints pass AWS WAF domains (stubhub, expedia) that block Chrome 136.
+ * Falls back to plain `fetch` when the binary is unavailable.
+ */
+async function rawGetImpersonate(url, reqHeaders, timeoutMs, maxBytes) {
+    let impersonate;
+    try {
+        impersonate = await import("@webcrawl/curl-impersonate");
+    }
+    catch {
+        return null;
+    }
+    // The chrome150 profile's own identity headers MUST survive (profileHeaders
+    // merges extras last, so a stale UA would override the profile UA and the
+    // UA/TLS mismatch is exactly what DataDome/AWS WAF block). Keep cookies and
+    // referer, drop UA + sec-ch/client-hint identity headers.
+    const extra = {};
+    for (const [key, value] of Object.entries(reqHeaders)) {
+        const lower = key.toLowerCase();
+        if (lower === "user-agent" ||
+            lower.startsWith("sec-ch-ua") ||
+            lower.startsWith("sec-fetch-") ||
+            lower === "upgrade-insecure-requests" ||
+            lower === "priority") {
+            continue;
+        }
+        extra[key] = value;
+    }
+    try {
+        const res = await impersonate.impersonateFetch(url, {
+            headers: extra,
+            timeoutMs,
+            maxBytes,
+        });
+        const headers = res.headers;
+        const isRedirect = res.status >= 300 && res.status < 400 && !!headers["location"];
+        if (!isRedirect && maxBytes !== undefined)
+            checkDeclaredLength(headers, maxBytes);
+        return {
+            status: res.status,
+            headers,
+            body: isRedirect ? new ArrayBuffer(0) : res.body.buffer,
+        };
+    }
+    catch (err) {
+        if (err instanceof UnsafeUrlError)
+            throw err;
+        return null; // tier unavailable or fetch failed — caller falls back
+    }
+}
+/**
+ * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)sonation)
  * so Cloudflare Bot Management and Vercel Security Checkpoint treat requests as real Chrome.
  * Falls back to standard `fetch` if `impit` is unavailable on the current platform.
  * Every redirect hop is checked against the SSRF guard.

package/dist/core/local-engine.js

@@ -10,7 +10,7 @@
  * powers both this local engine (`npx webcrawl`) and the hosted `webcrawl.app` API.
  */
 import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, mapWebsiteUrls, setCachedPage, sha256, } from "./engine.js";
-import { fetchResource } from "./engine/fetcher.js";
+import { fetchResource, lastFetchTier } from "./engine/fetcher.js";
 import { parseFile, publicParseMeta } from "./parse/index.js";
 import { isBinaryContentType, looksLikeBinaryUrl } from "./parse/fetch.js";
 import { maxDownloadBytes } from "./parse/limits.js";
@@ -174,7 +174,7 @@ export function createLocalEngineClient() {
                 },
                 evidence: {
                     ...targetInspection.evidence,
-                    tier: "impit-tls",
+                    tier: lastFetchTier() === "impersonate" ? "impersonate" : "impit-tls",
                     contentHash,
                     latencyMs: Date.now() - start,
                 },

package/dist/core/targets/types.d.ts

@@ -6,7 +6,7 @@
  * and returns structured results paired with verifiable Evidence & Provenance.
  */
 export type TargetCategory = "web" | "documents" | "media" | "commerce" | "travel" | "real-estate" | "local" | "security" | "developer" | "academic";
-export type TargetTier = "cache" | "static" | "impit-tls" | "cloudflare" | "stealth-browser" | "sandbox" | "native-parser" | "ocr" | "api";
+export type TargetTier = "cache" | "static" | "impit-tls" | "impersonate" | "cloudflare" | "stealth-browser" | "sandbox" | "native-parser" | "ocr" | "api";
 export interface TargetEvidence {
     /** Which target plugin handled the inspection */
     targetId: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.2.9",
+  "version": "1.3.1",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,7 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.2.9",
+    "@webcrawl/client": "^1.3.1",
+    "@webcrawl/curl-impersonate": "^1.3.1",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",