webcrawl 1.3.2 → 1.3.3

package/dist/core/engine/fetcher.d.ts

@@ -43,8 +43,11 @@ export declare function lastFetchTier(): "impit-tls" | "impersonate" | "proxy-im
 export declare function lastProxyBytesCount(): number;
 /**
  * Residential proxy URL from env, or null when the tier is disabled.
- * Accepts a full `DECODO_URL` (`http://user:pass@host:port`) or the split
- * `DECODO_USER` + `DECODO_PWD` pair (gateway defaults to `gate.decodo.com:7000`).
+ * Generic: a full `WEBCRAWL_PROXY_URL` (`http://user:pass@host:port`) or the
+ * split `WEBCRAWL_PROXY_USER` + `WEBCRAWL_PROXY_PWD` pair, with the gateway
+ * host/port overridable via `WEBCRAWL_PROXY_HOST` / `WEBCRAWL_PROXY_PORT`.
+ * `DECODO_*` equivalents are honored as a legacy fallback so existing
+ * deployments keep working (their default gateway is `gate.decodo.com:7000`).
  */
 export declare function residentialProxyUrl(): string | null;
 /**

package/dist/core/engine/fetcher.js

@@ -134,17 +134,22 @@ export function lastProxyBytesCount() {
 }
 /**
  * Residential proxy URL from env, or null when the tier is disabled.
- * Accepts a full `DECODO_URL` (`http://user:pass@host:port`) or the split
- * `DECODO_USER` + `DECODO_PWD` pair (gateway defaults to `gate.decodo.com:7000`).
+ * Generic: a full `WEBCRAWL_PROXY_URL` (`http://user:pass@host:port`) or the
+ * split `WEBCRAWL_PROXY_USER` + `WEBCRAWL_PROXY_PWD` pair, with the gateway
+ * host/port overridable via `WEBCRAWL_PROXY_HOST` / `WEBCRAWL_PROXY_PORT`.
+ * `DECODO_*` equivalents are honored as a legacy fallback so existing
+ * deployments keep working (their default gateway is `gate.decodo.com:7000`).
  */
 export function residentialProxyUrl() {
-    const direct = process.env.DECODO_URL?.trim();
+    const direct = process.env.WEBCRAWL_PROXY_URL?.trim() || process.env.DECODO_URL?.trim();
     if (direct)
         return direct;
-    const user = process.env.DECODO_USER?.trim();
-    const pwd = process.env.DECODO_PWD?.trim();
+    const user = process.env.WEBCRAWL_PROXY_USER?.trim() || process.env.DECODO_USER?.trim();
+    const pwd = process.env.WEBCRAWL_PROXY_PWD?.trim() || process.env.DECODO_PWD?.trim();
     if (user && pwd) {
-        return `http://${encodeURIComponent(user)}:${encodeURIComponent(pwd)}@gate.decodo.com:7000`;
+        const host = process.env.WEBCRAWL_PROXY_HOST?.trim() || "gate.decodo.com";
+        const port = process.env.WEBCRAWL_PROXY_PORT?.trim() || "7000";
+        return `http://${encodeURIComponent(user)}:${encodeURIComponent(pwd)}@${host}:${port}`;
     }
     return null;
 }

package/dist/core/interact/driver.d.ts

@@ -38,6 +38,15 @@ export type StepAction = Record<string, unknown> & {
  */
 export declare function interactBrowserAvailable(): boolean;
 export declare function interactBackend(): "local" | "sandbox" | "none";
+/**
+ * Path to a real, user-installed Chrome when one exists, else null.
+ * Anti-bot vendors (DataDome, Akamai) fingerprint the automation build of
+ * Chromium and serve degraded pages to it, while the same engine driving a
+ * real installed Chrome gets full content — practitioner reports confirm
+ * this on DataDome-protected sites. Priority: explicit env override, then
+ * the standard per-OS install locations.
+ */
+export declare function installedChromePath(): string | null;
 export declare function startBrowser(opts?: {
     timeoutMs?: number;
     userAgent?: string;

package/dist/core/interact/driver.js

@@ -9,7 +9,7 @@
  *            (Local World / `next dev`).
  */
 import { execFile, spawn } from "node:child_process";
-import { mkdtempSync, writeFileSync } from "node:fs";
+import { existsSync, mkdtempSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
 import { Sandbox } from "@vercel/sandbox";
@@ -131,6 +131,38 @@ async function startWarmSandboxBrowser(domain) {
     }
 }
 /* ------------------------------ local backend ----------------------------- */
+/**
+ * Path to a real, user-installed Chrome when one exists, else null.
+ * Anti-bot vendors (DataDome, Akamai) fingerprint the automation build of
+ * Chromium and serve degraded pages to it, while the same engine driving a
+ * real installed Chrome gets full content — practitioner reports confirm
+ * this on DataDome-protected sites. Priority: explicit env override, then
+ * the standard per-OS install locations.
+ */
+export function installedChromePath() {
+    if (process.env.INTERACT_CHROME_PATH)
+        return process.env.INTERACT_CHROME_PATH;
+    const candidates = [];
+    if (process.platform === "darwin") {
+        candidates.push("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", `${process.env.HOME || ""}/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`);
+    }
+    else if (process.platform === "win32") {
+        candidates.push(`${process.env["ProgramFiles"] || "C:\\Program Files"}\\Google\\Chrome\\Application\\chrome.exe`, `${process.env["ProgramFiles(x86)"] || "C:\\Program Files (x86)"}\\Google\\Chrome\\Application\\chrome.exe`);
+    }
+    else {
+        candidates.push("/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium", "/snap/bin/chromium");
+    }
+    for (const p of candidates) {
+        try {
+            if (existsSync(p))
+                return p;
+        }
+        catch {
+            /* not installed here */
+        }
+    }
+    return null;
+}
 async function resolveLocalPlaywright() {
     const candidates = [
         process.env.WC_PLAYWRIGHT,
@@ -140,7 +172,10 @@ async function resolveLocalPlaywright() {
     for (const p of candidates) {
         try {
             const mod = (await import(/* webpackIgnore: true */ p));
-            const chrome = process.env.INTERACT_CHROME_PATH || mod.chromium.executablePath();
+            // Prefer a real installed Chrome over playwright's bundled automation
+            // Chromium (see installedChromePath); fall back to the bundle when the
+            // host has no Chrome (CI, containers, minimal servers).
+            const chrome = installedChromePath() || mod.chromium.executablePath();
             return { playwright: p, chrome };
         }
         catch {

package/dist/core/interact/vm.js

@@ -188,6 +188,13 @@ try {
       // only needs the document, its scripts and the challenge sensor (the
       // sensor host is never on the ad blocklist). A byte cap aborts the rest
       // of the page once the billed total blows past the budget.
+      // Anti-bot sensor/challenge hosts are ALWAYS allowed, before any block
+      // rule: blocking them would make the DataDome / Cloudflare / AWS WAF
+      // challenge unsolvable — no cookie is minted, the interstitial loops
+      // forever, and the failed sensor POST is itself a bot signal. The
+      // sensor payload is a few KB of form-encoded text, so allowing it
+      // costs essentially nothing against the byte budget.
+      const allowRe = /(api-js\.datadome\.co|js\.datadome\.co|ct\.captcha-delivery\.com|geo\.captcha-delivery\.com|challenges\.cloudflare\.com|[a-z0-9-]+\.awswaf\.com)/i;
       let proxyBilled = 0;
       if (action.lightweight || action.proxyUser) {
         const skip = new Set(action.proxyUser ? ["image", "media", "font", "stylesheet"] : ["image", "media", "font"]);
@@ -197,6 +204,7 @@ try {
         await page.route("**/*", (r) => {
           try {
             const req = r.request();
+            if (allowRe.test(req.url())) return r.continue(); // never block a challenge sensor, even past the cap
             if (proxyBilled > cap) return r.abort();
             if (skip.has(req.resourceType())) return r.abort();
             if (adRe.test(req.url())) return r.abort();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.3.2",
+  "version": "1.3.3",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.3.2",
-    "@webcrawl/curl-impersonate": "^1.3.2",
+    "@webcrawl/client": "^1.3.3",
+    "@webcrawl/curl-impersonate": "^1.3.3",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",