webcrawl 1.2.5 → 1.2.8

package/dist/core/engine/awswaf.d.ts

@@ -0,0 +1,59 @@
+/**
+ * AWS WAF challenge handling (local prototype).
+ *
+ * `webcrawl`'s bot-check detector (`isAntiBotChallenge`) only matches
+ * Cloudflare and Vercel Security Checkpoint signatures. AWS WAF serves
+ * its invisible JS challenge as **HTTP 202** with an empty `<title>` and
+ * none of those markers, so such pages come back as
+ * `Empty page (HTTP 202)` with `blocked: false` and never reach the
+ * stealth browser. This module adds the missing detection plus a local
+ * stealth-routing path so those pages can be solved and read anyway.
+ *
+ * Detection markers are layered (no single one is proof):
+ *   - status 202 (documented Challenge interstitial status)
+ *   - `awsWafCookieDomainList` / `AwsWafIntegration` (documented JS API)
+ *   - `aws-waf-token` cookie reference
+ *   - `<div id="challenge-container">` + `/__challenge_<id>/.../challenge.js`
+ *
+ * The detection itself now lives in the shared detector
+ * (`isAwsWafChallenge` in `./fetcher.js`, also folded into
+ * `isAntiBotChallenge`); this module re-exports it and adds the local
+ * stealth-routing path so challenge pages can be solved and read anyway.
+ */
+import { isAwsWafChallenge } from "./fetcher.js";
+export { isAwsWafChallenge };
+/** True when a fetch response is any known bot check (upstream detector + AWS WAF). */
+export declare function isBlockedLike(html: string, status?: number): boolean;
+export interface AwsWafPage {
+    url: string;
+    title: string;
+    markdown: string;
+    linksCount: number;
+    cached: false;
+    cacheLayer: "miss";
+    structuredData: null;
+    statusCode?: number;
+    blocked?: boolean;
+    error?: string;
+    awsWaf?: boolean;
+    stealth?: boolean;
+}
+/**
+ * Opens `url` in a real local Chromium, waits out the AWS WAF challenge
+ * (the interstitial reloads itself once `AwsWafIntegration.getToken()`
+ * resolves), then returns the rendered page as a CrawledPage-shaped result.
+ * Never fabricates content: a page still showing the challenge comes back
+ * `blocked: true` with an honest error.
+ */
+export declare function awsWafStealthFetch(url: string, opts?: {
+    timeoutMs?: number;
+}): Promise<AwsWafPage>;
+/**
+ * Reads one page: plain TLS-impersonated fetch first; when the site answers
+ * with an AWS WAF challenge (which the upstream detector misses), routes
+ * through the local stealth browser instead.
+ */
+export declare function crawlPageWithAwsWaf(url: string, opts?: {
+    bypassCache?: boolean;
+    stealthTimeoutMs?: number;
+}): Promise<AwsWafPage>;

package/dist/core/engine/awswaf.js

@@ -0,0 +1,164 @@
+/**
+ * AWS WAF challenge handling (local prototype).
+ *
+ * `webcrawl`'s bot-check detector (`isAntiBotChallenge`) only matches
+ * Cloudflare and Vercel Security Checkpoint signatures. AWS WAF serves
+ * its invisible JS challenge as **HTTP 202** with an empty `<title>` and
+ * none of those markers, so such pages come back as
+ * `Empty page (HTTP 202)` with `blocked: false` and never reach the
+ * stealth browser. This module adds the missing detection plus a local
+ * stealth-routing path so those pages can be solved and read anyway.
+ *
+ * Detection markers are layered (no single one is proof):
+ *   - status 202 (documented Challenge interstitial status)
+ *   - `awsWafCookieDomainList` / `AwsWafIntegration` (documented JS API)
+ *   - `aws-waf-token` cookie reference
+ *   - `<div id="challenge-container">` + `/__challenge_<id>/.../challenge.js`
+ *
+ * The detection itself now lives in the shared detector
+ * (`isAwsWafChallenge` in `./fetcher.js`, also folded into
+ * `isAntiBotChallenge`); this module re-exports it and adds the local
+ * stealth-routing path so challenge pages can be solved and read anyway.
+ */
+import { fetchWithTlsImpersonation, isAntiBotChallenge, isAwsWafChallenge } from "./fetcher.js";
+import { fastHtmlToMarkdown } from "./markdown.js";
+export { isAwsWafChallenge };
+/** True when a fetch response is any known bot check (upstream detector + AWS WAF). */
+export function isBlockedLike(html, status) {
+    return isAntiBotChallenge(html ?? "", status ?? 200) || isAwsWafChallenge(html ?? "", status);
+}
+const MAC_CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
+const CHALLENGE_TIMEOUT_MS = 45_000;
+/** Waits for a dynamically-installed playwright-core (avoids a hard dep). */
+async function loadPlaywrightCore() {
+    // Non-literal specifier: playwright-core stays a soft, runtime-only dependency
+    // so `tsc --noEmit` passes without the package installed.
+    const specifier = "playwright-core";
+    return (await import(specifier));
+}
+function chromePath() {
+    return process.env.INTERACT_CHROME_PATH || MAC_CHROME;
+}
+/**
+ * Opens `url` in a real local Chromium, waits out the AWS WAF challenge
+ * (the interstitial reloads itself once `AwsWafIntegration.getToken()`
+ * resolves), then returns the rendered page as a CrawledPage-shaped result.
+ * Never fabricates content: a page still showing the challenge comes back
+ * `blocked: true` with an honest error.
+ */
+export async function awsWafStealthFetch(url, opts = {}) {
+    const fail = (error, blocked = true) => ({
+        url,
+        title: url,
+        markdown: "",
+        linksCount: 0,
+        cached: false,
+        cacheLayer: "miss",
+        structuredData: null,
+        blocked,
+        error,
+        awsWaf: true,
+        stealth: true,
+    });
+    const { chromium } = await loadPlaywrightCore();
+    const executablePath = chromePath();
+    const browser = await chromium.launch({
+        executablePath,
+        headless: true,
+        args: [
+            "--no-sandbox",
+            "--disable-gpu",
+            "--disable-dev-shm-usage",
+            "--no-first-run",
+            "--no-default-browser-check",
+            // Hide navigator.webdriver, which bot checks look for.
+            "--disable-blink-features=AutomationControlled",
+            `--window-size=1280,800`,
+        ],
+    });
+    try {
+        const ctx = await browser.newContext({
+            userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+            viewport: { width: 1280, height: 800 },
+            locale: "en-US",
+        });
+        await ctx.addInitScript("Object.defineProperty(Navigator.prototype,'webdriver',{get:()=>false});");
+        const page = await ctx.newPage();
+        let navStatus = null;
+        page.on("response", (r) => {
+            try {
+                if (r.request().isNavigationRequest() && r.frame() === page.mainFrame())
+                    navStatus = r.status();
+            }
+            catch { }
+        });
+        // The interstitial reloads itself after getToken(); domcontentloaded +
+        // marker polling survives that reload race.
+        await page.goto(url, { waitUntil: "domcontentloaded", timeout: Math.min(30_000, opts.timeoutMs ?? 30_000) });
+        const deadline = Date.now() + (opts.timeoutMs ?? CHALLENGE_TIMEOUT_MS);
+        let html = "";
+        for (;;) {
+            html = await page.content().catch(() => "");
+            if (!isAwsWafChallenge(html) && !isAntiBotChallenge(html, 200))
+                break;
+            if (Date.now() > deadline)
+                break;
+            await page.waitForTimeout(700).catch(() => { });
+        }
+        try {
+            await page.waitForLoadState("networkidle", { timeout: 2500 });
+        }
+        catch { }
+        html = await page.content().catch(() => html);
+        if (isAwsWafChallenge(html) || isAntiBotChallenge(html, 200)) {
+            return { ...fail("Still blocked by the site's AWS WAF challenge in local stealth mode"), statusCode: navStatus ?? undefined };
+        }
+        const finalUrl = page.url();
+        const parsed = fastHtmlToMarkdown(html, finalUrl || url, { retainImages: "all" });
+        if (parsed.markdown.length === 0) {
+            return {
+                ...fail(`Empty page after challenge (HTTP ${navStatus ?? "?"})`, false),
+                statusCode: navStatus ?? undefined,
+            };
+        }
+        return {
+            url,
+            title: parsed.title || finalUrl || url,
+            markdown: parsed.markdown,
+            linksCount: parsed.links.length,
+            cached: false,
+            cacheLayer: "miss",
+            structuredData: null,
+            statusCode: navStatus ?? 200,
+            awsWaf: true,
+            stealth: true,
+        };
+    }
+    finally {
+        await browser.close().catch(() => { });
+    }
+}
+/**
+ * Reads one page: plain TLS-impersonated fetch first; when the site answers
+ * with an AWS WAF challenge (which the upstream detector misses), routes
+ * through the local stealth browser instead.
+ */
+export async function crawlPageWithAwsWaf(url, opts = {}) {
+    const fetched = await fetchWithTlsImpersonation(url, 12_000);
+    if (!isAwsWafChallenge(fetched.html, fetched.status)) {
+        const parsed = fastHtmlToMarkdown(fetched.html, url, { retainImages: "all" });
+        return {
+            url,
+            title: parsed.title || url,
+            markdown: parsed.markdown,
+            linksCount: parsed.links.length,
+            cached: false,
+            cacheLayer: "miss",
+            structuredData: null,
+            statusCode: fetched.status,
+            ...(parsed.markdown.length === 0 ? { error: `Empty page (HTTP ${fetched.status})` } : {}),
+        };
+    }
+    console.info("[awswaf] challenge detected, routing through local stealth browser", { url, status: fetched.status });
+    return awsWafStealthFetch(url, { timeoutMs: opts.stealthTimeoutMs });
+}

package/dist/core/engine/fetcher.d.ts

@@ -23,6 +23,13 @@ export interface ScrapeResult {
 }
 export declare function getBrowserHeaders(targetUrl: string, extraHeaders?: Record<string, string>): Record<string, string>;
 export declare function setDomainCookie(hostname: string, cookieHeader: string): void;
+/**
+ * True when an HTML response (with optional status) is an AWS WAF challenge
+ * interstitial: a marker match proves it regardless of status; otherwise a
+ * documented HTTP 202 whose payload is a `<script>`-only HTML shell with zero
+ * visible text (APIs legitimately return 202 too, so status alone is not proof).
+ */
+export declare function isAwsWafChallenge(html: string, status?: number): boolean;
 export declare function isAntiBotChallenge(html: string, status: number): boolean;
 /**
  * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)

package/dist/core/engine/fetcher.js

@@ -47,18 +47,55 @@ function hashCode(str) {
     }
     return hash;
 }
+/** Markers that only appear on an AWS WAF challenge interstitial. */
+const AWS_WAF_CHALLENGE_MARKERS = [
+    "awswafcookiedomainlist",
+    "awswafintegration",
+    "aws-waf-token",
+    'id="challenge-container"',
+    "/__challenge_",
+];
+/**
+ * True when an HTML response (with optional status) is an AWS WAF challenge
+ * interstitial: a marker match proves it regardless of status; otherwise a
+ * documented HTTP 202 whose payload is a `<script>`-only HTML shell with zero
+ * visible text (APIs legitimately return 202 too, so status alone is not proof).
+ */
+export function isAwsWafChallenge(html, status) {
+    const lower = String(html ?? "").slice(0, 8000).toLowerCase();
+    if (AWS_WAF_CHALLENGE_MARKERS.some((m) => lower.includes(m)))
+        return true;
+    if (status === 202) {
+        if (!lower.includes("<html"))
+            return false;
+        const visibleText = lower
+            .slice(0, 8000)
+            .replace(/<script[\s\S]*?<\/script>/g, "")
+            .replace(/<noscript[\s\S]*?<\/noscript>/g, "")
+            .replace(/<style[\s\S]*?<\/style>/g, "")
+            .replace(/<[^>]+>/g, " ")
+            .trim();
+        return visibleText.length === 0;
+    }
+    return false;
+}
 export function isAntiBotChallenge(html, status) {
     if (status === 403 || status === 429 || status === 503) {
         return true;
     }
     const lower = html.slice(0, 6000).toLowerCase();
-    return (lower.includes("cf-browser-verification") ||
+    if (lower.includes("cf-browser-verification") ||
         lower.includes("challenge-platform") ||
         lower.includes("just a moment...") ||
         lower.includes("vercel security checkpoint") ||
         lower.includes("_vercel/security") ||
         lower.includes("enable javascript and cookies to continue") ||
-        lower.includes("_cf_chl_opt"));
+        lower.includes("_cf_chl_opt")) {
+        return true;
+    }
+    // AWS WAF serves its invisible JS challenge as HTTP 202 with none of the
+    // Cloudflare/Vercel markers, so those pages used to pass as real content.
+    return isAwsWafChallenge(html, status);
 }
 let sharedImpit = null;
 /** One request, redirects NOT followed (callers validate each hop via followSafely). */

package/dist/core/interact/vm.js

@@ -155,7 +155,7 @@ try {
       const PER_DOC = 1500000;
       const TOTAL = 10000000;
       const started = Date.now();
-      const markers = ["vercel security checkpoint", "just a moment...", "cf-browser-verification", "challenge-platform", "_cf_chl_opt", "enable javascript and cookies to continue", "_vercel/security", "attention required! | cloudflare"];
+      const markers = ["vercel security checkpoint", "just a moment...", "cf-browser-verification", "challenge-platform", "_cf_chl_opt", "enable javascript and cookies to continue", "_vercel/security", "attention required! | cloudflare", "awswafcookiedomainlist", "awswafintegration", "aws-waf-token", 'id="challenge-container"', "/__challenge_"];
       const looksBlocked = (s) => { const l = String(s || "").slice(0, 8000).toLowerCase(); return markers.some((m) => l.includes(m)); };
       let ua = "";
       try {

package/dist/core/sandbox.js

@@ -122,18 +122,22 @@ export async function escalateToSandboxOrBrowser(url, options = {}) {
                     // ignore cookie extraction failure
                 }
                 // A challenge page is not content: never return it as a successful escalation.
+                // An empty shell (e.g. an AWS WAF 202 interstitial) is not content either:
+                // parse first and fall through to the browser tier instead.
                 if (html && html.length > 100 && !isAntiBotChallenge(html, 200)) {
                     const parsed = fastHtmlToMarkdown(html, url, {
                         retainImages: options.retainImages,
                     });
-                    return {
-                        html,
-                        markdown: parsed.markdown,
-                        title: parsed.title,
-                        links: parsed.links,
-                        statusCode: 200,
-                        tier: "sandbox",
-                    };
+                    if (parsed.markdown.trim().length > 0) {
+                        return {
+                            html,
+                            markdown: parsed.markdown,
+                            title: parsed.title,
+                            links: parsed.links,
+                            statusCode: 200,
+                            tier: "sandbox",
+                        };
+                    }
                 }
             }
             finally {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.2.5",
+  "version": "1.2.8",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,7 +127,7 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.2.5",
+    "@webcrawl/client": "^1.2.8",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",