webcrawl 1.2.7 → 1.2.8

package/dist/core/engine/awswaf.d.ts

@@ -1,5 +1,27 @@
-/** True when an HTML response (with optional status) is an AWS WAF challenge interstitial. */
-export declare function isAwsWafChallenge(html: string, status?: number): boolean;
+/**
+ * AWS WAF challenge handling (local prototype).
+ *
+ * `webcrawl`'s bot-check detector (`isAntiBotChallenge`) only matches
+ * Cloudflare and Vercel Security Checkpoint signatures. AWS WAF serves
+ * its invisible JS challenge as **HTTP 202** with an empty `<title>` and
+ * none of those markers, so such pages come back as
+ * `Empty page (HTTP 202)` with `blocked: false` and never reach the
+ * stealth browser. This module adds the missing detection plus a local
+ * stealth-routing path so those pages can be solved and read anyway.
+ *
+ * Detection markers are layered (no single one is proof):
+ *   - status 202 (documented Challenge interstitial status)
+ *   - `awsWafCookieDomainList` / `AwsWafIntegration` (documented JS API)
+ *   - `aws-waf-token` cookie reference
+ *   - `<div id="challenge-container">` + `/__challenge_<id>/.../challenge.js`
+ *
+ * The detection itself now lives in the shared detector
+ * (`isAwsWafChallenge` in `./fetcher.js`, also folded into
+ * `isAntiBotChallenge`); this module re-exports it and adds the local
+ * stealth-routing path so challenge pages can be solved and read anyway.
+ */
+import { isAwsWafChallenge } from "./fetcher.js";
+export { isAwsWafChallenge };
 /** True when a fetch response is any known bot check (upstream detector + AWS WAF). */
 export declare function isBlockedLike(html: string, status?: number): boolean;
 export interface AwsWafPage {

package/dist/core/engine/awswaf.js

@@ -14,40 +14,15 @@
  *   - `awsWafCookieDomainList` / `AwsWafIntegration` (documented JS API)
  *   - `aws-waf-token` cookie reference
  *   - `<div id="challenge-container">` + `/__challenge_<id>/.../challenge.js`
+ *
+ * The detection itself now lives in the shared detector
+ * (`isAwsWafChallenge` in `./fetcher.js`, also folded into
+ * `isAntiBotChallenge`); this module re-exports it and adds the local
+ * stealth-routing path so challenge pages can be solved and read anyway.
  */
-import { fetchWithTlsImpersonation, fastHtmlToMarkdown, isAntiBotChallenge } from "../engine.js";
-/** Markers that only appear on an AWS WAF challenge interstitial. */
-const AWS_WAF_MARKERS = [
-    "awswafcookiedomainlist",
-    "awswafintegration",
-    "aws-waf-token",
-    "id=\"challenge-container\"",
-    "/__challenge_",
-];
-/** True when an HTML response (with optional status) is an AWS WAF challenge interstitial. */
-export function isAwsWafChallenge(html, status) {
-    const lower = String(html ?? "").slice(0, 8000).toLowerCase();
-    const hasMarker = AWS_WAF_MARKERS.some((m) => lower.includes(m));
-    if (hasMarker)
-        return true;
-    // 202 alone is the documented Challenge status, but APIs can legitimately
-    // return 202 - only treat it as a challenge when the body is empty/challenge-ish.
-    if (status === 202) {
-        // 202 is the documented Challenge interstitial status. The interstitial's
-        // entire payload is <script>/<noscript>, so visible text is empty.
-        if (!lower.includes("<html"))
-            return false;
-        const visibleText = lower
-            .slice(0, 8000)
-            .replace(/<script[\s\S]*?<\/script>/g, "")
-            .replace(/<noscript[\s\S]*?<\/noscript>/g, "")
-            .replace(/<style[\s\S]*?<\/style>/g, "")
-            .replace(/<[^>]+>/g, " ")
-            .trim();
-        return visibleText.length === 0;
-    }
-    return false;
-}
+import { fetchWithTlsImpersonation, isAntiBotChallenge, isAwsWafChallenge } from "./fetcher.js";
+import { fastHtmlToMarkdown } from "./markdown.js";
+export { isAwsWafChallenge };
 /** True when a fetch response is any known bot check (upstream detector + AWS WAF). */
 export function isBlockedLike(html, status) {
     return isAntiBotChallenge(html ?? "", status ?? 200) || isAwsWafChallenge(html ?? "", status);

package/dist/core/engine/fetcher.d.ts

@@ -23,6 +23,13 @@ export interface ScrapeResult {
 }
 export declare function getBrowserHeaders(targetUrl: string, extraHeaders?: Record<string, string>): Record<string, string>;
 export declare function setDomainCookie(hostname: string, cookieHeader: string): void;
+/**
+ * True when an HTML response (with optional status) is an AWS WAF challenge
+ * interstitial: a marker match proves it regardless of status; otherwise a
+ * documented HTTP 202 whose payload is a `<script>`-only HTML shell with zero
+ * visible text (APIs legitimately return 202 too, so status alone is not proof).
+ */
+export declare function isAwsWafChallenge(html: string, status?: number): boolean;
 export declare function isAntiBotChallenge(html: string, status: number): boolean;
 /**
  * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)

package/dist/core/engine/fetcher.js

@@ -47,18 +47,55 @@ function hashCode(str) {
     }
     return hash;
 }
+/** Markers that only appear on an AWS WAF challenge interstitial. */
+const AWS_WAF_CHALLENGE_MARKERS = [
+    "awswafcookiedomainlist",
+    "awswafintegration",
+    "aws-waf-token",
+    'id="challenge-container"',
+    "/__challenge_",
+];
+/**
+ * True when an HTML response (with optional status) is an AWS WAF challenge
+ * interstitial: a marker match proves it regardless of status; otherwise a
+ * documented HTTP 202 whose payload is a `<script>`-only HTML shell with zero
+ * visible text (APIs legitimately return 202 too, so status alone is not proof).
+ */
+export function isAwsWafChallenge(html, status) {
+    const lower = String(html ?? "").slice(0, 8000).toLowerCase();
+    if (AWS_WAF_CHALLENGE_MARKERS.some((m) => lower.includes(m)))
+        return true;
+    if (status === 202) {
+        if (!lower.includes("<html"))
+            return false;
+        const visibleText = lower
+            .slice(0, 8000)
+            .replace(/<script[\s\S]*?<\/script>/g, "")
+            .replace(/<noscript[\s\S]*?<\/noscript>/g, "")
+            .replace(/<style[\s\S]*?<\/style>/g, "")
+            .replace(/<[^>]+>/g, " ")
+            .trim();
+        return visibleText.length === 0;
+    }
+    return false;
+}
 export function isAntiBotChallenge(html, status) {
     if (status === 403 || status === 429 || status === 503) {
         return true;
     }
     const lower = html.slice(0, 6000).toLowerCase();
-    return (lower.includes("cf-browser-verification") ||
+    if (lower.includes("cf-browser-verification") ||
         lower.includes("challenge-platform") ||
         lower.includes("just a moment...") ||
         lower.includes("vercel security checkpoint") ||
         lower.includes("_vercel/security") ||
         lower.includes("enable javascript and cookies to continue") ||
-        lower.includes("_cf_chl_opt"));
+        lower.includes("_cf_chl_opt")) {
+        return true;
+    }
+    // AWS WAF serves its invisible JS challenge as HTTP 202 with none of the
+    // Cloudflare/Vercel markers, so those pages used to pass as real content.
+    return isAwsWafChallenge(html, status);
 }
 let sharedImpit = null;
 /** One request, redirects NOT followed (callers validate each hop via followSafely). */

package/dist/core/interact/vm.js

@@ -155,7 +155,7 @@ try {
       const PER_DOC = 1500000;
       const TOTAL = 10000000;
       const started = Date.now();
-      const markers = ["vercel security checkpoint", "just a moment...", "cf-browser-verification", "challenge-platform", "_cf_chl_opt", "enable javascript and cookies to continue", "_vercel/security", "attention required! | cloudflare"];
+      const markers = ["vercel security checkpoint", "just a moment...", "cf-browser-verification", "challenge-platform", "_cf_chl_opt", "enable javascript and cookies to continue", "_vercel/security", "attention required! | cloudflare", "awswafcookiedomainlist", "awswafintegration", "aws-waf-token", 'id="challenge-container"', "/__challenge_"];
       const looksBlocked = (s) => { const l = String(s || "").slice(0, 8000).toLowerCase(); return markers.some((m) => l.includes(m)); };
       let ua = "";
       try {

package/dist/core/sandbox.js

@@ -122,18 +122,22 @@ export async function escalateToSandboxOrBrowser(url, options = {}) {
                     // ignore cookie extraction failure
                 }
                 // A challenge page is not content: never return it as a successful escalation.
+                // An empty shell (e.g. an AWS WAF 202 interstitial) is not content either:
+                // parse first and fall through to the browser tier instead.
                 if (html && html.length > 100 && !isAntiBotChallenge(html, 200)) {
                     const parsed = fastHtmlToMarkdown(html, url, {
                         retainImages: options.retainImages,
                     });
-                    return {
-                        html,
-                        markdown: parsed.markdown,
-                        title: parsed.title,
-                        links: parsed.links,
-                        statusCode: 200,
-                        tier: "sandbox",
-                    };
+                    if (parsed.markdown.trim().length > 0) {
+                        return {
+                            html,
+                            markdown: parsed.markdown,
+                            title: parsed.title,
+                            links: parsed.links,
+                            statusCode: 200,
+                            tier: "sandbox",
+                        };
+                    }
                 }
             }
             finally {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.2.7",
+  "version": "1.2.8",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,7 +127,7 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.2.7",
+    "@webcrawl/client": "^1.2.8",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",