webcrawl 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/engine/fetcher.js +10 -0
- package/package.json +3 -3
|
@@ -104,6 +104,16 @@ export function isAntiBotChallenge(html, status) {
|
|
|
104
104
|
lower.includes("_cf_chl_opt")) {
|
|
105
105
|
return true;
|
|
106
106
|
}
|
|
107
|
+
// IP-reputation interstitials (stubhub, expedia) that pass the TLS gate
|
|
108
|
+
// but gate datacenter IPs behind a "enable JS / are you human" shell.
|
|
109
|
+
// These can arrive cached or served with status 200, so text markers are
|
|
110
|
+
// required on top of the 403/429 status check above.
|
|
111
|
+
if (lower.includes("enable javascript and disable any ad blocker") ||
|
|
112
|
+
lower.includes("show us your human side") ||
|
|
113
|
+
lower.includes("we can't tell if you're a human or a bot") ||
|
|
114
|
+
lower.includes("we can’t tell if you’re a human or a bot")) {
|
|
115
|
+
return true;
|
|
116
|
+
}
|
|
107
117
|
// AWS WAF serves its invisible JS challenge as HTTP 202 with none of the
|
|
108
118
|
// Cloudflare/Vercel markers, so those pages used to pass as real content.
|
|
109
119
|
return isAwsWafChallenge(html, status);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "webcrawl",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.1",
|
|
4
4
|
"description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -127,8 +127,8 @@
|
|
|
127
127
|
"@ai-sdk/gateway": "^4.0.123",
|
|
128
128
|
"@firecrawl/pdf-inspector": "^1.25.2",
|
|
129
129
|
"@vercel/sandbox": "^1.0.0",
|
|
130
|
-
"@webcrawl/client": "^1.3.
|
|
131
|
-
"@webcrawl/curl-impersonate": "^1.3.
|
|
130
|
+
"@webcrawl/client": "^1.3.1",
|
|
131
|
+
"@webcrawl/curl-impersonate": "^1.3.1",
|
|
132
132
|
"ai": "^7.0.150",
|
|
133
133
|
"impit": "^0.14.7",
|
|
134
134
|
"mammoth": "^1.13.0",
|