recker 1.0.79 → 1.0.80-next.70ea84d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/browser/mini.d.ts +2 -2
- package/dist/browser/browser/recker-mini.d.ts +8 -8
- package/dist/browser/browser/recker.d.ts +11 -8
- package/dist/browser/browser/recker.js +54 -6
- package/dist/browser/core/client.d.ts +15 -10
- package/dist/browser/core/client.js +54 -38
- package/dist/browser/index.iife.min.js +129 -130
- package/dist/browser/index.min.js +129 -130
- package/dist/browser/index.mini.iife.js +5697 -636
- package/dist/browser/index.mini.iife.min.js +47 -48
- package/dist/browser/index.mini.min.js +47 -48
- package/dist/browser/index.mini.umd.js +5697 -636
- package/dist/browser/index.mini.umd.min.js +47 -48
- package/dist/browser/index.umd.min.js +129 -130
- package/dist/browser/mini.d.ts +2 -2
- package/dist/browser/plugins/proxy-rotator.d.ts +2 -2
- package/dist/browser/plugins/proxy-rotator.js +6 -28
- package/dist/browser/recker-mini.d.ts +8 -8
- package/dist/browser/recker.d.ts +11 -8
- package/dist/browser/recker.js +54 -6
- package/dist/browser/scrape/document.js +2 -2
- package/dist/browser/scrape/element.js +7 -1
- package/dist/browser/scrape/parser/nodes/html.js +1 -1
- package/dist/browser/scrape/spider.d.ts +52 -0
- package/dist/browser/scrape/spider.js +620 -38
- package/dist/browser/scrape/types.d.ts +2 -0
- package/dist/browser/search/google.d.ts +26 -1
- package/dist/browser/search/google.js +427 -45
- package/dist/browser/seo/analyzer.d.ts +1 -0
- package/dist/browser/seo/analyzer.js +144 -1
- package/dist/browser/seo/index.d.ts +1 -1
- package/dist/browser/seo/keyword-campaign-analyzer.d.ts +2 -0
- package/dist/browser/seo/keyword-campaign-analyzer.js +538 -0
- package/dist/browser/seo/keyword-campaign-seed-advanced.d.ts +17 -0
- package/dist/browser/seo/keyword-campaign-seed-advanced.js +269 -0
- package/dist/browser/seo/keyword-campaign-seed-core.d.ts +29 -0
- package/dist/browser/seo/keyword-campaign-seed-core.js +525 -0
- package/dist/browser/seo/keyword-campaign-shared.d.ts +165 -0
- package/dist/browser/seo/keyword-campaign-shared.js +59 -0
- package/dist/browser/seo/keyword-campaign.d.ts +4 -107
- package/dist/browser/seo/keyword-campaign.js +2 -380
- package/dist/browser/seo/keywords.js +5 -22
- package/dist/browser/seo/types.d.ts +19 -0
- package/dist/browser/transport/curl.d.ts +5 -1
- package/dist/browser/transport/curl.js +207 -50
- package/dist/browser/transport/undici.d.ts +4 -3
- package/dist/browser/transport/undici.js +123 -49
- package/dist/browser/types/index.d.ts +9 -3
- package/dist/browser/utils/binary-manager.js +26 -3
- package/dist/browser/utils/block-detector.d.ts +8 -0
- package/dist/browser/utils/block-detector.js +542 -7
- package/dist/cli/commands/hls-runner.js +5 -4
- package/dist/cli/commands/live-runner.js +5 -4
- package/dist/cli/commands/loadtest-runner.js +3 -2
- package/dist/cli/commands/search.d.ts +2 -0
- package/dist/cli/commands/search.js +105 -0
- package/dist/cli/commands/seo-runner.js +9 -7
- package/dist/cli/commands/seo.js +140 -1
- package/dist/cli/commands/serve.js +75 -131
- package/dist/cli/commands/server-runner.js +59 -82
- package/dist/cli/commands/spider-runner.d.ts +37 -1
- package/dist/cli/commands/spider-runner.js +134 -10
- package/dist/cli/commands/spider.d.ts +18 -1
- package/dist/cli/commands/spider.js +457 -27
- package/dist/cli/events/handlers/cli.js +30 -1
- package/dist/cli/events/handlers/tui.js +26 -0
- package/dist/cli/events/types.d.ts +27 -0
- package/dist/cli/handler.d.ts +2 -12
- package/dist/cli/handler.js +20 -15
- package/dist/cli/handlers/protocols.js +39 -12
- package/dist/cli/handlers/search.d.ts +2 -0
- package/dist/cli/handlers/search.js +171 -0
- package/dist/cli/handlers/seo-analyze.d.ts +1 -0
- package/dist/cli/handlers/seo-analyze.js +666 -0
- package/dist/cli/handlers/seo-robots.d.ts +1 -0
- package/dist/cli/handlers/seo-robots.js +76 -0
- package/dist/cli/handlers/seo-serp.d.ts +54 -0
- package/dist/cli/handlers/seo-serp.js +243 -0
- package/dist/cli/handlers/seo-sitemap.d.ts +1 -0
- package/dist/cli/handlers/seo-sitemap.js +55 -0
- package/dist/cli/handlers/seo-spider.d.ts +1 -0
- package/dist/cli/handlers/seo-spider.js +334 -0
- package/dist/cli/handlers/seo.d.ts +8 -4
- package/dist/cli/handlers/seo.js +294 -442
- package/dist/cli/handlers/spider.js +94 -17
- package/dist/cli/handlers/streaming.js +5 -1
- package/dist/cli/index.js +11 -2
- package/dist/cli/presets.d.ts +1 -1
- package/dist/cli/presets.js +15 -4
- package/dist/cli/tui/app.js +6 -1
- package/dist/cli/tui/components/rich-response.d.ts +72 -0
- package/dist/cli/tui/components/rich-response.js +117 -0
- package/dist/cli/tui/executor-commands/background.js +30 -24
- package/dist/cli/tui/executor-commands/testing.js +3 -2
- package/dist/cli/tui/hooks/useDomains.d.ts +17 -0
- package/dist/cli/tui/hooks/useHelp.js +15 -2
- package/dist/cli/tui/job-manager.d.ts +4 -4
- package/dist/cli/tui/job-manager.js +5 -1
- package/dist/cli/tui/spider-tui.d.ts +63 -0
- package/dist/cli/tui/spider-tui.js +120 -2
- package/dist/cli/types.d.ts +12 -0
- package/dist/cli/types.js +1 -0
- package/dist/cli/utils/option-helpers.d.ts +10 -0
- package/dist/cli/utils/option-helpers.js +63 -0
- package/dist/cli/utils/score-color.d.ts +11 -0
- package/dist/cli/utils/score-color.js +11 -0
- package/dist/cli/utils/serp-campaign.d.ts +53 -0
- package/dist/cli/utils/serp-campaign.js +53 -0
- package/dist/cli/utils/serp-config.d.ts +26 -0
- package/dist/cli/utils/serp-config.js +125 -0
- package/dist/core/client.d.ts +15 -10
- package/dist/core/client.js +54 -38
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp/cli.js +35 -34
- package/dist/mcp/client.d.ts +2 -2
- package/dist/mcp/client.js +20 -2
- package/dist/mcp/contract.d.ts +1 -1
- package/dist/mcp/profiles.js +5 -1
- package/dist/mcp/prompts/index.js +8 -4
- package/dist/mcp/resources/index.js +46 -23
- package/dist/mcp/server.js +9 -6
- package/dist/mcp/tools/protocols.js +9 -2
- package/dist/mcp/tools/registry.js +13 -3
- package/dist/mcp/tools/seo.js +427 -2
- package/dist/mcp/types.d.ts +5 -1
- package/dist/plugins/proxy-rotator.d.ts +2 -2
- package/dist/plugins/proxy-rotator.js +6 -28
- package/dist/raffel/client.d.ts +38 -0
- package/dist/raffel/client.js +282 -0
- package/dist/raffel/index.d.ts +2 -0
- package/dist/raffel/index.js +2 -0
- package/dist/raffel/types.d.ts +40 -0
- package/dist/raffel/types.js +14 -0
- package/dist/recker.d.ts +13 -7
- package/dist/recker.js +58 -6
- package/dist/scrape/document.js +2 -2
- package/dist/scrape/element.js +7 -1
- package/dist/scrape/parser/nodes/html.js +1 -1
- package/dist/scrape/spider.d.ts +52 -0
- package/dist/scrape/spider.js +620 -38
- package/dist/scrape/types.d.ts +2 -0
- package/dist/search/google.d.ts +26 -1
- package/dist/search/google.js +427 -45
- package/dist/search/index.d.ts +1 -1
- package/dist/seo/analyzer.d.ts +1 -0
- package/dist/seo/analyzer.js +144 -1
- package/dist/seo/index.d.ts +1 -1
- package/dist/seo/keyword-campaign-analyzer.d.ts +2 -0
- package/dist/seo/keyword-campaign-analyzer.js +538 -0
- package/dist/seo/keyword-campaign-seed-advanced.d.ts +17 -0
- package/dist/seo/keyword-campaign-seed-advanced.js +269 -0
- package/dist/seo/keyword-campaign-seed-core.d.ts +29 -0
- package/dist/seo/keyword-campaign-seed-core.js +525 -0
- package/dist/seo/keyword-campaign-shared.d.ts +165 -0
- package/dist/seo/keyword-campaign-shared.js +59 -0
- package/dist/seo/keyword-campaign.d.ts +4 -107
- package/dist/seo/keyword-campaign.js +2 -380
- package/dist/seo/keywords.js +5 -22
- package/dist/seo/types.d.ts +19 -0
- package/dist/template/index.d.ts +1 -1
- package/dist/template/types.d.ts +0 -2
- package/dist/testing/index.d.ts +1 -22
- package/dist/testing/index.js +1 -11
- package/dist/transport/curl.d.ts +5 -1
- package/dist/transport/curl.js +207 -50
- package/dist/transport/undici.d.ts +4 -3
- package/dist/transport/undici.js +123 -49
- package/dist/types/index.d.ts +9 -3
- package/dist/utils/binary-manager.js +26 -3
- package/dist/utils/block-detector.d.ts +8 -0
- package/dist/utils/block-detector.js +542 -7
- package/dist/version.js +1 -1
- package/package.json +12 -1
- package/dist/testing/mock-dns-server.d.ts +0 -69
- package/dist/testing/mock-dns-server.js +0 -269
- package/dist/testing/mock-ftp-server.d.ts +0 -89
- package/dist/testing/mock-ftp-server.js +0 -562
- package/dist/testing/mock-hls-server.d.ts +0 -80
- package/dist/testing/mock-hls-server.js +0 -381
- package/dist/testing/mock-http-server.d.ts +0 -124
- package/dist/testing/mock-http-server.js +0 -343
- package/dist/testing/mock-proxy-server.d.ts +0 -108
- package/dist/testing/mock-proxy-server.js +0 -615
- package/dist/testing/mock-sse-server.d.ts +0 -76
- package/dist/testing/mock-sse-server.js +0 -291
- package/dist/testing/mock-telnet-server.d.ts +0 -59
- package/dist/testing/mock-telnet-server.js +0 -274
- package/dist/testing/mock-udp-server.d.ts +0 -43
- package/dist/testing/mock-udp-server.js +0 -188
- package/dist/testing/mock-websocket-server.d.ts +0 -76
- package/dist/testing/mock-websocket-server.js +0 -334
- package/dist/testing/mock-whois-server.d.ts +0 -56
- package/dist/testing/mock-whois-server.js +0 -234
- package/dist/testing/proxy-certs.d.ts +0 -19
- package/dist/testing/proxy-certs.js +0 -208
package/dist/scrape/spider.js
CHANGED
|
@@ -4,10 +4,64 @@ import { ScrapeDocument } from './document.js';
|
|
|
4
4
|
import { RequestPool } from '../utils/request-pool.js';
|
|
5
5
|
import { discoverSitemaps, fetchAndValidateSitemap, } from '../seo/validators/sitemap.js';
|
|
6
6
|
import { fetchAndValidateRobotsTxt, isPathAllowed, } from '../seo/validators/robots.js';
|
|
7
|
-
import { detectBlock, isProtectedDomain } from '../utils/block-detector.js';
|
|
7
|
+
import { detectBlock, detectCaptcha, isProtectedDomain, } from '../utils/block-detector.js';
|
|
8
8
|
import { hasImpersonate } from '../utils/binary-manager.js';
|
|
9
9
|
import { CurlTransport } from '../transport/curl.js';
|
|
10
10
|
import { HttpRequest } from '../core/request.js';
|
|
11
|
+
import { getRandomUserAgent } from '../utils/user-agent.js';
|
|
12
|
+
const FALLBACK_ACCEPT_LANGUAGES = [
|
|
13
|
+
'en-US,en;q=0.9',
|
|
14
|
+
'en-GB,en;q=0.9',
|
|
15
|
+
'en-CA,en;q=0.9',
|
|
16
|
+
'pt-BR,pt;q=0.9,en-US;q=0.8',
|
|
17
|
+
'es-ES,es;q=0.9,en;q=0.8',
|
|
18
|
+
];
|
|
19
|
+
const FALLBACK_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
|
|
20
|
+
const RETRIABLE_STATUS_CODES = new Set([403, 429, 500, 502, 503, 504]);
|
|
21
|
+
const SEC_CH_UA_HINTS = [
|
|
22
|
+
'"Not(A:Brand";v="99", "Chromium";v="120", "Google Chrome";v="120")',
|
|
23
|
+
'"Not(A:Brand";v="99", "Chromium";v="120", "Microsoft Edge";v="120")',
|
|
24
|
+
];
|
|
25
|
+
const SEC_CH_UA_MOBILE_HINTS = [
|
|
26
|
+
'"Not(A:Brand";v="99", "Chromium";v="120", "Google Chrome";v="120")',
|
|
27
|
+
'"Not(A:Brand";v="99", "Chromium";v="120", "Google Chrome M";v="120")',
|
|
28
|
+
];
|
|
29
|
+
const SEC_CH_PLATFORM_HINTS = [
|
|
30
|
+
'"Windows"',
|
|
31
|
+
'"macOS"',
|
|
32
|
+
'"Linux"',
|
|
33
|
+
'"Android"',
|
|
34
|
+
'"iOS"',
|
|
35
|
+
];
|
|
36
|
+
function getHostname(url) {
|
|
37
|
+
return new URL(url).hostname;
|
|
38
|
+
}
|
|
39
|
+
function sleep(ms) {
|
|
40
|
+
if (ms <= 0)
|
|
41
|
+
return Promise.resolve();
|
|
42
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
43
|
+
}
|
|
44
|
+
function getRetryAfterDelay(response) {
|
|
45
|
+
const retryAfter = response.headers.get('retry-after');
|
|
46
|
+
if (!retryAfter)
|
|
47
|
+
return 0;
|
|
48
|
+
const asSeconds = Number.parseInt(retryAfter, 10);
|
|
49
|
+
if (!Number.isNaN(asSeconds) && asSeconds >= 0) {
|
|
50
|
+
return asSeconds * 1000;
|
|
51
|
+
}
|
|
52
|
+
const asDate = Date.parse(retryAfter);
|
|
53
|
+
if (!Number.isNaN(asDate)) {
|
|
54
|
+
return Math.max(0, asDate - Date.now());
|
|
55
|
+
}
|
|
56
|
+
return 0;
|
|
57
|
+
}
|
|
58
|
+
function pickRandom(values) {
|
|
59
|
+
if (values.length === 0) {
|
|
60
|
+
throw new Error('Cannot pick from an empty array');
|
|
61
|
+
}
|
|
62
|
+
const index = Math.floor(Math.random() * values.length);
|
|
63
|
+
return values[index];
|
|
64
|
+
}
|
|
11
65
|
const TRACKING_PARAMS = new Set([
|
|
12
66
|
'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
|
|
13
67
|
'gclid', 'gclsrc', 'dclid',
|
|
@@ -76,6 +130,28 @@ function shouldCrawl(url, baseHost, options) {
|
|
|
76
130
|
return false;
|
|
77
131
|
}
|
|
78
132
|
}
|
|
133
|
+
function isHtmlContentType(contentType) {
|
|
134
|
+
const normalized = contentType.toLowerCase();
|
|
135
|
+
return (normalized.includes('text/html') ||
|
|
136
|
+
normalized.includes('application/xhtml+xml') ||
|
|
137
|
+
normalized.includes('application/html'));
|
|
138
|
+
}
|
|
139
|
+
function looksLikeHtmlDocument(content) {
|
|
140
|
+
const snippet = content.trimStart().slice(0, 2048).toLowerCase();
|
|
141
|
+
return (snippet.startsWith('<!doctype html') ||
|
|
142
|
+
snippet.startsWith('<html') ||
|
|
143
|
+
/<\s*(html|head|body|main|div|title|script|style|link|meta|header|footer)\b/.test(snippet));
|
|
144
|
+
}
|
|
145
|
+
function shouldReadResponseBody(status, contentType) {
|
|
146
|
+
return (!contentType ||
|
|
147
|
+
isHtmlContentType(contentType) ||
|
|
148
|
+
RETRIABLE_STATUS_CODES.has(status) ||
|
|
149
|
+
status === 403);
|
|
150
|
+
}
|
|
151
|
+
function getCharset(contentType) {
|
|
152
|
+
const match = /charset=([^;]+)/i.exec(contentType);
|
|
153
|
+
return match ? match[1].trim() : undefined;
|
|
154
|
+
}
|
|
79
155
|
function parseExtractSelectors(selectors) {
|
|
80
156
|
const schema = {};
|
|
81
157
|
for (const sel of selectors) {
|
|
@@ -107,6 +183,7 @@ export class Spider {
|
|
|
107
183
|
blockedDomains = new Set();
|
|
108
184
|
curlTransport = null;
|
|
109
185
|
curlAvailable = false;
|
|
186
|
+
domainStates = new Map();
|
|
110
187
|
sitemapUrls = [];
|
|
111
188
|
sitemapUrlSet = new Set();
|
|
112
189
|
robotsData = null;
|
|
@@ -137,6 +214,15 @@ export class Spider {
|
|
|
137
214
|
timeout: options.timeout ?? 10000,
|
|
138
215
|
delay: options.delay ?? 100,
|
|
139
216
|
userAgent: options.userAgent ?? 'Recker Spider/1.0',
|
|
217
|
+
rotateUserAgent: options.rotateUserAgent ?? true,
|
|
218
|
+
randomizeHeaders: options.randomizeHeaders ?? true,
|
|
219
|
+
maxRetryAttempts: options.maxRetryAttempts ?? 3,
|
|
220
|
+
baseRetryDelayMs: options.baseRetryDelayMs ?? 1000,
|
|
221
|
+
maxRetryDelayMs: options.maxRetryDelayMs ?? 12000,
|
|
222
|
+
retryBackoffMultiplier: options.retryBackoffMultiplier ?? 2,
|
|
223
|
+
retryJitterMs: options.retryJitterMs ?? 250,
|
|
224
|
+
maxDomainBlockStrikes: options.maxDomainBlockStrikes ?? 2,
|
|
225
|
+
preferCurlFirst: options.preferCurlFirst ?? true,
|
|
140
226
|
respectRobotsTxt: options.respectRobotsTxt ?? true,
|
|
141
227
|
useSitemap: options.useSitemap ?? false,
|
|
142
228
|
transport: options.transport ?? 'auto',
|
|
@@ -145,8 +231,10 @@ export class Spider {
|
|
|
145
231
|
include: options.include,
|
|
146
232
|
onPage: options.onPage,
|
|
147
233
|
onPageWithHtml: options.onPageWithHtml,
|
|
234
|
+
onCaptchaDetected: options.onCaptchaDetected,
|
|
148
235
|
onProgress: options.onProgress,
|
|
149
236
|
extract: extractSchema,
|
|
237
|
+
parserOptions: options.parserOptions,
|
|
150
238
|
};
|
|
151
239
|
this.client = createClient({
|
|
152
240
|
baseUrl: 'http://localhost',
|
|
@@ -185,6 +273,7 @@ export class Spider {
|
|
|
185
273
|
this.sitemapValidation = null;
|
|
186
274
|
this.robotsValidation = null;
|
|
187
275
|
this.blockedDomains.clear();
|
|
276
|
+
this.domainStates.clear();
|
|
188
277
|
if (this.options.transport !== 'undici') {
|
|
189
278
|
this.curlAvailable = await hasImpersonate();
|
|
190
279
|
if (this.options.transport === 'auto' && this.curlAvailable && isProtectedDomain(this.baseHost)) {
|
|
@@ -420,44 +509,252 @@ export class Spider {
|
|
|
420
509
|
};
|
|
421
510
|
}
|
|
422
511
|
async fetchPage(url) {
|
|
423
|
-
const hostname =
|
|
424
|
-
const
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
const
|
|
438
|
-
|
|
439
|
-
const timings = response.timings ? {
|
|
440
|
-
dns: response.timings.dns,
|
|
441
|
-
tcp: response.timings.tcp,
|
|
442
|
-
tls: response.timings.tls,
|
|
443
|
-
ttfb: response.timings.firstByte,
|
|
444
|
-
download: Math.round(downloadTime),
|
|
445
|
-
total: response.timings.firstByte ? Math.round(response.timings.firstByte + downloadTime) : undefined,
|
|
446
|
-
} : undefined;
|
|
447
|
-
if (this.options.transport === 'auto' && this.curlAvailable && this.curlTransport) {
|
|
448
|
-
const detection = detectBlock(response, body);
|
|
449
|
-
if (detection.blocked && detection.confidence > 0.7) {
|
|
450
|
-
this.blockedDomains.add(hostname);
|
|
512
|
+
const hostname = getHostname(url);
|
|
513
|
+
const maxAttempts = Math.max(1, this.options.maxRetryAttempts);
|
|
514
|
+
const hasCurl = this.options.transport !== 'undici' && this.curlAvailable && this.curlTransport !== null;
|
|
515
|
+
let lastResponse = null;
|
|
516
|
+
let lastBody = '';
|
|
517
|
+
let lastDetection = { blocked: false, confidence: 0 };
|
|
518
|
+
let lastCaptcha = { detected: false, confidence: 0 };
|
|
519
|
+
let lastTimings;
|
|
520
|
+
let lastUsedCurl = false;
|
|
521
|
+
let lastError;
|
|
522
|
+
let captchaReported = false;
|
|
523
|
+
let forcedTransport = null;
|
|
524
|
+
let attemptLog = [];
|
|
525
|
+
let lastRetryAfterMs = 0;
|
|
526
|
+
const executeRequest = async (useCurl) => {
|
|
527
|
+
if (useCurl && this.curlTransport) {
|
|
451
528
|
const req = new HttpRequest(url, {
|
|
452
529
|
method: 'GET',
|
|
453
|
-
headers:
|
|
530
|
+
headers: this.buildRequestHeaders(url, true),
|
|
454
531
|
});
|
|
455
|
-
const
|
|
456
|
-
const
|
|
457
|
-
|
|
532
|
+
const requestStart = performance.now();
|
|
533
|
+
const response = await this.curlTransport.dispatch(req);
|
|
534
|
+
const contentType = response.headers.get('content-type') || '';
|
|
535
|
+
const shouldReadCurlResponseBody = shouldReadResponseBody(response.status, contentType);
|
|
536
|
+
let body = '';
|
|
537
|
+
const responseStart = performance.now();
|
|
538
|
+
if (shouldReadCurlResponseBody) {
|
|
539
|
+
body = await response.text();
|
|
540
|
+
}
|
|
541
|
+
const totalMs = Math.round(performance.now() - requestStart);
|
|
542
|
+
const downloadMs = Math.round(performance.now() - responseStart);
|
|
543
|
+
const timings = response.timings ? {
|
|
544
|
+
dns: response.timings.dns,
|
|
545
|
+
tcp: response.timings.tcp,
|
|
546
|
+
tls: response.timings.tls,
|
|
547
|
+
ttfb: response.timings.firstByte,
|
|
548
|
+
download: shouldReadCurlResponseBody ? downloadMs : undefined,
|
|
549
|
+
total: response.timings.total ? Math.round(response.timings.total) : totalMs,
|
|
550
|
+
} : {
|
|
551
|
+
download: shouldReadCurlResponseBody ? downloadMs : undefined,
|
|
552
|
+
total: totalMs,
|
|
553
|
+
};
|
|
554
|
+
return {
|
|
555
|
+
response: response,
|
|
556
|
+
body,
|
|
557
|
+
usedCurl: true,
|
|
558
|
+
timings,
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
const response = await this.client.get(url, {
|
|
562
|
+
headers: this.buildRequestHeaders(url, false),
|
|
563
|
+
});
|
|
564
|
+
const contentType = response.headers.get('content-type') || '';
|
|
565
|
+
const shouldReadUndiciBody = !contentType ||
|
|
566
|
+
isHtmlContentType(contentType) ||
|
|
567
|
+
this.isRetryableStatus(response.status);
|
|
568
|
+
let body = '';
|
|
569
|
+
let downloadMs = 0;
|
|
570
|
+
const requestEnd = performance.now();
|
|
571
|
+
if (shouldReadUndiciBody) {
|
|
572
|
+
const downloadStart = performance.now();
|
|
573
|
+
body = await response.text();
|
|
574
|
+
downloadMs = Math.round(performance.now() - downloadStart);
|
|
575
|
+
}
|
|
576
|
+
const timings = response.timings ? {
|
|
577
|
+
dns: response.timings?.dns,
|
|
578
|
+
tcp: response.timings?.tcp,
|
|
579
|
+
tls: response.timings?.tls,
|
|
580
|
+
ttfb: response.timings?.firstByte,
|
|
581
|
+
download: shouldReadUndiciBody ? downloadMs : undefined,
|
|
582
|
+
total: response.timings?.total ? Math.round(response.timings.total) : Math.round(performance.now() - requestEnd),
|
|
583
|
+
} : {
|
|
584
|
+
download: shouldReadUndiciBody ? downloadMs : undefined,
|
|
585
|
+
total: Math.round(performance.now() - requestEnd),
|
|
586
|
+
};
|
|
587
|
+
return {
|
|
588
|
+
response: response,
|
|
589
|
+
body,
|
|
590
|
+
usedCurl: false,
|
|
591
|
+
timings,
|
|
592
|
+
};
|
|
593
|
+
};
|
|
594
|
+
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
595
|
+
await this.waitForDomainPenalty(hostname);
|
|
596
|
+
const useCurl = this.shouldUseCurlForHost(hostname, hasCurl, forcedTransport);
|
|
597
|
+
const transportForAttempt = useCurl ? 'curl' : 'undici';
|
|
598
|
+
forcedTransport = null;
|
|
599
|
+
const attemptStart = performance.now();
|
|
600
|
+
try {
|
|
601
|
+
const { response, body, usedCurl, timings } = await executeRequest(transportForAttempt === 'curl');
|
|
602
|
+
const transportUsed = usedCurl ? 'curl' : 'undici';
|
|
603
|
+
this.recordTransportResult(hostname, transportForAttempt, true);
|
|
604
|
+
const contentType = response.headers.get('content-type') || '';
|
|
605
|
+
const readBodyForDetection = shouldReadResponseBody(response.status, contentType);
|
|
606
|
+
const detection = detectBlock(response, readBodyForDetection ? body : undefined);
|
|
607
|
+
const captcha = detectCaptcha(response, readBodyForDetection ? body : undefined);
|
|
608
|
+
const hasCaptchaSignal = captcha.detected && captcha.confidence >= 0.6;
|
|
609
|
+
const highConfidenceCaptcha = captcha.detected && captcha.confidence >= 0.8;
|
|
610
|
+
const state = this.getOrCreateDomainState(hostname);
|
|
611
|
+
if (captcha.detected && this.options.onCaptchaDetected && !captchaReported) {
|
|
612
|
+
await this.options.onCaptchaDetected({
|
|
613
|
+
url,
|
|
614
|
+
status: response.status,
|
|
615
|
+
confidence: captcha.confidence,
|
|
616
|
+
provider: captcha.provider,
|
|
617
|
+
usedCurl,
|
|
618
|
+
});
|
|
619
|
+
captchaReported = true;
|
|
620
|
+
}
|
|
621
|
+
if (hasCaptchaSignal) {
|
|
622
|
+
this.registerDomainChallenge(hostname, captcha.provider, captcha.confidence);
|
|
623
|
+
}
|
|
624
|
+
lastResponse = response;
|
|
625
|
+
lastBody = body;
|
|
626
|
+
lastDetection = detection;
|
|
627
|
+
lastCaptcha = captcha;
|
|
628
|
+
lastUsedCurl = usedCurl;
|
|
629
|
+
lastTimings = timings;
|
|
630
|
+
const attemptReason = hasCaptchaSignal ? 'captcha' : detection.reason;
|
|
631
|
+
attemptLog.push({
|
|
632
|
+
attempt: attempt + 1,
|
|
633
|
+
transport: transportForAttempt,
|
|
634
|
+
status: response.status,
|
|
635
|
+
blocked: detection.blocked || hasCaptchaSignal,
|
|
636
|
+
captchaDetected: hasCaptchaSignal,
|
|
637
|
+
reason: attemptReason,
|
|
638
|
+
durationMs: Math.max(0, Math.round(performance.now() - attemptStart)),
|
|
639
|
+
});
|
|
640
|
+
const canRetryWithCurl = this.options.transport === 'auto' && hasCurl;
|
|
641
|
+
const retryOnCaptcha = hasCaptchaSignal && canRetryWithCurl && highConfidenceCaptcha;
|
|
642
|
+
const shouldRetry = (attempt < maxAttempts - 1 &&
|
|
643
|
+
(retryOnCaptcha ||
|
|
644
|
+
this.isRetryableStatus(response.status) ||
|
|
645
|
+
(detection.blocked && detection.confidence >= 0.6) ||
|
|
646
|
+
(hasCaptchaSignal && this.options.transport !== 'undici')));
|
|
647
|
+
const isHighQualitySuccess = response.status < 400 && !detection.blocked && !hasCaptchaSignal;
|
|
648
|
+
if (isHighQualitySuccess) {
|
|
649
|
+
this.registerDomainSuccess(hostname);
|
|
650
|
+
}
|
|
651
|
+
if (!shouldRetry || attempt === maxAttempts - 1) {
|
|
652
|
+
return {
|
|
653
|
+
response,
|
|
654
|
+
body,
|
|
655
|
+
usedCurl: transportUsed === 'curl',
|
|
656
|
+
timings,
|
|
657
|
+
detection,
|
|
658
|
+
captcha,
|
|
659
|
+
attempts: attempt + 1,
|
|
660
|
+
retryCount: attempt,
|
|
661
|
+
retryAfterMs: lastRetryAfterMs,
|
|
662
|
+
transportUsed: transportForAttempt,
|
|
663
|
+
attemptLog,
|
|
664
|
+
};
|
|
665
|
+
}
|
|
666
|
+
if (!hasCaptchaSignal && (detection.blocked || this.isRetryableStatus(response.status))) {
|
|
667
|
+
this.registerDomainBlock(hostname);
|
|
668
|
+
}
|
|
669
|
+
const retryAfter = getRetryAfterDelay(response);
|
|
670
|
+
state.retryAfterMs = Math.max(retryAfter, this.getRetryWait(hostname, attempt + 1, captcha.provider));
|
|
671
|
+
lastRetryAfterMs = state.retryAfterMs;
|
|
672
|
+
const waitMs = this.getRetryWait(hostname, attempt + 1, captcha.provider);
|
|
673
|
+
if (this.options.transport === 'auto' && hasCurl) {
|
|
674
|
+
if (transportUsed === 'curl' && state.consecutiveCurlFailures >= 2) {
|
|
675
|
+
forcedTransport = 'undici';
|
|
676
|
+
}
|
|
677
|
+
else if (transportUsed === 'undici' && (detection.blocked || hasCaptchaSignal)) {
|
|
678
|
+
forcedTransport = 'curl';
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
await sleep(waitMs);
|
|
682
|
+
continue;
|
|
683
|
+
}
|
|
684
|
+
catch (error) {
|
|
685
|
+
lastError = error;
|
|
686
|
+
this.recordTransportResult(hostname, transportForAttempt, false);
|
|
687
|
+
const state = this.getOrCreateDomainState(hostname);
|
|
688
|
+
state.retryAfterMs = this.getRetryWait(hostname, attempt + 1);
|
|
689
|
+
lastRetryAfterMs = state.retryAfterMs;
|
|
690
|
+
attemptLog.push({
|
|
691
|
+
attempt: attempt + 1,
|
|
692
|
+
transport: transportForAttempt,
|
|
693
|
+
status: 0,
|
|
694
|
+
blocked: false,
|
|
695
|
+
captchaDetected: false,
|
|
696
|
+
reason: 'status',
|
|
697
|
+
durationMs: Math.max(0, Math.round(performance.now() - attemptStart)),
|
|
698
|
+
});
|
|
699
|
+
if (attempt === maxAttempts - 1) {
|
|
700
|
+
throw error;
|
|
701
|
+
}
|
|
702
|
+
this.registerDomainBlock(hostname);
|
|
703
|
+
if (this.options.transport === 'auto' && hasCurl) {
|
|
704
|
+
if (transportForAttempt === 'curl') {
|
|
705
|
+
const state = this.getOrCreateDomainState(hostname);
|
|
706
|
+
forcedTransport = state.consecutiveCurlFailures >= 2 ? 'undici' : null;
|
|
707
|
+
}
|
|
708
|
+
else {
|
|
709
|
+
forcedTransport = 'curl';
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
const waitMs = this.getRetryWait(hostname, attempt + 1);
|
|
713
|
+
await sleep(waitMs);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
if (lastResponse) {
|
|
717
|
+
return {
|
|
718
|
+
response: lastResponse,
|
|
719
|
+
body: lastBody,
|
|
720
|
+
usedCurl: lastUsedCurl,
|
|
721
|
+
timings: lastTimings,
|
|
722
|
+
detection: lastDetection,
|
|
723
|
+
captcha: lastCaptcha,
|
|
724
|
+
attempts: attemptLog.length || 1,
|
|
725
|
+
retryCount: Math.max(0, attemptLog.length - 1),
|
|
726
|
+
retryAfterMs: lastRetryAfterMs,
|
|
727
|
+
transportUsed: lastUsedCurl ? 'curl' : 'undici',
|
|
728
|
+
attemptLog,
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
const attempts = attemptLog.length || 1;
|
|
732
|
+
const retryCount = Math.max(0, attempts - 1);
|
|
733
|
+
if (lastError) {
|
|
734
|
+
const wrapped = (lastError instanceof Error ? lastError : new Error(`Failed to fetch ${url}`));
|
|
735
|
+
wrapped.attempts = attempts;
|
|
736
|
+
wrapped.retryCount = retryCount;
|
|
737
|
+
wrapped.retryAfterMs = lastRetryAfterMs;
|
|
738
|
+
wrapped.transportUsed = lastUsedCurl ? 'curl' : 'undici';
|
|
739
|
+
if (lastResponse) {
|
|
740
|
+
wrapped.response = lastResponse;
|
|
741
|
+
wrapped.security = {
|
|
742
|
+
blocked: lastDetection.blocked,
|
|
743
|
+
reason: lastDetection.reason,
|
|
744
|
+
confidence: lastDetection.confidence,
|
|
745
|
+
captchaDetected: lastCaptcha.detected,
|
|
746
|
+
captchaProvider: lastCaptcha.provider,
|
|
747
|
+
attempts,
|
|
748
|
+
retryCount,
|
|
749
|
+
retryAfterMs: lastRetryAfterMs,
|
|
750
|
+
transport: lastUsedCurl ? 'curl' : 'undici',
|
|
751
|
+
blockedByCaptcha: lastCaptcha.detected,
|
|
752
|
+
lastStatus: lastResponse.status,
|
|
753
|
+
};
|
|
458
754
|
}
|
|
755
|
+
throw wrapped;
|
|
459
756
|
}
|
|
460
|
-
|
|
757
|
+
throw new Error(`Failed to fetch ${url}`);
|
|
461
758
|
}
|
|
462
759
|
async crawlPage(item) {
|
|
463
760
|
const startTime = performance.now();
|
|
@@ -471,13 +768,54 @@ export class Spider {
|
|
|
471
768
|
depth: item.depth,
|
|
472
769
|
});
|
|
473
770
|
try {
|
|
474
|
-
const { response, body: html, timings } = await this.fetchPage(item.url);
|
|
771
|
+
const { response, body: html, timings, detection, captcha, attempts, retryCount, retryAfterMs, transportUsed, } = await this.fetchPage(item.url);
|
|
475
772
|
const status = response.status;
|
|
476
773
|
const contentType = response.headers.get('content-type') || '';
|
|
477
|
-
|
|
774
|
+
const isHtml = isHtmlContentType(contentType) || (!contentType && looksLikeHtmlDocument(html));
|
|
775
|
+
const hasCaptcha = captcha.detected && captcha.confidence >= 0.7;
|
|
776
|
+
const contentLength = Number.parseInt(response.headers.get('content-length') || '', 10);
|
|
777
|
+
const normalizedHtmlSize = Number.isFinite(contentLength) ? contentLength : html.length;
|
|
778
|
+
const security = {
|
|
779
|
+
blocked: detection.blocked || hasCaptcha,
|
|
780
|
+
reason: hasCaptcha ? (detection.reason ?? 'captcha') : detection.reason,
|
|
781
|
+
confidence: Math.max(detection.confidence, hasCaptcha ? captcha.confidence : 0),
|
|
782
|
+
captchaDetected: hasCaptcha,
|
|
783
|
+
captchaProvider: captcha.provider,
|
|
784
|
+
attempts,
|
|
785
|
+
retryCount,
|
|
786
|
+
retryAfterMs,
|
|
787
|
+
transport: transportUsed,
|
|
788
|
+
blockedByCaptcha: hasCaptcha,
|
|
789
|
+
lastStatus: status,
|
|
790
|
+
};
|
|
791
|
+
if (!isHtml || hasCaptcha || (detection.blocked && detection.reason === 'captcha')) {
|
|
792
|
+
const nonHtmlResult = {
|
|
793
|
+
url: item.url,
|
|
794
|
+
status,
|
|
795
|
+
title: '',
|
|
796
|
+
depth: item.depth,
|
|
797
|
+
links: [],
|
|
798
|
+
duration: Math.round(performance.now() - startTime),
|
|
799
|
+
error: hasCaptcha ? 'CAPTCHA detected' : status >= 400 ? `HTTP ${status}` : 'Blocked by bot protection',
|
|
800
|
+
meta: {
|
|
801
|
+
charset: getCharset(contentType),
|
|
802
|
+
},
|
|
803
|
+
metrics: {
|
|
804
|
+
htmlSize: normalizedHtmlSize,
|
|
805
|
+
textLength: 0,
|
|
806
|
+
},
|
|
807
|
+
timings,
|
|
808
|
+
security,
|
|
809
|
+
fetchedAt,
|
|
810
|
+
};
|
|
811
|
+
this.results.push(nonHtmlResult);
|
|
812
|
+
this.options.onPage?.(nonHtmlResult);
|
|
478
813
|
return;
|
|
479
814
|
}
|
|
480
|
-
const doc = await ScrapeDocument.create(html, {
|
|
815
|
+
const doc = await ScrapeDocument.create(html, {
|
|
816
|
+
baseUrl: item.url,
|
|
817
|
+
parserOptions: this.options.parserOptions,
|
|
818
|
+
});
|
|
481
819
|
const title = doc.selectFirst('title').text() || '';
|
|
482
820
|
const links = doc.links({ absolute: true });
|
|
483
821
|
const meta = doc.meta();
|
|
@@ -486,6 +824,9 @@ export class Spider {
|
|
|
486
824
|
htmlSize: html.length,
|
|
487
825
|
textLength: doc.text('body').length,
|
|
488
826
|
};
|
|
827
|
+
const imageCount = doc.images().length;
|
|
828
|
+
const scriptCount = doc.scripts().length;
|
|
829
|
+
const stylesheetCount = doc.styles().length;
|
|
489
830
|
const social = {
|
|
490
831
|
ogTitle: og.title,
|
|
491
832
|
ogDescription: og.description,
|
|
@@ -517,6 +858,12 @@ export class Spider {
|
|
|
517
858
|
lang: doc.attr('html', 'lang')
|
|
518
859
|
},
|
|
519
860
|
metrics,
|
|
861
|
+
security,
|
|
862
|
+
resources: {
|
|
863
|
+
images: imageCount,
|
|
864
|
+
scripts: scriptCount,
|
|
865
|
+
stylesheets: stylesheetCount,
|
|
866
|
+
},
|
|
520
867
|
social,
|
|
521
868
|
timings,
|
|
522
869
|
fetchedAt,
|
|
@@ -542,8 +889,48 @@ export class Spider {
|
|
|
542
889
|
}
|
|
543
890
|
}
|
|
544
891
|
catch (error) {
|
|
545
|
-
const
|
|
892
|
+
const errAttempts = typeof error?.attempts === 'number' ? error.attempts : 1;
|
|
893
|
+
const errRetryCount = typeof error?.retryCount === 'number' ? error.retryCount : Math.max(0, errAttempts - 1);
|
|
894
|
+
const errRetryAfterMs = typeof error?.retryAfterMs === 'number' ? error?.retryAfterMs : 0;
|
|
895
|
+
const errTransport = error?.transportUsed === 'curl' || error?.transportUsed === 'undici'
|
|
896
|
+
? error.transportUsed
|
|
897
|
+
: this.options.transport === 'curl'
|
|
898
|
+
? 'curl'
|
|
899
|
+
: 'undici';
|
|
900
|
+
const errResponse = error.response;
|
|
901
|
+
const errStatus = errResponse?.status || 0;
|
|
902
|
+
const errDetection = errResponse ? detectBlock(errResponse, undefined) : { blocked: true, confidence: 0.75, reason: 'status' };
|
|
903
|
+
const errCaptcha = errResponse ? detectCaptcha(errResponse, undefined) : { detected: false, confidence: 0 };
|
|
904
|
+
const status = errStatus;
|
|
546
905
|
const message = error.message || 'Unknown error';
|
|
906
|
+
const errorResponse = errResponse;
|
|
907
|
+
const errorContentType = errorResponse?.headers?.get?.('content-type') || '';
|
|
908
|
+
const errorContentLength = Number.parseInt(errorResponse?.headers?.get?.('content-length') || '', 10);
|
|
909
|
+
const errorTimings = errorResponse?.timings
|
|
910
|
+
? {
|
|
911
|
+
dns: errorResponse.timings?.dns,
|
|
912
|
+
tcp: errorResponse.timings?.tcp,
|
|
913
|
+
tls: errorResponse.timings?.tls,
|
|
914
|
+
ttfb: errorResponse.timings?.firstByte,
|
|
915
|
+
total: errorResponse.timings?.total ? Math.round(errorResponse.timings.total) : Math.round(performance.now() - startTime),
|
|
916
|
+
}
|
|
917
|
+
: {
|
|
918
|
+
total: Math.round(performance.now() - startTime),
|
|
919
|
+
};
|
|
920
|
+
const hasCaptcha = errCaptcha.detected && errCaptcha.confidence >= 0.7;
|
|
921
|
+
const security = {
|
|
922
|
+
blocked: errDetection.blocked || hasCaptcha,
|
|
923
|
+
reason: hasCaptcha ? (errDetection.reason ?? 'captcha') : errDetection.reason,
|
|
924
|
+
confidence: Math.max(errDetection.confidence, hasCaptcha ? errCaptcha.confidence : 0),
|
|
925
|
+
captchaDetected: hasCaptcha,
|
|
926
|
+
captchaProvider: errCaptcha.provider,
|
|
927
|
+
attempts: errAttempts,
|
|
928
|
+
retryCount: errRetryCount,
|
|
929
|
+
retryAfterMs: errRetryAfterMs,
|
|
930
|
+
transport: errTransport,
|
|
931
|
+
blockedByCaptcha: hasCaptcha,
|
|
932
|
+
lastStatus: status,
|
|
933
|
+
};
|
|
547
934
|
const errorResult = {
|
|
548
935
|
url: item.url,
|
|
549
936
|
status: status,
|
|
@@ -552,6 +939,15 @@ export class Spider {
|
|
|
552
939
|
links: [],
|
|
553
940
|
duration: Math.round(performance.now() - startTime),
|
|
554
941
|
error: message,
|
|
942
|
+
meta: {
|
|
943
|
+
charset: getCharset(errorContentType),
|
|
944
|
+
},
|
|
945
|
+
metrics: {
|
|
946
|
+
htmlSize: Number.isFinite(errorContentLength) ? errorContentLength : 0,
|
|
947
|
+
textLength: 0,
|
|
948
|
+
},
|
|
949
|
+
timings: errorTimings,
|
|
950
|
+
security,
|
|
555
951
|
fetchedAt,
|
|
556
952
|
};
|
|
557
953
|
this.results.push(errorResult);
|
|
@@ -559,6 +955,192 @@ export class Spider {
|
|
|
559
955
|
this.options.onPage?.(errorResult);
|
|
560
956
|
}
|
|
561
957
|
}
|
|
958
|
+
getOrCreateDomainState(hostname) {
|
|
959
|
+
const current = this.domainStates.get(hostname);
|
|
960
|
+
if (current)
|
|
961
|
+
return current;
|
|
962
|
+
const next = {
|
|
963
|
+
strikes: 0,
|
|
964
|
+
retryAfterMs: 0,
|
|
965
|
+
penaltyUntil: 0,
|
|
966
|
+
challengeCooldownUntil: 0,
|
|
967
|
+
consecutiveCurlFailures: 0,
|
|
968
|
+
consecutiveUndiciFailures: 0,
|
|
969
|
+
lastTransport: 'undici',
|
|
970
|
+
lastCaptchaConfidence: 0,
|
|
971
|
+
};
|
|
972
|
+
this.domainStates.set(hostname, next);
|
|
973
|
+
return next;
|
|
974
|
+
}
|
|
975
|
+
recordTransportResult(hostname, transport, success) {
|
|
976
|
+
const state = this.getOrCreateDomainState(hostname);
|
|
977
|
+
state.lastTransport = transport;
|
|
978
|
+
if (success) {
|
|
979
|
+
if (transport === 'curl') {
|
|
980
|
+
state.consecutiveCurlFailures = 0;
|
|
981
|
+
}
|
|
982
|
+
else if (transport === 'undici') {
|
|
983
|
+
state.consecutiveUndiciFailures = 0;
|
|
984
|
+
}
|
|
985
|
+
return;
|
|
986
|
+
}
|
|
987
|
+
if (transport === 'curl') {
|
|
988
|
+
state.consecutiveCurlFailures += 1;
|
|
989
|
+
}
|
|
990
|
+
else {
|
|
991
|
+
state.consecutiveUndiciFailures += 1;
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
isRetryableStatus(status) {
|
|
995
|
+
return RETRIABLE_STATUS_CODES.has(status);
|
|
996
|
+
}
|
|
997
|
+
buildRequestHeaders(url, useCurl) {
|
|
998
|
+
const hostname = getHostname(url);
|
|
999
|
+
const randomizedHeaders = this.options.randomizeHeaders;
|
|
1000
|
+
const userAgent = this.options.rotateUserAgent
|
|
1001
|
+
? (Math.random() < 0.8 ? getRandomUserAgent('desktop') : getRandomUserAgent('mobile'))
|
|
1002
|
+
: this.options.userAgent;
|
|
1003
|
+
const isMobile = /mobile|android|iphone|ipad/i.test(userAgent);
|
|
1004
|
+
const baseHeaders = new Headers({
|
|
1005
|
+
'Accept': FALLBACK_ACCEPT,
|
|
1006
|
+
'Accept-Language': pickRandom(FALLBACK_ACCEPT_LANGUAGES),
|
|
1007
|
+
'Cache-Control': 'max-age=0',
|
|
1008
|
+
'Upgrade-Insecure-Requests': '1',
|
|
1009
|
+
'Sec-Fetch-Dest': 'document',
|
|
1010
|
+
'Sec-Fetch-Mode': 'navigate',
|
|
1011
|
+
'Sec-Fetch-Site': 'same-origin',
|
|
1012
|
+
'DNT': '1',
|
|
1013
|
+
'Connection': 'keep-alive',
|
|
1014
|
+
Referer: `${new URL(url).origin}/`,
|
|
1015
|
+
'User-Agent': userAgent,
|
|
1016
|
+
'sec-ch-ua': isMobile ? pickRandom(SEC_CH_UA_MOBILE_HINTS) : pickRandom(SEC_CH_UA_HINTS),
|
|
1017
|
+
'sec-ch-ua-platform': pickRandom(SEC_CH_PLATFORM_HINTS),
|
|
1018
|
+
'sec-ch-ua-mobile': isMobile ? '?1' : '?0',
|
|
1019
|
+
});
|
|
1020
|
+
if (useCurl) {
|
|
1021
|
+
baseHeaders.set('Referer', `${new URL(url).origin}/`);
|
|
1022
|
+
}
|
|
1023
|
+
if (!randomizedHeaders) {
|
|
1024
|
+
baseHeaders.delete('Sec-Fetch-Mode');
|
|
1025
|
+
baseHeaders.delete('Sec-Fetch-Dest');
|
|
1026
|
+
baseHeaders.delete('Sec-Fetch-Site');
|
|
1027
|
+
baseHeaders.delete('DNT');
|
|
1028
|
+
baseHeaders.delete('sec-ch-ua');
|
|
1029
|
+
baseHeaders.delete('sec-ch-ua-platform');
|
|
1030
|
+
baseHeaders.delete('sec-ch-ua-mobile');
|
|
1031
|
+
baseHeaders.set('Accept-Language', 'en-US,en;q=0.9');
|
|
1032
|
+
}
|
|
1033
|
+
if (this.options.rotateUserAgent && hostname.includes('.')) {
|
|
1034
|
+
baseHeaders.set('Accept-Language', pickRandom(FALLBACK_ACCEPT_LANGUAGES));
|
|
1035
|
+
}
|
|
1036
|
+
if (this.domainStates.get(hostname)?.strikes) {
|
|
1037
|
+
baseHeaders.set('Pragma', 'no-cache');
|
|
1038
|
+
}
|
|
1039
|
+
return baseHeaders;
|
|
1040
|
+
}
|
|
1041
|
+
shouldUseCurlForHost(hostname, hasCurl, forceTransport = null) {
|
|
1042
|
+
if (!hasCurl)
|
|
1043
|
+
return false;
|
|
1044
|
+
if (this.options.transport === 'curl')
|
|
1045
|
+
return true;
|
|
1046
|
+
if (this.options.transport !== 'auto')
|
|
1047
|
+
return false;
|
|
1048
|
+
const state = this.getOrCreateDomainState(hostname);
|
|
1049
|
+
if (forceTransport === 'curl') {
|
|
1050
|
+
return true;
|
|
1051
|
+
}
|
|
1052
|
+
if (forceTransport === 'undici') {
|
|
1053
|
+
return false;
|
|
1054
|
+
}
|
|
1055
|
+
const hasActiveChallengeCooldown = state.challengeCooldownUntil > Date.now();
|
|
1056
|
+
if (hasActiveChallengeCooldown || isProtectedDomain(hostname) || this.blockedDomains.has(hostname)) {
|
|
1057
|
+
return true;
|
|
1058
|
+
}
|
|
1059
|
+
if (state.consecutiveCurlFailures >= 2 && state.lastTransport === 'curl') {
|
|
1060
|
+
return false;
|
|
1061
|
+
}
|
|
1062
|
+
if (this.options.preferCurlFirst) {
|
|
1063
|
+
return true;
|
|
1064
|
+
}
|
|
1065
|
+
if (state.consecutiveUndiciFailures >= 2 && state.lastTransport === 'undici') {
|
|
1066
|
+
return true;
|
|
1067
|
+
}
|
|
1068
|
+
return false;
|
|
1069
|
+
}
|
|
1070
|
+
async waitForDomainPenalty(hostname) {
|
|
1071
|
+
const state = this.getOrCreateDomainState(hostname);
|
|
1072
|
+
const now = Date.now();
|
|
1073
|
+
const delay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
|
|
1074
|
+
if (delay > 0) {
|
|
1075
|
+
await sleep(delay);
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
registerDomainBlock(hostname) {
|
|
1079
|
+
const state = this.getOrCreateDomainState(hostname);
|
|
1080
|
+
state.strikes += 1;
|
|
1081
|
+
const baseDelay = this.options.baseRetryDelayMs;
|
|
1082
|
+
const cappedDelay = Math.min(this.options.maxRetryDelayMs, baseDelay * Math.pow(this.options.retryBackoffMultiplier, Math.max(state.strikes - 1, 0)));
|
|
1083
|
+
state.penaltyUntil = Date.now() + cappedDelay;
|
|
1084
|
+
state.retryAfterMs = cappedDelay;
|
|
1085
|
+
if (this.options.transport === 'auto' && state.strikes >= this.options.maxDomainBlockStrikes) {
|
|
1086
|
+
this.blockedDomains.add(hostname);
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
registerDomainSuccess(hostname) {
|
|
1090
|
+
const state = this.domainStates.get(hostname);
|
|
1091
|
+
if (!state)
|
|
1092
|
+
return;
|
|
1093
|
+
state.strikes = Math.max(0, state.strikes - 1);
|
|
1094
|
+
state.penaltyUntil = 0;
|
|
1095
|
+
state.challengeCooldownUntil = 0;
|
|
1096
|
+
if (state.strikes === 0) {
|
|
1097
|
+
state.lastCaptchaConfidence = 0;
|
|
1098
|
+
state.lastCaptchaProvider = undefined;
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
getCaptchaRetryMultiplier(provider) {
|
|
1102
|
+
if (!provider)
|
|
1103
|
+
return 1.2;
|
|
1104
|
+
switch (provider) {
|
|
1105
|
+
case 'cloudflare':
|
|
1106
|
+
case 'datadome':
|
|
1107
|
+
return 1.9;
|
|
1108
|
+
case 'funcaptcha':
|
|
1109
|
+
return 1.7;
|
|
1110
|
+
case 'recaptcha':
|
|
1111
|
+
case 'hcaptcha':
|
|
1112
|
+
return 1.5;
|
|
1113
|
+
case 'turnstile':
|
|
1114
|
+
return 1.4;
|
|
1115
|
+
default:
|
|
1116
|
+
return 1.25;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
registerDomainChallenge(hostname, provider, confidence = 0) {
|
|
1120
|
+
const state = this.getOrCreateDomainState(hostname);
|
|
1121
|
+
const providerMultiplier = this.getCaptchaRetryMultiplier(provider);
|
|
1122
|
+
const confidenceMultiplier = 1 + Math.min(confidence, 0.95);
|
|
1123
|
+
const challengePenalty = Math.round(this.options.baseRetryDelayMs
|
|
1124
|
+
* this.options.retryBackoffMultiplier
|
|
1125
|
+
* providerMultiplier
|
|
1126
|
+
* confidenceMultiplier);
|
|
1127
|
+
state.strikes += 1;
|
|
1128
|
+
state.lastCaptchaProvider = provider;
|
|
1129
|
+
state.lastCaptchaConfidence = Math.max(state.lastCaptchaConfidence, confidence);
|
|
1130
|
+
state.challengeCooldownUntil = Date.now() + Math.min(this.options.maxRetryDelayMs, challengePenalty);
|
|
1131
|
+
state.penaltyUntil = state.challengeCooldownUntil;
|
|
1132
|
+
state.retryAfterMs = Math.min(this.options.maxRetryDelayMs, challengePenalty);
|
|
1133
|
+
this.blockedDomains.add(hostname);
|
|
1134
|
+
}
|
|
1135
|
+
getRetryWait(hostname, attempt, provider) {
|
|
1136
|
+
const state = this.getOrCreateDomainState(hostname);
|
|
1137
|
+
const providerMultiplier = this.getCaptchaRetryMultiplier(provider ?? state.lastCaptchaProvider);
|
|
1138
|
+
const baseDelay = this.options.baseRetryDelayMs;
|
|
1139
|
+
const attemptFactor = Math.pow(this.options.retryBackoffMultiplier, attempt);
|
|
1140
|
+
const retryAfter = state.retryAfterMs;
|
|
1141
|
+
const computed = Math.min(this.options.maxRetryDelayMs, Math.max(baseDelay * attemptFactor * providerMultiplier, baseDelay));
|
|
1142
|
+
return Math.max(computed, retryAfter) + (this.options.retryJitterMs ? Math.random() * this.options.retryJitterMs : 0);
|
|
1143
|
+
}
|
|
562
1144
|
abort() {
|
|
563
1145
|
this.aborted = true;
|
|
564
1146
|
}
|