recker 1.0.79 → 1.0.80-next.70ea84d

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/dist/browser/browser/mini.d.ts +2 -2
  2. package/dist/browser/browser/recker-mini.d.ts +8 -8
  3. package/dist/browser/browser/recker.d.ts +11 -8
  4. package/dist/browser/browser/recker.js +54 -6
  5. package/dist/browser/core/client.d.ts +15 -10
  6. package/dist/browser/core/client.js +54 -38
  7. package/dist/browser/index.iife.min.js +129 -130
  8. package/dist/browser/index.min.js +129 -130
  9. package/dist/browser/index.mini.iife.js +5697 -636
  10. package/dist/browser/index.mini.iife.min.js +47 -48
  11. package/dist/browser/index.mini.min.js +47 -48
  12. package/dist/browser/index.mini.umd.js +5697 -636
  13. package/dist/browser/index.mini.umd.min.js +47 -48
  14. package/dist/browser/index.umd.min.js +129 -130
  15. package/dist/browser/mini.d.ts +2 -2
  16. package/dist/browser/plugins/proxy-rotator.d.ts +2 -2
  17. package/dist/browser/plugins/proxy-rotator.js +6 -28
  18. package/dist/browser/recker-mini.d.ts +8 -8
  19. package/dist/browser/recker.d.ts +11 -8
  20. package/dist/browser/recker.js +54 -6
  21. package/dist/browser/scrape/document.js +2 -2
  22. package/dist/browser/scrape/element.js +7 -1
  23. package/dist/browser/scrape/parser/nodes/html.js +1 -1
  24. package/dist/browser/scrape/spider.d.ts +52 -0
  25. package/dist/browser/scrape/spider.js +620 -38
  26. package/dist/browser/scrape/types.d.ts +2 -0
  27. package/dist/browser/search/google.d.ts +26 -1
  28. package/dist/browser/search/google.js +427 -45
  29. package/dist/browser/seo/analyzer.d.ts +1 -0
  30. package/dist/browser/seo/analyzer.js +144 -1
  31. package/dist/browser/seo/index.d.ts +1 -1
  32. package/dist/browser/seo/keyword-campaign-analyzer.d.ts +2 -0
  33. package/dist/browser/seo/keyword-campaign-analyzer.js +538 -0
  34. package/dist/browser/seo/keyword-campaign-seed-advanced.d.ts +17 -0
  35. package/dist/browser/seo/keyword-campaign-seed-advanced.js +269 -0
  36. package/dist/browser/seo/keyword-campaign-seed-core.d.ts +29 -0
  37. package/dist/browser/seo/keyword-campaign-seed-core.js +525 -0
  38. package/dist/browser/seo/keyword-campaign-shared.d.ts +165 -0
  39. package/dist/browser/seo/keyword-campaign-shared.js +59 -0
  40. package/dist/browser/seo/keyword-campaign.d.ts +4 -107
  41. package/dist/browser/seo/keyword-campaign.js +2 -380
  42. package/dist/browser/seo/keywords.js +5 -22
  43. package/dist/browser/seo/types.d.ts +19 -0
  44. package/dist/browser/transport/curl.d.ts +5 -1
  45. package/dist/browser/transport/curl.js +207 -50
  46. package/dist/browser/transport/undici.d.ts +4 -3
  47. package/dist/browser/transport/undici.js +123 -49
  48. package/dist/browser/types/index.d.ts +9 -3
  49. package/dist/browser/utils/binary-manager.js +26 -3
  50. package/dist/browser/utils/block-detector.d.ts +8 -0
  51. package/dist/browser/utils/block-detector.js +542 -7
  52. package/dist/cli/commands/hls-runner.js +5 -4
  53. package/dist/cli/commands/live-runner.js +5 -4
  54. package/dist/cli/commands/loadtest-runner.js +3 -2
  55. package/dist/cli/commands/search.d.ts +2 -0
  56. package/dist/cli/commands/search.js +105 -0
  57. package/dist/cli/commands/seo-runner.js +9 -7
  58. package/dist/cli/commands/seo.js +140 -1
  59. package/dist/cli/commands/serve.js +75 -131
  60. package/dist/cli/commands/server-runner.js +59 -82
  61. package/dist/cli/commands/spider-runner.d.ts +37 -1
  62. package/dist/cli/commands/spider-runner.js +134 -10
  63. package/dist/cli/commands/spider.d.ts +18 -1
  64. package/dist/cli/commands/spider.js +457 -27
  65. package/dist/cli/events/handlers/cli.js +30 -1
  66. package/dist/cli/events/handlers/tui.js +26 -0
  67. package/dist/cli/events/types.d.ts +27 -0
  68. package/dist/cli/handler.d.ts +2 -12
  69. package/dist/cli/handler.js +20 -15
  70. package/dist/cli/handlers/protocols.js +39 -12
  71. package/dist/cli/handlers/search.d.ts +2 -0
  72. package/dist/cli/handlers/search.js +171 -0
  73. package/dist/cli/handlers/seo-analyze.d.ts +1 -0
  74. package/dist/cli/handlers/seo-analyze.js +666 -0
  75. package/dist/cli/handlers/seo-robots.d.ts +1 -0
  76. package/dist/cli/handlers/seo-robots.js +76 -0
  77. package/dist/cli/handlers/seo-serp.d.ts +54 -0
  78. package/dist/cli/handlers/seo-serp.js +243 -0
  79. package/dist/cli/handlers/seo-sitemap.d.ts +1 -0
  80. package/dist/cli/handlers/seo-sitemap.js +55 -0
  81. package/dist/cli/handlers/seo-spider.d.ts +1 -0
  82. package/dist/cli/handlers/seo-spider.js +334 -0
  83. package/dist/cli/handlers/seo.d.ts +8 -4
  84. package/dist/cli/handlers/seo.js +294 -442
  85. package/dist/cli/handlers/spider.js +94 -17
  86. package/dist/cli/handlers/streaming.js +5 -1
  87. package/dist/cli/index.js +11 -2
  88. package/dist/cli/presets.d.ts +1 -1
  89. package/dist/cli/presets.js +15 -4
  90. package/dist/cli/tui/app.js +6 -1
  91. package/dist/cli/tui/components/rich-response.d.ts +72 -0
  92. package/dist/cli/tui/components/rich-response.js +117 -0
  93. package/dist/cli/tui/executor-commands/background.js +30 -24
  94. package/dist/cli/tui/executor-commands/testing.js +3 -2
  95. package/dist/cli/tui/hooks/useDomains.d.ts +17 -0
  96. package/dist/cli/tui/hooks/useHelp.js +15 -2
  97. package/dist/cli/tui/job-manager.d.ts +4 -4
  98. package/dist/cli/tui/job-manager.js +5 -1
  99. package/dist/cli/tui/spider-tui.d.ts +63 -0
  100. package/dist/cli/tui/spider-tui.js +120 -2
  101. package/dist/cli/types.d.ts +12 -0
  102. package/dist/cli/types.js +1 -0
  103. package/dist/cli/utils/option-helpers.d.ts +10 -0
  104. package/dist/cli/utils/option-helpers.js +63 -0
  105. package/dist/cli/utils/score-color.d.ts +11 -0
  106. package/dist/cli/utils/score-color.js +11 -0
  107. package/dist/cli/utils/serp-campaign.d.ts +53 -0
  108. package/dist/cli/utils/serp-campaign.js +53 -0
  109. package/dist/cli/utils/serp-config.d.ts +26 -0
  110. package/dist/cli/utils/serp-config.js +125 -0
  111. package/dist/core/client.d.ts +15 -10
  112. package/dist/core/client.js +54 -38
  113. package/dist/index.d.ts +1 -0
  114. package/dist/index.js +1 -0
  115. package/dist/mcp/cli.js +35 -34
  116. package/dist/mcp/client.d.ts +2 -2
  117. package/dist/mcp/client.js +20 -2
  118. package/dist/mcp/contract.d.ts +1 -1
  119. package/dist/mcp/profiles.js +5 -1
  120. package/dist/mcp/prompts/index.js +8 -4
  121. package/dist/mcp/resources/index.js +46 -23
  122. package/dist/mcp/server.js +9 -6
  123. package/dist/mcp/tools/protocols.js +9 -2
  124. package/dist/mcp/tools/registry.js +13 -3
  125. package/dist/mcp/tools/seo.js +427 -2
  126. package/dist/mcp/types.d.ts +5 -1
  127. package/dist/plugins/proxy-rotator.d.ts +2 -2
  128. package/dist/plugins/proxy-rotator.js +6 -28
  129. package/dist/raffel/client.d.ts +38 -0
  130. package/dist/raffel/client.js +282 -0
  131. package/dist/raffel/index.d.ts +2 -0
  132. package/dist/raffel/index.js +2 -0
  133. package/dist/raffel/types.d.ts +40 -0
  134. package/dist/raffel/types.js +14 -0
  135. package/dist/recker.d.ts +13 -7
  136. package/dist/recker.js +58 -6
  137. package/dist/scrape/document.js +2 -2
  138. package/dist/scrape/element.js +7 -1
  139. package/dist/scrape/parser/nodes/html.js +1 -1
  140. package/dist/scrape/spider.d.ts +52 -0
  141. package/dist/scrape/spider.js +620 -38
  142. package/dist/scrape/types.d.ts +2 -0
  143. package/dist/search/google.d.ts +26 -1
  144. package/dist/search/google.js +427 -45
  145. package/dist/search/index.d.ts +1 -1
  146. package/dist/seo/analyzer.d.ts +1 -0
  147. package/dist/seo/analyzer.js +144 -1
  148. package/dist/seo/index.d.ts +1 -1
  149. package/dist/seo/keyword-campaign-analyzer.d.ts +2 -0
  150. package/dist/seo/keyword-campaign-analyzer.js +538 -0
  151. package/dist/seo/keyword-campaign-seed-advanced.d.ts +17 -0
  152. package/dist/seo/keyword-campaign-seed-advanced.js +269 -0
  153. package/dist/seo/keyword-campaign-seed-core.d.ts +29 -0
  154. package/dist/seo/keyword-campaign-seed-core.js +525 -0
  155. package/dist/seo/keyword-campaign-shared.d.ts +165 -0
  156. package/dist/seo/keyword-campaign-shared.js +59 -0
  157. package/dist/seo/keyword-campaign.d.ts +4 -107
  158. package/dist/seo/keyword-campaign.js +2 -380
  159. package/dist/seo/keywords.js +5 -22
  160. package/dist/seo/types.d.ts +19 -0
  161. package/dist/template/index.d.ts +1 -1
  162. package/dist/template/types.d.ts +0 -2
  163. package/dist/testing/index.d.ts +1 -22
  164. package/dist/testing/index.js +1 -11
  165. package/dist/transport/curl.d.ts +5 -1
  166. package/dist/transport/curl.js +207 -50
  167. package/dist/transport/undici.d.ts +4 -3
  168. package/dist/transport/undici.js +123 -49
  169. package/dist/types/index.d.ts +9 -3
  170. package/dist/utils/binary-manager.js +26 -3
  171. package/dist/utils/block-detector.d.ts +8 -0
  172. package/dist/utils/block-detector.js +542 -7
  173. package/dist/version.js +1 -1
  174. package/package.json +12 -1
  175. package/dist/testing/mock-dns-server.d.ts +0 -69
  176. package/dist/testing/mock-dns-server.js +0 -269
  177. package/dist/testing/mock-ftp-server.d.ts +0 -89
  178. package/dist/testing/mock-ftp-server.js +0 -562
  179. package/dist/testing/mock-hls-server.d.ts +0 -80
  180. package/dist/testing/mock-hls-server.js +0 -381
  181. package/dist/testing/mock-http-server.d.ts +0 -124
  182. package/dist/testing/mock-http-server.js +0 -343
  183. package/dist/testing/mock-proxy-server.d.ts +0 -108
  184. package/dist/testing/mock-proxy-server.js +0 -615
  185. package/dist/testing/mock-sse-server.d.ts +0 -76
  186. package/dist/testing/mock-sse-server.js +0 -291
  187. package/dist/testing/mock-telnet-server.d.ts +0 -59
  188. package/dist/testing/mock-telnet-server.js +0 -274
  189. package/dist/testing/mock-udp-server.d.ts +0 -43
  190. package/dist/testing/mock-udp-server.js +0 -188
  191. package/dist/testing/mock-websocket-server.d.ts +0 -76
  192. package/dist/testing/mock-websocket-server.js +0 -334
  193. package/dist/testing/mock-whois-server.d.ts +0 -56
  194. package/dist/testing/mock-whois-server.js +0 -234
  195. package/dist/testing/proxy-certs.d.ts +0 -19
  196. package/dist/testing/proxy-certs.js +0 -208
@@ -4,14 +4,22 @@ export interface BlockDetectionResult {
4
4
  confidence: number;
5
5
  description?: string;
6
6
  }
7
+ export interface CaptchaDetectionResult {
8
+ detected: boolean;
9
+ confidence: number;
10
+ provider?: 'recaptcha' | 'hcaptcha' | 'turnstile' | 'funcaptcha' | 'datadome' | 'cloudflare' | 'generic';
11
+ description?: string;
12
+ }
7
13
  interface ResponseLike {
8
14
  status: number;
9
15
  headers: {
10
16
  get(name: string): string | null;
11
17
  has?(name: string): boolean;
18
+ getAll?(name: string): string[] | undefined;
12
19
  };
13
20
  }
14
21
  export declare function detectBlock(response: ResponseLike, body?: string): BlockDetectionResult;
22
+ export declare function detectCaptcha(response: ResponseLike, body?: string): CaptchaDetectionResult;
15
23
  export declare function isProtectedDomain(hostname: string): boolean;
16
24
  export declare function isCloudflareChallenge(response: ResponseLike, body?: string): boolean;
17
25
  export {};
@@ -7,6 +7,364 @@ const BLOCK_STATUS_CODES = new Set([
7
7
  const CLOUDFLARE_HEADERS = ['cf-ray', 'cf-cache-status', 'cf-mitigated'];
8
8
  const AKAMAI_HEADERS = ['x-akamai-transformed', 'akamai-grn', 'x-akamai-request-id'];
9
9
  const DATADOME_HEADERS = ['x-datadome', 'x-dd-b', 'x-dd-type'];
10
+ const CAPTCHA_HEADERS = [
11
+ 'x-captcha',
12
+ 'x-captcha-result',
13
+ 'x-captcha-badge',
14
+ 'x-recaptcha',
15
+ 'x-hcaptcha',
16
+ 'x-cloudflare-challenge',
17
+ 'x-cf-challenge',
18
+ 'cf-challenge',
19
+ 'x-cf-chl-bypass',
20
+ 'cf-chl-bypass',
21
+ 'x-akamai-challenge',
22
+ 'akamai-challenge',
23
+ 'x-arkose',
24
+ 'x-arkoseapi-session-id',
25
+ 'x-detect-captcha',
26
+ 'x-perimeterx-request-id',
27
+ 'x-px-request-id',
28
+ 'x-px-config',
29
+ 'x-sucuri-id',
30
+ 'x-vercel-protection-id',
31
+ 'x-recaptcha-token',
32
+ 'x-turnstile',
33
+ 'x-hcaptcha-response',
34
+ 'x-px-action',
35
+ 'cf-chl-bot-score',
36
+ 'x-datadome-request-id',
37
+ 'x-cf-bot-score',
38
+ 'cf-bot-score',
39
+ 'x-akamai-bms-correlation-id',
40
+ 'x-akamai-edgescape',
41
+ 'x-akamai-request-id',
42
+ 'x-akamai-rid',
43
+ 'x-turnstile-score',
44
+ ];
45
+ const CAPTCHA_REDIRECT_HINTS = [
46
+ /httpservice\/retry\/enablejs/i,
47
+ /\/cdn-cgi\/(?:challenge|challenge-platform|images|ray)\//i,
48
+ /turnstile\/(?:v0|v3)\//i,
49
+ /g-recaptcha|recaptcha\/api2/i,
50
+ /hcaptcha\.com\/(?:1\/|verify)/i,
51
+ /funcaptcha\.com|arkoselabs\.com|perimeterx\.com/i,
52
+ /challenge[-_]platform/i,
53
+ /cloudflare[-/]?(?:challenge|bot)|access denied|blocked/i,
54
+ /challenge\.js|\bjschl_vc\b|\bjschl_answer\b/i,
55
+ ];
56
+ const CAPTCHA_COOKIE_PATTERNS = [
57
+ {
58
+ pattern: /__cf_bm|cf_clearance/i,
59
+ provider: 'cloudflare',
60
+ confidence: 0.7,
61
+ description: 'Cloudflare anti-bot cookie detected',
62
+ },
63
+ {
64
+ pattern: /datadome/i,
65
+ provider: 'datadome',
66
+ confidence: 0.9,
67
+ description: 'DataDome cookie detected',
68
+ },
69
+ {
70
+ pattern: /_dd_s|_dd_nuid|dd_session|ak_bmsc|visid_incap|incap_ses_/i,
71
+ provider: 'generic',
72
+ confidence: 0.68,
73
+ description: 'CAPTCHA/anti-bot anti-bot cookie detected',
74
+ },
75
+ {
76
+ pattern: /cf_chl_|cf_clearance|cf_chl2/i,
77
+ provider: 'cloudflare',
78
+ confidence: 0.82,
79
+ description: 'Cloudflare challenge cookie marker detected',
80
+ },
81
+ ];
82
+ const CAPTCHA_HEADER_PATTERNS = [
83
+ {
84
+ pattern: /cloudflare/i,
85
+ provider: 'cloudflare',
86
+ confidence: 0.45,
87
+ description: 'Cloudflare server signature detected',
88
+ header: 'server',
89
+ },
90
+ {
91
+ pattern: /akamai/i,
92
+ provider: 'generic',
93
+ confidence: 0.38,
94
+ description: 'Akamai server signature detected',
95
+ header: 'server',
96
+ },
97
+ {
98
+ pattern: /datadome/i,
99
+ provider: 'datadome',
100
+ confidence: 0.42,
101
+ description: 'DataDome transport header signature detected',
102
+ header: 'x-datadome',
103
+ },
104
+ {
105
+ pattern: /perimeterx|px/i,
106
+ provider: 'funcaptcha',
107
+ confidence: 0.4,
108
+ description: 'PerimeterX-like header signature detected',
109
+ header: 'x-px-id',
110
+ },
111
+ ];
112
+ const CAPTCHA_FORM_PATTERNS = [
113
+ {
114
+ pattern: /<input[^>]+name=["']g-recaptcha-response["'][^>]*>/i,
115
+ provider: 'recaptcha',
116
+ confidence: 0.95,
117
+ description: 'reCAPTCHA response field found',
118
+ },
119
+ {
120
+ pattern: /<div[^>]+class=["'][^"']*g-recaptcha[^"']*["'][^>]*data-sitekey/i,
121
+ provider: 'recaptcha',
122
+ confidence: 0.85,
123
+ description: 'reCAPTCHA widget marker found',
124
+ },
125
+ {
126
+ pattern: /<input[^>]+name=["']hcaptcha-response["'][^>]*>/i,
127
+ provider: 'hcaptcha',
128
+ confidence: 0.95,
129
+ description: 'hCaptcha response field found',
130
+ },
131
+ {
132
+ pattern: /<div[^>]+class=["'][^"']*h-captcha[^"']*["'][^>]*data-sitekey/i,
133
+ provider: 'hcaptcha',
134
+ confidence: 0.85,
135
+ description: 'hCaptcha widget marker found',
136
+ },
137
+ {
138
+ pattern: /<input[^>]+name=["']cf-turnstile-response["'][^>]*>/i,
139
+ provider: 'turnstile',
140
+ confidence: 0.95,
141
+ description: 'Cloudflare Turnstile response field found',
142
+ },
143
+ {
144
+ pattern: /<input[^>]+name=["']turnstile_response["'][^>]*>/i,
145
+ provider: 'turnstile',
146
+ confidence: 0.88,
147
+ description: 'Turnstile response field found',
148
+ },
149
+ {
150
+ pattern: /data-.*captcha|name=["'][^"']*captcha[^"']*["'][^>]*>/i,
151
+ provider: 'generic',
152
+ confidence: 0.6,
153
+ description: 'Generic captcha-related form field detected',
154
+ },
155
+ {
156
+ pattern: /data-sitekey|data-site-key/i,
157
+ provider: 'generic',
158
+ confidence: 0.62,
159
+ description: 'Captcha sitekey attribute detected',
160
+ },
161
+ {
162
+ pattern: /id=["'][^"']*(?:cf-turnstile|hcaptcha|recaptcha|g-recaptcha|turnstile-response|cf-bm|cf_clearance)[^"']*["'][^>]*>/i,
163
+ provider: 'generic',
164
+ confidence: 0.7,
165
+ description: 'Challenge-related DOM id detected',
166
+ },
167
+ {
168
+ pattern: /aria-label=["'][^"']*(?:I am not a robot|human verification|verify you are human|bot protection|captcha)[^"']*["']/i,
169
+ provider: 'generic',
170
+ confidence: 0.55,
171
+ description: 'Accessible challenge label detected',
172
+ },
173
+ ];
174
+ const CAPTCHA_SCRIPT_PATTERNS = [
175
+ {
176
+ pattern: /<script[^>]*src=["'][^"']*(?:gstatic\.com\/recaptcha|google\.com\/recaptcha|recaptcha\/api\.js)/i,
177
+ provider: 'recaptcha',
178
+ confidence: 0.95,
179
+ description: 'reCAPTCHA script bundle detected',
180
+ },
181
+ {
182
+ pattern: /grecaptcha\.render|grecaptcha\.execute|___grecaptcha_cfg|g-recaptcha|g-recaptcha-response/i,
183
+ provider: 'recaptcha',
184
+ confidence: 0.85,
185
+ description: 'reCAPTCHA runtime marker detected',
186
+ },
187
+ {
188
+ pattern: /<script[^>]*src=["'][^"']*hcaptcha\.com\/1\/api\.js/i,
189
+ provider: 'hcaptcha',
190
+ confidence: 0.95,
191
+ description: 'hCaptcha script bundle detected',
192
+ },
193
+ {
194
+ pattern: /hcaptcha\.render|h-captcha|hcaptcha-response/i,
195
+ provider: 'hcaptcha',
196
+ confidence: 0.8,
197
+ description: 'hCaptcha runtime marker detected',
198
+ },
199
+ {
200
+ pattern: /<script[^>]*src=["'][^"']*(?:challenges\.arkoselabs\.com|arkoselabs\.com\/\w+|funcaptcha\.com)/i,
201
+ provider: 'funcaptcha',
202
+ confidence: 0.9,
203
+ description: 'FunCaptcha / Arkose script bundle detected',
204
+ },
205
+ {
206
+ pattern: /arkose|funcaptcha|px-captcha|perimeterx|px\._init/i,
207
+ provider: 'funcaptcha',
208
+ confidence: 0.78,
209
+ description: 'Anti-bot challenge script marker detected',
210
+ },
211
+ {
212
+ pattern: /<script[^>]*src=["'][^"']*challenges\.cloudflare\.com\/turnstile/i,
213
+ provider: 'turnstile',
214
+ confidence: 0.95,
215
+ description: 'Cloudflare Turnstile script bundle detected',
216
+ },
217
+ {
218
+ pattern: /turnstile\.render|cf-turnstile-response|cf-turnstile/i,
219
+ provider: 'turnstile',
220
+ confidence: 0.8,
221
+ description: 'Turnstile challenge marker detected',
222
+ },
223
+ {
224
+ pattern: /__cf_chl_|cf_chl_2|cf_chl_prog|checking your browser/i,
225
+ provider: 'cloudflare',
226
+ confidence: 0.84,
227
+ description: 'Cloudflare challenge JS marker detected',
228
+ },
229
+ {
230
+ pattern: /challenges\.cloudflare\.com\/cdn-cgi\/challenge-platform\/|__cf_chl|chals\.cloudflare/i,
231
+ provider: 'cloudflare',
232
+ confidence: 0.9,
233
+ description: 'Cloudflare challenge script bundle marker detected',
234
+ },
235
+ {
236
+ pattern: /grecaptcha\.enterprise|recaptcha\/enterprise\.js|reCAPTCHA.*enterprise/i,
237
+ provider: 'recaptcha',
238
+ confidence: 0.88,
239
+ description: 'reCAPTCHA Enterprise marker detected',
240
+ },
241
+ {
242
+ pattern: /<script[^>]*src=["'][^"']*hcaptcha\.com\/[\w.-]+\/api\.js["'][^>]*>/i,
243
+ provider: 'hcaptcha',
244
+ confidence: 0.66,
245
+ description: 'hCaptcha script bundle detected',
246
+ },
247
+ {
248
+ pattern: /<script[^>]*src=["'][^"']*challenges\.perimeterx\.[^"']*["'][^>]*>/i,
249
+ provider: 'funcaptcha',
250
+ confidence: 0.72,
251
+ description: 'PerimeterX challenge script detected',
252
+ },
253
+ {
254
+ pattern: /<script[^>]*src=["'][^"']*recaptcha\/api\.js(\?.*)?/i,
255
+ provider: 'recaptcha',
256
+ confidence: 0.88,
257
+ description: 'reCAPTCHA API entrypoint script detected',
258
+ },
259
+ {
260
+ pattern: /<script[^>]*src=["'][^"']*hcaptcha\.com\/[\w\-.]+\/api\.js[\w\-.]*(\?.*)?/i,
261
+ provider: 'hcaptcha',
262
+ confidence: 0.85,
263
+ description: 'hCaptcha API script detected',
264
+ },
265
+ {
266
+ pattern: /<script[^>]*src=["'][^"']*challenges\.cloudflare\.com\/cdn-cgi\/challenge-platform[\w\-/%.?=&]*["'][^>]*>/i,
267
+ provider: 'cloudflare',
268
+ confidence: 0.9,
269
+ description: 'Cloudflare challenge platform script bundle detected',
270
+ },
271
+ ];
272
+ const CAPTCHA_PATTERNS = [
273
+ {
274
+ pattern: /httpservice\/retry\/enablejs/i,
275
+ provider: 'generic',
276
+ confidence: 0.86,
277
+ description: 'Google JS challenge retry endpoint detected',
278
+ },
279
+ {
280
+ pattern: /recaptcha|g-recaptcha|google\.com\/recaptcha/i,
281
+ provider: 'recaptcha',
282
+ confidence: 0.95,
283
+ description: 'reCAPTCHA challenge detected',
284
+ },
285
+ {
286
+ pattern: /hcaptcha|h-captcha/i,
287
+ provider: 'hcaptcha',
288
+ confidence: 0.9,
289
+ description: 'hCaptcha challenge detected',
290
+ },
291
+ {
292
+ pattern: /turnstile|cf-turnstile/i,
293
+ provider: 'turnstile',
294
+ confidence: 0.92,
295
+ description: 'Cloudflare Turnstile challenge detected',
296
+ },
297
+ {
298
+ pattern: /funcaptcha|arkose|arkoselabs|perimeterx|px-captcha|bot-management|human-verification/i,
299
+ provider: 'funcaptcha',
300
+ confidence: 0.9,
301
+ description: 'Bot-management CAPTCHA challenge detected',
302
+ },
303
+ {
304
+ pattern: /captcha|bot.*verification|human.*verification|verify you are human/i,
305
+ provider: 'generic',
306
+ confidence: 0.75,
307
+ description: 'Generic CAPTCHA challenge detected',
308
+ },
309
+ ];
310
+ const CAPTCHA_DOM_MARKERS = [
311
+ {
312
+ pattern: /<title>[^<]*(just a moment|attention required|human verification|verify you are human|security check)[^<]*<\/title>/i,
313
+ provider: 'cloudflare',
314
+ confidence: 0.9,
315
+ description: 'Challenge-like page title detected',
316
+ },
317
+ {
318
+ pattern: /<body[^>]*class=["'][^"']*\b(cf-browser-verification|cf-stage|cf-im-under-attack|cf-nojs|cf-turnstile|hcaptcha|g-recaptcha|recaptcha)[^"']*\b[^"']*["'][^>]*>/i,
319
+ provider: 'cloudflare',
320
+ confidence: 0.7,
321
+ description: 'Challenge-like body class/markup detected',
322
+ },
323
+ {
324
+ pattern: /<iframe[^>]+src=["'][^"']*captcha|<iframe[^>]+src=["'][^"']*turnstile|window\.__cf_chl_|_cf_chl_|cf_chl_/i,
325
+ provider: 'cloudflare',
326
+ confidence: 0.75,
327
+ description: 'Cloudflare challenge DOM marker detected',
328
+ },
329
+ {
330
+ pattern: /id=["'][^"']*(captcha|challenge|hcaptcha|recaptcha|turnstile)[^"']*["'][^>]*>/i,
331
+ provider: 'generic',
332
+ confidence: 0.65,
333
+ description: 'Generic challenge element id detected',
334
+ },
335
+ {
336
+ pattern: /data-challenge|data-sitekey|aria-label=["'][^"']*(captcha|challenge|verify|human|bot)[^"']*["']/i,
337
+ provider: 'generic',
338
+ confidence: 0.6,
339
+ description: 'Generic challenge markup attribute detected',
340
+ },
341
+ ];
342
+ const CAPTCHA_NOSCRIPT_PATTERNS = [
343
+ {
344
+ pattern: /<noscript>[\s\S]*?data-sitekey[\s\S]*?<\/noscript>/i,
345
+ provider: 'recaptcha',
346
+ confidence: 0.78,
347
+ description: 'reCAPTCHA noscript fallback marker detected',
348
+ },
349
+ {
350
+ pattern: /<noscript>[\s\S]*?turnstile[\s\S]*?<\/noscript>/i,
351
+ provider: 'turnstile',
352
+ confidence: 0.76,
353
+ description: 'Turnstile noscript fallback marker detected',
354
+ },
355
+ {
356
+ pattern: /<noscript>[\s\S]*?recaptcha[\s\S]*?<\/noscript>/i,
357
+ provider: 'recaptcha',
358
+ confidence: 0.7,
359
+ description: 'reCAPTCHA noscript fallback marker detected',
360
+ },
361
+ {
362
+ pattern: /httpservice\/retry\/enablejs/i,
363
+ provider: 'generic',
364
+ confidence: 0.7,
365
+ description: 'Google JS challenge retry marker detected',
366
+ },
367
+ ];
10
368
  const BLOCK_PATTERNS = [
11
369
  {
12
370
  pattern: /checking your browser|just a moment|enable javascript and cookies|cloudflare ray id/i,
@@ -21,7 +379,7 @@ const BLOCK_PATTERNS = [
21
379
  description: 'Cloudflare protection page',
22
380
  },
23
381
  {
24
- pattern: /access denied|reference #[\da-f]+|akamai/i,
382
+ pattern: /access denied|reference #[\da-f]+/i,
25
383
  reason: 'akamai',
26
384
  confidence: 0.9,
27
385
  description: 'Akamai Bot Manager block',
@@ -33,19 +391,25 @@ const BLOCK_PATTERNS = [
33
391
  description: 'DataDome bot detection',
34
392
  },
35
393
  {
36
- pattern: /captcha|recaptcha|hcaptcha|funcaptcha|arkose/i,
394
+ pattern: /captcha|recaptcha|hcaptcha|funcaptcha|arkose|human verification|please verify you.re human/i,
37
395
  reason: 'captcha',
38
396
  confidence: 0.9,
39
397
  description: 'CAPTCHA challenge required',
40
398
  },
41
399
  {
42
- pattern: /rate limit|too many requests|slow down|try again later/i,
400
+ pattern: /httpservice\/retry\/enablejs|enable javascript and cookies/i,
401
+ reason: 'captcha',
402
+ confidence: 0.92,
403
+ description: 'Google browser challenge endpoint detected',
404
+ },
405
+ {
406
+ pattern: /too many requests|slow down|try again later|rate\s+limits?|rate\s+limited|rate\s+limiting/i,
43
407
  reason: 'rate-limit',
44
- confidence: 0.85,
408
+ confidence: 0.8,
45
409
  description: 'Rate limiting detected',
46
410
  },
47
411
  {
48
- pattern: /blocked|forbidden|security check|bot detected|suspicious activity/i,
412
+ pattern: /(?:your request has been )?(?:blocked|denied|forbidden)|suspicious activity|security check required/i,
49
413
  reason: 'waf',
50
414
  confidence: 0.7,
51
415
  description: 'Generic WAF/security block',
@@ -63,8 +427,60 @@ const BLOCK_PATTERNS = [
63
427
  description: 'Imperva/Incapsula protection',
64
428
  },
65
429
  ];
430
+ function addMatch(matches, provider, confidence, description) {
431
+ matches.push({
432
+ provider,
433
+ confidence,
434
+ description,
435
+ });
436
+ }
437
+ function scoreByProvider(matches) {
438
+ const providerScores = new Map();
439
+ for (const match of matches) {
440
+ if (!match.provider)
441
+ continue;
442
+ const current = providerScores.get(match.provider) ?? { score: 0, descriptions: [] };
443
+ const addition = match.confidence * (1 - current.score);
444
+ const updatedScore = current.score + addition;
445
+ providerScores.set(match.provider, {
446
+ score: Math.min(0.99, updatedScore),
447
+ descriptions: [...current.descriptions, match.description ?? ''],
448
+ });
449
+ }
450
+ const best = Array.from(providerScores.entries()).sort((a, b) => b[1].score - a[1].score)[0];
451
+ if (!best) {
452
+ return {
453
+ provider: undefined,
454
+ confidence: 0,
455
+ description: undefined,
456
+ };
457
+ }
458
+ return {
459
+ provider: best[0],
460
+ confidence: Math.round(best[1].score * 100) / 100,
461
+ description: best[1].descriptions.slice(-3).join(' | '),
462
+ };
463
+ }
66
464
  export function detectBlock(response, body) {
67
465
  const results = [];
466
+ const server = response.headers.get('server') || '';
467
+ const location = response.headers.get('location');
468
+ if (location && CAPTCHA_REDIRECT_HINTS.some((pattern) => pattern.test(location))) {
469
+ results.push({
470
+ blocked: true,
471
+ reason: 'waf',
472
+ confidence: 0.9,
473
+ description: `Redirect to challenge endpoint (${location})`,
474
+ });
475
+ }
476
+ if (location && response.status >= 300 && response.status < 400) {
477
+ results.push({
478
+ blocked: true,
479
+ reason: 'waf',
480
+ confidence: 0.35,
481
+ description: 'HTTP redirect is often used for challenge interstitials',
482
+ });
483
+ }
68
484
  if (BLOCK_STATUS_CODES.has(response.status)) {
69
485
  results.push({
70
486
  blocked: true,
@@ -73,7 +489,8 @@ export function detectBlock(response, body) {
73
489
  description: `HTTP ${response.status} status code`,
74
490
  });
75
491
  }
76
- const hasCloudflare = CLOUDFLARE_HEADERS.some(h => response.headers.get(h) !== null);
492
+ const hasCloudflare = CLOUDFLARE_HEADERS.some(h => response.headers.get(h) !== null)
493
+ || /cloudflare/i.test(server);
77
494
  if (hasCloudflare && (response.status === 403 || response.status === 503)) {
78
495
  results.push({
79
496
  blocked: true,
@@ -101,8 +518,13 @@ export function detectBlock(response, body) {
101
518
  });
102
519
  }
103
520
  if (body) {
104
- const checkBody = body.length < 100_000 ? body : body.slice(0, 100_000);
521
+ const isLongBody = body.length > 100_000;
522
+ const checkBody = isLongBody ? body.slice(0, 30_000) : body;
523
+ const challengeHint = /just a moment|attention required|access denied|checking your browser|security check|human verification|suspicious activity|captcha|recaptcha|hcaptcha|turnstile|datadome|cloudflare/i.test(checkBody.slice(0, 8000));
105
524
  for (const { pattern, reason, confidence, description } of BLOCK_PATTERNS) {
525
+ if (isLongBody && response.status === 200 && confidence < 0.85 && !challengeHint) {
526
+ continue;
527
+ }
106
528
  if (pattern.test(checkBody)) {
107
529
  results.push({
108
530
  blocked: true,
@@ -123,6 +545,14 @@ export function detectBlock(response, body) {
123
545
  description: 'Suspicious redirect page',
124
546
  });
125
547
  }
548
+ if (CAPTCHA_REDIRECT_HINTS.some((pattern) => pattern.test(location ?? '')) && response.status === 200) {
549
+ results.push({
550
+ blocked: true,
551
+ reason: 'captcha',
552
+ confidence: 0.85,
553
+ description: 'Challenge URL pattern detected in redirect location',
554
+ });
555
+ }
126
556
  }
127
557
  if (results.length === 0) {
128
558
  return {
@@ -133,6 +563,111 @@ export function detectBlock(response, body) {
133
563
  results.sort((a, b) => b.confidence - a.confidence);
134
564
  return results[0];
135
565
  }
566
+ export function detectCaptcha(response, body) {
567
+ const matches = [];
568
+ const location = response.headers.get('location');
569
+ const server = response.headers.get('server') || '';
570
+ for (const { header, pattern, provider, confidence, description } of CAPTCHA_HEADER_PATTERNS) {
571
+ const value = response.headers.get(header);
572
+ if (typeof value === 'string' && pattern.test(value)) {
573
+ addMatch(matches, provider, confidence, `${description} (${header})`);
574
+ }
575
+ }
576
+ const hasCaptchaHeader = CAPTCHA_HEADERS.some((header) => {
577
+ const value = response.headers.get(header);
578
+ const hasValue = typeof value === 'string' && value.length > 0;
579
+ if (hasValue) {
580
+ addMatch(matches, 'generic', 0.95, `CAPTCHA-related response header detected: ${header}`);
581
+ }
582
+ return hasValue;
583
+ });
584
+ if (response.status === 403 || response.status === 503) {
585
+ const cfRay = response.headers.get('cf-ray');
586
+ const cfRequestId = response.headers.get('cf-request-id');
587
+ if (cfRay || cfRequestId) {
588
+ addMatch(matches, 'cloudflare', 0.7, 'Cloudflare challenge headers detected');
589
+ }
590
+ }
591
+ if (body) {
592
+ const checkBody = body.length < 120_000 ? body : body.slice(0, 120_000);
593
+ const isTinyBody = checkBody.length > 0 && checkBody.length < 12000;
594
+ const hasHtmlTags = /<html|<head|<body|<script|<meta/i.test(checkBody);
595
+ const challengeTitleOrText = /just a moment|attention required|human verification|verify you are human|security check/i.test(checkBody);
596
+ if (isTinyBody && challengeTitleOrText && hasHtmlTags) {
597
+ addMatch(matches, 'generic', 0.6, 'Tiny HTML response with challenge-like text');
598
+ }
599
+ const setCookieHeader = response.headers.get('set-cookie') || '';
600
+ const setCookieHeaders = response.headers.getAll?.('set-cookie') ?? [];
601
+ const combinedCookieHeader = [setCookieHeader, ...setCookieHeaders].filter(Boolean).join('\n');
602
+ for (const { pattern, provider, confidence, description } of CAPTCHA_COOKIE_PATTERNS) {
603
+ if (pattern.test(combinedCookieHeader)) {
604
+ addMatch(matches, provider, confidence, description);
605
+ }
606
+ }
607
+ for (const { pattern, provider, confidence, description } of CAPTCHA_SCRIPT_PATTERNS) {
608
+ if (pattern.test(checkBody)) {
609
+ addMatch(matches, provider, confidence, description);
610
+ }
611
+ }
612
+ for (const { pattern, provider, confidence, description } of CAPTCHA_FORM_PATTERNS) {
613
+ if (pattern.test(checkBody)) {
614
+ addMatch(matches, provider, confidence, description);
615
+ }
616
+ }
617
+ for (const { pattern, provider, confidence, description } of CAPTCHA_DOM_MARKERS) {
618
+ if (pattern.test(checkBody)) {
619
+ addMatch(matches, provider, confidence, description);
620
+ }
621
+ }
622
+ for (const { pattern, provider, confidence, description } of CAPTCHA_NOSCRIPT_PATTERNS) {
623
+ if (pattern.test(checkBody)) {
624
+ addMatch(matches, provider, confidence, description);
625
+ }
626
+ }
627
+ if (response.status === 403 || response.status === 503) {
628
+ addMatch(matches, 'generic', 0.35, 'Bot protection status with captcha-like response body');
629
+ }
630
+ if (location && CAPTCHA_REDIRECT_HINTS.some((pattern) => pattern.test(location))) {
631
+ addMatch(matches, 'generic', 0.72, `Challenge redirect target detected (${location})`);
632
+ }
633
+ if (hasCaptchaHeader && response.status >= 300 && response.status < 500) {
634
+ addMatch(matches, 'cloudflare', 0.68, 'Challenge-related headers with status response');
635
+ }
636
+ if (/cloudflare/i.test(server)) {
637
+ addMatch(matches, 'cloudflare', 0.45, 'Cloudflare server signature detected');
638
+ }
639
+ if (response.status >= 300 &&
640
+ response.status < 400 &&
641
+ /refresh|challenge|verify|captcha|javascript challenge|cloudflare|bot/i.test((location || ''))) {
642
+ addMatch(matches, 'generic', 0.67, 'Redirect location indicates bot challenge flow');
643
+ }
644
+ if (/<script[^>]*src=["'][^"']*(?:challenges\.cloudflare\.com|recaptcha\/api\.js|hcaptcha\.com\/1\/api\.js|challenges\.perimeterx\.)[^"']*["'][^>]*>/i.test(checkBody.slice(0, 120_000))) {
645
+ addMatch(matches, 'cloudflare', 0.6, 'Known challenge script reference found in HTML');
646
+ }
647
+ if ((response.status >= 500 && response.status < 600) &&
648
+ /cloudflare|challenge|verification|blocked|security/i.test(checkBody.slice(0, 3000))) {
649
+ addMatch(matches, 'generic', 0.2, 'Server challenge-like text on error response');
650
+ }
651
+ for (const { pattern, provider, confidence, description } of CAPTCHA_PATTERNS) {
652
+ if (pattern.test(checkBody)) {
653
+ addMatch(matches, provider, confidence, description);
654
+ }
655
+ }
656
+ }
657
+ const result = scoreByProvider(matches);
658
+ if (result.confidence >= 0.5 || hasCaptchaHeader) {
659
+ return {
660
+ detected: true,
661
+ confidence: result.confidence,
662
+ provider: result.provider,
663
+ description: result.description,
664
+ };
665
+ }
666
+ return {
667
+ detected: false,
668
+ confidence: 0,
669
+ };
670
+ }
136
671
  export function isProtectedDomain(hostname) {
137
672
  const protectedPatterns = [
138
673
  /cloudflare/i,
package/dist/version.js CHANGED
@@ -1,4 +1,4 @@
1
- const VERSION = '1.0.79';
1
+ const VERSION = '1.0.80-next.70ea84d';
2
2
  let _version = null;
3
3
  export async function getVersion() {
4
4
  if (_version)