webvac 0.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. webvac/__init__.py +3 -0
  2. webvac/__main__.py +6 -0
  3. webvac/auth/__init__.py +0 -0
  4. webvac/auth/auth.py +774 -0
  5. webvac/auth/cookie_audit.py +61 -0
  6. webvac/auth/credentials.py +39 -0
  7. webvac/auth/default_creds.py +589 -0
  8. webvac/auth/manager.py +471 -0
  9. webvac/auth/mfa.py +73 -0
  10. webvac/auth/popups.py +206 -0
  11. webvac/auth/profile.py +98 -0
  12. webvac/auth/session_store.py +150 -0
  13. webvac/auth/steps.py +210 -0
  14. webvac/auth/wall.py +140 -0
  15. webvac/captcha/__init__.py +24 -0
  16. webvac/captcha/config.py +145 -0
  17. webvac/captcha/detect.py +423 -0
  18. webvac/captcha/extract.py +465 -0
  19. webvac/captcha/inject.py +235 -0
  20. webvac/captcha/manager.py +208 -0
  21. webvac/captcha/models.py +65 -0
  22. webvac/captcha/network_watch.py +255 -0
  23. webvac/captcha/providers/__init__.py +15 -0
  24. webvac/captcha/providers/capsolver.py +258 -0
  25. webvac/cli/__init__.py +1 -0
  26. webvac/cli/captcha_demo.py +441 -0
  27. webvac/cli/interactive.py +882 -0
  28. webvac/cli/scraper.py +1157 -0
  29. webvac/config/__init__.py +0 -0
  30. webvac/config/config.py +126 -0
  31. webvac/core/__init__.py +0 -0
  32. webvac/core/crawler.py +833 -0
  33. webvac/core/page_scrape_flow.py +641 -0
  34. webvac/core/pipeline.py +37 -0
  35. webvac/data/__init__.py +0 -0
  36. webvac/data/html_parser.py +394 -0
  37. webvac/data/page_record.py +82 -0
  38. webvac/data/storage.py +1132 -0
  39. webvac/models/__init__.py +8 -0
  40. webvac/models/origin.py +70 -0
  41. webvac/models/scan.py +70 -0
  42. webvac/store/__init__.py +5 -0
  43. webvac/store/scan_session.py +126 -0
  44. webvac/utils/__init__.py +0 -0
  45. webvac/utils/asset_downloader.py +111 -0
  46. webvac/utils/browser.py +1071 -0
  47. webvac/utils/browser_pool.py +41 -0
  48. webvac/utils/consent.py +245 -0
  49. webvac/utils/detection.py +240 -0
  50. webvac/utils/humanize.py +344 -0
  51. webvac/utils/network_debug.py +334 -0
  52. webvac/utils/network_listener.py +210 -0
  53. webvac/utils/origin_probe.py +146 -0
  54. webvac/utils/proxy.py +643 -0
  55. webvac/utils/proxy_playbook.py +75 -0
  56. webvac/utils/robots.py +213 -0
  57. webvac/utils/screenshot.py +107 -0
  58. webvac/vapt/__init__.py +5 -0
  59. webvac/vapt/decaffeinator.py +160 -0
  60. webvac-0.3.1.dist-info/METADATA +508 -0
  61. webvac-0.3.1.dist-info/RECORD +65 -0
  62. webvac-0.3.1.dist-info/WHEEL +5 -0
  63. webvac-0.3.1.dist-info/entry_points.txt +3 -0
  64. webvac-0.3.1.dist-info/licenses/LICENSE +21 -0
  65. webvac-0.3.1.dist-info/top_level.txt +1 -0
webvac/auth/auth.py ADDED
@@ -0,0 +1,774 @@
1
+ """
2
+ auth.py — Login handler for username/password protected sites.
3
+
4
+ Features
5
+ --------
6
+ - Auto-detects common username / password / submit selectors.
7
+ - Explicit-selector override via login_with_selectors().
8
+ - Human-like typing delays to avoid bot detection.
9
+ - Smart post-login success/failure detection:
10
+ * Avoids false-negatives caused by login-related words in redirect URLs.
11
+ * Detects CAPTCHA / 2FA challenges and warns the user.
12
+ * Waits for an actual navigation away from the login page, not just networkidle.
13
+ - Session persistence: save_session() / restore_session() dump and reload cookies
14
+ so a crawl can resume without re-authenticating.
15
+ - Full error handling in both auto and manual selector paths.
16
+ """
17
+
18
+ import asyncio
19
+ import json
20
+ import os
21
+ import re
22
+ from typing import Optional
23
+
24
+ from patchright.async_api import Page, BrowserContext
25
+ from webvac.config.config import DEFAULT_CONFIG
26
+ from webvac.auth.popups import dismiss_popups_patchright
27
+
28
+
29
+ # ── Common selector patterns ──────────────────────────────────────────────────
30
+
31
+ USERNAME_SELECTORS = [
32
+ '#login-username', # common visible alias (e.g. Vue dual-field forms)
33
+ 'input[name="_username"]', # Symfony / dual-field forms
34
+ 'input[type="email"]',
35
+ 'input[name="email"]',
36
+ 'input[name="username"]',
37
+ 'input[name="user"]',
38
+ 'input[name="login"]',
39
+ 'input[name="log"]',
40
+ 'input[name="identifier"]',
41
+ 'input[id="email"]',
42
+ 'input[id="username"]',
43
+ 'input[id="login_field"]', # GitHub
44
+ 'input[id="ap_email"]', # Amazon
45
+ 'input[type="text"][name*="user"]',
46
+ 'input[type="text"][name*="email"]',
47
+ 'input[type="text"][name*="login"]',
48
+ 'input[id*="user"]',
49
+ 'input[id*="email"]',
50
+ 'input[id*="login"]',
51
+ 'input[placeholder*="email" i]',
52
+ 'input[placeholder*="username" i]',
53
+ 'input[autocomplete="username"]',
54
+ 'input[autocomplete="email"]',
55
+ ]
56
+
57
+ PASSWORD_SELECTORS = [
58
+ '#login-password', # chess.com visible field
59
+ 'input[name="_password"]',
60
+ 'input[type="password"]',
61
+ 'input[autocomplete="current-password"]',
62
+ 'input[name="password"]',
63
+ 'input[id="password"]',
64
+ 'input[id="ap_password"]',
65
+ ]
66
+
67
+ # First-step "Continue / Next" when password is not yet on the page
68
+ CONTINUE_SELECTORS = [
69
+ 'button:has-text("Continue")',
70
+ 'button:has-text("Next")',
71
+ 'button:has-text("Weiter")',
72
+ '[role="button"]:has-text("Continue")',
73
+ '[role="button"]:has-text("Next")',
74
+ 'form button[type="submit"]',
75
+ 'button[type="submit"]',
76
+ 'input[type="submit"]',
77
+ ]
78
+
79
+ SUBMIT_SELECTORS = [
80
+ 'button#login', # chess.com
81
+ '#authentication-login-form button[type="submit"]',
82
+ 'form button[type="submit"]',
83
+ 'form input[type="submit"]',
84
+ 'button[type="submit"]',
85
+ 'input[type="submit"]',
86
+ 'button:has-text("Log In")',
87
+ 'button:has-text("Log in")',
88
+ 'button:has-text("Login")',
89
+ 'button:has-text("Sign in")',
90
+ 'button:has-text("Sign In")',
91
+ 'button:has-text("Continue")',
92
+ '[role="button"]:has-text("Log in")',
93
+ '[role="button"]:has-text("Sign in")',
94
+ ]
95
+
96
+ # Never treat these as the site's password-login submit button
97
+ _SSO_TEXT_RE = re.compile(
98
+ r"google|gmail|facebook|github|apple|microsoft|linkedin|"
99
+ r"sso|oauth|continue with|sign in with",
100
+ re.I,
101
+ )
102
+
103
+ # Solvable CAPTCHA widgets + invisible/embed signals (chess.com Turnstile has no visible box)
104
+ CAPTCHA_WIDGET_SELECTORS = [
105
+ 'iframe[src*="recaptcha"]',
106
+ 'iframe[src*="hcaptcha"]',
107
+ 'iframe[src*="turnstile"]',
108
+ 'iframe[src*="challenges.cloudflare.com"]',
109
+ '.g-recaptcha',
110
+ '.h-captcha',
111
+ '.cf-turnstile',
112
+ '[data-sitekey]',
113
+ '#turnstile-login-form',
114
+ '#turnstile_token',
115
+ 'input[name="cf-turnstile-response"]',
116
+ 'input[name="turnstile_token"]',
117
+ ]
118
+
119
+ # Legacy alias
120
+ CAPTCHA_SELECTORS = CAPTCHA_WIDGET_SELECTORS
121
+
122
+ # Login failure — only match elements with actual credential-failure text,
123
+ # not generic [class*="error"] / [role="alert"] which fire on many valid pages.
124
+ ERROR_SELECTORS = [
125
+ '[class*="alert-danger"]:visible',
126
+ '[class*="alert-error"]:visible',
127
+ ':is(p, span, div):has-text("incorrect password")',
128
+ ':is(p, span, div):has-text("invalid password")',
129
+ ':is(p, span, div):has-text("wrong password")',
130
+ ':is(p, span, div):has-text("account not found")',
131
+ ':is(p, span, div):has-text("invalid credentials")',
132
+ ':is(p, span, div):has-text("invalid username")',
133
+ ':is(p, span, div):has-text("login failed")',
134
+ ':is(p, span, div):has-text("authentication failed")',
135
+ ':is(p, span, div):has-text("incorrect email")',
136
+ ]
137
+
138
+ # Words in the *path component* (not query/fragment) that suggest we're still on
139
+ # a login page. We only check the path so /account/settings?from=signin doesn't
140
+ # trigger a false negative.
141
+ LOGIN_PATH_KEYWORDS = ["login", "signin", "sign-in", "log-in", "auth/login"]
142
+
143
+
144
+ class AuthHandler:
145
+ def __init__(
146
+ self,
147
+ typing_delay: int = DEFAULT_CONFIG["typing_delay"],
148
+ field_delay: float = DEFAULT_CONFIG["field_delay"],
149
+ ):
150
+ self.typing_delay = typing_delay
151
+ self.field_delay = field_delay
152
+
153
+ # ── Public API ────────────────────────────────────────────────────────────
154
+
155
+ async def login(
156
+ self,
157
+ page: Page,
158
+ login_url: str,
159
+ username: str,
160
+ password: str,
161
+ timeout: int = DEFAULT_CONFIG["timeout"],
162
+ wait_until: str = DEFAULT_CONFIG["wait_until"],
163
+ *,
164
+ already_on_page: bool = False,
165
+ ) -> bool:
166
+ """
167
+ Navigate to login_url (unless already_on_page) and fill credentials using
168
+ auto-detected selectors. Supports email→Continue→password wizards.
169
+ """
170
+ if not already_on_page:
171
+ print(f"[Auth] Navigating to login page: {login_url}")
172
+ if not await self._goto_safe(page, login_url, timeout, wait_until):
173
+ return False
174
+ n = await dismiss_popups_patchright(page, rounds=4)
175
+ if n:
176
+ print(f"[Auth] Dismissed {n} cookie/privacy popup control(s)")
177
+ else:
178
+ print(f"[Auth] Filling credentials on current page: {page.url}")
179
+
180
+ return await self._auto_fill_and_submit(
181
+ page, login_url, username, password, timeout, wait_until
182
+ )
183
+
184
+ async def login_with_selectors(
185
+ self,
186
+ page: Page,
187
+ login_url: str,
188
+ username: str,
189
+ password: str,
190
+ username_selector: str,
191
+ password_selector: str,
192
+ submit_selector: Optional[str] = None,
193
+ timeout: int = DEFAULT_CONFIG["timeout"],
194
+ wait_until: str = DEFAULT_CONFIG["wait_until"],
195
+ *,
196
+ already_on_page: bool = False,
197
+ ) -> bool:
198
+ """Login using explicitly provided CSS selectors."""
199
+ if not already_on_page:
200
+ print(f"[Auth] Logging in with custom selectors at: {login_url}")
201
+ if not await self._goto_safe(page, login_url, timeout, wait_until):
202
+ return False
203
+ n = await dismiss_popups_patchright(page, rounds=4)
204
+ if n:
205
+ print(f"[Auth] Dismissed {n} cookie/privacy popup control(s)")
206
+ else:
207
+ print(f"[Auth] Filling custom selectors on: {page.url}")
208
+
209
+ try:
210
+ user_loc = page.locator(username_selector).first
211
+ await user_loc.wait_for(state="visible", timeout=min(timeout, 15000))
212
+ await self._fill_locator(user_loc, username, "username")
213
+ await asyncio.sleep(self.field_delay)
214
+
215
+ pass_loc = page.locator(password_selector).first
216
+ # Multi-step: password may appear after Continue
217
+ try:
218
+ await pass_loc.wait_for(state="visible", timeout=4000)
219
+ except Exception:
220
+ print("[Auth] Password field not visible yet — clicking Continue/Next…")
221
+ cont = await self._find_element(page, CONTINUE_SELECTORS, timeout=2000)
222
+ if cont:
223
+ await cont.click()
224
+ await asyncio.sleep(1.0)
225
+ await pass_loc.wait_for(state="visible", timeout=min(timeout, 15000))
226
+
227
+ await self._fill_locator(pass_loc, password, "password")
228
+ await asyncio.sleep(self.field_delay)
229
+ except Exception as e:
230
+ print(f"[Auth] Error filling in login fields: {e}")
231
+ return False
232
+
233
+ await self._sync_hidden_credential_fields(page)
234
+
235
+ # Solve any CAPTCHA on the login form BEFORE submitting
236
+ await self._solve_pre_submit_captcha(page)
237
+
238
+ if submit_selector:
239
+ try:
240
+ await page.click(submit_selector)
241
+ except Exception as e:
242
+ print(f"[Auth] Error clicking submit ({submit_selector}): {e}")
243
+ try:
244
+ await page.press(password_selector, "Enter")
245
+ except Exception:
246
+ return False
247
+ else:
248
+ try:
249
+ # Prefer the real login button over Enter (chess.com rejects Enter).
250
+ submit = await self._find_element(page, SUBMIT_SELECTORS, timeout=2000)
251
+ if submit:
252
+ await submit.click()
253
+ else:
254
+ await page.press(password_selector, "Enter")
255
+ except Exception as e:
256
+ print(f"[Auth] Could not submit form: {e}")
257
+ return False
258
+
259
+ await asyncio.sleep(1.2)
260
+ if self._path_looks_like_login(page.url, login_url):
261
+ if await self._force_native_form_submit(page):
262
+ print("[Auth] Client-side submit gate detected — forced native form submit.")
263
+ await asyncio.sleep(1.5)
264
+
265
+ return await self._verify_login(page, login_url, timeout, wait_until)
266
+
267
+ async def _auto_fill_and_submit(
268
+ self,
269
+ page: Page,
270
+ login_url: str,
271
+ username: str,
272
+ password: str,
273
+ timeout: int,
274
+ wait_until: str,
275
+ ) -> bool:
276
+ """Auto-detect fields, handle email→Continue→password, submit, verify."""
277
+ if not await self._wait_for_login_form(page, timeout_ms=min(timeout, 20000)):
278
+ print("[Auth] Login form did not appear in time (SPA / iframe / wrong URL?).")
279
+ return False
280
+
281
+ username_field = await self._find_element(page, USERNAME_SELECTORS, timeout=5000)
282
+ if not username_field:
283
+ print("[Auth] Could not find username/email field.")
284
+ print(
285
+ "[Auth] Tip: pass --login-url to the real login page, or set "
286
+ "--username-selector / --password-selector."
287
+ )
288
+ return False
289
+
290
+ if not await self._fill_locator(username_field, username, "username"):
291
+ return False
292
+
293
+ password_field = await self._find_element(page, PASSWORD_SELECTORS, timeout=2500)
294
+ if not password_field:
295
+ print("[Auth] Password not on this step — advancing with Continue/Next…")
296
+ cont = await self._find_element(page, CONTINUE_SELECTORS, timeout=3000)
297
+ if not cont:
298
+ print("[Auth] Could not find password field or Continue button.")
299
+ return False
300
+ try:
301
+ await cont.click()
302
+ except Exception as e:
303
+ print(f"[Auth] Failed to click Continue: {e}")
304
+ return False
305
+ await asyncio.sleep(1.2)
306
+ n = await dismiss_popups_patchright(page, rounds=2)
307
+ if n:
308
+ print(f"[Auth] Dismissed {n} popup(s) after Continue")
309
+ password_field = await self._find_element(page, PASSWORD_SELECTORS, timeout=8000)
310
+ if not password_field:
311
+ print("[Auth] Could not find password field after Continue.")
312
+ return False
313
+
314
+ if not await self._fill_locator(password_field, password, "password"):
315
+ return False
316
+
317
+ await self._sync_hidden_credential_fields(page)
318
+
319
+ # Solve any CAPTCHA on the login form BEFORE submitting
320
+ await self._solve_pre_submit_captcha(page)
321
+
322
+ return await self._submit_and_verify(
323
+ page, password_field, login_url, timeout, wait_until
324
+ )
325
+
326
+ async def save_session(self, context: BrowserContext, path: str) -> None:
327
+ """
328
+ Persist cookies from the browser context to a JSON file so the session
329
+ can be restored later without re-authenticating.
330
+ """
331
+ try:
332
+ cookies = await context.cookies()
333
+ os.makedirs(os.path.dirname(path) if os.path.dirname(path) else ".", exist_ok=True)
334
+ with open(path, "w", encoding="utf-8") as f:
335
+ json.dump(cookies, f, indent=2)
336
+ print(f"[Auth] Session saved -> {path} ({len(cookies)} cookies)")
337
+ except Exception as e:
338
+ print(f"[Auth] Warning: could not save session: {e}")
339
+
340
+ async def restore_session(self, context: BrowserContext, path: str) -> bool:
341
+ """
342
+ Load cookies from a previously saved session file back into the context.
343
+ Returns True if cookies were loaded successfully.
344
+ """
345
+ if not os.path.isfile(path):
346
+ print(f"[Auth] No session file found at {path}")
347
+ return False
348
+ try:
349
+ with open(path, "r", encoding="utf-8") as f:
350
+ cookies = json.load(f)
351
+ await context.add_cookies(cookies)
352
+ print(f"[Auth] Session restored <- {path} ({len(cookies)} cookies)")
353
+ return True
354
+ except Exception as e:
355
+ print(f"[Auth] Warning: could not restore session: {e}")
356
+ return False
357
+
358
+ # ── Internal helpers ──────────────────────────────────────────────────────
359
+
360
+ async def _goto_safe(
361
+ self, page: Page, url: str, timeout: int, wait_until: str
362
+ ) -> bool:
363
+ """Navigate to url, returning False on failure instead of raising."""
364
+ try:
365
+ await page.goto(url, wait_until=wait_until, timeout=timeout)
366
+ return True
367
+ except Exception as e:
368
+ print(f"[Auth] Failed to load page ({url}): {e}")
369
+ return False
370
+
371
+ async def _fill_field(self, page: Page, locator, value: str, label: str) -> bool:
372
+ """Back-compat alias — prefers locator.fill so focus cannot be lost."""
373
+ return await self._fill_locator(locator, value, label)
374
+
375
+ async def _fill_locator(self, locator, value: str, label: str) -> bool:
376
+ """Focus, clear, and set the field value (fill first; typed fallback)."""
377
+ try:
378
+ await locator.scroll_into_view_if_needed(timeout=3000)
379
+ await locator.click(timeout=5000)
380
+ await asyncio.sleep(0.15)
381
+ # fill() is reliable; keyboard.type often types into the wrong target
382
+ # if focus was stolen by a cookie banner / overlay.
383
+ try:
384
+ await locator.fill(value, timeout=5000)
385
+ except Exception:
386
+ await locator.fill("")
387
+ await asyncio.sleep(0.05)
388
+ await locator.type(value, delay=max(20, int(self.typing_delay or 40)))
389
+ # Verify something landed
390
+ try:
391
+ current = await locator.input_value(timeout=1000)
392
+ if not current:
393
+ await locator.fill(value, timeout=5000)
394
+ except Exception:
395
+ pass
396
+ await asyncio.sleep(self.field_delay)
397
+ print(f"[Auth] Filled {label} field.")
398
+ return True
399
+ except Exception as e:
400
+ print(f"[Auth] Error filling {label} field: {e}")
401
+ return False
402
+
403
+ async def _solve_pre_submit_captcha(self, page: Page) -> bool:
404
+ """
405
+ If a CAPTCHA is present on the login form (visible widget OR invisible
406
+ Turnstile fields / scripts), solve via CapSolver before submitting.
407
+
408
+ While CapSolver runs (often 10–40s), also watch for the page leaving the
409
+ login URL — headed users may complete Turnstile + Log In manually.
410
+ """
411
+ from webvac.captcha.detect import detect_captcha_raw
412
+
413
+ raw = await detect_captcha_raw(page)
414
+ if not raw:
415
+ if not await self._any_present(page, CAPTCHA_WIDGET_SELECTORS):
416
+ return False
417
+
418
+ print("[Auth] CAPTCHA detected on login form — solving before submit...")
419
+ try:
420
+ from webvac.captcha import solver_from_config
421
+ from webvac.captcha.config import CaptchaSolverConfig
422
+
423
+ cfg = CaptchaSolverConfig.from_mapping()
424
+ if not cfg.api_key or not cfg.enabled:
425
+ print("[Auth] No CapSolver API key — cannot auto-solve login CAPTCHA.")
426
+ return False
427
+
428
+ mgr = solver_from_config()
429
+ try:
430
+ mgr.attach_network_watcher(page)
431
+ except Exception:
432
+ pass
433
+ url = getattr(page, "url", "") or ""
434
+ login_url = url
435
+ ua = ""
436
+ try:
437
+ ua = await page.evaluate("navigator.userAgent") or ""
438
+ except Exception:
439
+ pass
440
+
441
+ solve_task = asyncio.create_task(
442
+ mgr.try_solve_on_page(page, url=url, user_agent=ua)
443
+ )
444
+ while not solve_task.done():
445
+ await asyncio.sleep(0.6)
446
+ try:
447
+ cur = page.url
448
+ except Exception:
449
+ cur = ""
450
+ if cur and not self._path_looks_like_login(cur, login_url):
451
+ print(f"[Auth] Page left login URL during CAPTCHA solve → {cur}")
452
+ solve_task.cancel()
453
+ try:
454
+ await solve_task
455
+ except (asyncio.CancelledError, Exception):
456
+ pass
457
+ return True
458
+
459
+ result = solve_task.result()
460
+ if result.success:
461
+ print("[Auth] CAPTCHA solved — proceeding to submit.")
462
+ await self._sync_hidden_credential_fields(page)
463
+ await asyncio.sleep(0.5)
464
+ return True
465
+
466
+ # Inject can fail if the page already navigated (manual login)
467
+ try:
468
+ cur = page.url
469
+ except Exception:
470
+ cur = ""
471
+ if cur and not self._path_looks_like_login(cur, login_url):
472
+ print(f"[Auth] Login already completed during CAPTCHA solve → {cur}")
473
+ return True
474
+
475
+ print(f"[Auth] CAPTCHA auto-solve failed: {result.error}")
476
+ return False
477
+ except Exception as exc:
478
+ try:
479
+ cur = getattr(page, "url", "") or ""
480
+ if cur and "login" not in cur.lower():
481
+ print(f"[Auth] Login already completed (CAPTCHA error ignored) → {cur}")
482
+ return True
483
+ except Exception:
484
+ pass
485
+ print(f"[Auth] CAPTCHA solve error: {exc}")
486
+ return False
487
+
488
+ async def _sync_hidden_credential_fields(self, page: Page) -> None:
489
+ """Copy visible login fields into hidden submit fields (chess.com Vue form)."""
490
+ try:
491
+ await page.evaluate(
492
+ """() => {
493
+ const pairs = [
494
+ ['#login-username', '#username, input[name="_username"]'],
495
+ ['#login-password', '#password, input[name="_password"]'],
496
+ ];
497
+ for (const [visSel, hidSel] of pairs) {
498
+ const vis = document.querySelector(visSel);
499
+ const hid = document.querySelector(hidSel);
500
+ if (vis && hid && vis.value) {
501
+ hid.value = vis.value;
502
+ hid.dispatchEvent(new Event('input', { bubbles: true }));
503
+ hid.dispatchEvent(new Event('change', { bubbles: true }));
504
+ }
505
+ }
506
+ }"""
507
+ )
508
+ except Exception:
509
+ pass
510
+
511
+ async def _submit_and_verify(
512
+ self,
513
+ page: Page,
514
+ password_field,
515
+ login_url: str,
516
+ timeout: int,
517
+ wait_until: str,
518
+ ) -> bool:
519
+ """Click the submit button (or press Enter) then verify login success."""
520
+ # Already logged in (manual captcha / late navigation during CapSolver)
521
+ try:
522
+ if not self._path_looks_like_login(page.url, login_url):
523
+ print(f"[Auth] Login successful. Now at: {page.url}")
524
+ return True
525
+ except Exception:
526
+ pass
527
+
528
+ await self._sync_hidden_credential_fields(page)
529
+ pre_url = page.url
530
+ clicked = await self._click_login_submit(page, password_field)
531
+ if not clicked:
532
+ # Page may have navigated during the click attempt
533
+ try:
534
+ if not self._path_looks_like_login(page.url, login_url):
535
+ print(f"[Auth] Login successful. Now at: {page.url}")
536
+ return True
537
+ except Exception:
538
+ pass
539
+ print("[Auth] Could not click login submit control.")
540
+ return False
541
+
542
+ await asyncio.sleep(1.2)
543
+ try:
544
+ still_here = page.url == pre_url or self._path_looks_like_login(page.url, login_url)
545
+ except Exception:
546
+ still_here = True
547
+ if still_here:
548
+ forced = await self._force_native_form_submit(page)
549
+ if forced:
550
+ print("[Auth] Client-side submit gate detected — forced native form submit.")
551
+ await asyncio.sleep(1.5)
552
+
553
+ return await self._verify_login(page, login_url, timeout, wait_until)
554
+
555
+ async def _click_login_submit(self, page: Page, password_field) -> bool:
556
+ """Click the primary login button; fall back to Enter only as last resort."""
557
+ # Direct, high-signal selectors first (chess.com uses button#login)
558
+ preferred = [
559
+ "button#login",
560
+ "#login",
561
+ "#authentication-login-form button[type='submit']",
562
+ "button:has-text('Log In')",
563
+ "button:has-text('Log in')",
564
+ "button:has-text('Sign in')",
565
+ "button[type='submit']",
566
+ ]
567
+ for sel in preferred:
568
+ try:
569
+ loc = page.locator(sel).first
570
+ await loc.wait_for(state="visible", timeout=2500)
571
+ if await loc.is_enabled(timeout=500):
572
+ await loc.click(timeout=5000)
573
+ print(f"[Auth] Clicked submit ({sel}).")
574
+ return True
575
+ except Exception:
576
+ continue
577
+
578
+ submit = await self._find_element(page, SUBMIT_SELECTORS, timeout=3000)
579
+ if submit:
580
+ try:
581
+ await submit.click(timeout=5000)
582
+ print("[Auth] Clicked submit (auto-detected).")
583
+ return True
584
+ except Exception as e:
585
+ print(f"[Auth] Submit click failed: {e}")
586
+
587
+ try:
588
+ print("[Auth] No submit button found; pressing Enter in password field.")
589
+ await password_field.press("Enter")
590
+ return True
591
+ except Exception as e:
592
+ print(f"[Auth] Error submitting login form: {e}")
593
+ return False
594
+
595
+ def _path_looks_like_login(self, current_url: str, login_url: str) -> bool:
596
+ from urllib.parse import urlparse
597
+ try:
598
+ return (
599
+ urlparse(current_url).path.lower().rstrip("/")
600
+ == urlparse(login_url).path.lower().rstrip("/")
601
+ )
602
+ except Exception:
603
+ return False
604
+
605
+ async def _force_native_form_submit(self, page: Page) -> bool:
606
+ """
607
+ Call HTMLFormElement.prototype.submit() to bypass page submit listeners
608
+ that block when Turnstile is mid-challenge (chess.com) or require a
609
+ specific submitter id.
610
+ """
611
+ try:
612
+ return bool(
613
+ await page.evaluate(
614
+ """() => {
615
+ const form = document.querySelector(
616
+ '#authentication-login-form, form.authentication-login-form, form.login-form, form'
617
+ );
618
+ if (!form) return false;
619
+ const user = form.querySelector('#username, input[name="_username"], input[type="email"]');
620
+ const pass = form.querySelector('#password, input[name="_password"], input[type="password"]');
621
+ const tok = form.querySelector(
622
+ '#turnstile_token, [name="turnstile_token"], [name="cf-turnstile-response"]'
623
+ );
624
+ // Only force when credentials look filled (and token if present).
625
+ if (user && !(user.value || '').trim()) return false;
626
+ if (pass && !(pass.value || '').trim()) return false;
627
+ if (tok && !(tok.value || '').trim()) return false;
628
+ HTMLFormElement.prototype.submit.call(form);
629
+ return true;
630
+ }"""
631
+ )
632
+ )
633
+ except Exception:
634
+ return False
635
+
636
+ async def _verify_login(
637
+ self, page: Page, login_url: str, timeout: int, wait_until: str
638
+ ) -> bool:
639
+ """
640
+ Wait for the page to change after submit, then decide if login succeeded.
641
+
642
+ Strategy:
643
+ 1. Wait for navigation / load state.
644
+ 2. If an OTP/MFA field appeared → return False (caller handles MFA).
645
+ 3. If a CAPTCHA widget appeared → return False (caller tries CapSolver).
646
+ 4. If we're still on the login page path with error text → failed.
647
+ 5. If we navigated away → success.
648
+ """
649
+ try:
650
+ await page.wait_for_load_state(wait_until, timeout=timeout)
651
+ except Exception:
652
+ pass
653
+
654
+ await asyncio.sleep(1.2)
655
+
656
+ try:
657
+ current_url = page.url
658
+ except Exception:
659
+ current_url = ""
660
+
661
+ from urllib.parse import urlparse
662
+ current_path = urlparse(current_url).path.lower().rstrip("/")
663
+ login_path = urlparse(login_url).path.lower().rstrip("/")
664
+
665
+ # Already away from login — success (even if CAPTCHA nodes briefly linger)
666
+ if current_path and current_path != login_path:
667
+ if await self._any_visible(page, ERROR_SELECTORS, timeout=400):
668
+ print(f"[Auth] Login failed — credential error visible at {current_url}.")
669
+ return False
670
+ print(f"[Auth] Login successful. Now at: {current_url}")
671
+ return True
672
+
673
+ # Still on login page: CAPTCHA present → caller may retry CapSolver
674
+ from webvac.captcha.detect import detect_captcha_raw
675
+
676
+ if await detect_captcha_raw(page) or await self._any_present(page, CAPTCHA_WIDGET_SELECTORS):
677
+ # Interactive checkbox visible without a token often means solve incomplete
678
+ print("[Auth] CAPTCHA detected after submit — will attempt auto-solve.")
679
+ return False
680
+
681
+ if await self._any_visible(page, ERROR_SELECTORS, timeout=800):
682
+ print("[Auth] Login failed — credential error visible on page.")
683
+ else:
684
+ print("[Auth] Login may have failed — still on the login page after submit.")
685
+ return False
686
+
687
+ async def _wait_for_login_form(self, page: Page, *, timeout_ms: int = 15000) -> bool:
688
+ """Wait until a username or password field is visible (main page or iframe)."""
689
+ probes = USERNAME_SELECTORS[:6] + PASSWORD_SELECTORS
690
+ per_try = max(1500, timeout_ms // max(len(probes), 1))
691
+ roots = [page] + [f for f in page.frames if f != page.main_frame]
692
+ for root in roots:
693
+ for sel in probes:
694
+ try:
695
+ loc = root.locator(sel).first
696
+ await loc.wait_for(state="visible", timeout=per_try)
697
+ if root is not page:
698
+ print(f"[Auth] Login form detected in iframe.")
699
+ return True
700
+ except Exception:
701
+ continue
702
+ await asyncio.sleep(1.0)
703
+ return bool(await self._find_element(page, USERNAME_SELECTORS + PASSWORD_SELECTORS, timeout=2000))
704
+
705
+ async def _find_element(self, page: Page, selectors: list, timeout: int = 2000):
706
+ """Try each selector on the main page, then inside child iframes."""
707
+ el = await self._find_element_in_root(page, selectors, timeout)
708
+ if el:
709
+ return el
710
+ for frame in page.frames:
711
+ if frame == page.main_frame:
712
+ continue
713
+ try:
714
+ el = await self._find_element_in_root(frame, selectors, min(timeout, 4000))
715
+ if el:
716
+ print(f"[Auth] Matched login control inside iframe.")
717
+ return el
718
+ except Exception:
719
+ continue
720
+ return None
721
+
722
+ async def _find_element_in_root(self, root, selectors: list, timeout: int = 2000):
723
+ """Try each selector and return the first visible, enabled, non-SSO element."""
724
+ # Per-candidate visibility probe stays short so we can try many selectors.
725
+ probe = min(800, max(250, timeout // 4))
726
+ for sel in selectors:
727
+ try:
728
+ loc = root.locator(sel)
729
+ count = await loc.count()
730
+ for i in range(min(count, 8)):
731
+ el = loc.nth(i)
732
+ try:
733
+ if not await el.is_visible(timeout=probe):
734
+ continue
735
+ if not await el.is_enabled(timeout=probe):
736
+ continue
737
+ except Exception:
738
+ continue
739
+ try:
740
+ text = (await el.inner_text(timeout=400) or "").strip()
741
+ except Exception:
742
+ text = ""
743
+ try:
744
+ aria = (await el.get_attribute("aria-label")) or ""
745
+ except Exception:
746
+ aria = ""
747
+ blob = f"{text} {aria} {sel}"
748
+ if _SSO_TEXT_RE.search(blob):
749
+ continue
750
+ return el
751
+ except Exception:
752
+ continue
753
+ return None
754
+
755
+ async def _any_visible(self, page: Page, selectors: list, timeout: int = 500) -> bool:
756
+ """Return True if any selector in the list matches a visible element."""
757
+ for sel in selectors:
758
+ try:
759
+ el = page.locator(sel).first
760
+ if await el.is_visible(timeout=timeout):
761
+ return True
762
+ except Exception:
763
+ continue
764
+ return False
765
+
766
+ async def _any_present(self, page: Page, selectors: list) -> bool:
767
+ """Return True if any selector matches in the DOM (including hidden inputs)."""
768
+ for sel in selectors:
769
+ try:
770
+ if await page.locator(sel).count() > 0:
771
+ return True
772
+ except Exception:
773
+ continue
774
+ return False