webvac 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webvac/__init__.py +3 -0
- webvac/__main__.py +6 -0
- webvac/auth/__init__.py +0 -0
- webvac/auth/auth.py +774 -0
- webvac/auth/cookie_audit.py +61 -0
- webvac/auth/credentials.py +39 -0
- webvac/auth/default_creds.py +589 -0
- webvac/auth/manager.py +471 -0
- webvac/auth/mfa.py +73 -0
- webvac/auth/popups.py +206 -0
- webvac/auth/profile.py +98 -0
- webvac/auth/session_store.py +150 -0
- webvac/auth/steps.py +210 -0
- webvac/auth/wall.py +140 -0
- webvac/captcha/__init__.py +24 -0
- webvac/captcha/config.py +145 -0
- webvac/captcha/detect.py +423 -0
- webvac/captcha/extract.py +465 -0
- webvac/captcha/inject.py +235 -0
- webvac/captcha/manager.py +208 -0
- webvac/captcha/models.py +65 -0
- webvac/captcha/network_watch.py +255 -0
- webvac/captcha/providers/__init__.py +15 -0
- webvac/captcha/providers/capsolver.py +258 -0
- webvac/cli/__init__.py +1 -0
- webvac/cli/captcha_demo.py +441 -0
- webvac/cli/interactive.py +882 -0
- webvac/cli/scraper.py +1157 -0
- webvac/config/__init__.py +0 -0
- webvac/config/config.py +126 -0
- webvac/core/__init__.py +0 -0
- webvac/core/crawler.py +833 -0
- webvac/core/page_scrape_flow.py +641 -0
- webvac/core/pipeline.py +37 -0
- webvac/data/__init__.py +0 -0
- webvac/data/html_parser.py +394 -0
- webvac/data/page_record.py +82 -0
- webvac/data/storage.py +1132 -0
- webvac/models/__init__.py +8 -0
- webvac/models/origin.py +70 -0
- webvac/models/scan.py +70 -0
- webvac/store/__init__.py +5 -0
- webvac/store/scan_session.py +126 -0
- webvac/utils/__init__.py +0 -0
- webvac/utils/asset_downloader.py +111 -0
- webvac/utils/browser.py +1071 -0
- webvac/utils/browser_pool.py +41 -0
- webvac/utils/consent.py +245 -0
- webvac/utils/detection.py +240 -0
- webvac/utils/humanize.py +344 -0
- webvac/utils/network_debug.py +334 -0
- webvac/utils/network_listener.py +210 -0
- webvac/utils/origin_probe.py +146 -0
- webvac/utils/proxy.py +643 -0
- webvac/utils/proxy_playbook.py +75 -0
- webvac/utils/robots.py +213 -0
- webvac/utils/screenshot.py +107 -0
- webvac/vapt/__init__.py +5 -0
- webvac/vapt/decaffeinator.py +160 -0
- webvac-0.3.1.dist-info/METADATA +508 -0
- webvac-0.3.1.dist-info/RECORD +65 -0
- webvac-0.3.1.dist-info/WHEEL +5 -0
- webvac-0.3.1.dist-info/entry_points.txt +3 -0
- webvac-0.3.1.dist-info/licenses/LICENSE +21 -0
- webvac-0.3.1.dist-info/top_level.txt +1 -0
webvac/auth/auth.py
ADDED
|
@@ -0,0 +1,774 @@
|
|
|
1
|
+
"""
|
|
2
|
+
auth.py — Login handler for username/password protected sites.
|
|
3
|
+
|
|
4
|
+
Features
|
|
5
|
+
--------
|
|
6
|
+
- Auto-detects common username / password / submit selectors.
|
|
7
|
+
- Explicit-selector override via login_with_selectors().
|
|
8
|
+
- Human-like typing delays to avoid bot detection.
|
|
9
|
+
- Smart post-login success/failure detection:
|
|
10
|
+
* Avoids false-negatives caused by login-related words in redirect URLs.
|
|
11
|
+
* Detects CAPTCHA / 2FA challenges and warns the user.
|
|
12
|
+
* Waits for an actual navigation away from the login page, not just networkidle.
|
|
13
|
+
- Session persistence: save_session() / restore_session() dump and reload cookies
|
|
14
|
+
so a crawl can resume without re-authenticating.
|
|
15
|
+
- Full error handling in both auto and manual selector paths.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import asyncio
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
from typing import Optional
|
|
23
|
+
|
|
24
|
+
from patchright.async_api import Page, BrowserContext
|
|
25
|
+
from webvac.config.config import DEFAULT_CONFIG
|
|
26
|
+
from webvac.auth.popups import dismiss_popups_patchright
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# ── Common selector patterns ──────────────────────────────────────────────────
|
|
30
|
+
|
|
31
|
+
USERNAME_SELECTORS = [
|
|
32
|
+
'#login-username', # common visible alias (e.g. Vue dual-field forms)
|
|
33
|
+
'input[name="_username"]', # Symfony / dual-field forms
|
|
34
|
+
'input[type="email"]',
|
|
35
|
+
'input[name="email"]',
|
|
36
|
+
'input[name="username"]',
|
|
37
|
+
'input[name="user"]',
|
|
38
|
+
'input[name="login"]',
|
|
39
|
+
'input[name="log"]',
|
|
40
|
+
'input[name="identifier"]',
|
|
41
|
+
'input[id="email"]',
|
|
42
|
+
'input[id="username"]',
|
|
43
|
+
'input[id="login_field"]', # GitHub
|
|
44
|
+
'input[id="ap_email"]', # Amazon
|
|
45
|
+
'input[type="text"][name*="user"]',
|
|
46
|
+
'input[type="text"][name*="email"]',
|
|
47
|
+
'input[type="text"][name*="login"]',
|
|
48
|
+
'input[id*="user"]',
|
|
49
|
+
'input[id*="email"]',
|
|
50
|
+
'input[id*="login"]',
|
|
51
|
+
'input[placeholder*="email" i]',
|
|
52
|
+
'input[placeholder*="username" i]',
|
|
53
|
+
'input[autocomplete="username"]',
|
|
54
|
+
'input[autocomplete="email"]',
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
PASSWORD_SELECTORS = [
|
|
58
|
+
'#login-password', # chess.com visible field
|
|
59
|
+
'input[name="_password"]',
|
|
60
|
+
'input[type="password"]',
|
|
61
|
+
'input[autocomplete="current-password"]',
|
|
62
|
+
'input[name="password"]',
|
|
63
|
+
'input[id="password"]',
|
|
64
|
+
'input[id="ap_password"]',
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
# First-step "Continue / Next" when password is not yet on the page
|
|
68
|
+
CONTINUE_SELECTORS = [
|
|
69
|
+
'button:has-text("Continue")',
|
|
70
|
+
'button:has-text("Next")',
|
|
71
|
+
'button:has-text("Weiter")',
|
|
72
|
+
'[role="button"]:has-text("Continue")',
|
|
73
|
+
'[role="button"]:has-text("Next")',
|
|
74
|
+
'form button[type="submit"]',
|
|
75
|
+
'button[type="submit"]',
|
|
76
|
+
'input[type="submit"]',
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
SUBMIT_SELECTORS = [
|
|
80
|
+
'button#login', # chess.com
|
|
81
|
+
'#authentication-login-form button[type="submit"]',
|
|
82
|
+
'form button[type="submit"]',
|
|
83
|
+
'form input[type="submit"]',
|
|
84
|
+
'button[type="submit"]',
|
|
85
|
+
'input[type="submit"]',
|
|
86
|
+
'button:has-text("Log In")',
|
|
87
|
+
'button:has-text("Log in")',
|
|
88
|
+
'button:has-text("Login")',
|
|
89
|
+
'button:has-text("Sign in")',
|
|
90
|
+
'button:has-text("Sign In")',
|
|
91
|
+
'button:has-text("Continue")',
|
|
92
|
+
'[role="button"]:has-text("Log in")',
|
|
93
|
+
'[role="button"]:has-text("Sign in")',
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
# Never treat these as the site's password-login submit button
|
|
97
|
+
_SSO_TEXT_RE = re.compile(
|
|
98
|
+
r"google|gmail|facebook|github|apple|microsoft|linkedin|"
|
|
99
|
+
r"sso|oauth|continue with|sign in with",
|
|
100
|
+
re.I,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
# Solvable CAPTCHA widgets + invisible/embed signals (chess.com Turnstile has no visible box)
|
|
104
|
+
CAPTCHA_WIDGET_SELECTORS = [
|
|
105
|
+
'iframe[src*="recaptcha"]',
|
|
106
|
+
'iframe[src*="hcaptcha"]',
|
|
107
|
+
'iframe[src*="turnstile"]',
|
|
108
|
+
'iframe[src*="challenges.cloudflare.com"]',
|
|
109
|
+
'.g-recaptcha',
|
|
110
|
+
'.h-captcha',
|
|
111
|
+
'.cf-turnstile',
|
|
112
|
+
'[data-sitekey]',
|
|
113
|
+
'#turnstile-login-form',
|
|
114
|
+
'#turnstile_token',
|
|
115
|
+
'input[name="cf-turnstile-response"]',
|
|
116
|
+
'input[name="turnstile_token"]',
|
|
117
|
+
]
|
|
118
|
+
|
|
119
|
+
# Legacy alias
|
|
120
|
+
CAPTCHA_SELECTORS = CAPTCHA_WIDGET_SELECTORS
|
|
121
|
+
|
|
122
|
+
# Login failure — only match elements with actual credential-failure text,
|
|
123
|
+
# not generic [class*="error"] / [role="alert"] which fire on many valid pages.
|
|
124
|
+
ERROR_SELECTORS = [
|
|
125
|
+
'[class*="alert-danger"]:visible',
|
|
126
|
+
'[class*="alert-error"]:visible',
|
|
127
|
+
':is(p, span, div):has-text("incorrect password")',
|
|
128
|
+
':is(p, span, div):has-text("invalid password")',
|
|
129
|
+
':is(p, span, div):has-text("wrong password")',
|
|
130
|
+
':is(p, span, div):has-text("account not found")',
|
|
131
|
+
':is(p, span, div):has-text("invalid credentials")',
|
|
132
|
+
':is(p, span, div):has-text("invalid username")',
|
|
133
|
+
':is(p, span, div):has-text("login failed")',
|
|
134
|
+
':is(p, span, div):has-text("authentication failed")',
|
|
135
|
+
':is(p, span, div):has-text("incorrect email")',
|
|
136
|
+
]
|
|
137
|
+
|
|
138
|
+
# Words in the *path component* (not query/fragment) that suggest we're still on
|
|
139
|
+
# a login page. We only check the path so /account/settings?from=signin doesn't
|
|
140
|
+
# trigger a false negative.
|
|
141
|
+
LOGIN_PATH_KEYWORDS = ["login", "signin", "sign-in", "log-in", "auth/login"]
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class AuthHandler:
|
|
145
|
+
def __init__(
|
|
146
|
+
self,
|
|
147
|
+
typing_delay: int = DEFAULT_CONFIG["typing_delay"],
|
|
148
|
+
field_delay: float = DEFAULT_CONFIG["field_delay"],
|
|
149
|
+
):
|
|
150
|
+
self.typing_delay = typing_delay
|
|
151
|
+
self.field_delay = field_delay
|
|
152
|
+
|
|
153
|
+
# ── Public API ────────────────────────────────────────────────────────────
|
|
154
|
+
|
|
155
|
+
async def login(
|
|
156
|
+
self,
|
|
157
|
+
page: Page,
|
|
158
|
+
login_url: str,
|
|
159
|
+
username: str,
|
|
160
|
+
password: str,
|
|
161
|
+
timeout: int = DEFAULT_CONFIG["timeout"],
|
|
162
|
+
wait_until: str = DEFAULT_CONFIG["wait_until"],
|
|
163
|
+
*,
|
|
164
|
+
already_on_page: bool = False,
|
|
165
|
+
) -> bool:
|
|
166
|
+
"""
|
|
167
|
+
Navigate to login_url (unless already_on_page) and fill credentials using
|
|
168
|
+
auto-detected selectors. Supports email→Continue→password wizards.
|
|
169
|
+
"""
|
|
170
|
+
if not already_on_page:
|
|
171
|
+
print(f"[Auth] Navigating to login page: {login_url}")
|
|
172
|
+
if not await self._goto_safe(page, login_url, timeout, wait_until):
|
|
173
|
+
return False
|
|
174
|
+
n = await dismiss_popups_patchright(page, rounds=4)
|
|
175
|
+
if n:
|
|
176
|
+
print(f"[Auth] Dismissed {n} cookie/privacy popup control(s)")
|
|
177
|
+
else:
|
|
178
|
+
print(f"[Auth] Filling credentials on current page: {page.url}")
|
|
179
|
+
|
|
180
|
+
return await self._auto_fill_and_submit(
|
|
181
|
+
page, login_url, username, password, timeout, wait_until
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
async def login_with_selectors(
|
|
185
|
+
self,
|
|
186
|
+
page: Page,
|
|
187
|
+
login_url: str,
|
|
188
|
+
username: str,
|
|
189
|
+
password: str,
|
|
190
|
+
username_selector: str,
|
|
191
|
+
password_selector: str,
|
|
192
|
+
submit_selector: Optional[str] = None,
|
|
193
|
+
timeout: int = DEFAULT_CONFIG["timeout"],
|
|
194
|
+
wait_until: str = DEFAULT_CONFIG["wait_until"],
|
|
195
|
+
*,
|
|
196
|
+
already_on_page: bool = False,
|
|
197
|
+
) -> bool:
|
|
198
|
+
"""Login using explicitly provided CSS selectors."""
|
|
199
|
+
if not already_on_page:
|
|
200
|
+
print(f"[Auth] Logging in with custom selectors at: {login_url}")
|
|
201
|
+
if not await self._goto_safe(page, login_url, timeout, wait_until):
|
|
202
|
+
return False
|
|
203
|
+
n = await dismiss_popups_patchright(page, rounds=4)
|
|
204
|
+
if n:
|
|
205
|
+
print(f"[Auth] Dismissed {n} cookie/privacy popup control(s)")
|
|
206
|
+
else:
|
|
207
|
+
print(f"[Auth] Filling custom selectors on: {page.url}")
|
|
208
|
+
|
|
209
|
+
try:
|
|
210
|
+
user_loc = page.locator(username_selector).first
|
|
211
|
+
await user_loc.wait_for(state="visible", timeout=min(timeout, 15000))
|
|
212
|
+
await self._fill_locator(user_loc, username, "username")
|
|
213
|
+
await asyncio.sleep(self.field_delay)
|
|
214
|
+
|
|
215
|
+
pass_loc = page.locator(password_selector).first
|
|
216
|
+
# Multi-step: password may appear after Continue
|
|
217
|
+
try:
|
|
218
|
+
await pass_loc.wait_for(state="visible", timeout=4000)
|
|
219
|
+
except Exception:
|
|
220
|
+
print("[Auth] Password field not visible yet — clicking Continue/Next…")
|
|
221
|
+
cont = await self._find_element(page, CONTINUE_SELECTORS, timeout=2000)
|
|
222
|
+
if cont:
|
|
223
|
+
await cont.click()
|
|
224
|
+
await asyncio.sleep(1.0)
|
|
225
|
+
await pass_loc.wait_for(state="visible", timeout=min(timeout, 15000))
|
|
226
|
+
|
|
227
|
+
await self._fill_locator(pass_loc, password, "password")
|
|
228
|
+
await asyncio.sleep(self.field_delay)
|
|
229
|
+
except Exception as e:
|
|
230
|
+
print(f"[Auth] Error filling in login fields: {e}")
|
|
231
|
+
return False
|
|
232
|
+
|
|
233
|
+
await self._sync_hidden_credential_fields(page)
|
|
234
|
+
|
|
235
|
+
# Solve any CAPTCHA on the login form BEFORE submitting
|
|
236
|
+
await self._solve_pre_submit_captcha(page)
|
|
237
|
+
|
|
238
|
+
if submit_selector:
|
|
239
|
+
try:
|
|
240
|
+
await page.click(submit_selector)
|
|
241
|
+
except Exception as e:
|
|
242
|
+
print(f"[Auth] Error clicking submit ({submit_selector}): {e}")
|
|
243
|
+
try:
|
|
244
|
+
await page.press(password_selector, "Enter")
|
|
245
|
+
except Exception:
|
|
246
|
+
return False
|
|
247
|
+
else:
|
|
248
|
+
try:
|
|
249
|
+
# Prefer the real login button over Enter (chess.com rejects Enter).
|
|
250
|
+
submit = await self._find_element(page, SUBMIT_SELECTORS, timeout=2000)
|
|
251
|
+
if submit:
|
|
252
|
+
await submit.click()
|
|
253
|
+
else:
|
|
254
|
+
await page.press(password_selector, "Enter")
|
|
255
|
+
except Exception as e:
|
|
256
|
+
print(f"[Auth] Could not submit form: {e}")
|
|
257
|
+
return False
|
|
258
|
+
|
|
259
|
+
await asyncio.sleep(1.2)
|
|
260
|
+
if self._path_looks_like_login(page.url, login_url):
|
|
261
|
+
if await self._force_native_form_submit(page):
|
|
262
|
+
print("[Auth] Client-side submit gate detected — forced native form submit.")
|
|
263
|
+
await asyncio.sleep(1.5)
|
|
264
|
+
|
|
265
|
+
return await self._verify_login(page, login_url, timeout, wait_until)
|
|
266
|
+
|
|
267
|
+
async def _auto_fill_and_submit(
|
|
268
|
+
self,
|
|
269
|
+
page: Page,
|
|
270
|
+
login_url: str,
|
|
271
|
+
username: str,
|
|
272
|
+
password: str,
|
|
273
|
+
timeout: int,
|
|
274
|
+
wait_until: str,
|
|
275
|
+
) -> bool:
|
|
276
|
+
"""Auto-detect fields, handle email→Continue→password, submit, verify."""
|
|
277
|
+
if not await self._wait_for_login_form(page, timeout_ms=min(timeout, 20000)):
|
|
278
|
+
print("[Auth] Login form did not appear in time (SPA / iframe / wrong URL?).")
|
|
279
|
+
return False
|
|
280
|
+
|
|
281
|
+
username_field = await self._find_element(page, USERNAME_SELECTORS, timeout=5000)
|
|
282
|
+
if not username_field:
|
|
283
|
+
print("[Auth] Could not find username/email field.")
|
|
284
|
+
print(
|
|
285
|
+
"[Auth] Tip: pass --login-url to the real login page, or set "
|
|
286
|
+
"--username-selector / --password-selector."
|
|
287
|
+
)
|
|
288
|
+
return False
|
|
289
|
+
|
|
290
|
+
if not await self._fill_locator(username_field, username, "username"):
|
|
291
|
+
return False
|
|
292
|
+
|
|
293
|
+
password_field = await self._find_element(page, PASSWORD_SELECTORS, timeout=2500)
|
|
294
|
+
if not password_field:
|
|
295
|
+
print("[Auth] Password not on this step — advancing with Continue/Next…")
|
|
296
|
+
cont = await self._find_element(page, CONTINUE_SELECTORS, timeout=3000)
|
|
297
|
+
if not cont:
|
|
298
|
+
print("[Auth] Could not find password field or Continue button.")
|
|
299
|
+
return False
|
|
300
|
+
try:
|
|
301
|
+
await cont.click()
|
|
302
|
+
except Exception as e:
|
|
303
|
+
print(f"[Auth] Failed to click Continue: {e}")
|
|
304
|
+
return False
|
|
305
|
+
await asyncio.sleep(1.2)
|
|
306
|
+
n = await dismiss_popups_patchright(page, rounds=2)
|
|
307
|
+
if n:
|
|
308
|
+
print(f"[Auth] Dismissed {n} popup(s) after Continue")
|
|
309
|
+
password_field = await self._find_element(page, PASSWORD_SELECTORS, timeout=8000)
|
|
310
|
+
if not password_field:
|
|
311
|
+
print("[Auth] Could not find password field after Continue.")
|
|
312
|
+
return False
|
|
313
|
+
|
|
314
|
+
if not await self._fill_locator(password_field, password, "password"):
|
|
315
|
+
return False
|
|
316
|
+
|
|
317
|
+
await self._sync_hidden_credential_fields(page)
|
|
318
|
+
|
|
319
|
+
# Solve any CAPTCHA on the login form BEFORE submitting
|
|
320
|
+
await self._solve_pre_submit_captcha(page)
|
|
321
|
+
|
|
322
|
+
return await self._submit_and_verify(
|
|
323
|
+
page, password_field, login_url, timeout, wait_until
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
async def save_session(self, context: BrowserContext, path: str) -> None:
|
|
327
|
+
"""
|
|
328
|
+
Persist cookies from the browser context to a JSON file so the session
|
|
329
|
+
can be restored later without re-authenticating.
|
|
330
|
+
"""
|
|
331
|
+
try:
|
|
332
|
+
cookies = await context.cookies()
|
|
333
|
+
os.makedirs(os.path.dirname(path) if os.path.dirname(path) else ".", exist_ok=True)
|
|
334
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
335
|
+
json.dump(cookies, f, indent=2)
|
|
336
|
+
print(f"[Auth] Session saved -> {path} ({len(cookies)} cookies)")
|
|
337
|
+
except Exception as e:
|
|
338
|
+
print(f"[Auth] Warning: could not save session: {e}")
|
|
339
|
+
|
|
340
|
+
async def restore_session(self, context: BrowserContext, path: str) -> bool:
|
|
341
|
+
"""
|
|
342
|
+
Load cookies from a previously saved session file back into the context.
|
|
343
|
+
Returns True if cookies were loaded successfully.
|
|
344
|
+
"""
|
|
345
|
+
if not os.path.isfile(path):
|
|
346
|
+
print(f"[Auth] No session file found at {path}")
|
|
347
|
+
return False
|
|
348
|
+
try:
|
|
349
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
350
|
+
cookies = json.load(f)
|
|
351
|
+
await context.add_cookies(cookies)
|
|
352
|
+
print(f"[Auth] Session restored <- {path} ({len(cookies)} cookies)")
|
|
353
|
+
return True
|
|
354
|
+
except Exception as e:
|
|
355
|
+
print(f"[Auth] Warning: could not restore session: {e}")
|
|
356
|
+
return False
|
|
357
|
+
|
|
358
|
+
# ── Internal helpers ──────────────────────────────────────────────────────
|
|
359
|
+
|
|
360
|
+
async def _goto_safe(
|
|
361
|
+
self, page: Page, url: str, timeout: int, wait_until: str
|
|
362
|
+
) -> bool:
|
|
363
|
+
"""Navigate to url, returning False on failure instead of raising."""
|
|
364
|
+
try:
|
|
365
|
+
await page.goto(url, wait_until=wait_until, timeout=timeout)
|
|
366
|
+
return True
|
|
367
|
+
except Exception as e:
|
|
368
|
+
print(f"[Auth] Failed to load page ({url}): {e}")
|
|
369
|
+
return False
|
|
370
|
+
|
|
371
|
+
async def _fill_field(self, page: Page, locator, value: str, label: str) -> bool:
|
|
372
|
+
"""Back-compat alias — prefers locator.fill so focus cannot be lost."""
|
|
373
|
+
return await self._fill_locator(locator, value, label)
|
|
374
|
+
|
|
375
|
+
async def _fill_locator(self, locator, value: str, label: str) -> bool:
|
|
376
|
+
"""Focus, clear, and set the field value (fill first; typed fallback)."""
|
|
377
|
+
try:
|
|
378
|
+
await locator.scroll_into_view_if_needed(timeout=3000)
|
|
379
|
+
await locator.click(timeout=5000)
|
|
380
|
+
await asyncio.sleep(0.15)
|
|
381
|
+
# fill() is reliable; keyboard.type often types into the wrong target
|
|
382
|
+
# if focus was stolen by a cookie banner / overlay.
|
|
383
|
+
try:
|
|
384
|
+
await locator.fill(value, timeout=5000)
|
|
385
|
+
except Exception:
|
|
386
|
+
await locator.fill("")
|
|
387
|
+
await asyncio.sleep(0.05)
|
|
388
|
+
await locator.type(value, delay=max(20, int(self.typing_delay or 40)))
|
|
389
|
+
# Verify something landed
|
|
390
|
+
try:
|
|
391
|
+
current = await locator.input_value(timeout=1000)
|
|
392
|
+
if not current:
|
|
393
|
+
await locator.fill(value, timeout=5000)
|
|
394
|
+
except Exception:
|
|
395
|
+
pass
|
|
396
|
+
await asyncio.sleep(self.field_delay)
|
|
397
|
+
print(f"[Auth] Filled {label} field.")
|
|
398
|
+
return True
|
|
399
|
+
except Exception as e:
|
|
400
|
+
print(f"[Auth] Error filling {label} field: {e}")
|
|
401
|
+
return False
|
|
402
|
+
|
|
403
|
+
async def _solve_pre_submit_captcha(self, page: Page) -> bool:
|
|
404
|
+
"""
|
|
405
|
+
If a CAPTCHA is present on the login form (visible widget OR invisible
|
|
406
|
+
Turnstile fields / scripts), solve via CapSolver before submitting.
|
|
407
|
+
|
|
408
|
+
While CapSolver runs (often 10–40s), also watch for the page leaving the
|
|
409
|
+
login URL — headed users may complete Turnstile + Log In manually.
|
|
410
|
+
"""
|
|
411
|
+
from webvac.captcha.detect import detect_captcha_raw
|
|
412
|
+
|
|
413
|
+
raw = await detect_captcha_raw(page)
|
|
414
|
+
if not raw:
|
|
415
|
+
if not await self._any_present(page, CAPTCHA_WIDGET_SELECTORS):
|
|
416
|
+
return False
|
|
417
|
+
|
|
418
|
+
print("[Auth] CAPTCHA detected on login form — solving before submit...")
|
|
419
|
+
try:
|
|
420
|
+
from webvac.captcha import solver_from_config
|
|
421
|
+
from webvac.captcha.config import CaptchaSolverConfig
|
|
422
|
+
|
|
423
|
+
cfg = CaptchaSolverConfig.from_mapping()
|
|
424
|
+
if not cfg.api_key or not cfg.enabled:
|
|
425
|
+
print("[Auth] No CapSolver API key — cannot auto-solve login CAPTCHA.")
|
|
426
|
+
return False
|
|
427
|
+
|
|
428
|
+
mgr = solver_from_config()
|
|
429
|
+
try:
|
|
430
|
+
mgr.attach_network_watcher(page)
|
|
431
|
+
except Exception:
|
|
432
|
+
pass
|
|
433
|
+
url = getattr(page, "url", "") or ""
|
|
434
|
+
login_url = url
|
|
435
|
+
ua = ""
|
|
436
|
+
try:
|
|
437
|
+
ua = await page.evaluate("navigator.userAgent") or ""
|
|
438
|
+
except Exception:
|
|
439
|
+
pass
|
|
440
|
+
|
|
441
|
+
solve_task = asyncio.create_task(
|
|
442
|
+
mgr.try_solve_on_page(page, url=url, user_agent=ua)
|
|
443
|
+
)
|
|
444
|
+
while not solve_task.done():
|
|
445
|
+
await asyncio.sleep(0.6)
|
|
446
|
+
try:
|
|
447
|
+
cur = page.url
|
|
448
|
+
except Exception:
|
|
449
|
+
cur = ""
|
|
450
|
+
if cur and not self._path_looks_like_login(cur, login_url):
|
|
451
|
+
print(f"[Auth] Page left login URL during CAPTCHA solve → {cur}")
|
|
452
|
+
solve_task.cancel()
|
|
453
|
+
try:
|
|
454
|
+
await solve_task
|
|
455
|
+
except (asyncio.CancelledError, Exception):
|
|
456
|
+
pass
|
|
457
|
+
return True
|
|
458
|
+
|
|
459
|
+
result = solve_task.result()
|
|
460
|
+
if result.success:
|
|
461
|
+
print("[Auth] CAPTCHA solved — proceeding to submit.")
|
|
462
|
+
await self._sync_hidden_credential_fields(page)
|
|
463
|
+
await asyncio.sleep(0.5)
|
|
464
|
+
return True
|
|
465
|
+
|
|
466
|
+
# Inject can fail if the page already navigated (manual login)
|
|
467
|
+
try:
|
|
468
|
+
cur = page.url
|
|
469
|
+
except Exception:
|
|
470
|
+
cur = ""
|
|
471
|
+
if cur and not self._path_looks_like_login(cur, login_url):
|
|
472
|
+
print(f"[Auth] Login already completed during CAPTCHA solve → {cur}")
|
|
473
|
+
return True
|
|
474
|
+
|
|
475
|
+
print(f"[Auth] CAPTCHA auto-solve failed: {result.error}")
|
|
476
|
+
return False
|
|
477
|
+
except Exception as exc:
|
|
478
|
+
try:
|
|
479
|
+
cur = getattr(page, "url", "") or ""
|
|
480
|
+
if cur and "login" not in cur.lower():
|
|
481
|
+
print(f"[Auth] Login already completed (CAPTCHA error ignored) → {cur}")
|
|
482
|
+
return True
|
|
483
|
+
except Exception:
|
|
484
|
+
pass
|
|
485
|
+
print(f"[Auth] CAPTCHA solve error: {exc}")
|
|
486
|
+
return False
|
|
487
|
+
|
|
488
|
+
async def _sync_hidden_credential_fields(self, page: Page) -> None:
|
|
489
|
+
"""Copy visible login fields into hidden submit fields (chess.com Vue form)."""
|
|
490
|
+
try:
|
|
491
|
+
await page.evaluate(
|
|
492
|
+
"""() => {
|
|
493
|
+
const pairs = [
|
|
494
|
+
['#login-username', '#username, input[name="_username"]'],
|
|
495
|
+
['#login-password', '#password, input[name="_password"]'],
|
|
496
|
+
];
|
|
497
|
+
for (const [visSel, hidSel] of pairs) {
|
|
498
|
+
const vis = document.querySelector(visSel);
|
|
499
|
+
const hid = document.querySelector(hidSel);
|
|
500
|
+
if (vis && hid && vis.value) {
|
|
501
|
+
hid.value = vis.value;
|
|
502
|
+
hid.dispatchEvent(new Event('input', { bubbles: true }));
|
|
503
|
+
hid.dispatchEvent(new Event('change', { bubbles: true }));
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
}"""
|
|
507
|
+
)
|
|
508
|
+
except Exception:
|
|
509
|
+
pass
|
|
510
|
+
|
|
511
|
+
async def _submit_and_verify(
|
|
512
|
+
self,
|
|
513
|
+
page: Page,
|
|
514
|
+
password_field,
|
|
515
|
+
login_url: str,
|
|
516
|
+
timeout: int,
|
|
517
|
+
wait_until: str,
|
|
518
|
+
) -> bool:
|
|
519
|
+
"""Click the submit button (or press Enter) then verify login success."""
|
|
520
|
+
# Already logged in (manual captcha / late navigation during CapSolver)
|
|
521
|
+
try:
|
|
522
|
+
if not self._path_looks_like_login(page.url, login_url):
|
|
523
|
+
print(f"[Auth] Login successful. Now at: {page.url}")
|
|
524
|
+
return True
|
|
525
|
+
except Exception:
|
|
526
|
+
pass
|
|
527
|
+
|
|
528
|
+
await self._sync_hidden_credential_fields(page)
|
|
529
|
+
pre_url = page.url
|
|
530
|
+
clicked = await self._click_login_submit(page, password_field)
|
|
531
|
+
if not clicked:
|
|
532
|
+
# Page may have navigated during the click attempt
|
|
533
|
+
try:
|
|
534
|
+
if not self._path_looks_like_login(page.url, login_url):
|
|
535
|
+
print(f"[Auth] Login successful. Now at: {page.url}")
|
|
536
|
+
return True
|
|
537
|
+
except Exception:
|
|
538
|
+
pass
|
|
539
|
+
print("[Auth] Could not click login submit control.")
|
|
540
|
+
return False
|
|
541
|
+
|
|
542
|
+
await asyncio.sleep(1.2)
|
|
543
|
+
try:
|
|
544
|
+
still_here = page.url == pre_url or self._path_looks_like_login(page.url, login_url)
|
|
545
|
+
except Exception:
|
|
546
|
+
still_here = True
|
|
547
|
+
if still_here:
|
|
548
|
+
forced = await self._force_native_form_submit(page)
|
|
549
|
+
if forced:
|
|
550
|
+
print("[Auth] Client-side submit gate detected — forced native form submit.")
|
|
551
|
+
await asyncio.sleep(1.5)
|
|
552
|
+
|
|
553
|
+
return await self._verify_login(page, login_url, timeout, wait_until)
|
|
554
|
+
|
|
555
|
+
async def _click_login_submit(self, page: Page, password_field) -> bool:
|
|
556
|
+
"""Click the primary login button; fall back to Enter only as last resort."""
|
|
557
|
+
# Direct, high-signal selectors first (chess.com uses button#login)
|
|
558
|
+
preferred = [
|
|
559
|
+
"button#login",
|
|
560
|
+
"#login",
|
|
561
|
+
"#authentication-login-form button[type='submit']",
|
|
562
|
+
"button:has-text('Log In')",
|
|
563
|
+
"button:has-text('Log in')",
|
|
564
|
+
"button:has-text('Sign in')",
|
|
565
|
+
"button[type='submit']",
|
|
566
|
+
]
|
|
567
|
+
for sel in preferred:
|
|
568
|
+
try:
|
|
569
|
+
loc = page.locator(sel).first
|
|
570
|
+
await loc.wait_for(state="visible", timeout=2500)
|
|
571
|
+
if await loc.is_enabled(timeout=500):
|
|
572
|
+
await loc.click(timeout=5000)
|
|
573
|
+
print(f"[Auth] Clicked submit ({sel}).")
|
|
574
|
+
return True
|
|
575
|
+
except Exception:
|
|
576
|
+
continue
|
|
577
|
+
|
|
578
|
+
submit = await self._find_element(page, SUBMIT_SELECTORS, timeout=3000)
|
|
579
|
+
if submit:
|
|
580
|
+
try:
|
|
581
|
+
await submit.click(timeout=5000)
|
|
582
|
+
print("[Auth] Clicked submit (auto-detected).")
|
|
583
|
+
return True
|
|
584
|
+
except Exception as e:
|
|
585
|
+
print(f"[Auth] Submit click failed: {e}")
|
|
586
|
+
|
|
587
|
+
try:
|
|
588
|
+
print("[Auth] No submit button found; pressing Enter in password field.")
|
|
589
|
+
await password_field.press("Enter")
|
|
590
|
+
return True
|
|
591
|
+
except Exception as e:
|
|
592
|
+
print(f"[Auth] Error submitting login form: {e}")
|
|
593
|
+
return False
|
|
594
|
+
|
|
595
|
+
def _path_looks_like_login(self, current_url: str, login_url: str) -> bool:
|
|
596
|
+
from urllib.parse import urlparse
|
|
597
|
+
try:
|
|
598
|
+
return (
|
|
599
|
+
urlparse(current_url).path.lower().rstrip("/")
|
|
600
|
+
== urlparse(login_url).path.lower().rstrip("/")
|
|
601
|
+
)
|
|
602
|
+
except Exception:
|
|
603
|
+
return False
|
|
604
|
+
|
|
605
|
+
async def _force_native_form_submit(self, page: Page) -> bool:
|
|
606
|
+
"""
|
|
607
|
+
Call HTMLFormElement.prototype.submit() to bypass page submit listeners
|
|
608
|
+
that block when Turnstile is mid-challenge (chess.com) or require a
|
|
609
|
+
specific submitter id.
|
|
610
|
+
"""
|
|
611
|
+
try:
|
|
612
|
+
return bool(
|
|
613
|
+
await page.evaluate(
|
|
614
|
+
"""() => {
|
|
615
|
+
const form = document.querySelector(
|
|
616
|
+
'#authentication-login-form, form.authentication-login-form, form.login-form, form'
|
|
617
|
+
);
|
|
618
|
+
if (!form) return false;
|
|
619
|
+
const user = form.querySelector('#username, input[name="_username"], input[type="email"]');
|
|
620
|
+
const pass = form.querySelector('#password, input[name="_password"], input[type="password"]');
|
|
621
|
+
const tok = form.querySelector(
|
|
622
|
+
'#turnstile_token, [name="turnstile_token"], [name="cf-turnstile-response"]'
|
|
623
|
+
);
|
|
624
|
+
// Only force when credentials look filled (and token if present).
|
|
625
|
+
if (user && !(user.value || '').trim()) return false;
|
|
626
|
+
if (pass && !(pass.value || '').trim()) return false;
|
|
627
|
+
if (tok && !(tok.value || '').trim()) return false;
|
|
628
|
+
HTMLFormElement.prototype.submit.call(form);
|
|
629
|
+
return true;
|
|
630
|
+
}"""
|
|
631
|
+
)
|
|
632
|
+
)
|
|
633
|
+
except Exception:
|
|
634
|
+
return False
|
|
635
|
+
|
|
636
|
+
async def _verify_login(
|
|
637
|
+
self, page: Page, login_url: str, timeout: int, wait_until: str
|
|
638
|
+
) -> bool:
|
|
639
|
+
"""
|
|
640
|
+
Wait for the page to change after submit, then decide if login succeeded.
|
|
641
|
+
|
|
642
|
+
Strategy:
|
|
643
|
+
1. Wait for navigation / load state.
|
|
644
|
+
2. If an OTP/MFA field appeared → return False (caller handles MFA).
|
|
645
|
+
3. If a CAPTCHA widget appeared → return False (caller tries CapSolver).
|
|
646
|
+
4. If we're still on the login page path with error text → failed.
|
|
647
|
+
5. If we navigated away → success.
|
|
648
|
+
"""
|
|
649
|
+
try:
|
|
650
|
+
await page.wait_for_load_state(wait_until, timeout=timeout)
|
|
651
|
+
except Exception:
|
|
652
|
+
pass
|
|
653
|
+
|
|
654
|
+
await asyncio.sleep(1.2)
|
|
655
|
+
|
|
656
|
+
try:
|
|
657
|
+
current_url = page.url
|
|
658
|
+
except Exception:
|
|
659
|
+
current_url = ""
|
|
660
|
+
|
|
661
|
+
from urllib.parse import urlparse
|
|
662
|
+
current_path = urlparse(current_url).path.lower().rstrip("/")
|
|
663
|
+
login_path = urlparse(login_url).path.lower().rstrip("/")
|
|
664
|
+
|
|
665
|
+
# Already away from login — success (even if CAPTCHA nodes briefly linger)
|
|
666
|
+
if current_path and current_path != login_path:
|
|
667
|
+
if await self._any_visible(page, ERROR_SELECTORS, timeout=400):
|
|
668
|
+
print(f"[Auth] Login failed — credential error visible at {current_url}.")
|
|
669
|
+
return False
|
|
670
|
+
print(f"[Auth] Login successful. Now at: {current_url}")
|
|
671
|
+
return True
|
|
672
|
+
|
|
673
|
+
# Still on login page: CAPTCHA present → caller may retry CapSolver
|
|
674
|
+
from webvac.captcha.detect import detect_captcha_raw
|
|
675
|
+
|
|
676
|
+
if await detect_captcha_raw(page) or await self._any_present(page, CAPTCHA_WIDGET_SELECTORS):
|
|
677
|
+
# Interactive checkbox visible without a token often means solve incomplete
|
|
678
|
+
print("[Auth] CAPTCHA detected after submit — will attempt auto-solve.")
|
|
679
|
+
return False
|
|
680
|
+
|
|
681
|
+
if await self._any_visible(page, ERROR_SELECTORS, timeout=800):
|
|
682
|
+
print("[Auth] Login failed — credential error visible on page.")
|
|
683
|
+
else:
|
|
684
|
+
print("[Auth] Login may have failed — still on the login page after submit.")
|
|
685
|
+
return False
|
|
686
|
+
|
|
687
|
+
async def _wait_for_login_form(self, page: Page, *, timeout_ms: int = 15000) -> bool:
|
|
688
|
+
"""Wait until a username or password field is visible (main page or iframe)."""
|
|
689
|
+
probes = USERNAME_SELECTORS[:6] + PASSWORD_SELECTORS
|
|
690
|
+
per_try = max(1500, timeout_ms // max(len(probes), 1))
|
|
691
|
+
roots = [page] + [f for f in page.frames if f != page.main_frame]
|
|
692
|
+
for root in roots:
|
|
693
|
+
for sel in probes:
|
|
694
|
+
try:
|
|
695
|
+
loc = root.locator(sel).first
|
|
696
|
+
await loc.wait_for(state="visible", timeout=per_try)
|
|
697
|
+
if root is not page:
|
|
698
|
+
print(f"[Auth] Login form detected in iframe.")
|
|
699
|
+
return True
|
|
700
|
+
except Exception:
|
|
701
|
+
continue
|
|
702
|
+
await asyncio.sleep(1.0)
|
|
703
|
+
return bool(await self._find_element(page, USERNAME_SELECTORS + PASSWORD_SELECTORS, timeout=2000))
|
|
704
|
+
|
|
705
|
+
async def _find_element(self, page: Page, selectors: list, timeout: int = 2000):
|
|
706
|
+
"""Try each selector on the main page, then inside child iframes."""
|
|
707
|
+
el = await self._find_element_in_root(page, selectors, timeout)
|
|
708
|
+
if el:
|
|
709
|
+
return el
|
|
710
|
+
for frame in page.frames:
|
|
711
|
+
if frame == page.main_frame:
|
|
712
|
+
continue
|
|
713
|
+
try:
|
|
714
|
+
el = await self._find_element_in_root(frame, selectors, min(timeout, 4000))
|
|
715
|
+
if el:
|
|
716
|
+
print(f"[Auth] Matched login control inside iframe.")
|
|
717
|
+
return el
|
|
718
|
+
except Exception:
|
|
719
|
+
continue
|
|
720
|
+
return None
|
|
721
|
+
|
|
722
|
+
async def _find_element_in_root(self, root, selectors: list, timeout: int = 2000):
|
|
723
|
+
"""Try each selector and return the first visible, enabled, non-SSO element."""
|
|
724
|
+
# Per-candidate visibility probe stays short so we can try many selectors.
|
|
725
|
+
probe = min(800, max(250, timeout // 4))
|
|
726
|
+
for sel in selectors:
|
|
727
|
+
try:
|
|
728
|
+
loc = root.locator(sel)
|
|
729
|
+
count = await loc.count()
|
|
730
|
+
for i in range(min(count, 8)):
|
|
731
|
+
el = loc.nth(i)
|
|
732
|
+
try:
|
|
733
|
+
if not await el.is_visible(timeout=probe):
|
|
734
|
+
continue
|
|
735
|
+
if not await el.is_enabled(timeout=probe):
|
|
736
|
+
continue
|
|
737
|
+
except Exception:
|
|
738
|
+
continue
|
|
739
|
+
try:
|
|
740
|
+
text = (await el.inner_text(timeout=400) or "").strip()
|
|
741
|
+
except Exception:
|
|
742
|
+
text = ""
|
|
743
|
+
try:
|
|
744
|
+
aria = (await el.get_attribute("aria-label")) or ""
|
|
745
|
+
except Exception:
|
|
746
|
+
aria = ""
|
|
747
|
+
blob = f"{text} {aria} {sel}"
|
|
748
|
+
if _SSO_TEXT_RE.search(blob):
|
|
749
|
+
continue
|
|
750
|
+
return el
|
|
751
|
+
except Exception:
|
|
752
|
+
continue
|
|
753
|
+
return None
|
|
754
|
+
|
|
755
|
+
async def _any_visible(self, page: Page, selectors: list, timeout: int = 500) -> bool:
|
|
756
|
+
"""Return True if any selector in the list matches a visible element."""
|
|
757
|
+
for sel in selectors:
|
|
758
|
+
try:
|
|
759
|
+
el = page.locator(sel).first
|
|
760
|
+
if await el.is_visible(timeout=timeout):
|
|
761
|
+
return True
|
|
762
|
+
except Exception:
|
|
763
|
+
continue
|
|
764
|
+
return False
|
|
765
|
+
|
|
766
|
+
async def _any_present(self, page: Page, selectors: list) -> bool:
|
|
767
|
+
"""Return True if any selector matches in the DOM (including hidden inputs)."""
|
|
768
|
+
for sel in selectors:
|
|
769
|
+
try:
|
|
770
|
+
if await page.locator(sel).count() > 0:
|
|
771
|
+
return True
|
|
772
|
+
except Exception:
|
|
773
|
+
continue
|
|
774
|
+
return False
|