ricibrowser 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ricibrowser-0.2.1/PKG-INFO +107 -0
- ricibrowser-0.2.1/README.md +78 -0
- ricibrowser-0.2.1/pyproject.toml +53 -0
- ricibrowser-0.2.1/setup.cfg +4 -0
- ricibrowser-0.2.1/src/ricibrowser/__init__.py +38 -0
- ricibrowser-0.2.1/src/ricibrowser/captcha.py +395 -0
- ricibrowser-0.2.1/src/ricibrowser/cdp_client.py +261 -0
- ricibrowser-0.2.1/src/ricibrowser/chrome_launcher.py +183 -0
- ricibrowser-0.2.1/src/ricibrowser/config.py +92 -0
- ricibrowser-0.2.1/src/ricibrowser/cookie_jar.py +116 -0
- ricibrowser-0.2.1/src/ricibrowser/engine.py +244 -0
- ricibrowser-0.2.1/src/ricibrowser/fingerprint.py +180 -0
- ricibrowser-0.2.1/src/ricibrowser/geolocation.py +217 -0
- ricibrowser-0.2.1/src/ricibrowser/headers.py +157 -0
- ricibrowser-0.2.1/src/ricibrowser/input.py +234 -0
- ricibrowser-0.2.1/src/ricibrowser/lightpanda.py +245 -0
- ricibrowser-0.2.1/src/ricibrowser/network.py +159 -0
- ricibrowser-0.2.1/src/ricibrowser/proxy_integration.py +171 -0
- ricibrowser-0.2.1/src/ricibrowser/py.typed +0 -0
- ricibrowser-0.2.1/src/ricibrowser/session.py +354 -0
- ricibrowser-0.2.1/src/ricibrowser/stealth.py +77 -0
- ricibrowser-0.2.1/src/ricibrowser/utils.py +111 -0
- ricibrowser-0.2.1/src/ricibrowser/wait.py +137 -0
- ricibrowser-0.2.1/src/ricibrowser.egg-info/PKG-INFO +107 -0
- ricibrowser-0.2.1/src/ricibrowser.egg-info/SOURCES.txt +32 -0
- ricibrowser-0.2.1/src/ricibrowser.egg-info/dependency_links.txt +1 -0
- ricibrowser-0.2.1/src/ricibrowser.egg-info/requires.txt +8 -0
- ricibrowser-0.2.1/src/ricibrowser.egg-info/top_level.txt +1 -0
- ricibrowser-0.2.1/tests/test_cdp_client.py +48 -0
- ricibrowser-0.2.1/tests/test_cookie_jar.py +69 -0
- ricibrowser-0.2.1/tests/test_engine.py +83 -0
- ricibrowser-0.2.1/tests/test_extensions.py +301 -0
- ricibrowser-0.2.1/tests/test_utils.py +109 -0
- ricibrowser-0.2.1/tests/test_wait.py +58 -0
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ricibrowser
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: A lightweight two-engine browser automation module built on CDP — no Playwright, no Puppeteer
|
|
5
|
+
Author-email: Riciplay <dev@riciplay.xyz>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Zaidux/ricibrowser
|
|
8
|
+
Project-URL: Repository, https://github.com/Zaidux/ricibrowser
|
|
9
|
+
Project-URL: Documentation, https://github.com/Zaidux/ricibrowser/blob/main/ARCHITECTURE.md
|
|
10
|
+
Keywords: cdp,browser-automation,headless,chrome-devtools-protocol,security
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
|
|
19
|
+
Classifier: Topic :: Security
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
Requires-Dist: websockets>=12.0
|
|
23
|
+
Requires-Dist: httpx>=0.27
|
|
24
|
+
Requires-Dist: aiofiles>=23.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-cov>=4.1; extra == "dev"
|
|
29
|
+
|
|
30
|
+
# ricibrowser
|
|
31
|
+
|
|
32
|
+
A lightweight two-engine browser automation module built entirely on the Chrome DevTools Protocol (CDP). No Playwright, no Puppeteer, no selenium.
|
|
33
|
+
|
|
34
|
+
## Engines
|
|
35
|
+
|
|
36
|
+
| Engine | Use case | Technology |
|
|
37
|
+
|--------|----------|------------|
|
|
38
|
+
| **Lightpanda** (fast path) | crawl, recon, endpoint discovery, non-JS-heavy targets | Zig-based headless engine, CDP at `ws://127.0.0.1:9222` |
|
|
39
|
+
| **CDP-Chrome** (thorough path) | DAST, JS-heavy targets, auth flows, anti-bot | Custom CDP client driving the user's real installed Chrome |
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install ricibrowser
|
|
45
|
+
|
|
46
|
+
# For the fast path (optional):
|
|
47
|
+
bash scripts/install_lightpanda.sh
|
|
48
|
+
lightpanda serve --host 127.0.0.1 --port 9222
|
|
49
|
+
|
|
50
|
+
# For the thorough path:
|
|
51
|
+
# Just have Google Chrome installed on your system.
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Quick start
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
import asyncio
|
|
58
|
+
from ricibrowser import Engine, EngineConfig
|
|
59
|
+
|
|
60
|
+
async def main():
|
|
61
|
+
engine = Engine(EngineConfig())
|
|
62
|
+
|
|
63
|
+
# Fast path (Lightpanda) — crawl/recon
|
|
64
|
+
page = await engine.fast_browse("https://example.com")
|
|
65
|
+
print(f"Title: {page.title}")
|
|
66
|
+
print(f"Text: {page.text[:200]}")
|
|
67
|
+
print(f"Links: {len(page.links)}")
|
|
68
|
+
|
|
69
|
+
# Thorough path (CDP-Chrome) — DAST/auth flows
|
|
70
|
+
session = await engine.create_session()
|
|
71
|
+
await session.navigate("https://example.com/login")
|
|
72
|
+
await session.fill("#username", "admin")
|
|
73
|
+
await session.fill("#password", "pass")
|
|
74
|
+
await session.click("#login-btn")
|
|
75
|
+
|
|
76
|
+
# Cookies persist across sessions via CookieJar
|
|
77
|
+
await session.navigate("https://example.com/dashboard") # authenticated!
|
|
78
|
+
|
|
79
|
+
# JS evaluation in isolated world (never Runtime.enable on main world)
|
|
80
|
+
count = await session.evaluate("document.querySelectorAll('script').length")
|
|
81
|
+
|
|
82
|
+
# Network capture (opt-in, off by default)
|
|
83
|
+
engine2 = Engine(EngineConfig(debug_network=True))
|
|
84
|
+
session2 = await engine2.create_session()
|
|
85
|
+
# ... browse ...
|
|
86
|
+
flows = engine2.network.to_dict()
|
|
87
|
+
|
|
88
|
+
await engine.close()
|
|
89
|
+
|
|
90
|
+
asyncio.run(main())
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Stealth
|
|
94
|
+
|
|
95
|
+
- `navigator.webdriver` suppressed via `--disable-blink-features=AutomationControlled` (Blink-level, not JS injection)
|
|
96
|
+
- Uses the user's real installed Chrome (not bundled Chromium) — TLS/JA3 fingerprint matches a real Chrome release
|
|
97
|
+
- Never calls `Runtime.enable` on the main world — isolated worlds only
|
|
98
|
+
- `Console.enable` off by default — only enabled in explicit debug mode
|
|
99
|
+
- `Network.enable` off by default — known CDP detection vector
|
|
100
|
+
|
|
101
|
+
## Architecture
|
|
102
|
+
|
|
103
|
+
See [ARCHITECTURE.md](ARCHITECTURE.md) for the full design.
|
|
104
|
+
|
|
105
|
+
## License
|
|
106
|
+
|
|
107
|
+
MIT
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# ricibrowser
|
|
2
|
+
|
|
3
|
+
A lightweight two-engine browser automation module built entirely on the Chrome DevTools Protocol (CDP). No Playwright, no Puppeteer, no selenium.
|
|
4
|
+
|
|
5
|
+
## Engines
|
|
6
|
+
|
|
7
|
+
| Engine | Use case | Technology |
|
|
8
|
+
|--------|----------|------------|
|
|
9
|
+
| **Lightpanda** (fast path) | crawl, recon, endpoint discovery, non-JS-heavy targets | Zig-based headless engine, CDP at `ws://127.0.0.1:9222` |
|
|
10
|
+
| **CDP-Chrome** (thorough path) | DAST, JS-heavy targets, auth flows, anti-bot | Custom CDP client driving the user's real installed Chrome |
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install ricibrowser
|
|
16
|
+
|
|
17
|
+
# For the fast path (optional):
|
|
18
|
+
bash scripts/install_lightpanda.sh
|
|
19
|
+
lightpanda serve --host 127.0.0.1 --port 9222
|
|
20
|
+
|
|
21
|
+
# For the thorough path:
|
|
22
|
+
# Just have Google Chrome installed on your system.
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Quick start
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
import asyncio
|
|
29
|
+
from ricibrowser import Engine, EngineConfig
|
|
30
|
+
|
|
31
|
+
async def main():
|
|
32
|
+
engine = Engine(EngineConfig())
|
|
33
|
+
|
|
34
|
+
# Fast path (Lightpanda) — crawl/recon
|
|
35
|
+
page = await engine.fast_browse("https://example.com")
|
|
36
|
+
print(f"Title: {page.title}")
|
|
37
|
+
print(f"Text: {page.text[:200]}")
|
|
38
|
+
print(f"Links: {len(page.links)}")
|
|
39
|
+
|
|
40
|
+
# Thorough path (CDP-Chrome) — DAST/auth flows
|
|
41
|
+
session = await engine.create_session()
|
|
42
|
+
await session.navigate("https://example.com/login")
|
|
43
|
+
await session.fill("#username", "admin")
|
|
44
|
+
await session.fill("#password", "pass")
|
|
45
|
+
await session.click("#login-btn")
|
|
46
|
+
|
|
47
|
+
# Cookies persist across sessions via CookieJar
|
|
48
|
+
await session.navigate("https://example.com/dashboard") # authenticated!
|
|
49
|
+
|
|
50
|
+
# JS evaluation in isolated world (never Runtime.enable on main world)
|
|
51
|
+
count = await session.evaluate("document.querySelectorAll('script').length")
|
|
52
|
+
|
|
53
|
+
# Network capture (opt-in, off by default)
|
|
54
|
+
engine2 = Engine(EngineConfig(debug_network=True))
|
|
55
|
+
session2 = await engine2.create_session()
|
|
56
|
+
# ... browse ...
|
|
57
|
+
flows = engine2.network.to_dict()
|
|
58
|
+
|
|
59
|
+
await engine.close()
|
|
60
|
+
|
|
61
|
+
asyncio.run(main())
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Stealth
|
|
65
|
+
|
|
66
|
+
- `navigator.webdriver` suppressed via `--disable-blink-features=AutomationControlled` (Blink-level, not JS injection)
|
|
67
|
+
- Uses the user's real installed Chrome (not bundled Chromium) — TLS/JA3 fingerprint matches a real Chrome release
|
|
68
|
+
- Never calls `Runtime.enable` on the main world — isolated worlds only
|
|
69
|
+
- `Console.enable` off by default — only enabled in explicit debug mode
|
|
70
|
+
- `Network.enable` off by default — known CDP detection vector
|
|
71
|
+
|
|
72
|
+
## Architecture
|
|
73
|
+
|
|
74
|
+
See [ARCHITECTURE.md](ARCHITECTURE.md) for the full design.
|
|
75
|
+
|
|
76
|
+
## License
|
|
77
|
+
|
|
78
|
+
MIT
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ricibrowser"
|
|
7
|
+
version = "0.2.1"
|
|
8
|
+
description = "A lightweight two-engine browser automation module built on CDP — no Playwright, no Puppeteer"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = {text = "MIT"}
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Riciplay", email = "dev@riciplay.xyz"},
|
|
14
|
+
]
|
|
15
|
+
keywords = ["cdp", "browser-automation", "headless", "chrome-devtools-protocol", "security"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Topic :: Internet :: WWW/HTTP :: Browsers",
|
|
25
|
+
"Topic :: Security",
|
|
26
|
+
]
|
|
27
|
+
dependencies = [
|
|
28
|
+
"websockets>=12.0",
|
|
29
|
+
"httpx>=0.27",
|
|
30
|
+
"aiofiles>=23.0",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
dev = [
|
|
35
|
+
"pytest>=8.0",
|
|
36
|
+
"pytest-asyncio>=0.23",
|
|
37
|
+
"pytest-cov>=4.1",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://github.com/Zaidux/ricibrowser"
|
|
42
|
+
Repository = "https://github.com/Zaidux/ricibrowser"
|
|
43
|
+
Documentation = "https://github.com/Zaidux/ricibrowser/blob/main/ARCHITECTURE.md"
|
|
44
|
+
|
|
45
|
+
[tool.setuptools.packages.find]
|
|
46
|
+
where = ["src"]
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.package-data]
|
|
49
|
+
ricibrowser = ["py.typed"]
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
asyncio_mode = "auto"
|
|
53
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""ricibrowser — a lightweight two-engine browser automation module.
|
|
2
|
+
|
|
3
|
+
No Playwright, no Puppeteer, no selenium. Built entirely on the public Chrome
|
|
4
|
+
DevTools Protocol specification (chromedevtools.github.io/devtools-protocol/).
|
|
5
|
+
|
|
6
|
+
Two engines:
|
|
7
|
+
- **Lightpanda** (fast path): Zig-based headless engine, CDP at ws://127.0.0.1:9222.
|
|
8
|
+
For crawl/recon/endpoint discovery — ~16× less memory, ~9× faster than Chromium.
|
|
9
|
+
- **CDP-Chrome** (thorough path): custom CDP client driving the user's real
|
|
10
|
+
installed Chrome. For DAST/JS-heavy targets/auth flow testing/anti-bot.
|
|
11
|
+
|
|
12
|
+
Extensions:
|
|
13
|
+
- **fingerprint**: Canvas/WebGL/audio fingerprint randomization (opt-in).
|
|
14
|
+
- **geolocation**: Timezone/locale/geolocation override + proxy rotation pool.
|
|
15
|
+
- **captcha**: CAPTCHA detection, Cloudflare auto-resolve, solver hooks.
|
|
16
|
+
- **input**: Human-like mouse movement simulation (bezier curves + timing).
|
|
17
|
+
- **headers**: Sec-CH-UA consistency + request header ordering.
|
|
18
|
+
|
|
19
|
+
Usage::
|
|
20
|
+
|
|
21
|
+
from ricibrowser import Engine, EngineConfig, EngineType
|
|
22
|
+
|
|
23
|
+
engine = Engine(EngineConfig(fast_engine=EngineType.AUTO))
|
|
24
|
+
page = await engine.fast_browse("https://example.com")
|
|
25
|
+
print(page.title, page.text[:200])
|
|
26
|
+
|
|
27
|
+
session = await engine.create_session()
|
|
28
|
+
await session.navigate("https://example.com")
|
|
29
|
+
result = await session.evaluate("document.title")
|
|
30
|
+
await session.close()
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from ricibrowser.config import EngineConfig, EngineType
|
|
34
|
+
from ricibrowser.engine import Engine
|
|
35
|
+
from ricibrowser.session import Page, Session
|
|
36
|
+
|
|
37
|
+
__version__ = "0.2.0"
|
|
38
|
+
__all__ = ["Engine", "EngineConfig", "EngineType", "Page", "Session"]
|
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
"""CAPTCHA detection, Cloudflare auto-resolve, and solver hook framework.
|
|
2
|
+
|
|
3
|
+
Detection: identifies which CAPTCHA/anti-bot system is blocking navigation
|
|
4
|
+
(Cloudflare JS challenge, Cloudflare Turnstile, reCAPTCHA v2/v3, hCaptcha).
|
|
5
|
+
|
|
6
|
+
Auto-resolve: for Cloudflare JS challenges ("Just a moment..."), waits for
|
|
7
|
+
the challenge to auto-resolve in real Chrome (the JavaScript runs natively
|
|
8
|
+
and often passes within 5-10 seconds). Captures the ``cf_clearance`` cookie
|
|
9
|
+
for persistence.
|
|
10
|
+
|
|
11
|
+
Solver hooks: for CAPTCHAs that can't be auto-resolved (Turnstile,
|
|
12
|
+
reCAPTCHA, hCaptcha), provides a hook interface for external solver services.
|
|
13
|
+
The operator registers a ``CaptchaSolver`` implementation and ricibrowser
|
|
14
|
+
calls it when a CAPTCHA is detected.
|
|
15
|
+
|
|
16
|
+
Usage::
|
|
17
|
+
|
|
18
|
+
from ricibrowser.captcha import CaptCHAHandler, CloudflareAutoSolver
|
|
19
|
+
|
|
20
|
+
handler = CaptchaHandler(auto_solver=CloudflareAutoSolver())
|
|
21
|
+
result = await handler.detect_and_solve(session)
|
|
22
|
+
if result.solved:
|
|
23
|
+
print("Challenge resolved!") # cf_clearance now in cookie jar
|
|
24
|
+
elif result.captcha_type:
|
|
25
|
+
print(f"Unsupported: {result.captcha_type} — needs external solver")
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import asyncio
|
|
31
|
+
import logging
|
|
32
|
+
import time
|
|
33
|
+
from dataclasses import dataclass
|
|
34
|
+
from enum import Enum
|
|
35
|
+
from typing import Any, Protocol, runtime_checkable
|
|
36
|
+
|
|
37
|
+
from ricibrowser.cdp_client import CDPClient, CDPError
|
|
38
|
+
|
|
39
|
+
logger = logging.getLogger(__name__)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class CaptchaType(Enum):
|
|
43
|
+
"""Detected CAPTCHA / anti-bot challenge type."""
|
|
44
|
+
|
|
45
|
+
NONE = "none"
|
|
46
|
+
"""No CAPTCHA detected."""
|
|
47
|
+
|
|
48
|
+
CLOUDFLARE_JS = "cloudflare_js"
|
|
49
|
+
"""Cloudflare JS challenge ("Just a moment..."). Often auto-resolves."""
|
|
50
|
+
|
|
51
|
+
CLOUDFLARE_TURNSTILE = "cloudflare_turnstile"
|
|
52
|
+
"""Cloudflare Turnstile widget. Needs external solver."""
|
|
53
|
+
|
|
54
|
+
RECAPTCHA_V2 = "recaptcha_v2"
|
|
55
|
+
"""Google reCAPTCHA v2 (checkbox or invisible). Needs external solver."""
|
|
56
|
+
|
|
57
|
+
RECAPTCHA_V3 = "recaptcha_v3"
|
|
58
|
+
"""Google reCAPTCHA v3 (score-based, invisible). Needs external solver."""
|
|
59
|
+
|
|
60
|
+
HCAPTCHA = "hcaptcha"
|
|
61
|
+
"""hCaptcha widget. Needs external solver."""
|
|
62
|
+
|
|
63
|
+
GENERIC = "generic"
|
|
64
|
+
"""Unknown anti-bot challenge."""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass
|
|
68
|
+
class CaptchaResult:
|
|
69
|
+
"""Result of a CAPTCHA detection / solve attempt."""
|
|
70
|
+
|
|
71
|
+
captcha_type: CaptchaType
|
|
72
|
+
solved: bool
|
|
73
|
+
cookie_name: str | None = None
|
|
74
|
+
"""The cookie that proves the challenge was solved (e.g. cf_clearance)."""
|
|
75
|
+
solver_used: str | None = None
|
|
76
|
+
"""Which solver was used (e.g. 'cloudflare_auto', 'external')."""
|
|
77
|
+
error: str | None = None
|
|
78
|
+
duration_seconds: float = 0.0
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@runtime_checkable
|
|
82
|
+
class CaptchaSolver(Protocol):
|
|
83
|
+
"""Protocol for external CAPTCHA solver implementations.
|
|
84
|
+
|
|
85
|
+
Operators implement this to integrate third-party solver services
|
|
86
|
+
(2captcha, anti-captcha, capsolver, etc.) for CAPTCHAs that can't
|
|
87
|
+
be auto-resolved.
|
|
88
|
+
|
|
89
|
+
Example implementation::
|
|
90
|
+
|
|
91
|
+
class TwoCaptchaSolver:
|
|
92
|
+
async def solve(self, captcha_type, site_key, url):
|
|
93
|
+
# Call the 2captcha API
|
|
94
|
+
token = await call_service(site_key, url)
|
|
95
|
+
return CaptchaToken(token=token)
|
|
96
|
+
|
|
97
|
+
handler = CaptCHAHandler(solver=TwoCaptchaSolver())
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
async def solve(
|
|
101
|
+
self,
|
|
102
|
+
captcha_type: CaptchaType,
|
|
103
|
+
site_key: str | None,
|
|
104
|
+
url: str,
|
|
105
|
+
) -> str | None:
|
|
106
|
+
"""Solve a CAPTCHA and return the token.
|
|
107
|
+
|
|
108
|
+
Args:
|
|
109
|
+
captcha_type: The type of CAPTCHA detected.
|
|
110
|
+
site_key: The site key (for reCAPTCHA/hCaptcha/Turnstile).
|
|
111
|
+
url: The page URL where the CAPTCHA appears.
|
|
112
|
+
|
|
113
|
+
Returns:
|
|
114
|
+
The solver token, or None if the solver can't handle this type.
|
|
115
|
+
"""
|
|
116
|
+
...
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass
|
|
120
|
+
class CaptchaToken:
|
|
121
|
+
"""A solved CAPTCHA token."""
|
|
122
|
+
token: str
|
|
123
|
+
captcha_type: CaptchaType
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# ── Detection ──────────────────────────────────────────────────────────
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
async def detect_captcha(session) -> CaptchaType:
|
|
130
|
+
"""Detect what type of CAPTCHA or anti-bot challenge is on the current page.
|
|
131
|
+
|
|
132
|
+
Checks the DOM for known CAPTCHA widget signatures and the page text
|
|
133
|
+
for challenge messages.
|
|
134
|
+
"""
|
|
135
|
+
# Check for Cloudflare JS challenge ("Just a moment...")
|
|
136
|
+
title = await session.evaluate("document.title") or ""
|
|
137
|
+
body_text = await session.evaluate(
|
|
138
|
+
"document.body ? document.body.innerText.substring(0, 500) : ''"
|
|
139
|
+
) or ""
|
|
140
|
+
|
|
141
|
+
title_lower = title.lower()
|
|
142
|
+
text_lower = (body_text or "").lower()
|
|
143
|
+
|
|
144
|
+
if any(p in title_lower or p in text_lower for p in(
|
|
145
|
+
"just a moment", "checking your browser", "attention required",
|
|
146
|
+
"cf-challenge", "challenge-platform",
|
|
147
|
+
)):
|
|
148
|
+
return CaptchaType.CLOUDFLARE_JS
|
|
149
|
+
|
|
150
|
+
# Check for Cloudflare Turnstile widget
|
|
151
|
+
turnstile = await session.evaluate_bool(
|
|
152
|
+
"document.querySelector('.cf-turnstile, [data-sitekey]') !== null"
|
|
153
|
+
)
|
|
154
|
+
if turnstile:
|
|
155
|
+
# Verify it's actually a Turnstile widget (not reCAPTCHA)
|
|
156
|
+
is_turnstile = await session.evaluate_bool(
|
|
157
|
+
"document.querySelector('script[src*=\"challenges.cloudflare.com/turnstile\"]') !== null"
|
|
158
|
+
)
|
|
159
|
+
if is_turnstile:
|
|
160
|
+
return CaptchaType.CLOUDFLARE_TURNSTILE
|
|
161
|
+
|
|
162
|
+
# Check for reCAPTCHA
|
|
163
|
+
recaptcha = await session.evaluate_bool(
|
|
164
|
+
"document.querySelector('.g-recaptcha, [data-sitekey], "
|
|
165
|
+
"iframe[src*=\"recaptcha\"]') !== null"
|
|
166
|
+
)
|
|
167
|
+
if recaptcha:
|
|
168
|
+
# Check if it's v2 (visible checkbox) or v3 (invisible)
|
|
169
|
+
visible = await session.evaluate_bool(
|
|
170
|
+
"document.querySelector('.g-recaptcha') && "
|
|
171
|
+
"getComputedStyle(document.querySelector('.g-recaptcha')).display !== 'none'"
|
|
172
|
+
)
|
|
173
|
+
if visible:
|
|
174
|
+
return CaptchaType.RECAPTCHA_V2
|
|
175
|
+
return CaptchaType.RECAPTCHA_V3
|
|
176
|
+
|
|
177
|
+
# Check for hCaptcha
|
|
178
|
+
hcaptcha = await session.evaluate_bool(
|
|
179
|
+
"document.querySelector('.h-captcha, iframe[src*=\"hcaptcha\"]') !== null"
|
|
180
|
+
)
|
|
181
|
+
if hcaptcha:
|
|
182
|
+
return CaptchaType.HCAPTCHA
|
|
183
|
+
|
|
184
|
+
# Generic anti-bot check
|
|
185
|
+
if any(p in text_lower for p in(
|
|
186
|
+
"enable javascript and cookies", "please complete the security check",
|
|
187
|
+
"verify you are human", "are you a robot",
|
|
188
|
+
)):
|
|
189
|
+
return CaptchaType.GENERIC
|
|
190
|
+
|
|
191
|
+
return CaptchaType.NONE
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
# ── Cloudflare Auto-Solver ────────────────────────────────────────────
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class CloudflareAutoSolver:
|
|
198
|
+
"""Auto-resolves Cloudflare JS challenges by waiting for native JS execution.
|
|
199
|
+
|
|
200
|
+
Cloudflare's "Just a moment..." challenge runs JavaScript that computes
|
|
201
|
+
a token and auto-submits a form. In a real Chrome browser with real V8
|
|
202
|
+
engine (which ricibrowser uses), this JavaScript executes natively and
|
|
203
|
+
the challenge resolves automatically — usually within 5-10 seconds.
|
|
204
|
+
|
|
205
|
+
This solver:
|
|
206
|
+
1. Waits for the challenge page to load
|
|
207
|
+
2. Polls every 1s for the challenge to clear (body text changes,
|
|
208
|
+
challenge elements disappear, content loads)
|
|
209
|
+
3. Checks for the ``cf_clearance`` cookie which proves the challenge passed
|
|
210
|
+
4. Returns the result with the cookie name
|
|
211
|
+
|
|
212
|
+
Limitations:
|
|
213
|
+
- Only works with real Chrome (not Lightpanda — Lightpanda's V8 may
|
|
214
|
+
not handle the obfuscated CF JS)
|
|
215
|
+
- Doesn't work if the CF challenge requires a Turnstile click
|
|
216
|
+
- If the IP address or User-Agent is too suspicious, CF may loop forever
|
|
217
|
+
"""
|
|
218
|
+
|
|
219
|
+
def __init__(self, max_wait: float = 15.0, poll_interval: float = 1.0):
|
|
220
|
+
self.max_wait = max_wait
|
|
221
|
+
self.poll_interval = poll_interval
|
|
222
|
+
|
|
223
|
+
async def solve(self, session) -> CaptchaResult:
|
|
224
|
+
"""Wait for the Cloudflare JS challenge to auto-resolve.
|
|
225
|
+
|
|
226
|
+
Args:
|
|
227
|
+
session: A ricibrowser Session on the challenge page.
|
|
228
|
+
|
|
229
|
+
Returns:
|
|
230
|
+
CaptchaResult with solved=True if the challenge cleared.
|
|
231
|
+
"""
|
|
232
|
+
start = time.monotonic()
|
|
233
|
+
|
|
234
|
+
for _ in range(int(self.max_wait / self.poll_interval)):
|
|
235
|
+
elapsed = time.monotonic() - start
|
|
236
|
+
|
|
237
|
+
# Check if the challenge has cleared
|
|
238
|
+
body_text = await session.evaluate(
|
|
239
|
+
"document.body ? document.body.innerText.substring(0, 200) : ''"
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
# If evaluate returned None (context not ready, CDP error),
|
|
243
|
+
# treat as "unknown — keep waiting" rather than "cleared".
|
|
244
|
+
if body_text is None:
|
|
245
|
+
await asyncio.sleep(self.poll_interval)
|
|
246
|
+
continue
|
|
247
|
+
|
|
248
|
+
text_lower = body_text.lower()
|
|
249
|
+
|
|
250
|
+
# Challenge cleared if the "Just a moment" / "Checking" text is gone
|
|
251
|
+
# AND there is some actual body content (not empty).
|
|
252
|
+
challenge_cleared = (
|
|
253
|
+
bool(body_text.strip())
|
|
254
|
+
and not any(p in text_lower for p in(
|
|
255
|
+
"just a moment", "checking your browser", "attention required",
|
|
256
|
+
"enable javascript and cookies",
|
|
257
|
+
))
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
if challenge_cleared:
|
|
261
|
+
# Verify cf_clearance cookie was set
|
|
262
|
+
cookies = await session.get_cookies()
|
|
263
|
+
cf_clearance = [c for c in cookies if c.get("name") == "cf_clearance"]
|
|
264
|
+
if cf_clearance:
|
|
265
|
+
return CaptchaResult(
|
|
266
|
+
captcha_type=CaptchaType.CLOUDFLARE_JS,
|
|
267
|
+
solved=True,
|
|
268
|
+
cookie_name="cf_clearance",
|
|
269
|
+
solver_used="cloudflare_auto",
|
|
270
|
+
duration_seconds=elapsed,
|
|
271
|
+
)
|
|
272
|
+
# Challenge cleared but no cookie — might be a lighter check
|
|
273
|
+
return CaptchaResult(
|
|
274
|
+
captcha_type=CaptchaType.CLOUDFLARE_JS,
|
|
275
|
+
solved=True,
|
|
276
|
+
cookie_name=None,
|
|
277
|
+
solver_used="cloudflare_auto",
|
|
278
|
+
duration_seconds=elapsed,
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
await asyncio.sleep(self.poll_interval)
|
|
282
|
+
|
|
283
|
+
# Timed out
|
|
284
|
+
return CaptchaResult(
|
|
285
|
+
captcha_type=CaptchaType.CLOUDFLARE_JS,
|
|
286
|
+
solved=False,
|
|
287
|
+
error=f"Cloudflare challenge did not auto-resolve within {self.max_wait}s. "
|
|
288
|
+
"The IP may be flagged or a Turnstile click may be required.",
|
|
289
|
+
duration_seconds=time.monotonic() - start,
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
# ── Handler ───────────────────────────────────────────────────────────
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
class CaptchaHandler:
|
|
297
|
+
"""Detect and attempt to solve CAPTCHAs automatically.
|
|
298
|
+
|
|
299
|
+
Priority:
|
|
300
|
+
1. Cloudflare JS challenge → CloudflareAutoSolver (native JS execution)
|
|
301
|
+
2. Turnstile / reCAPTCHA / hCaptcha → external solver (if registered)
|
|
302
|
+
3. Generic → report and suggest manual intervention
|
|
303
|
+
|
|
304
|
+
Usage::
|
|
305
|
+
|
|
306
|
+
handler = CaptchaHandler(
|
|
307
|
+
auto_solver=CloudflareAutoSolver(),
|
|
308
|
+
external_solver=MySolver(), # optional
|
|
309
|
+
)
|
|
310
|
+
result = await handler.detect_and_solve(session)
|
|
311
|
+
"""
|
|
312
|
+
|
|
313
|
+
def __init__(
|
|
314
|
+
self,
|
|
315
|
+
auto_solver: CloudflareAutoSolver | None = None,
|
|
316
|
+
external_solver: CaptchaSolver | None = None,
|
|
317
|
+
):
|
|
318
|
+
self.auto_solver = auto_solver or CloudflareAutoSolver()
|
|
319
|
+
self.external_solver = external_solver
|
|
320
|
+
|
|
321
|
+
async def detect_and_solve(self, session) -> CaptchaResult:
|
|
322
|
+
"""Detect the CAPTCHA type and attempt resolution.
|
|
323
|
+
|
|
324
|
+
Returns a CaptchaResult. If solved, the session is ready to continue.
|
|
325
|
+
If not solved, the result contains the type and an error message.
|
|
326
|
+
"""
|
|
327
|
+
start = time.monotonic()
|
|
328
|
+
|
|
329
|
+
# ── Detect ──────────────────────────────────────────────────
|
|
330
|
+
captcha_type = await detect_captcha(session)
|
|
331
|
+
|
|
332
|
+
if captcha_type == CaptchaType.NONE:
|
|
333
|
+
return CaptchaResult(
|
|
334
|
+
captcha_type=CaptchaType.NONE,
|
|
335
|
+
solved=True,
|
|
336
|
+
duration_seconds=time.monotonic() - start,
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
logger.info("CAPTCHA detected: %s", captcha_type.value)
|
|
340
|
+
|
|
341
|
+
# ── Cloudflare JS challenge → auto-solve ───────────────────
|
|
342
|
+
if captcha_type == CaptchaType.CLOUDFLARE_JS:
|
|
343
|
+
result = await self.auto_solver.solve(session)
|
|
344
|
+
return result
|
|
345
|
+
|
|
346
|
+
# ── External solver for reCAPTCHA/hCaptcha/Turnstile ───────
|
|
347
|
+
if self.external_solver and captcha_type in(
|
|
348
|
+
CaptchaType.RECAPTCHA_V2,
|
|
349
|
+
CaptchaType.RECAPTCHA_V3,
|
|
350
|
+
CaptchaType.HCAPTCHA,
|
|
351
|
+
CaptchaType.CLOUDFLARE_TURNSTILE,
|
|
352
|
+
):
|
|
353
|
+
# Extract the site key
|
|
354
|
+
site_key = await session.evaluate(
|
|
355
|
+
"document.querySelector('[data-sitekey]')?.getAttribute('data-sitekey') || "
|
|
356
|
+
"document.querySelector('iframe[src*=\"recaptcha\"]')?.src.match(/render=([^&]+)/)?.[1] || "
|
|
357
|
+
"document.querySelector('iframe[src*=\"hcaptcha\"]')?.src.match(/sitekey=([^&]+)/)?.[1] || ''"
|
|
358
|
+
) or ""
|
|
359
|
+
|
|
360
|
+
url = session._current_url
|
|
361
|
+
|
|
362
|
+
try:
|
|
363
|
+
token = await self.external_solver.solve(captcha_type, site_key or None, url)
|
|
364
|
+
if token:
|
|
365
|
+
# Inject the token into the page
|
|
366
|
+
await session.evaluate(
|
|
367
|
+
f"document.getElementById('g-recaptcha-response').value = '{token}';"
|
|
368
|
+
if captcha_type == CaptchaType.RECAPTCHA_V2 else
|
|
369
|
+
f"window.__captcha_token__ = '{token}';"
|
|
370
|
+
)
|
|
371
|
+
return CaptchaResult(
|
|
372
|
+
captcha_type=captcha_type,
|
|
373
|
+
solved=True,
|
|
374
|
+
solver_used="external",
|
|
375
|
+
duration_seconds=time.monotonic() - start,
|
|
376
|
+
)
|
|
377
|
+
except Exception as exc:
|
|
378
|
+
logger.warning("External solver failed: %s", exc)
|
|
379
|
+
return CaptchaResult(
|
|
380
|
+
captcha_type=captcha_type,
|
|
381
|
+
solved=False,
|
|
382
|
+
error=f"External solver error: {exc}",
|
|
383
|
+
duration_seconds=time.monotonic() - start,
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
# ── No solver available for this type ──────────────────────
|
|
387
|
+
solver_name = "external" if self.external_solver else "none"
|
|
388
|
+
return CaptchaResult(
|
|
389
|
+
captcha_type=captcha_type,
|
|
390
|
+
solved=False,
|
|
391
|
+
error=f"No solver available for {captcha_type.value} "
|
|
392
|
+
f"(registered: {solver_name}). For Cloudflare JS challenges, "
|
|
393
|
+
f"ensure real Chrome is used (not Lightpanda).",
|
|
394
|
+
duration_seconds=time.monotonic() - start,
|
|
395
|
+
)
|