stealthhttp 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/CHANGELOG.md +40 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/PKG-INFO +1 -1
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/pyproject.toml +1 -1
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/__init__.py +71 -1
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/async_session.py +1 -2
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/backends/native_backend.py +3 -2
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/cdp_solver.py +2 -1
- stealthhttp-0.4.0/python/stealthhttp/challenge_vm.py +460 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/clienthints.py +1 -1
- stealthhttp-0.4.0/python/stealthhttp/coherence.py +187 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/collectors/__init__.py +5 -1
- stealthhttp-0.4.0/python/stealthhttp/cookie_intel.py +211 -0
- stealthhttp-0.4.0/python/stealthhttp/cover_traffic.py +187 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/detection.py +3 -2
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/device_db.py +3 -3
- stealthhttp-0.4.0/python/stealthhttp/immortal.py +210 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/js_sandbox.py +3 -1
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/jsenv.py +6 -5
- stealthhttp-0.4.0/python/stealthhttp/mesh.py +316 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/middleware.py +10 -9
- stealthhttp-0.4.0/python/stealthhttp/poison_detect.py +350 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/pow.py +6 -4
- stealthhttp-0.4.0/python/stealthhttp/predictive_rate.py +94 -0
- stealthhttp-0.4.0/python/stealthhttp/proto_chameleon.py +252 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/proxy.py +0 -2
- stealthhttp-0.4.0/python/stealthhttp/resp_diff.py +285 -0
- stealthhttp-0.4.0/python/stealthhttp/rhythm.py +92 -0
- stealthhttp-0.4.0/python/stealthhttp/self_audit.py +370 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/session_coord.py +2 -2
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/session_ext.py +1 -1
- stealthhttp-0.4.0/python/stealthhttp/smart_retry.py +168 -0
- stealthhttp-0.4.0/python/stealthhttp/synth_fingerprint.py +324 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/tests/test_features_v3.py +1 -1
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/tests/test_v030_features.py +1 -1
- stealthhttp-0.4.0/python/stealthhttp/tests/test_v040_features.py +979 -0
- stealthhttp-0.4.0/python/stealthhttp/waf_recon.py +276 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/websocket.py +2 -1
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/.cargo/config.toml +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/.github/workflows/release.yml +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/.gitignore +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/Cargo.lock +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/Cargo.toml +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/LICENSE +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/README.md +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/benchmarks/bench.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/benchmarks/bench_native.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/examples/basic.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/_fallback.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/_loader.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/_rotation.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/ambient.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/antidetect.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/auto_bypass.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/auto_profile.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/backends/__init__.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/backends/base.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/backends/curl_backend.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/backends/wreq_backend.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/behavior.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/browser_cookies.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/browser_pool.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/browsing.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/__init__.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/akamai.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/captcha.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/cloudflare.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/datadome.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/flow.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/imperva.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/kasada.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/perimeterx.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/shape.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/bypasses/solver.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/capabilities.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/captcha.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/captcha_farm.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/challenge.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/collectors/akamai.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/collectors/datadome.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/collectors/imperva.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/collectors/kasada.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/collectors/perimeterx.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/collectors/shape.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/consistency.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/cookies.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/dns_intel.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/doh.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/error_analysis.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/exceptions.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/fingerprint_diff.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/forms.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/geo.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/graphql.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/h2_frames.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/har.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/header_canon.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/health_monitor.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/hooks.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/hsts.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/httpcache.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/humanize.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/identity.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/jsdb.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/lifecycle.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/lint.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/multipart.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/parsing.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/prepare.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/proxy_scoring.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/record_replay.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/redirect_intel.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/request_graph.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/response.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/response_decoder.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/response_intel.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/retry.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/rotation.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/selfcheck.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/sensors.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/session.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/site_recipes.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/smart_proxy.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/solving/__init__.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/solving/audio.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/solving/base.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/solving/browser.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/solving/local.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/solving/orchestrator.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/solving/providers.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/solving/vision.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/stealth_audit.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/strategy.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/tcp_tuning.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/tests/test_cdp_solver.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/tests/test_features_v3_live.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/tls_ext.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/tls_intel.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/tls_session.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/python/stealthhttp/traffic.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/fingerprint/headers.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/fingerprint/ja3.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/fingerprint/ja4.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/fingerprint/mod.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/fingerprint/profile.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/lib.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/proxypool.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/rotation.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/transport/decode.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/transport/engine.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/transport/mod.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/src/transport/tls.rs +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/conftest.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_audio.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_bypass.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_bypasses.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_bypasses_live.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_emulation.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_features_v2.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_features_v2_live.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_fingerprint.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_humanize.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_jsemu.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_jsemu_live.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_live_bypass.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_live_detection.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_new_features.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_policies.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_routing.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_session.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_solving.py +0 -0
- {stealthhttp-0.3.0 → stealthhttp-0.4.0}/tests/test_vision.py +0 -0
|
@@ -1,5 +1,45 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.4.0] - 2026-08-10
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- **synth_fingerprint** -- SynthProfile and FingerprintSynthesizer for generating realistic browser fingerprints from statistical models
|
|
8
|
+
- **waf_recon** -- WafInfo, recon_sync/recon for identifying WAF vendors and versions from response signatures
|
|
9
|
+
- **poison_detect** -- PoisonScore, PoisonDetector, poison_check for detecting cache poisoning and response manipulation
|
|
10
|
+
- **resp_diff** -- DiffReport, diff_json, diff_html_structure, compare_bodies for structural comparison of HTTP responses
|
|
11
|
+
- **predictive_rate** -- PredictiveRateLimiter for anticipating rate limit windows and pre-throttling requests
|
|
12
|
+
- **smart_retry** -- FailureDiagnosis, RetryStrategy, SmartRetry for intelligent retry with root-cause failure analysis
|
|
13
|
+
- **rhythm** -- HumanRhythm, weibull_delay for Weibull-distributed inter-request timing that mimics human browsing patterns
|
|
14
|
+
- **coherence** -- CoherentProfile, coherent_headers, check_coherence, country_to_locale/timezone/accept_language for ensuring header-locale-timezone consistency
|
|
15
|
+
- **cookie_intel** -- CookieClassification, CookieIntelligence for classifying cookies by vendor and purpose (tracking, auth, antibot)
|
|
16
|
+
- **cover_traffic** -- SubResource, CoverTrafficGenerator for generating plausible background requests (CSS, JS, images) alongside primary requests
|
|
17
|
+
- **immortal** -- ImmortalSession for sessions that survive proxy failures, identity rotation, and connection drops without losing state
|
|
18
|
+
- **mesh** -- SessionMesh, MeshState for coordinating multiple sessions with shared state and identity distribution
|
|
19
|
+
- **challenge_vm** -- ChallengeVM, ChallengeResult for executing JavaScript challenge scripts in an isolated VM environment
|
|
20
|
+
- **proto_chameleon** -- ProtocolChameleon, NegotiationResult for dynamic HTTP/1.1 vs HTTP/2 vs HTTP/3 negotiation matching target server preferences
|
|
21
|
+
- **self_audit** -- RequestAuditor, AuditReport, AuditFinding for runtime self-inspection of outgoing requests against known detection vectors
|
|
22
|
+
|
|
23
|
+
## [0.3.1] - 2026-08-10
|
|
24
|
+
|
|
25
|
+
### Fixed
|
|
26
|
+
|
|
27
|
+
- **proxy**: remove false-positive 403 ban detection (bare 403 is routine for APIs)
|
|
28
|
+
- **async_session**: remove semaphore from `each()._do` preventing deadlock
|
|
29
|
+
- **js_sandbox**: `return None` moved outside for-loop (was unreachable)
|
|
30
|
+
- **session_ext**: use `urlsplit().hostname` instead of `netloc.split(':')[0]` for credential URLs
|
|
31
|
+
- **native_backend**: add `time.monotonic()` for elapsed measurement
|
|
32
|
+
- **detection**: priority header check now accepts Firefox
|
|
33
|
+
- **device_db**: Android platform string corrected to 'Linux armv81'
|
|
34
|
+
- **clienthints**: thread-safe iteration with `list(policy.accepted)`
|
|
35
|
+
- **jsenv**: timing values chained sequentially
|
|
36
|
+
- **websocket**: CLOSE frame sent under lock
|
|
37
|
+
- **session_coord**: adaptive cutoff `now - max(1.0, 1/rps)` instead of hardcoded
|
|
38
|
+
- **middleware**: DedupeMiddleware wrapped in `threading.Lock()`
|
|
39
|
+
- **cdp_solver**: handle proxy URL without port
|
|
40
|
+
- **pow**: validate `nonce_size < 8`
|
|
41
|
+
- **collectors**: add KasadaCollector and ShapeCollector exports
|
|
42
|
+
|
|
3
43
|
## [0.3.0] - 2026-08-05
|
|
4
44
|
|
|
5
45
|
### Added — Killer Features
|
|
@@ -118,7 +118,23 @@ from .browser_pool import BrowserPool
|
|
|
118
118
|
from .record_replay import HttpRecorder, SessionReplayer, RecordedEntry, record_session
|
|
119
119
|
from .captcha_farm import CaptchaFarm, SolveResult
|
|
120
120
|
|
|
121
|
-
|
|
121
|
+
from .synth_fingerprint import SynthProfile, FingerprintSynthesizer
|
|
122
|
+
from .waf_recon import WafInfo, recon_sync, recon, WAF_SIGNATURES
|
|
123
|
+
from .poison_detect import PoisonScore, PoisonDetector, poison_check
|
|
124
|
+
from .resp_diff import DiffReport, diff_json, diff_html_structure, compare_bodies
|
|
125
|
+
from .predictive_rate import PredictiveRateLimiter
|
|
126
|
+
from .smart_retry import FailureDiagnosis, RetryStrategy, SmartRetry
|
|
127
|
+
from .rhythm import HumanRhythm, weibull_delay
|
|
128
|
+
from .coherence import CoherentProfile, coherent_headers, check_coherence, country_to_locale, country_to_timezone, country_to_accept_language
|
|
129
|
+
from .cookie_intel import CookieClassification, CookieIntelligence
|
|
130
|
+
from .cover_traffic import SubResource, CoverTrafficGenerator
|
|
131
|
+
from .immortal import ImmortalSession
|
|
132
|
+
from .mesh import SessionMesh, MeshState
|
|
133
|
+
from .challenge_vm import ChallengeVM, ChallengeResult
|
|
134
|
+
from .proto_chameleon import ProtocolChameleon, NegotiationResult
|
|
135
|
+
from .self_audit import RequestAuditor, AuditReport, AuditFinding
|
|
136
|
+
|
|
137
|
+
__version__ = "0.4.0"
|
|
122
138
|
|
|
123
139
|
__all__ = [
|
|
124
140
|
"Session",
|
|
@@ -390,6 +406,60 @@ __all__ = [
|
|
|
390
406
|
"HttpRecorder", "SessionReplayer", "RecordedEntry", "record_session",
|
|
391
407
|
# captcha farm
|
|
392
408
|
"CaptchaFarm", "SolveResult",
|
|
409
|
+
# synthetic fingerprint generation
|
|
410
|
+
"SynthProfile",
|
|
411
|
+
"FingerprintSynthesizer",
|
|
412
|
+
# WAF reconnaissance
|
|
413
|
+
"WafInfo",
|
|
414
|
+
"recon_sync",
|
|
415
|
+
"recon",
|
|
416
|
+
"WAF_SIGNATURES",
|
|
417
|
+
# cache/response poison detection
|
|
418
|
+
"PoisonScore",
|
|
419
|
+
"PoisonDetector",
|
|
420
|
+
"poison_check",
|
|
421
|
+
# response diffing
|
|
422
|
+
"DiffReport",
|
|
423
|
+
"diff_json",
|
|
424
|
+
"diff_html_structure",
|
|
425
|
+
"compare_bodies",
|
|
426
|
+
# predictive rate limiting
|
|
427
|
+
"PredictiveRateLimiter",
|
|
428
|
+
# smart retry with failure diagnosis
|
|
429
|
+
"FailureDiagnosis",
|
|
430
|
+
"RetryStrategy",
|
|
431
|
+
"SmartRetry",
|
|
432
|
+
# human-like request rhythm
|
|
433
|
+
"HumanRhythm",
|
|
434
|
+
"weibull_delay",
|
|
435
|
+
# header/locale coherence
|
|
436
|
+
"CoherentProfile",
|
|
437
|
+
"coherent_headers",
|
|
438
|
+
"check_coherence",
|
|
439
|
+
"country_to_locale",
|
|
440
|
+
"country_to_timezone",
|
|
441
|
+
"country_to_accept_language",
|
|
442
|
+
# cookie intelligence
|
|
443
|
+
"CookieClassification",
|
|
444
|
+
"CookieIntelligence",
|
|
445
|
+
# cover traffic generation
|
|
446
|
+
"SubResource",
|
|
447
|
+
"CoverTrafficGenerator",
|
|
448
|
+
# immortal session
|
|
449
|
+
"ImmortalSession",
|
|
450
|
+
# session mesh
|
|
451
|
+
"SessionMesh",
|
|
452
|
+
"MeshState",
|
|
453
|
+
# challenge VM
|
|
454
|
+
"ChallengeVM",
|
|
455
|
+
"ChallengeResult",
|
|
456
|
+
# protocol chameleon
|
|
457
|
+
"ProtocolChameleon",
|
|
458
|
+
"NegotiationResult",
|
|
459
|
+
# self-audit
|
|
460
|
+
"RequestAuditor",
|
|
461
|
+
"AuditReport",
|
|
462
|
+
"AuditFinding",
|
|
393
463
|
# module-level verbs
|
|
394
464
|
"request",
|
|
395
465
|
"get",
|
|
@@ -116,8 +116,7 @@ class AsyncSession:
|
|
|
116
116
|
) -> list[Any]:
|
|
117
117
|
"""Apply an async function to each item with concurrency limiting."""
|
|
118
118
|
async def _do(item: Any) -> Any:
|
|
119
|
-
|
|
120
|
-
return await fn(item, self)
|
|
119
|
+
return await fn(item, self)
|
|
121
120
|
return list(await asyncio.gather(*[_do(it) for it in items]))
|
|
122
121
|
|
|
123
122
|
# -- stats -------------------------------------------------------------
|
|
@@ -23,6 +23,7 @@ to see each hop.
|
|
|
23
23
|
|
|
24
24
|
from __future__ import annotations
|
|
25
25
|
|
|
26
|
+
import time
|
|
26
27
|
from typing import Any
|
|
27
28
|
from urllib.parse import urljoin, urlsplit
|
|
28
29
|
|
|
@@ -131,10 +132,12 @@ class NativeBackend:
|
|
|
131
132
|
url = f"{url}{sep}{urlencode(dict(req.params))}"
|
|
132
133
|
|
|
133
134
|
elapsed = 0.0
|
|
135
|
+
t0 = time.monotonic()
|
|
134
136
|
for hop in range(max_redirects + 1):
|
|
135
137
|
raw = self._engine.fetch_one(self._payload(
|
|
136
138
|
method, url, headers, body, req, verify, resolve
|
|
137
139
|
))
|
|
140
|
+
elapsed = (time.monotonic() - t0) * 1000
|
|
138
141
|
err = raw["error"]
|
|
139
142
|
if err:
|
|
140
143
|
raise _translate(err, method, url)
|
|
@@ -149,8 +152,6 @@ class NativeBackend:
|
|
|
149
152
|
method, body = "GET", None
|
|
150
153
|
headers.pop("content-type", None)
|
|
151
154
|
headers.pop("content-length", None)
|
|
152
|
-
# A redirect to another origin must not carry the first one's
|
|
153
|
-
# cookies or Host-specific headers.
|
|
154
155
|
continue
|
|
155
156
|
if req.allow_redirects and location and 300 <= status < 400:
|
|
156
157
|
raise TooManyRedirects(
|
|
@@ -290,7 +290,8 @@ class StealthBrowser:
|
|
|
290
290
|
args.append(f"--user-agent={ua}")
|
|
291
291
|
if self._proxy:
|
|
292
292
|
p = urlparse(self._proxy)
|
|
293
|
-
|
|
293
|
+
port_part = f":{p.port}" if p.port else ""
|
|
294
|
+
args.append(f"--proxy-server={p.scheme}://{p.hostname}{port_part}")
|
|
294
295
|
args.append("--host-resolver-rules=MAP * ~NOTFOUND , EXCLUDE 127.0.0.1")
|
|
295
296
|
if p.username:
|
|
296
297
|
self._proxy_creds = (p.username, p.password or "")
|
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import time
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Any
|
|
7
|
+
from urllib.parse import urlparse, urljoin, quote
|
|
8
|
+
|
|
9
|
+
try:
|
|
10
|
+
from .js_sandbox import JSSandbox, match_challenge, Deobfuscator
|
|
11
|
+
except ImportError:
|
|
12
|
+
JSSandbox = None # type: ignore[assignment,misc]
|
|
13
|
+
match_challenge = None # type: ignore[assignment]
|
|
14
|
+
Deobfuscator = None # type: ignore[assignment,misc]
|
|
15
|
+
|
|
16
|
+
_CF_PATTERNS = (
|
|
17
|
+
re.compile(r'cf-browser-verification', re.I),
|
|
18
|
+
re.compile(r'cf_chl_opt', re.I),
|
|
19
|
+
re.compile(r'/cdn-cgi/challenge-platform', re.I),
|
|
20
|
+
re.compile(r'jschl_vc', re.I),
|
|
21
|
+
re.compile(r'jschl_answer', re.I),
|
|
22
|
+
)
|
|
23
|
+
_AK_PATTERNS = (
|
|
24
|
+
re.compile(r'ak_bmsc', re.I),
|
|
25
|
+
re.compile(r'_abck', re.I),
|
|
26
|
+
re.compile(r'sensor_data', re.I),
|
|
27
|
+
re.compile(r'akamai[^a-z]', re.I),
|
|
28
|
+
)
|
|
29
|
+
_DD_PATTERNS = (
|
|
30
|
+
re.compile(r'datadome', re.I),
|
|
31
|
+
re.compile(r'dd\.js', re.I),
|
|
32
|
+
re.compile(r'geo\.captcha-delivery\.com', re.I),
|
|
33
|
+
)
|
|
34
|
+
_GENERIC_PATTERNS = (
|
|
35
|
+
re.compile(r'<meta[^>]+http-equiv\s*=\s*["\']?refresh["\']?[^>]+content\s*=\s*["\']?\s*\d+\s*;', re.I),
|
|
36
|
+
re.compile(r'id\s*=\s*["\']challenge["\']', re.I),
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
_SCRIPT_TAG = re.compile(r'<script[^>]*>([\s\S]*?)</script>', re.I)
|
|
40
|
+
_CF_JSCHL_MATH = re.compile(
|
|
41
|
+
r'setTimeout\s*\(\s*function\s*\(\)\s*\{([\s\S]{20,2000}?jschl_answer[\s\S]{0,500}?)\}', re.I)
|
|
42
|
+
_CF_JSCHL_VC = re.compile(r'name="jschl_vc"\s+value="([^"]+)"', re.I)
|
|
43
|
+
_CF_JSCHL_PASS = re.compile(r'name="pass"\s+value="([^"]+)"', re.I)
|
|
44
|
+
_CF_R_VALUE = re.compile(r'name="r"\s+value="([^"]*)"', re.I)
|
|
45
|
+
_CF_FORM_ACTION = re.compile(r'id="challenge-form"[^>]*action="([^"]+)"', re.I)
|
|
46
|
+
_MATH_EXPR = re.compile(r'^[\s\d+\-*/().%^&|~<>]+$')
|
|
47
|
+
_ASSIGNMENT_CHAIN = re.compile(r'([a-zA-Z_$][\w.$]*)\s*([+\-*/])?=\s*([^;]+);')
|
|
48
|
+
_CHAR_CODE_AT = re.compile(r'\.charCodeAt\s*\(\s*(\d+)\s*\)')
|
|
49
|
+
_TO_FIXED = re.compile(r'\.toFixed\s*\(\s*(\d+)\s*\)')
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class ChallengeResult:
|
|
54
|
+
__slots__ = ("success", "token", "challenge_type", "elapsed_ms", "error")
|
|
55
|
+
success: bool
|
|
56
|
+
token: str
|
|
57
|
+
challenge_type: str
|
|
58
|
+
elapsed_ms: float
|
|
59
|
+
error: str
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ChallengeVM:
|
|
63
|
+
__slots__ = ("_timeout", "_sandbox", "_deobfuscator")
|
|
64
|
+
|
|
65
|
+
def __init__(self, timeout: float = 10.0) -> None:
|
|
66
|
+
self._timeout = timeout
|
|
67
|
+
self._sandbox: Any = None
|
|
68
|
+
self._deobfuscator: Any = None
|
|
69
|
+
if JSSandbox is not None:
|
|
70
|
+
self._sandbox = JSSandbox(timeout_ms=int(timeout * 1000))
|
|
71
|
+
if Deobfuscator is not None:
|
|
72
|
+
self._deobfuscator = Deobfuscator()
|
|
73
|
+
|
|
74
|
+
def solve(self, html: str, url: str = "") -> ChallengeResult:
|
|
75
|
+
start = time.monotonic()
|
|
76
|
+
ctype = self.detect(html)
|
|
77
|
+
if ctype is None:
|
|
78
|
+
return ChallengeResult(
|
|
79
|
+
success=False, token="", challenge_type="none",
|
|
80
|
+
elapsed_ms=(time.monotonic() - start) * 1000,
|
|
81
|
+
error="no_challenge_detected",
|
|
82
|
+
)
|
|
83
|
+
solvers = {
|
|
84
|
+
"cloudflare": self._solve_cloudflare,
|
|
85
|
+
"akamai": self._solve_akamai,
|
|
86
|
+
"datadome": self._solve_datadome,
|
|
87
|
+
}
|
|
88
|
+
solver = solvers.get(ctype, self._solve_generic)
|
|
89
|
+
try:
|
|
90
|
+
result = solver(html, url)
|
|
91
|
+
except Exception as exc:
|
|
92
|
+
result = ChallengeResult(
|
|
93
|
+
success=False, token="", challenge_type=ctype,
|
|
94
|
+
elapsed_ms=0.0, error=str(exc),
|
|
95
|
+
)
|
|
96
|
+
result.elapsed_ms = (time.monotonic() - start) * 1000
|
|
97
|
+
return result
|
|
98
|
+
|
|
99
|
+
def detect(self, html: str) -> str | None:
|
|
100
|
+
for pat in _CF_PATTERNS:
|
|
101
|
+
if pat.search(html):
|
|
102
|
+
return "cloudflare"
|
|
103
|
+
for pat in _AK_PATTERNS:
|
|
104
|
+
if pat.search(html):
|
|
105
|
+
return "akamai"
|
|
106
|
+
for pat in _DD_PATTERNS:
|
|
107
|
+
if pat.search(html):
|
|
108
|
+
return "datadome"
|
|
109
|
+
if match_challenge is not None:
|
|
110
|
+
tmpl = match_challenge(html)
|
|
111
|
+
if tmpl is not None:
|
|
112
|
+
return tmpl.vendor
|
|
113
|
+
for pat in _GENERIC_PATTERNS:
|
|
114
|
+
if pat.search(html):
|
|
115
|
+
return "generic"
|
|
116
|
+
return None
|
|
117
|
+
|
|
118
|
+
def _solve_cloudflare(self, html: str, url: str) -> ChallengeResult:
|
|
119
|
+
vc_m = _CF_JSCHL_VC.search(html)
|
|
120
|
+
pass_m = _CF_JSCHL_PASS.search(html)
|
|
121
|
+
if not vc_m:
|
|
122
|
+
return ChallengeResult(
|
|
123
|
+
success=False, token="", challenge_type="cloudflare",
|
|
124
|
+
elapsed_ms=0.0, error="requires_cdp",
|
|
125
|
+
)
|
|
126
|
+
jschl_vc = vc_m.group(1)
|
|
127
|
+
jschl_pass = pass_m.group(1) if pass_m else ""
|
|
128
|
+
r_val = ""
|
|
129
|
+
r_m = _CF_R_VALUE.search(html)
|
|
130
|
+
if r_m:
|
|
131
|
+
r_val = r_m.group(1)
|
|
132
|
+
|
|
133
|
+
parsed = urlparse(url) if url else None
|
|
134
|
+
hostname = parsed.hostname or "" if parsed else ""
|
|
135
|
+
host_len = len(hostname)
|
|
136
|
+
|
|
137
|
+
answer = self._eval_cf_math(html, hostname)
|
|
138
|
+
if answer is None:
|
|
139
|
+
return ChallengeResult(
|
|
140
|
+
success=False, token="", challenge_type="cloudflare",
|
|
141
|
+
elapsed_ms=0.0, error="requires_cdp",
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
action_m = _CF_FORM_ACTION.search(html)
|
|
145
|
+
action = action_m.group(1) if action_m else "/cdn-cgi/l/chk_jschl"
|
|
146
|
+
base = f"{parsed.scheme}://{parsed.netloc}" if parsed else ""
|
|
147
|
+
submit_url = urljoin(base, action)
|
|
148
|
+
params = f"jschl_vc={quote(jschl_vc)}&pass={quote(jschl_pass)}&jschl_answer={answer}"
|
|
149
|
+
if r_val:
|
|
150
|
+
params += f"&r={quote(r_val)}"
|
|
151
|
+
token = f"{submit_url}?{params}"
|
|
152
|
+
|
|
153
|
+
return ChallengeResult(
|
|
154
|
+
success=True, token=token, challenge_type="cloudflare",
|
|
155
|
+
elapsed_ms=0.0, error="",
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
def _solve_akamai(self, html: str, url: str) -> ChallengeResult:
|
|
159
|
+
script = self._extract_challenge_script(html)
|
|
160
|
+
if not script:
|
|
161
|
+
return ChallengeResult(
|
|
162
|
+
success=False, token="", challenge_type="akamai",
|
|
163
|
+
elapsed_ms=0.0, error="requires_cdp",
|
|
164
|
+
)
|
|
165
|
+
if self._deobfuscator is not None:
|
|
166
|
+
script = self._deobfuscator.deobfuscate(script)
|
|
167
|
+
|
|
168
|
+
if self._sandbox is not None:
|
|
169
|
+
env = self._build_browser_env(url)
|
|
170
|
+
for key, val in env.items():
|
|
171
|
+
self._sandbox.set_global(key, val)
|
|
172
|
+
try:
|
|
173
|
+
result = self._sandbox.exec(script)
|
|
174
|
+
sensor = result.get("sensor_data") or result.get("bmak")
|
|
175
|
+
if sensor:
|
|
176
|
+
token = sensor if isinstance(sensor, str) else str(sensor)
|
|
177
|
+
return ChallengeResult(
|
|
178
|
+
success=True, token=token, challenge_type="akamai",
|
|
179
|
+
elapsed_ms=0.0, error="",
|
|
180
|
+
)
|
|
181
|
+
except (RuntimeError, TimeoutError, SyntaxError):
|
|
182
|
+
pass
|
|
183
|
+
|
|
184
|
+
return ChallengeResult(
|
|
185
|
+
success=False, token="", challenge_type="akamai",
|
|
186
|
+
elapsed_ms=0.0, error="requires_cdp",
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
def _solve_datadome(self, html: str, url: str) -> ChallengeResult:
|
|
190
|
+
cid_m = re.search(r'cid["\']?\s*[:=]\s*["\']([a-zA-Z0-9]+)', html)
|
|
191
|
+
dd_m = re.search(r'datadome["\']?\s*[:=]\s*["\']([^"\']+)', html)
|
|
192
|
+
if cid_m and "captcha-delivery" in html:
|
|
193
|
+
return ChallengeResult(
|
|
194
|
+
success=False, token="", challenge_type="datadome",
|
|
195
|
+
elapsed_ms=0.0, error="requires_captcha",
|
|
196
|
+
)
|
|
197
|
+
script = self._extract_challenge_script(html)
|
|
198
|
+
if not script:
|
|
199
|
+
return ChallengeResult(
|
|
200
|
+
success=False, token="", challenge_type="datadome",
|
|
201
|
+
elapsed_ms=0.0, error="requires_cdp",
|
|
202
|
+
)
|
|
203
|
+
if self._sandbox is not None:
|
|
204
|
+
env = self._build_browser_env(url)
|
|
205
|
+
for key, val in env.items():
|
|
206
|
+
self._sandbox.set_global(key, val)
|
|
207
|
+
try:
|
|
208
|
+
result = self._sandbox.exec(script)
|
|
209
|
+
dd_cookie = result.get("datadome") or result.get("dd")
|
|
210
|
+
if dd_cookie and isinstance(dd_cookie, str):
|
|
211
|
+
return ChallengeResult(
|
|
212
|
+
success=True, token=dd_cookie, challenge_type="datadome",
|
|
213
|
+
elapsed_ms=0.0, error="",
|
|
214
|
+
)
|
|
215
|
+
except (RuntimeError, TimeoutError, SyntaxError):
|
|
216
|
+
pass
|
|
217
|
+
|
|
218
|
+
return ChallengeResult(
|
|
219
|
+
success=False, token="", challenge_type="datadome",
|
|
220
|
+
elapsed_ms=0.0, error="requires_cdp",
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
def _solve_generic(self, html: str, url: str) -> ChallengeResult:
|
|
224
|
+
refresh_m = re.search(
|
|
225
|
+
r'<meta[^>]+http-equiv\s*=\s*["\']?refresh["\']?[^>]*'
|
|
226
|
+
r'content\s*=\s*["\']?\s*\d+\s*;\s*url\s*=\s*([^\s"\'>;]+)',
|
|
227
|
+
html, re.I,
|
|
228
|
+
)
|
|
229
|
+
if refresh_m:
|
|
230
|
+
target = refresh_m.group(1).strip().strip("'\"")
|
|
231
|
+
base = url or ""
|
|
232
|
+
resolved = urljoin(base, target) if base else target
|
|
233
|
+
return ChallengeResult(
|
|
234
|
+
success=True, token=resolved, challenge_type="generic",
|
|
235
|
+
elapsed_ms=0.0, error="",
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
script = self._extract_challenge_script(html)
|
|
239
|
+
if script and self._sandbox is not None:
|
|
240
|
+
env = self._build_browser_env(url)
|
|
241
|
+
for key, val in env.items():
|
|
242
|
+
self._sandbox.set_global(key, val)
|
|
243
|
+
try:
|
|
244
|
+
result = self._sandbox.exec(script)
|
|
245
|
+
for key in ("answer", "token", "result", "solution"):
|
|
246
|
+
if key in result and result[key] is not None:
|
|
247
|
+
return ChallengeResult(
|
|
248
|
+
success=True, token=str(result[key]),
|
|
249
|
+
challenge_type="generic",
|
|
250
|
+
elapsed_ms=0.0, error="",
|
|
251
|
+
)
|
|
252
|
+
except (RuntimeError, TimeoutError, SyntaxError):
|
|
253
|
+
pass
|
|
254
|
+
|
|
255
|
+
if script:
|
|
256
|
+
answer = self._eval_simple_math(script)
|
|
257
|
+
if answer is not None:
|
|
258
|
+
return ChallengeResult(
|
|
259
|
+
success=True, token=str(answer),
|
|
260
|
+
challenge_type="generic",
|
|
261
|
+
elapsed_ms=0.0, error="",
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
return ChallengeResult(
|
|
265
|
+
success=False, token="", challenge_type="generic",
|
|
266
|
+
elapsed_ms=0.0, error="requires_cdp",
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
def _extract_challenge_script(self, html: str) -> str | None:
|
|
270
|
+
scripts: list[str] = []
|
|
271
|
+
for m in _SCRIPT_TAG.finditer(html):
|
|
272
|
+
body = m.group(1).strip()
|
|
273
|
+
if not body:
|
|
274
|
+
continue
|
|
275
|
+
scripts.append(body)
|
|
276
|
+
if not scripts:
|
|
277
|
+
return None
|
|
278
|
+
challenge_keywords = (
|
|
279
|
+
"challenge", "jschl", "cf_chl", "sensor_data", "bmak",
|
|
280
|
+
"_abck", "ak_bmsc", "datadome", "dd.", "captcha",
|
|
281
|
+
"getElementById", "createElement", "location.href",
|
|
282
|
+
)
|
|
283
|
+
for script in scripts:
|
|
284
|
+
lower = script.lower()
|
|
285
|
+
if any(kw in lower for kw in challenge_keywords):
|
|
286
|
+
return script
|
|
287
|
+
for script in scripts:
|
|
288
|
+
if "=" in script and ("+" in script or "-" in script or "*" in script):
|
|
289
|
+
return script
|
|
290
|
+
return scripts[0] if len(scripts) == 1 else None
|
|
291
|
+
|
|
292
|
+
def _build_browser_env(self, url: str) -> dict[str, Any]:
|
|
293
|
+
p = urlparse(url) if url else None
|
|
294
|
+
_nop = lambda *a: None
|
|
295
|
+
host = p.hostname or "example.com" if p else "example.com"
|
|
296
|
+
location: dict[str, Any] = {
|
|
297
|
+
"href": url or "https://example.com/", "hostname": host,
|
|
298
|
+
"pathname": p.path or "/" if p else "/",
|
|
299
|
+
"protocol": (p.scheme + ":") if p else "https:",
|
|
300
|
+
"host": p.netloc or "example.com" if p else "example.com",
|
|
301
|
+
"origin": f"{p.scheme}://{p.netloc}" if p else "https://example.com",
|
|
302
|
+
"search": f"?{p.query}" if p and p.query else "", "hash": "",
|
|
303
|
+
"port": str(p.port) if p and p.port else "",
|
|
304
|
+
"replace": _nop, "assign": _nop, "reload": _nop,
|
|
305
|
+
}
|
|
306
|
+
_elems: dict[str, dict[str, Any]] = {}
|
|
307
|
+
|
|
308
|
+
def _create_el(tag: str) -> dict[str, Any]:
|
|
309
|
+
el: dict[str, Any] = {
|
|
310
|
+
"tagName": tag.upper(), "innerHTML": "", "innerText": "",
|
|
311
|
+
"textContent": "", "style": {}, "className": "", "id": "",
|
|
312
|
+
"setAttribute": lambda k, v: el.update({k: v}),
|
|
313
|
+
"getAttribute": lambda k: el.get(k),
|
|
314
|
+
"appendChild": _nop, "removeChild": _nop,
|
|
315
|
+
}
|
|
316
|
+
if tag.lower() == "a": el["href"] = ""
|
|
317
|
+
if tag.lower() == "div": el["offsetHeight"] = el["offsetWidth"] = 0
|
|
318
|
+
return el
|
|
319
|
+
|
|
320
|
+
_child_stub = {"appendChild": _nop, "removeChild": _nop}
|
|
321
|
+
document: dict[str, Any] = {
|
|
322
|
+
"createElement": _create_el, "getElementById": lambda eid: _elems.get(eid),
|
|
323
|
+
"getElementsByTagName": lambda t: [], "getElementsByClassName": lambda c: [],
|
|
324
|
+
"querySelector": lambda s: None, "querySelectorAll": lambda s: [],
|
|
325
|
+
"cookie": "", "location": location,
|
|
326
|
+
"body": dict(_child_stub), "head": {"appendChild": _nop},
|
|
327
|
+
"readyState": "complete", "title": "", "domain": host,
|
|
328
|
+
}
|
|
329
|
+
_ua = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
330
|
+
"(KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36")
|
|
331
|
+
navigator: dict[str, Any] = {
|
|
332
|
+
"userAgent": _ua, "language": "en-US", "languages": ["en-US", "en"],
|
|
333
|
+
"platform": "Win32", "vendor": "Google Inc.", "appName": "Netscape",
|
|
334
|
+
"appVersion": "5.0", "cookieEnabled": True, "onLine": True,
|
|
335
|
+
"hardwareConcurrency": 8, "maxTouchPoints": 0, "webdriver": False,
|
|
336
|
+
}
|
|
337
|
+
screen: dict[str, Any] = {
|
|
338
|
+
"width": 1920, "height": 1080, "availWidth": 1920,
|
|
339
|
+
"availHeight": 1040, "colorDepth": 24, "pixelDepth": 24,
|
|
340
|
+
}
|
|
341
|
+
window: dict[str, Any] = {
|
|
342
|
+
"location": location, "document": document, "navigator": navigator,
|
|
343
|
+
"screen": screen, "innerWidth": 1920, "innerHeight": 969,
|
|
344
|
+
"outerWidth": 1920, "outerHeight": 1080, "devicePixelRatio": 1,
|
|
345
|
+
"setTimeout": lambda fn, ms: 0, "setInterval": lambda fn, ms: 0,
|
|
346
|
+
"clearTimeout": _nop, "clearInterval": _nop,
|
|
347
|
+
"XMLHttpRequest": lambda: {"open": _nop, "send": _nop, "setRequestHeader": _nop},
|
|
348
|
+
}
|
|
349
|
+
return {"window": window, "document": document, "navigator": navigator,
|
|
350
|
+
"screen": screen, "location": location}
|
|
351
|
+
|
|
352
|
+
def _eval_cf_math(self, html: str, hostname: str) -> str | None:
|
|
353
|
+
math_m = _CF_JSCHL_MATH.search(html)
|
|
354
|
+
if not math_m:
|
|
355
|
+
return self._eval_cf_via_sandbox(html, hostname)
|
|
356
|
+
|
|
357
|
+
block = math_m.group(1)
|
|
358
|
+
host_len = len(hostname)
|
|
359
|
+
|
|
360
|
+
if self._sandbox is not None:
|
|
361
|
+
env = self._build_browser_env(f"https://{hostname}/")
|
|
362
|
+
for key, val in env.items():
|
|
363
|
+
self._sandbox.set_global(key, val)
|
|
364
|
+
try:
|
|
365
|
+
cleaned = self._prepare_cf_script(block, hostname)
|
|
366
|
+
result = self._sandbox.exec(cleaned)
|
|
367
|
+
for key in ("jschl_answer", "answer", "a"):
|
|
368
|
+
if key in result and result[key] is not None:
|
|
369
|
+
val = result[key]
|
|
370
|
+
if isinstance(val, float):
|
|
371
|
+
return f"{val:.10f}"
|
|
372
|
+
return str(val)
|
|
373
|
+
except (RuntimeError, TimeoutError, SyntaxError):
|
|
374
|
+
pass
|
|
375
|
+
|
|
376
|
+
return self._eval_cf_math_python(block, hostname)
|
|
377
|
+
|
|
378
|
+
def _eval_cf_via_sandbox(self, html: str, hostname: str) -> str | None:
|
|
379
|
+
if self._sandbox is None:
|
|
380
|
+
return None
|
|
381
|
+
script = self._extract_challenge_script(html)
|
|
382
|
+
if not script:
|
|
383
|
+
return None
|
|
384
|
+
env = self._build_browser_env(f"https://{hostname}/")
|
|
385
|
+
for key, val in env.items():
|
|
386
|
+
self._sandbox.set_global(key, val)
|
|
387
|
+
try:
|
|
388
|
+
result = self._sandbox.exec(script)
|
|
389
|
+
for key in ("jschl_answer", "answer"):
|
|
390
|
+
if key in result and result[key] is not None:
|
|
391
|
+
val = result[key]
|
|
392
|
+
if isinstance(val, float):
|
|
393
|
+
return f"{val:.10f}"
|
|
394
|
+
return str(val)
|
|
395
|
+
except (RuntimeError, TimeoutError, SyntaxError):
|
|
396
|
+
pass
|
|
397
|
+
return None
|
|
398
|
+
|
|
399
|
+
def _prepare_cf_script(self, block: str, hostname: str) -> str:
|
|
400
|
+
block = block.replace("document.getElementById('challenge-form')", "{}")
|
|
401
|
+
block = re.sub(r'(\w+)\.action\b', '""', block)
|
|
402
|
+
block = re.sub(r'\.submit\s*\(\s*\)', '', block)
|
|
403
|
+
href_obj = f'{{href: "https://{hostname}/"}}'
|
|
404
|
+
block = re.sub(r't\s*=\s*document\.createElement\([^\)]+\)', f't = {href_obj}', block)
|
|
405
|
+
block = re.sub(r't\.href\s*=\s*["\']([^"\']+)["\']', f't.href = "https://{hostname}/"', block)
|
|
406
|
+
return block
|
|
407
|
+
|
|
408
|
+
def _eval_cf_math_python(self, block: str, hostname: str) -> str | None:
|
|
409
|
+
host_len = len(hostname)
|
|
410
|
+
accum = 0.0
|
|
411
|
+
for m in _ASSIGNMENT_CHAIN.finditer(block):
|
|
412
|
+
op = m.group(2) or ""
|
|
413
|
+
expr_raw = m.group(3).strip()
|
|
414
|
+
|
|
415
|
+
expr_raw = _CHAR_CODE_AT.sub(
|
|
416
|
+
lambda cm: str(ord(hostname[int(cm.group(1))]) if int(cm.group(1)) < host_len else 0),
|
|
417
|
+
expr_raw,
|
|
418
|
+
)
|
|
419
|
+
val = self._eval_simple_math(expr_raw)
|
|
420
|
+
if val is None:
|
|
421
|
+
continue
|
|
422
|
+
fval = float(val)
|
|
423
|
+
if op == "+":
|
|
424
|
+
accum += fval
|
|
425
|
+
elif op == "-":
|
|
426
|
+
accum -= fval
|
|
427
|
+
elif op == "*":
|
|
428
|
+
accum *= fval
|
|
429
|
+
elif op == "/":
|
|
430
|
+
if fval != 0:
|
|
431
|
+
accum /= fval
|
|
432
|
+
else:
|
|
433
|
+
accum = fval
|
|
434
|
+
|
|
435
|
+
to_fixed_m = _TO_FIXED.search(block)
|
|
436
|
+
precision = int(to_fixed_m.group(1)) if to_fixed_m else 10
|
|
437
|
+
|
|
438
|
+
accum += host_len
|
|
439
|
+
return f"{accum:.{precision}f}"
|
|
440
|
+
|
|
441
|
+
def _eval_simple_math(self, expr: str) -> int | float | None:
|
|
442
|
+
expr = expr.strip()
|
|
443
|
+
if not expr:
|
|
444
|
+
return None
|
|
445
|
+
cleaned = re.sub(r'\s+', '', expr)
|
|
446
|
+
cleaned = re.sub(r'\+\+', '+', cleaned)
|
|
447
|
+
cleaned = cleaned.replace("!+[]", "1").replace("!![]", "1")
|
|
448
|
+
cleaned = cleaned.replace("+!![]", "+1").replace("+!+[]", "+1")
|
|
449
|
+
cleaned = cleaned.replace("![]", "0").replace("+[]", "+0")
|
|
450
|
+
cleaned = cleaned.replace("[]", "0")
|
|
451
|
+
|
|
452
|
+
if not _MATH_EXPR.match(cleaned):
|
|
453
|
+
return None
|
|
454
|
+
try:
|
|
455
|
+
val = eval(cleaned, {"__builtins__": {}}, {}) # noqa: S307
|
|
456
|
+
if isinstance(val, (int, float)):
|
|
457
|
+
return val
|
|
458
|
+
except Exception:
|
|
459
|
+
pass
|
|
460
|
+
return None
|