dataapp_crawler 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataapp_crawler/__init__.py +87 -0
- dataapp_crawler/__main__.py +35 -0
- dataapp_crawler/_logging.py +148 -0
- dataapp_crawler/_network_diagnostics.py +193 -0
- dataapp_crawler/browser_tools/__init__.py +23 -0
- dataapp_crawler/browser_tools/_common.py +66 -0
- dataapp_crawler/browser_tools/dom.py +49 -0
- dataapp_crawler/browser_tools/lists.py +125 -0
- dataapp_crawler/browser_tools/navigation.py +243 -0
- dataapp_crawler/browser_tools/readiness.py +42 -0
- dataapp_crawler/captcha.py +912 -0
- dataapp_crawler/checkpoint.py +487 -0
- dataapp_crawler/cloudflare.py +523 -0
- dataapp_crawler/context.py +231 -0
- dataapp_crawler/cookies.py +207 -0
- dataapp_crawler/crawler.py +1392 -0
- dataapp_crawler/errors.py +184 -0
- dataapp_crawler/http.py +223 -0
- dataapp_crawler/images.py +267 -0
- dataapp_crawler/local_auth.py +77 -0
- dataapp_crawler/models.py +285 -0
- dataapp_crawler/pagination.py +172 -0
- dataapp_crawler/parsing.py +13 -0
- dataapp_crawler/providers/__init__.py +20 -0
- dataapp_crawler/providers/browser.py +292 -0
- dataapp_crawler/providers/camoufox.py +184 -0
- dataapp_crawler/providers/dataos.py +174 -0
- dataapp_crawler/providers/pool.py +451 -0
- dataapp_crawler/providers/proxy.py +209 -0
- dataapp_crawler/py.typed +0 -0
- dataapp_crawler/routing.py +67 -0
- dataapp_crawler/scheduling.py +425 -0
- dataapp_crawler/sessions.py +735 -0
- dataapp_crawler-0.1.0.dist-info/METADATA +211 -0
- dataapp_crawler-0.1.0.dist-info/RECORD +36 -0
- dataapp_crawler-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Small error vocabulary; messages never include upstream exception text or credentials."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from typing import Any, TypedDict
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
|
|
8
|
+
_BROWSER_ERROR_CODES = frozenset(
|
|
9
|
+
{
|
|
10
|
+
"ERR_PROXY_CONNECTION_FAILED", "ERR_TUNNEL_CONNECTION_FAILED",
|
|
11
|
+
"ERR_INVALID_AUTH_CREDENTIALS", "ERR_PROXY_AUTH_UNSUPPORTED", "ERR_NO_SUPPORTED_PROXIES",
|
|
12
|
+
"ERR_CONNECTION_RESET", "ERR_CONNECTION_CLOSED", "ERR_CONNECTION_REFUSED",
|
|
13
|
+
"ERR_NAME_NOT_RESOLVED", "ERR_CERT_AUTHORITY_INVALID", "ERR_PROXY_CERTIFICATE_INVALID",
|
|
14
|
+
"ERR_TIMED_OUT", "ERR_ABORTED",
|
|
15
|
+
"ERR_HTTP_RESPONSE_CODE_FAILURE",
|
|
16
|
+
}
|
|
17
|
+
)
|
|
18
|
+
_BROWSER_ERROR = re.compile(
|
|
19
|
+
r"net::(" + "|".join(sorted(_BROWSER_ERROR_CODES)) + r")(?![A-Za-z0-9_])"
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
_HTTP_ERROR_TYPES: tuple[tuple[type[httpx.HTTPError], str], ...] = (
|
|
23
|
+
(httpx.ConnectTimeout, "HTTPX_CONNECT_TIMEOUT"),
|
|
24
|
+
(httpx.ReadTimeout, "HTTPX_READ_TIMEOUT"),
|
|
25
|
+
(httpx.WriteTimeout, "HTTPX_WRITE_TIMEOUT"),
|
|
26
|
+
(httpx.PoolTimeout, "HTTPX_POOL_TIMEOUT"),
|
|
27
|
+
(httpx.ConnectError, "HTTPX_CONNECT_ERROR"),
|
|
28
|
+
(httpx.ReadError, "HTTPX_READ_ERROR"),
|
|
29
|
+
(httpx.WriteError, "HTTPX_WRITE_ERROR"),
|
|
30
|
+
(httpx.CloseError, "HTTPX_CLOSE_ERROR"),
|
|
31
|
+
(httpx.ProxyError, "HTTPX_PROXY_ERROR"),
|
|
32
|
+
(httpx.LocalProtocolError, "HTTPX_LOCAL_PROTOCOL_ERROR"),
|
|
33
|
+
(httpx.RemoteProtocolError, "HTTPX_REMOTE_PROTOCOL_ERROR"),
|
|
34
|
+
(httpx.DecodingError, "HTTPX_DECODING_ERROR"),
|
|
35
|
+
(httpx.TooManyRedirects, "HTTPX_TOO_MANY_REDIRECTS"),
|
|
36
|
+
(httpx.UnsupportedProtocol, "HTTPX_UNSUPPORTED_PROTOCOL"),
|
|
37
|
+
(httpx.HTTPStatusError, "HTTPX_HTTP_STATUS_ERROR"),
|
|
38
|
+
(httpx.TimeoutException, "HTTPX_TIMEOUT"),
|
|
39
|
+
(httpx.NetworkError, "HTTPX_NETWORK_ERROR"),
|
|
40
|
+
(httpx.ProtocolError, "HTTPX_PROTOCOL_ERROR"),
|
|
41
|
+
(httpx.TransportError, "HTTPX_TRANSPORT_ERROR"),
|
|
42
|
+
(httpx.RequestError, "HTTPX_REQUEST_ERROR"),
|
|
43
|
+
(httpx.HTTPError, "HTTPX_HTTP_ERROR"),
|
|
44
|
+
)
|
|
45
|
+
_HTTP_ERROR_CODES = frozenset(code for _, code in _HTTP_ERROR_TYPES) | {
|
|
46
|
+
"HTTP_REDIRECT_LIMIT_REACHED"
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class NetworkDiagnosticsSummary(TypedDict):
|
|
51
|
+
events: list[dict[str, Any]]
|
|
52
|
+
events_dropped: int
|
|
53
|
+
main_document_status: int | None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def browser_error_code(error: Exception | str) -> str | None:
|
|
57
|
+
"""Keep only fixed Chromium codes; never retain arbitrary upstream error text."""
|
|
58
|
+
code = getattr(error, "browser_error_code", None)
|
|
59
|
+
if isinstance(code, str) and code in _BROWSER_ERROR_CODES:
|
|
60
|
+
return code
|
|
61
|
+
match = _BROWSER_ERROR.search(str(error))
|
|
62
|
+
return match.group(1) if match else None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def http_error_code(error: Exception) -> str | None:
|
|
66
|
+
"""Map HTTPX exception types to fixed codes without inspecting exception text."""
|
|
67
|
+
code = getattr(error, "http_error_code", None)
|
|
68
|
+
if isinstance(code, str) and code in _HTTP_ERROR_CODES:
|
|
69
|
+
return code
|
|
70
|
+
return next(
|
|
71
|
+
(code for error_type, code in _HTTP_ERROR_TYPES if isinstance(error, error_type)),
|
|
72
|
+
None,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class CrawlerError(Exception):
|
|
77
|
+
kind = "crawler"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class FetchError(CrawlerError):
|
|
81
|
+
kind = "network"
|
|
82
|
+
|
|
83
|
+
def __init__(
|
|
84
|
+
self,
|
|
85
|
+
message: str,
|
|
86
|
+
*,
|
|
87
|
+
retryable: bool = False,
|
|
88
|
+
phase: str | None = None,
|
|
89
|
+
browser_error_code: str | None = None,
|
|
90
|
+
http_error_code: str | None = None,
|
|
91
|
+
network_diagnostics: NetworkDiagnosticsSummary | None = None,
|
|
92
|
+
main_document_status: int | None = None,
|
|
93
|
+
) -> None:
|
|
94
|
+
super().__init__(message)
|
|
95
|
+
self.retryable = retryable
|
|
96
|
+
self.phase = phase
|
|
97
|
+
self.retry_events: list[dict[str, Any]] = []
|
|
98
|
+
self.browser_diagnostics: dict[str, Any] = {}
|
|
99
|
+
self.network_diagnostics = network_diagnostics
|
|
100
|
+
self.main_document_status = (
|
|
101
|
+
main_document_status
|
|
102
|
+
if isinstance(main_document_status, int)
|
|
103
|
+
and not isinstance(main_document_status, bool)
|
|
104
|
+
and 100 <= main_document_status <= 599
|
|
105
|
+
else None
|
|
106
|
+
)
|
|
107
|
+
self.browser_error_code = (
|
|
108
|
+
browser_error_code
|
|
109
|
+
if isinstance(browser_error_code, str) and browser_error_code in _BROWSER_ERROR_CODES
|
|
110
|
+
else None
|
|
111
|
+
)
|
|
112
|
+
self.http_error_code = (
|
|
113
|
+
http_error_code
|
|
114
|
+
if isinstance(http_error_code, str) and http_error_code in _HTTP_ERROR_CODES
|
|
115
|
+
else None
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class ResponseTooLarge(FetchError):
|
|
120
|
+
kind = "response_too_large"
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class ConfigurationError(CrawlerError):
|
|
124
|
+
kind = "configuration"
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class CrawlCancelled(CrawlerError):
|
|
128
|
+
kind = "cancelled"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class LimitReached(CrawlerError):
|
|
132
|
+
kind = "limit"
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class QueueFull(CrawlerError):
|
|
136
|
+
kind = "queue_full"
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class ProcessingError(CrawlerError):
|
|
140
|
+
kind = "process"
|
|
141
|
+
phase = "process"
|
|
142
|
+
|
|
143
|
+
def __init__(
|
|
144
|
+
self, message: str, *, capture: Any = None, process_error_code: str | None = None,
|
|
145
|
+
) -> None:
|
|
146
|
+
super().__init__(message)
|
|
147
|
+
self.capture = capture
|
|
148
|
+
self.process_error_code = (
|
|
149
|
+
process_error_code
|
|
150
|
+
if isinstance(process_error_code, str) and process_error_code in _PROCESS_ERROR_CODES
|
|
151
|
+
else None
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
_PROCESS_ERROR_TYPES: tuple[tuple[type[Exception], str], ...] = (
|
|
156
|
+
(ResponseTooLarge, "PROCESS_RESPONSE_TOO_LARGE"),
|
|
157
|
+
(FetchError, "PROCESS_FETCH_ERROR"),
|
|
158
|
+
(TimeoutError, "PROCESS_TIMEOUT"),
|
|
159
|
+
(KeyError, "PROCESS_KEY_ERROR"),
|
|
160
|
+
(IndexError, "PROCESS_INDEX_ERROR"),
|
|
161
|
+
(AttributeError, "PROCESS_ATTRIBUTE_ERROR"),
|
|
162
|
+
(TypeError, "PROCESS_TYPE_ERROR"),
|
|
163
|
+
(ValueError, "PROCESS_VALUE_ERROR"),
|
|
164
|
+
(AssertionError, "PROCESS_ASSERTION_ERROR"),
|
|
165
|
+
)
|
|
166
|
+
_PROCESS_ERROR_CODES = frozenset(code for _, code in _PROCESS_ERROR_TYPES) | {"PROCESS_ERROR"}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def process_error_code(error: Exception) -> str:
|
|
170
|
+
"""Describe a callback failure using fixed codes, without its message or traceback."""
|
|
171
|
+
if (
|
|
172
|
+
isinstance(error, ProcessingError)
|
|
173
|
+
and isinstance(error.process_error_code, str)
|
|
174
|
+
and error.process_error_code in _PROCESS_ERROR_CODES
|
|
175
|
+
):
|
|
176
|
+
return error.process_error_code
|
|
177
|
+
# Optional browser libraries define their own TimeoutError. No provider import
|
|
178
|
+
# or arbitrary exception class name is needed in the public diagnostic.
|
|
179
|
+
if type(error).__name__ == "TimeoutError":
|
|
180
|
+
return "PROCESS_TIMEOUT"
|
|
181
|
+
return next(
|
|
182
|
+
(code for error_type, code in _PROCESS_ERROR_TYPES if isinstance(error, error_type)),
|
|
183
|
+
"PROCESS_ERROR",
|
|
184
|
+
)
|
dataapp_crawler/http.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""HTTPX session with bounded reads and explicit per-origin redirect handling."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import http.cookiejar
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import Callable
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
|
|
11
|
+
from .context import CrawlContext, SiteGate
|
|
12
|
+
from .cookies import (
|
|
13
|
+
cookies_from_jar,
|
|
14
|
+
is_partitioned_jar_cookie,
|
|
15
|
+
replace_jar,
|
|
16
|
+
validate_shareable_cookies,
|
|
17
|
+
)
|
|
18
|
+
from .errors import ConfigurationError, FetchError, ResponseTooLarge, http_error_code
|
|
19
|
+
from .models import CaptureResult, Identity, Limits, Request, origin, safe_url
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class _BorrowedAsyncTransport(httpx.AsyncBaseTransport):
|
|
23
|
+
"""Let multiple short-lived clients share one caller-owned transport."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, transport: httpx.AsyncBaseTransport) -> None:
|
|
26
|
+
self._transport = transport
|
|
27
|
+
|
|
28
|
+
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
29
|
+
return await self._transport.handle_async_request(request)
|
|
30
|
+
|
|
31
|
+
async def aclose(self) -> None:
|
|
32
|
+
# The Crawler closes the injected transport once after all generations retire.
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class HttpSession:
|
|
37
|
+
def __init__(
|
|
38
|
+
self,
|
|
39
|
+
limits: Limits,
|
|
40
|
+
identity: Identity,
|
|
41
|
+
*,
|
|
42
|
+
proxy_url: str | None = None,
|
|
43
|
+
transport: httpx.AsyncBaseTransport | None = None,
|
|
44
|
+
close_transport: bool = True,
|
|
45
|
+
reject_partitioned: bool = False,
|
|
46
|
+
) -> None:
|
|
47
|
+
self.limits = limits
|
|
48
|
+
self.identity = identity
|
|
49
|
+
self.reject_partitioned = reject_partitioned
|
|
50
|
+
cookies = [*(identity.storage_state or {}).get("cookies", []), *identity.cookies]
|
|
51
|
+
if reject_partitioned:
|
|
52
|
+
validate_shareable_cookies(cookies)
|
|
53
|
+
client_transport = (
|
|
54
|
+
_BorrowedAsyncTransport(transport)
|
|
55
|
+
if transport is not None and not close_transport
|
|
56
|
+
else transport
|
|
57
|
+
)
|
|
58
|
+
self.client = httpx.AsyncClient(
|
|
59
|
+
timeout=httpx.Timeout(limits.request_timeout),
|
|
60
|
+
follow_redirects=False,
|
|
61
|
+
trust_env=False,
|
|
62
|
+
proxy=proxy_url,
|
|
63
|
+
transport=client_transport,
|
|
64
|
+
)
|
|
65
|
+
self.client.cookies.jar.set_policy(
|
|
66
|
+
http.cookiejar.DefaultCookiePolicy(
|
|
67
|
+
strict_ns_domain=http.cookiejar.DefaultCookiePolicy.DomainStrictNonDomain
|
|
68
|
+
)
|
|
69
|
+
)
|
|
70
|
+
# CookieJar preserves Secure, path, domain, expiry and supported metadata.
|
|
71
|
+
replace_jar(
|
|
72
|
+
self.client.cookies.jar,
|
|
73
|
+
cookies,
|
|
74
|
+
ignore_partitioned=not reject_partitioned,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
async def close(self) -> None:
|
|
78
|
+
await self.client.aclose()
|
|
79
|
+
|
|
80
|
+
def export_cookies(self) -> list[dict[str, object]]:
|
|
81
|
+
"""Return the current jar as browser-compatible, non-header cookie records."""
|
|
82
|
+
return cookies_from_jar(self.client.cookies.jar)
|
|
83
|
+
|
|
84
|
+
def replace_cookies(self, cookies: list[dict[str, object]]) -> None:
|
|
85
|
+
"""Replace this session's jar from a canonical session snapshot."""
|
|
86
|
+
replace_jar(self.client.cookies.jar, cookies)
|
|
87
|
+
|
|
88
|
+
async def fetch(
|
|
89
|
+
self, request: Request, context: CrawlContext, gate: Callable[[str], SiteGate]
|
|
90
|
+
) -> CaptureResult:
|
|
91
|
+
initial_origin = origin(request.url)
|
|
92
|
+
headers = {**request.headers, **self.identity.headers_for(request.url)}
|
|
93
|
+
current = self.client.build_request(
|
|
94
|
+
request.method,
|
|
95
|
+
request.url,
|
|
96
|
+
params=request.params,
|
|
97
|
+
content=request.content,
|
|
98
|
+
json=request.json,
|
|
99
|
+
headers=headers,
|
|
100
|
+
)
|
|
101
|
+
# HTTPX build_request merges cookies through a fresh CookieJar. Reapply the
|
|
102
|
+
# session jar's host-only policy instead of inheriting that copy's policy.
|
|
103
|
+
current.headers.pop("cookie", None)
|
|
104
|
+
self.client.cookies.set_cookie_header(current)
|
|
105
|
+
for name, value in request.headers.items():
|
|
106
|
+
if name.lower() == "cookie":
|
|
107
|
+
current.headers[name] = value
|
|
108
|
+
started = time.monotonic()
|
|
109
|
+
for redirects in range(self.limits.max_redirects + 1):
|
|
110
|
+
context.reserve_request()
|
|
111
|
+
response: httpx.Response | None = None
|
|
112
|
+
failure: FetchError | None = None
|
|
113
|
+
try:
|
|
114
|
+
async with gate(request.site_key or origin(str(current.url))):
|
|
115
|
+
response = await self.client.send(current, stream=True, follow_redirects=False)
|
|
116
|
+
self._narrow_response_cookies(response)
|
|
117
|
+
body = bytearray()
|
|
118
|
+
async for chunk in response.aiter_bytes():
|
|
119
|
+
if len(body) + len(chunk) > self.limits.max_response_bytes:
|
|
120
|
+
raise ResponseTooLarge("Response exceeds configured byte limit")
|
|
121
|
+
body.extend(chunk)
|
|
122
|
+
next_request = response.next_request
|
|
123
|
+
if next_request is None:
|
|
124
|
+
return CaptureResult(
|
|
125
|
+
request,
|
|
126
|
+
str(response.url),
|
|
127
|
+
response.status_code,
|
|
128
|
+
dict(response.headers),
|
|
129
|
+
bytes(body),
|
|
130
|
+
elapsed=time.monotonic() - started,
|
|
131
|
+
diagnostics={"url": safe_url(str(response.url)), "mode": "http"},
|
|
132
|
+
)
|
|
133
|
+
if redirects == self.limits.max_redirects:
|
|
134
|
+
raise FetchError(
|
|
135
|
+
"Redirect limit reached",
|
|
136
|
+
phase="http_redirect",
|
|
137
|
+
http_error_code="HTTP_REDIRECT_LIMIT_REACHED",
|
|
138
|
+
main_document_status=response.status_code,
|
|
139
|
+
)
|
|
140
|
+
# Remove all caller custom headers on an origin change. Rebuild Cookie from
|
|
141
|
+
# the jar; add website credentials only for the next URL's explicit origin.
|
|
142
|
+
next_origin = origin(str(next_request.url))
|
|
143
|
+
target_host = httpx.Request("GET", next_request.url).headers["host"]
|
|
144
|
+
for name in set(request.headers) | {
|
|
145
|
+
k for values in self.identity.headers.values() for k in values
|
|
146
|
+
}:
|
|
147
|
+
next_request.headers.pop(name, None)
|
|
148
|
+
for name in ("authorization", "cookie", "proxy-authorization", "host"):
|
|
149
|
+
next_request.headers.pop(name, None)
|
|
150
|
+
self.client.cookies.set_cookie_header(next_request)
|
|
151
|
+
if next_origin == initial_origin:
|
|
152
|
+
next_request.headers.update(request.headers)
|
|
153
|
+
next_request.headers.update(self.identity.headers_for(str(next_request.url)))
|
|
154
|
+
if next_origin != initial_origin or "host" not in next_request.headers:
|
|
155
|
+
next_request.headers["host"] = target_host
|
|
156
|
+
current = next_request
|
|
157
|
+
except (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError) as exc:
|
|
158
|
+
failure = FetchError(
|
|
159
|
+
"HTTP transport failed",
|
|
160
|
+
retryable=True,
|
|
161
|
+
phase="http_transport",
|
|
162
|
+
http_error_code=http_error_code(exc),
|
|
163
|
+
)
|
|
164
|
+
except httpx.HTTPError as exc:
|
|
165
|
+
failure = FetchError(
|
|
166
|
+
"HTTP request failed",
|
|
167
|
+
phase="http_request",
|
|
168
|
+
http_error_code=http_error_code(exc),
|
|
169
|
+
)
|
|
170
|
+
finally:
|
|
171
|
+
if response is not None:
|
|
172
|
+
await response.aclose()
|
|
173
|
+
if failure is not None:
|
|
174
|
+
# Raise after leaving the HTTPX handler so the raw exception (which can
|
|
175
|
+
# contain URLs, headers, or proxy credentials) is not retained as context.
|
|
176
|
+
raise failure
|
|
177
|
+
raise FetchError(
|
|
178
|
+
"Redirect limit reached",
|
|
179
|
+
phase="http_redirect",
|
|
180
|
+
http_error_code="HTTP_REDIRECT_LIMIT_REACHED",
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
def _narrow_response_cookies(self, response: httpx.Response) -> None:
|
|
184
|
+
"""Keep accepted response cookies, but scope them to the setting host."""
|
|
185
|
+
host = response.url.host
|
|
186
|
+
for cookie in response.cookies.jar:
|
|
187
|
+
try:
|
|
188
|
+
self.client.cookies.jar.clear(cookie.domain, cookie.path, cookie.name)
|
|
189
|
+
except KeyError:
|
|
190
|
+
pass
|
|
191
|
+
try:
|
|
192
|
+
self.client.cookies.jar.clear(host, cookie.path, cookie.name)
|
|
193
|
+
except KeyError:
|
|
194
|
+
pass
|
|
195
|
+
if is_partitioned_jar_cookie(cookie):
|
|
196
|
+
if self.reject_partitioned:
|
|
197
|
+
raise ConfigurationError(
|
|
198
|
+
"Partitioned cookies cannot be shared across engines"
|
|
199
|
+
)
|
|
200
|
+
continue
|
|
201
|
+
if cookie.expires is not None and cookie.expires <= time.time():
|
|
202
|
+
continue
|
|
203
|
+
self.client.cookies.jar.set_cookie(
|
|
204
|
+
http.cookiejar.Cookie(
|
|
205
|
+
version=cookie.version,
|
|
206
|
+
name=cookie.name,
|
|
207
|
+
value=cookie.value,
|
|
208
|
+
port=cookie.port,
|
|
209
|
+
port_specified=cookie.port_specified,
|
|
210
|
+
domain=host,
|
|
211
|
+
domain_specified=False,
|
|
212
|
+
domain_initial_dot=False,
|
|
213
|
+
path=cookie.path,
|
|
214
|
+
path_specified=cookie.path_specified,
|
|
215
|
+
secure=cookie.secure,
|
|
216
|
+
expires=cookie.expires,
|
|
217
|
+
discard=cookie.discard,
|
|
218
|
+
comment=cookie.comment,
|
|
219
|
+
comment_url=cookie.comment_url,
|
|
220
|
+
rest=dict(getattr(cookie, "_rest", {})),
|
|
221
|
+
rfc2109=cookie.rfc2109,
|
|
222
|
+
)
|
|
223
|
+
)
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""Capture the currently rendered image without fetching its source URL again."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import base64
|
|
7
|
+
import binascii
|
|
8
|
+
import hashlib
|
|
9
|
+
import math
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from typing import Any, Literal
|
|
12
|
+
|
|
13
|
+
from .errors import ConfigurationError, FetchError, ResponseTooLarge
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True, slots=True)
|
|
17
|
+
class CapturedImage:
|
|
18
|
+
"""Private PNG bytes with encodings accepted by solver and vision resources.
|
|
19
|
+
|
|
20
|
+
Treat ``image_body`` and ``data_url`` as sensitive page content. They are not
|
|
21
|
+
included in the representation. A matching hash identifies captured bytes,
|
|
22
|
+
not a guarantee that the website still accepts the same challenge.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
_png: bytes = field(repr=False)
|
|
26
|
+
|
|
27
|
+
def __post_init__(self) -> None:
|
|
28
|
+
if not isinstance(self._png, bytes) or not self._png.startswith(b"\x89PNG\r\n\x1a\n"):
|
|
29
|
+
raise ConfigurationError("Captured image must contain PNG bytes")
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def image_body(self) -> str:
|
|
33
|
+
"""Return base64 PNG for ``solve_captcha(image_body=...)``."""
|
|
34
|
+
return base64.b64encode(self._png).decode("ascii")
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def data_url(self) -> str:
|
|
38
|
+
"""Return an inline PNG URL for a model's ``image_url`` content part."""
|
|
39
|
+
return "data:image/png;base64," + self.image_body
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def sha256(self) -> str:
|
|
43
|
+
"""Return a content fingerprint without exposing the image."""
|
|
44
|
+
return hashlib.sha256(self._png).hexdigest()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def capture_image(
|
|
48
|
+
locator: Any,
|
|
49
|
+
*,
|
|
50
|
+
timeout: float = 10_000,
|
|
51
|
+
max_bytes: int = 50_000,
|
|
52
|
+
source: Literal["rendered", "loaded_image"] = "rendered",
|
|
53
|
+
) -> CapturedImage:
|
|
54
|
+
"""Capture one native Playwright Locator without fetching its source again.
|
|
55
|
+
|
|
56
|
+
``timeout`` uses Playwright milliseconds and bounds capture work. Closing a
|
|
57
|
+
CDP session gets up to 250 additional milliseconds after timeout or caller
|
|
58
|
+
cancellation; the original TimeoutError or cancellation is then preserved.
|
|
59
|
+
Wrap this coroutine in ``context.run(...)`` to share a crawl's cancellation
|
|
60
|
+
and duration budget.
|
|
61
|
+
The default size fits Resource Proxy's image-to-text task limit. A vision
|
|
62
|
+
model may accept larger images; opt into a larger ``max_bytes`` explicitly.
|
|
63
|
+
|
|
64
|
+
The default ``rendered`` source captures a visible element. On Chromium it
|
|
65
|
+
uses a clipped CDP screenshot to avoid Playwright's document-font wait, then
|
|
66
|
+
falls back to the native Locator screenshot when CDP is unavailable. A
|
|
67
|
+
zero-size element is rejected instead of silently changing capture semantics.
|
|
68
|
+
|
|
69
|
+
``loaded_image`` explicitly copies pixels from an already loaded ``IMG`` via
|
|
70
|
+
canvas. It can capture an intentionally hidden image, but may fail for a
|
|
71
|
+
cross-origin image whose pixels are protected by the browser. Neither source
|
|
72
|
+
requests the element's ``src`` URL. Capture again only when starting a new
|
|
73
|
+
logical task.
|
|
74
|
+
"""
|
|
75
|
+
if (
|
|
76
|
+
isinstance(timeout, bool)
|
|
77
|
+
or not isinstance(timeout, (int, float))
|
|
78
|
+
or not math.isfinite(timeout)
|
|
79
|
+
or timeout <= 0
|
|
80
|
+
):
|
|
81
|
+
raise ConfigurationError("Image capture timeout must be finite and positive")
|
|
82
|
+
if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes <= 0:
|
|
83
|
+
raise ConfigurationError("Image capture max_bytes must be a positive integer")
|
|
84
|
+
if not isinstance(source, str) or source not in {"rendered", "loaded_image"}:
|
|
85
|
+
raise ConfigurationError("Image capture source must be rendered or loaded_image")
|
|
86
|
+
deadline = asyncio.get_running_loop().time() + timeout / 1000
|
|
87
|
+
try:
|
|
88
|
+
async with asyncio.timeout_at(deadline):
|
|
89
|
+
if source == "loaded_image":
|
|
90
|
+
png = await _capture_loaded_image(locator, max_bytes=max_bytes)
|
|
91
|
+
else:
|
|
92
|
+
png = await _capture_rendered(
|
|
93
|
+
locator, timeout=timeout, max_bytes=max_bytes, deadline=deadline,
|
|
94
|
+
)
|
|
95
|
+
except Exception as error:
|
|
96
|
+
# Playwright has its own TimeoutError; do not retain its locator/URL or
|
|
97
|
+
# arbitrary upstream exception messages in the public diagnostic.
|
|
98
|
+
if isinstance(error, TimeoutError) or type(error).__name__ == "TimeoutError":
|
|
99
|
+
raise TimeoutError("Image capture timed out") from None
|
|
100
|
+
if isinstance(error, FetchError):
|
|
101
|
+
raise
|
|
102
|
+
raise FetchError("Image capture failed", phase="image_capture") from None
|
|
103
|
+
if not isinstance(png, bytes):
|
|
104
|
+
raise FetchError("Browser returned invalid image bytes", phase="image_capture")
|
|
105
|
+
if len(png) > max_bytes:
|
|
106
|
+
raise ResponseTooLarge("Captured image exceeds max_bytes", phase="image_capture")
|
|
107
|
+
return CapturedImage(png)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
async def _capture_rendered(
|
|
111
|
+
locator: Any, *, timeout: float, max_bytes: int, deadline: float,
|
|
112
|
+
) -> bytes:
|
|
113
|
+
await _require_visible_area(locator)
|
|
114
|
+
page = getattr(locator, "page", None)
|
|
115
|
+
context = getattr(page, "context", None)
|
|
116
|
+
if page is not None and context is not None and hasattr(context, "new_cdp_session"):
|
|
117
|
+
try:
|
|
118
|
+
return await _capture_rendered_cdp(
|
|
119
|
+
locator,
|
|
120
|
+
page,
|
|
121
|
+
context,
|
|
122
|
+
timeout=timeout,
|
|
123
|
+
max_bytes=max_bytes,
|
|
124
|
+
)
|
|
125
|
+
except FetchError:
|
|
126
|
+
raise
|
|
127
|
+
except Exception as error:
|
|
128
|
+
# CDP is Chromium-only. Other engines and partial adapters retain
|
|
129
|
+
# the browser-neutral Locator screenshot path.
|
|
130
|
+
if isinstance(error, TimeoutError) or asyncio.get_running_loop().time() >= deadline:
|
|
131
|
+
raise
|
|
132
|
+
return await locator.screenshot(type="png", animations="disabled", timeout=timeout)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
async def _require_visible_area(locator: Any) -> None:
|
|
136
|
+
state = await locator.evaluate(
|
|
137
|
+
"""element => {
|
|
138
|
+
const rect = element.getBoundingClientRect();
|
|
139
|
+
const style = getComputedStyle(element);
|
|
140
|
+
return {
|
|
141
|
+
connected: element.isConnected,
|
|
142
|
+
width: rect.width,
|
|
143
|
+
height: rect.height,
|
|
144
|
+
display: style.display,
|
|
145
|
+
visibility: style.visibility
|
|
146
|
+
};
|
|
147
|
+
}"""
|
|
148
|
+
)
|
|
149
|
+
if not isinstance(state, dict):
|
|
150
|
+
raise FetchError("Browser returned invalid image bounds", phase="image_capture")
|
|
151
|
+
try:
|
|
152
|
+
width = float(state["width"])
|
|
153
|
+
height = float(state["height"])
|
|
154
|
+
except (KeyError, TypeError, ValueError, OverflowError):
|
|
155
|
+
raise FetchError("Browser returned invalid image bounds", phase="image_capture") from None
|
|
156
|
+
if (
|
|
157
|
+
state.get("connected") is not True
|
|
158
|
+
or state.get("display") == "none"
|
|
159
|
+
or state.get("visibility") in {"hidden", "collapse"}
|
|
160
|
+
or not math.isfinite(width)
|
|
161
|
+
or not math.isfinite(height)
|
|
162
|
+
or width <= 0
|
|
163
|
+
or height <= 0
|
|
164
|
+
):
|
|
165
|
+
raise FetchError("Rendered image has no visible area", phase="image_capture")
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
async def _capture_rendered_cdp(
|
|
169
|
+
locator: Any,
|
|
170
|
+
page: Any,
|
|
171
|
+
context: Any,
|
|
172
|
+
*,
|
|
173
|
+
timeout: float,
|
|
174
|
+
max_bytes: int,
|
|
175
|
+
) -> bytes:
|
|
176
|
+
await locator.scroll_into_view_if_needed(timeout=timeout)
|
|
177
|
+
box = await locator.bounding_box(timeout=timeout)
|
|
178
|
+
if not isinstance(box, dict):
|
|
179
|
+
raise FetchError("Rendered image is not visible", phase="image_capture")
|
|
180
|
+
try:
|
|
181
|
+
x = float(box["x"])
|
|
182
|
+
y = float(box["y"])
|
|
183
|
+
width = float(box["width"])
|
|
184
|
+
height = float(box["height"])
|
|
185
|
+
except (KeyError, TypeError, ValueError, OverflowError):
|
|
186
|
+
raise FetchError("Browser returned invalid image bounds", phase="image_capture") from None
|
|
187
|
+
if (
|
|
188
|
+
not all(math.isfinite(value) for value in (x, y, width, height))
|
|
189
|
+
or width <= 0
|
|
190
|
+
or height <= 0
|
|
191
|
+
):
|
|
192
|
+
raise FetchError("Rendered image has no visible area", phase="image_capture")
|
|
193
|
+
|
|
194
|
+
offset = await page.evaluate("() => ({x: window.scrollX, y: window.scrollY})")
|
|
195
|
+
if not isinstance(offset, dict):
|
|
196
|
+
raise FetchError("Browser returned invalid page position", phase="image_capture")
|
|
197
|
+
try:
|
|
198
|
+
page_x = float(offset["x"])
|
|
199
|
+
page_y = float(offset["y"])
|
|
200
|
+
except (KeyError, TypeError, ValueError, OverflowError):
|
|
201
|
+
raise FetchError("Browser returned invalid page position", phase="image_capture") from None
|
|
202
|
+
if not math.isfinite(page_x) or not math.isfinite(page_y):
|
|
203
|
+
raise FetchError("Browser returned invalid page position", phase="image_capture")
|
|
204
|
+
|
|
205
|
+
session = await context.new_cdp_session(page)
|
|
206
|
+
try:
|
|
207
|
+
result = await session.send(
|
|
208
|
+
"Page.captureScreenshot",
|
|
209
|
+
{
|
|
210
|
+
"format": "png",
|
|
211
|
+
"captureBeyondViewport": True,
|
|
212
|
+
"fromSurface": True,
|
|
213
|
+
"clip": {
|
|
214
|
+
"x": x + page_x,
|
|
215
|
+
"y": y + page_y,
|
|
216
|
+
"width": width,
|
|
217
|
+
"height": height,
|
|
218
|
+
"scale": 1,
|
|
219
|
+
},
|
|
220
|
+
},
|
|
221
|
+
)
|
|
222
|
+
finally:
|
|
223
|
+
await _detach_session(session)
|
|
224
|
+
if not isinstance(result, dict) or not isinstance(result.get("data"), str):
|
|
225
|
+
raise FetchError("Browser returned invalid image bytes", phase="image_capture")
|
|
226
|
+
return _decode_base64(result["data"], max_bytes=max_bytes)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
async def _capture_loaded_image(locator: Any, *, max_bytes: int) -> bytes:
|
|
230
|
+
encoded = await locator.evaluate(
|
|
231
|
+
"""image => {
|
|
232
|
+
if (!(image instanceof HTMLImageElement)) throw new Error('not an image');
|
|
233
|
+
if (!image.complete || !image.naturalWidth || !image.naturalHeight) {
|
|
234
|
+
throw new Error('image not loaded');
|
|
235
|
+
}
|
|
236
|
+
const canvas = document.createElement('canvas');
|
|
237
|
+
canvas.width = image.naturalWidth;
|
|
238
|
+
canvas.height = image.naturalHeight;
|
|
239
|
+
const context = canvas.getContext('2d');
|
|
240
|
+
if (!context) throw new Error('canvas unavailable');
|
|
241
|
+
context.drawImage(image, 0, 0);
|
|
242
|
+
return canvas.toDataURL('image/png').split(',', 2)[1];
|
|
243
|
+
}"""
|
|
244
|
+
)
|
|
245
|
+
if not isinstance(encoded, str):
|
|
246
|
+
raise FetchError("Browser returned invalid image bytes", phase="image_capture")
|
|
247
|
+
return _decode_base64(encoded, max_bytes=max_bytes)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
async def _detach_session(session: Any) -> None:
|
|
251
|
+
try:
|
|
252
|
+
async with asyncio.timeout(0.25):
|
|
253
|
+
await session.detach()
|
|
254
|
+
except TimeoutError:
|
|
255
|
+
pass
|
|
256
|
+
except Exception:
|
|
257
|
+
pass
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _decode_base64(encoded: str, *, max_bytes: int) -> bytes:
|
|
261
|
+
max_encoded = 4 * ((max_bytes + 2) // 3)
|
|
262
|
+
if len(encoded) > max_encoded:
|
|
263
|
+
raise ResponseTooLarge("Captured image exceeds max_bytes", phase="image_capture")
|
|
264
|
+
try:
|
|
265
|
+
return base64.b64decode(encoded, validate=True)
|
|
266
|
+
except (binascii.Error, ValueError):
|
|
267
|
+
raise FetchError("Browser returned invalid image bytes", phase="image_capture") from None
|