dataapp_crawler 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,184 @@
1
+ """Small error vocabulary; messages never include upstream exception text or credentials."""
2
+
3
+ import re
4
+ from typing import Any, TypedDict
5
+
6
+ import httpx
7
+
8
+ _BROWSER_ERROR_CODES = frozenset(
9
+ {
10
+ "ERR_PROXY_CONNECTION_FAILED", "ERR_TUNNEL_CONNECTION_FAILED",
11
+ "ERR_INVALID_AUTH_CREDENTIALS", "ERR_PROXY_AUTH_UNSUPPORTED", "ERR_NO_SUPPORTED_PROXIES",
12
+ "ERR_CONNECTION_RESET", "ERR_CONNECTION_CLOSED", "ERR_CONNECTION_REFUSED",
13
+ "ERR_NAME_NOT_RESOLVED", "ERR_CERT_AUTHORITY_INVALID", "ERR_PROXY_CERTIFICATE_INVALID",
14
+ "ERR_TIMED_OUT", "ERR_ABORTED",
15
+ "ERR_HTTP_RESPONSE_CODE_FAILURE",
16
+ }
17
+ )
18
+ _BROWSER_ERROR = re.compile(
19
+ r"net::(" + "|".join(sorted(_BROWSER_ERROR_CODES)) + r")(?![A-Za-z0-9_])"
20
+ )
21
+
22
+ _HTTP_ERROR_TYPES: tuple[tuple[type[httpx.HTTPError], str], ...] = (
23
+ (httpx.ConnectTimeout, "HTTPX_CONNECT_TIMEOUT"),
24
+ (httpx.ReadTimeout, "HTTPX_READ_TIMEOUT"),
25
+ (httpx.WriteTimeout, "HTTPX_WRITE_TIMEOUT"),
26
+ (httpx.PoolTimeout, "HTTPX_POOL_TIMEOUT"),
27
+ (httpx.ConnectError, "HTTPX_CONNECT_ERROR"),
28
+ (httpx.ReadError, "HTTPX_READ_ERROR"),
29
+ (httpx.WriteError, "HTTPX_WRITE_ERROR"),
30
+ (httpx.CloseError, "HTTPX_CLOSE_ERROR"),
31
+ (httpx.ProxyError, "HTTPX_PROXY_ERROR"),
32
+ (httpx.LocalProtocolError, "HTTPX_LOCAL_PROTOCOL_ERROR"),
33
+ (httpx.RemoteProtocolError, "HTTPX_REMOTE_PROTOCOL_ERROR"),
34
+ (httpx.DecodingError, "HTTPX_DECODING_ERROR"),
35
+ (httpx.TooManyRedirects, "HTTPX_TOO_MANY_REDIRECTS"),
36
+ (httpx.UnsupportedProtocol, "HTTPX_UNSUPPORTED_PROTOCOL"),
37
+ (httpx.HTTPStatusError, "HTTPX_HTTP_STATUS_ERROR"),
38
+ (httpx.TimeoutException, "HTTPX_TIMEOUT"),
39
+ (httpx.NetworkError, "HTTPX_NETWORK_ERROR"),
40
+ (httpx.ProtocolError, "HTTPX_PROTOCOL_ERROR"),
41
+ (httpx.TransportError, "HTTPX_TRANSPORT_ERROR"),
42
+ (httpx.RequestError, "HTTPX_REQUEST_ERROR"),
43
+ (httpx.HTTPError, "HTTPX_HTTP_ERROR"),
44
+ )
45
+ _HTTP_ERROR_CODES = frozenset(code for _, code in _HTTP_ERROR_TYPES) | {
46
+ "HTTP_REDIRECT_LIMIT_REACHED"
47
+ }
48
+
49
+
50
+ class NetworkDiagnosticsSummary(TypedDict):
51
+ events: list[dict[str, Any]]
52
+ events_dropped: int
53
+ main_document_status: int | None
54
+
55
+
56
+ def browser_error_code(error: Exception | str) -> str | None:
57
+ """Keep only fixed Chromium codes; never retain arbitrary upstream error text."""
58
+ code = getattr(error, "browser_error_code", None)
59
+ if isinstance(code, str) and code in _BROWSER_ERROR_CODES:
60
+ return code
61
+ match = _BROWSER_ERROR.search(str(error))
62
+ return match.group(1) if match else None
63
+
64
+
65
+ def http_error_code(error: Exception) -> str | None:
66
+ """Map HTTPX exception types to fixed codes without inspecting exception text."""
67
+ code = getattr(error, "http_error_code", None)
68
+ if isinstance(code, str) and code in _HTTP_ERROR_CODES:
69
+ return code
70
+ return next(
71
+ (code for error_type, code in _HTTP_ERROR_TYPES if isinstance(error, error_type)),
72
+ None,
73
+ )
74
+
75
+
76
+ class CrawlerError(Exception):
77
+ kind = "crawler"
78
+
79
+
80
+ class FetchError(CrawlerError):
81
+ kind = "network"
82
+
83
+ def __init__(
84
+ self,
85
+ message: str,
86
+ *,
87
+ retryable: bool = False,
88
+ phase: str | None = None,
89
+ browser_error_code: str | None = None,
90
+ http_error_code: str | None = None,
91
+ network_diagnostics: NetworkDiagnosticsSummary | None = None,
92
+ main_document_status: int | None = None,
93
+ ) -> None:
94
+ super().__init__(message)
95
+ self.retryable = retryable
96
+ self.phase = phase
97
+ self.retry_events: list[dict[str, Any]] = []
98
+ self.browser_diagnostics: dict[str, Any] = {}
99
+ self.network_diagnostics = network_diagnostics
100
+ self.main_document_status = (
101
+ main_document_status
102
+ if isinstance(main_document_status, int)
103
+ and not isinstance(main_document_status, bool)
104
+ and 100 <= main_document_status <= 599
105
+ else None
106
+ )
107
+ self.browser_error_code = (
108
+ browser_error_code
109
+ if isinstance(browser_error_code, str) and browser_error_code in _BROWSER_ERROR_CODES
110
+ else None
111
+ )
112
+ self.http_error_code = (
113
+ http_error_code
114
+ if isinstance(http_error_code, str) and http_error_code in _HTTP_ERROR_CODES
115
+ else None
116
+ )
117
+
118
+
119
+ class ResponseTooLarge(FetchError):
120
+ kind = "response_too_large"
121
+
122
+
123
+ class ConfigurationError(CrawlerError):
124
+ kind = "configuration"
125
+
126
+
127
+ class CrawlCancelled(CrawlerError):
128
+ kind = "cancelled"
129
+
130
+
131
+ class LimitReached(CrawlerError):
132
+ kind = "limit"
133
+
134
+
135
+ class QueueFull(CrawlerError):
136
+ kind = "queue_full"
137
+
138
+
139
+ class ProcessingError(CrawlerError):
140
+ kind = "process"
141
+ phase = "process"
142
+
143
+ def __init__(
144
+ self, message: str, *, capture: Any = None, process_error_code: str | None = None,
145
+ ) -> None:
146
+ super().__init__(message)
147
+ self.capture = capture
148
+ self.process_error_code = (
149
+ process_error_code
150
+ if isinstance(process_error_code, str) and process_error_code in _PROCESS_ERROR_CODES
151
+ else None
152
+ )
153
+
154
+
155
+ _PROCESS_ERROR_TYPES: tuple[tuple[type[Exception], str], ...] = (
156
+ (ResponseTooLarge, "PROCESS_RESPONSE_TOO_LARGE"),
157
+ (FetchError, "PROCESS_FETCH_ERROR"),
158
+ (TimeoutError, "PROCESS_TIMEOUT"),
159
+ (KeyError, "PROCESS_KEY_ERROR"),
160
+ (IndexError, "PROCESS_INDEX_ERROR"),
161
+ (AttributeError, "PROCESS_ATTRIBUTE_ERROR"),
162
+ (TypeError, "PROCESS_TYPE_ERROR"),
163
+ (ValueError, "PROCESS_VALUE_ERROR"),
164
+ (AssertionError, "PROCESS_ASSERTION_ERROR"),
165
+ )
166
+ _PROCESS_ERROR_CODES = frozenset(code for _, code in _PROCESS_ERROR_TYPES) | {"PROCESS_ERROR"}
167
+
168
+
169
+ def process_error_code(error: Exception) -> str:
170
+ """Describe a callback failure using fixed codes, without its message or traceback."""
171
+ if (
172
+ isinstance(error, ProcessingError)
173
+ and isinstance(error.process_error_code, str)
174
+ and error.process_error_code in _PROCESS_ERROR_CODES
175
+ ):
176
+ return error.process_error_code
177
+ # Optional browser libraries define their own TimeoutError. No provider import
178
+ # or arbitrary exception class name is needed in the public diagnostic.
179
+ if type(error).__name__ == "TimeoutError":
180
+ return "PROCESS_TIMEOUT"
181
+ return next(
182
+ (code for error_type, code in _PROCESS_ERROR_TYPES if isinstance(error, error_type)),
183
+ "PROCESS_ERROR",
184
+ )
@@ -0,0 +1,223 @@
1
+ """HTTPX session with bounded reads and explicit per-origin redirect handling."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import http.cookiejar
6
+ import time
7
+ from collections.abc import Callable
8
+
9
+ import httpx
10
+
11
+ from .context import CrawlContext, SiteGate
12
+ from .cookies import (
13
+ cookies_from_jar,
14
+ is_partitioned_jar_cookie,
15
+ replace_jar,
16
+ validate_shareable_cookies,
17
+ )
18
+ from .errors import ConfigurationError, FetchError, ResponseTooLarge, http_error_code
19
+ from .models import CaptureResult, Identity, Limits, Request, origin, safe_url
20
+
21
+
22
+ class _BorrowedAsyncTransport(httpx.AsyncBaseTransport):
23
+ """Let multiple short-lived clients share one caller-owned transport."""
24
+
25
+ def __init__(self, transport: httpx.AsyncBaseTransport) -> None:
26
+ self._transport = transport
27
+
28
+ async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
29
+ return await self._transport.handle_async_request(request)
30
+
31
+ async def aclose(self) -> None:
32
+ # The Crawler closes the injected transport once after all generations retire.
33
+ return None
34
+
35
+
36
+ class HttpSession:
37
+ def __init__(
38
+ self,
39
+ limits: Limits,
40
+ identity: Identity,
41
+ *,
42
+ proxy_url: str | None = None,
43
+ transport: httpx.AsyncBaseTransport | None = None,
44
+ close_transport: bool = True,
45
+ reject_partitioned: bool = False,
46
+ ) -> None:
47
+ self.limits = limits
48
+ self.identity = identity
49
+ self.reject_partitioned = reject_partitioned
50
+ cookies = [*(identity.storage_state or {}).get("cookies", []), *identity.cookies]
51
+ if reject_partitioned:
52
+ validate_shareable_cookies(cookies)
53
+ client_transport = (
54
+ _BorrowedAsyncTransport(transport)
55
+ if transport is not None and not close_transport
56
+ else transport
57
+ )
58
+ self.client = httpx.AsyncClient(
59
+ timeout=httpx.Timeout(limits.request_timeout),
60
+ follow_redirects=False,
61
+ trust_env=False,
62
+ proxy=proxy_url,
63
+ transport=client_transport,
64
+ )
65
+ self.client.cookies.jar.set_policy(
66
+ http.cookiejar.DefaultCookiePolicy(
67
+ strict_ns_domain=http.cookiejar.DefaultCookiePolicy.DomainStrictNonDomain
68
+ )
69
+ )
70
+ # CookieJar preserves Secure, path, domain, expiry and supported metadata.
71
+ replace_jar(
72
+ self.client.cookies.jar,
73
+ cookies,
74
+ ignore_partitioned=not reject_partitioned,
75
+ )
76
+
77
+ async def close(self) -> None:
78
+ await self.client.aclose()
79
+
80
+ def export_cookies(self) -> list[dict[str, object]]:
81
+ """Return the current jar as browser-compatible, non-header cookie records."""
82
+ return cookies_from_jar(self.client.cookies.jar)
83
+
84
+ def replace_cookies(self, cookies: list[dict[str, object]]) -> None:
85
+ """Replace this session's jar from a canonical session snapshot."""
86
+ replace_jar(self.client.cookies.jar, cookies)
87
+
88
+ async def fetch(
89
+ self, request: Request, context: CrawlContext, gate: Callable[[str], SiteGate]
90
+ ) -> CaptureResult:
91
+ initial_origin = origin(request.url)
92
+ headers = {**request.headers, **self.identity.headers_for(request.url)}
93
+ current = self.client.build_request(
94
+ request.method,
95
+ request.url,
96
+ params=request.params,
97
+ content=request.content,
98
+ json=request.json,
99
+ headers=headers,
100
+ )
101
+ # HTTPX build_request merges cookies through a fresh CookieJar. Reapply the
102
+ # session jar's host-only policy instead of inheriting that copy's policy.
103
+ current.headers.pop("cookie", None)
104
+ self.client.cookies.set_cookie_header(current)
105
+ for name, value in request.headers.items():
106
+ if name.lower() == "cookie":
107
+ current.headers[name] = value
108
+ started = time.monotonic()
109
+ for redirects in range(self.limits.max_redirects + 1):
110
+ context.reserve_request()
111
+ response: httpx.Response | None = None
112
+ failure: FetchError | None = None
113
+ try:
114
+ async with gate(request.site_key or origin(str(current.url))):
115
+ response = await self.client.send(current, stream=True, follow_redirects=False)
116
+ self._narrow_response_cookies(response)
117
+ body = bytearray()
118
+ async for chunk in response.aiter_bytes():
119
+ if len(body) + len(chunk) > self.limits.max_response_bytes:
120
+ raise ResponseTooLarge("Response exceeds configured byte limit")
121
+ body.extend(chunk)
122
+ next_request = response.next_request
123
+ if next_request is None:
124
+ return CaptureResult(
125
+ request,
126
+ str(response.url),
127
+ response.status_code,
128
+ dict(response.headers),
129
+ bytes(body),
130
+ elapsed=time.monotonic() - started,
131
+ diagnostics={"url": safe_url(str(response.url)), "mode": "http"},
132
+ )
133
+ if redirects == self.limits.max_redirects:
134
+ raise FetchError(
135
+ "Redirect limit reached",
136
+ phase="http_redirect",
137
+ http_error_code="HTTP_REDIRECT_LIMIT_REACHED",
138
+ main_document_status=response.status_code,
139
+ )
140
+ # Remove all caller custom headers on an origin change. Rebuild Cookie from
141
+ # the jar; add website credentials only for the next URL's explicit origin.
142
+ next_origin = origin(str(next_request.url))
143
+ target_host = httpx.Request("GET", next_request.url).headers["host"]
144
+ for name in set(request.headers) | {
145
+ k for values in self.identity.headers.values() for k in values
146
+ }:
147
+ next_request.headers.pop(name, None)
148
+ for name in ("authorization", "cookie", "proxy-authorization", "host"):
149
+ next_request.headers.pop(name, None)
150
+ self.client.cookies.set_cookie_header(next_request)
151
+ if next_origin == initial_origin:
152
+ next_request.headers.update(request.headers)
153
+ next_request.headers.update(self.identity.headers_for(str(next_request.url)))
154
+ if next_origin != initial_origin or "host" not in next_request.headers:
155
+ next_request.headers["host"] = target_host
156
+ current = next_request
157
+ except (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError) as exc:
158
+ failure = FetchError(
159
+ "HTTP transport failed",
160
+ retryable=True,
161
+ phase="http_transport",
162
+ http_error_code=http_error_code(exc),
163
+ )
164
+ except httpx.HTTPError as exc:
165
+ failure = FetchError(
166
+ "HTTP request failed",
167
+ phase="http_request",
168
+ http_error_code=http_error_code(exc),
169
+ )
170
+ finally:
171
+ if response is not None:
172
+ await response.aclose()
173
+ if failure is not None:
174
+ # Raise after leaving the HTTPX handler so the raw exception (which can
175
+ # contain URLs, headers, or proxy credentials) is not retained as context.
176
+ raise failure
177
+ raise FetchError(
178
+ "Redirect limit reached",
179
+ phase="http_redirect",
180
+ http_error_code="HTTP_REDIRECT_LIMIT_REACHED",
181
+ )
182
+
183
+ def _narrow_response_cookies(self, response: httpx.Response) -> None:
184
+ """Keep accepted response cookies, but scope them to the setting host."""
185
+ host = response.url.host
186
+ for cookie in response.cookies.jar:
187
+ try:
188
+ self.client.cookies.jar.clear(cookie.domain, cookie.path, cookie.name)
189
+ except KeyError:
190
+ pass
191
+ try:
192
+ self.client.cookies.jar.clear(host, cookie.path, cookie.name)
193
+ except KeyError:
194
+ pass
195
+ if is_partitioned_jar_cookie(cookie):
196
+ if self.reject_partitioned:
197
+ raise ConfigurationError(
198
+ "Partitioned cookies cannot be shared across engines"
199
+ )
200
+ continue
201
+ if cookie.expires is not None and cookie.expires <= time.time():
202
+ continue
203
+ self.client.cookies.jar.set_cookie(
204
+ http.cookiejar.Cookie(
205
+ version=cookie.version,
206
+ name=cookie.name,
207
+ value=cookie.value,
208
+ port=cookie.port,
209
+ port_specified=cookie.port_specified,
210
+ domain=host,
211
+ domain_specified=False,
212
+ domain_initial_dot=False,
213
+ path=cookie.path,
214
+ path_specified=cookie.path_specified,
215
+ secure=cookie.secure,
216
+ expires=cookie.expires,
217
+ discard=cookie.discard,
218
+ comment=cookie.comment,
219
+ comment_url=cookie.comment_url,
220
+ rest=dict(getattr(cookie, "_rest", {})),
221
+ rfc2109=cookie.rfc2109,
222
+ )
223
+ )
@@ -0,0 +1,267 @@
1
+ """Capture the currently rendered image without fetching its source URL again."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import base64
7
+ import binascii
8
+ import hashlib
9
+ import math
10
+ from dataclasses import dataclass, field
11
+ from typing import Any, Literal
12
+
13
+ from .errors import ConfigurationError, FetchError, ResponseTooLarge
14
+
15
+
16
+ @dataclass(frozen=True, slots=True)
17
+ class CapturedImage:
18
+ """Private PNG bytes with encodings accepted by solver and vision resources.
19
+
20
+ Treat ``image_body`` and ``data_url`` as sensitive page content. They are not
21
+ included in the representation. A matching hash identifies captured bytes,
22
+ not a guarantee that the website still accepts the same challenge.
23
+ """
24
+
25
+ _png: bytes = field(repr=False)
26
+
27
+ def __post_init__(self) -> None:
28
+ if not isinstance(self._png, bytes) or not self._png.startswith(b"\x89PNG\r\n\x1a\n"):
29
+ raise ConfigurationError("Captured image must contain PNG bytes")
30
+
31
+ @property
32
+ def image_body(self) -> str:
33
+ """Return base64 PNG for ``solve_captcha(image_body=...)``."""
34
+ return base64.b64encode(self._png).decode("ascii")
35
+
36
+ @property
37
+ def data_url(self) -> str:
38
+ """Return an inline PNG URL for a model's ``image_url`` content part."""
39
+ return "data:image/png;base64," + self.image_body
40
+
41
+ @property
42
+ def sha256(self) -> str:
43
+ """Return a content fingerprint without exposing the image."""
44
+ return hashlib.sha256(self._png).hexdigest()
45
+
46
+
47
+ async def capture_image(
48
+ locator: Any,
49
+ *,
50
+ timeout: float = 10_000,
51
+ max_bytes: int = 50_000,
52
+ source: Literal["rendered", "loaded_image"] = "rendered",
53
+ ) -> CapturedImage:
54
+ """Capture one native Playwright Locator without fetching its source again.
55
+
56
+ ``timeout`` uses Playwright milliseconds and bounds capture work. Closing a
57
+ CDP session gets up to 250 additional milliseconds after timeout or caller
58
+ cancellation; the original TimeoutError or cancellation is then preserved.
59
+ Wrap this coroutine in ``context.run(...)`` to share a crawl's cancellation
60
+ and duration budget.
61
+ The default size fits Resource Proxy's image-to-text task limit. A vision
62
+ model may accept larger images; opt into a larger ``max_bytes`` explicitly.
63
+
64
+ The default ``rendered`` source captures a visible element. On Chromium it
65
+ uses a clipped CDP screenshot to avoid Playwright's document-font wait, then
66
+ falls back to the native Locator screenshot when CDP is unavailable. A
67
+ zero-size element is rejected instead of silently changing capture semantics.
68
+
69
+ ``loaded_image`` explicitly copies pixels from an already loaded ``IMG`` via
70
+ canvas. It can capture an intentionally hidden image, but may fail for a
71
+ cross-origin image whose pixels are protected by the browser. Neither source
72
+ requests the element's ``src`` URL. Capture again only when starting a new
73
+ logical task.
74
+ """
75
+ if (
76
+ isinstance(timeout, bool)
77
+ or not isinstance(timeout, (int, float))
78
+ or not math.isfinite(timeout)
79
+ or timeout <= 0
80
+ ):
81
+ raise ConfigurationError("Image capture timeout must be finite and positive")
82
+ if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes <= 0:
83
+ raise ConfigurationError("Image capture max_bytes must be a positive integer")
84
+ if not isinstance(source, str) or source not in {"rendered", "loaded_image"}:
85
+ raise ConfigurationError("Image capture source must be rendered or loaded_image")
86
+ deadline = asyncio.get_running_loop().time() + timeout / 1000
87
+ try:
88
+ async with asyncio.timeout_at(deadline):
89
+ if source == "loaded_image":
90
+ png = await _capture_loaded_image(locator, max_bytes=max_bytes)
91
+ else:
92
+ png = await _capture_rendered(
93
+ locator, timeout=timeout, max_bytes=max_bytes, deadline=deadline,
94
+ )
95
+ except Exception as error:
96
+ # Playwright has its own TimeoutError; do not retain its locator/URL or
97
+ # arbitrary upstream exception messages in the public diagnostic.
98
+ if isinstance(error, TimeoutError) or type(error).__name__ == "TimeoutError":
99
+ raise TimeoutError("Image capture timed out") from None
100
+ if isinstance(error, FetchError):
101
+ raise
102
+ raise FetchError("Image capture failed", phase="image_capture") from None
103
+ if not isinstance(png, bytes):
104
+ raise FetchError("Browser returned invalid image bytes", phase="image_capture")
105
+ if len(png) > max_bytes:
106
+ raise ResponseTooLarge("Captured image exceeds max_bytes", phase="image_capture")
107
+ return CapturedImage(png)
108
+
109
+
110
+ async def _capture_rendered(
111
+ locator: Any, *, timeout: float, max_bytes: int, deadline: float,
112
+ ) -> bytes:
113
+ await _require_visible_area(locator)
114
+ page = getattr(locator, "page", None)
115
+ context = getattr(page, "context", None)
116
+ if page is not None and context is not None and hasattr(context, "new_cdp_session"):
117
+ try:
118
+ return await _capture_rendered_cdp(
119
+ locator,
120
+ page,
121
+ context,
122
+ timeout=timeout,
123
+ max_bytes=max_bytes,
124
+ )
125
+ except FetchError:
126
+ raise
127
+ except Exception as error:
128
+ # CDP is Chromium-only. Other engines and partial adapters retain
129
+ # the browser-neutral Locator screenshot path.
130
+ if isinstance(error, TimeoutError) or asyncio.get_running_loop().time() >= deadline:
131
+ raise
132
+ return await locator.screenshot(type="png", animations="disabled", timeout=timeout)
133
+
134
+
135
+ async def _require_visible_area(locator: Any) -> None:
136
+ state = await locator.evaluate(
137
+ """element => {
138
+ const rect = element.getBoundingClientRect();
139
+ const style = getComputedStyle(element);
140
+ return {
141
+ connected: element.isConnected,
142
+ width: rect.width,
143
+ height: rect.height,
144
+ display: style.display,
145
+ visibility: style.visibility
146
+ };
147
+ }"""
148
+ )
149
+ if not isinstance(state, dict):
150
+ raise FetchError("Browser returned invalid image bounds", phase="image_capture")
151
+ try:
152
+ width = float(state["width"])
153
+ height = float(state["height"])
154
+ except (KeyError, TypeError, ValueError, OverflowError):
155
+ raise FetchError("Browser returned invalid image bounds", phase="image_capture") from None
156
+ if (
157
+ state.get("connected") is not True
158
+ or state.get("display") == "none"
159
+ or state.get("visibility") in {"hidden", "collapse"}
160
+ or not math.isfinite(width)
161
+ or not math.isfinite(height)
162
+ or width <= 0
163
+ or height <= 0
164
+ ):
165
+ raise FetchError("Rendered image has no visible area", phase="image_capture")
166
+
167
+
168
+ async def _capture_rendered_cdp(
169
+ locator: Any,
170
+ page: Any,
171
+ context: Any,
172
+ *,
173
+ timeout: float,
174
+ max_bytes: int,
175
+ ) -> bytes:
176
+ await locator.scroll_into_view_if_needed(timeout=timeout)
177
+ box = await locator.bounding_box(timeout=timeout)
178
+ if not isinstance(box, dict):
179
+ raise FetchError("Rendered image is not visible", phase="image_capture")
180
+ try:
181
+ x = float(box["x"])
182
+ y = float(box["y"])
183
+ width = float(box["width"])
184
+ height = float(box["height"])
185
+ except (KeyError, TypeError, ValueError, OverflowError):
186
+ raise FetchError("Browser returned invalid image bounds", phase="image_capture") from None
187
+ if (
188
+ not all(math.isfinite(value) for value in (x, y, width, height))
189
+ or width <= 0
190
+ or height <= 0
191
+ ):
192
+ raise FetchError("Rendered image has no visible area", phase="image_capture")
193
+
194
+ offset = await page.evaluate("() => ({x: window.scrollX, y: window.scrollY})")
195
+ if not isinstance(offset, dict):
196
+ raise FetchError("Browser returned invalid page position", phase="image_capture")
197
+ try:
198
+ page_x = float(offset["x"])
199
+ page_y = float(offset["y"])
200
+ except (KeyError, TypeError, ValueError, OverflowError):
201
+ raise FetchError("Browser returned invalid page position", phase="image_capture") from None
202
+ if not math.isfinite(page_x) or not math.isfinite(page_y):
203
+ raise FetchError("Browser returned invalid page position", phase="image_capture")
204
+
205
+ session = await context.new_cdp_session(page)
206
+ try:
207
+ result = await session.send(
208
+ "Page.captureScreenshot",
209
+ {
210
+ "format": "png",
211
+ "captureBeyondViewport": True,
212
+ "fromSurface": True,
213
+ "clip": {
214
+ "x": x + page_x,
215
+ "y": y + page_y,
216
+ "width": width,
217
+ "height": height,
218
+ "scale": 1,
219
+ },
220
+ },
221
+ )
222
+ finally:
223
+ await _detach_session(session)
224
+ if not isinstance(result, dict) or not isinstance(result.get("data"), str):
225
+ raise FetchError("Browser returned invalid image bytes", phase="image_capture")
226
+ return _decode_base64(result["data"], max_bytes=max_bytes)
227
+
228
+
229
+ async def _capture_loaded_image(locator: Any, *, max_bytes: int) -> bytes:
230
+ encoded = await locator.evaluate(
231
+ """image => {
232
+ if (!(image instanceof HTMLImageElement)) throw new Error('not an image');
233
+ if (!image.complete || !image.naturalWidth || !image.naturalHeight) {
234
+ throw new Error('image not loaded');
235
+ }
236
+ const canvas = document.createElement('canvas');
237
+ canvas.width = image.naturalWidth;
238
+ canvas.height = image.naturalHeight;
239
+ const context = canvas.getContext('2d');
240
+ if (!context) throw new Error('canvas unavailable');
241
+ context.drawImage(image, 0, 0);
242
+ return canvas.toDataURL('image/png').split(',', 2)[1];
243
+ }"""
244
+ )
245
+ if not isinstance(encoded, str):
246
+ raise FetchError("Browser returned invalid image bytes", phase="image_capture")
247
+ return _decode_base64(encoded, max_bytes=max_bytes)
248
+
249
+
250
+ async def _detach_session(session: Any) -> None:
251
+ try:
252
+ async with asyncio.timeout(0.25):
253
+ await session.detach()
254
+ except TimeoutError:
255
+ pass
256
+ except Exception:
257
+ pass
258
+
259
+
260
+ def _decode_base64(encoded: str, *, max_bytes: int) -> bytes:
261
+ max_encoded = 4 * ((max_bytes + 2) // 3)
262
+ if len(encoded) > max_encoded:
263
+ raise ResponseTooLarge("Captured image exceeds max_bytes", phase="image_capture")
264
+ try:
265
+ return base64.b64decode(encoded, validate=True)
266
+ except (binascii.Error, ValueError):
267
+ raise FetchError("Browser returned invalid image bytes", phase="image_capture") from None