scrapely-python 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of scrapely-python might be problematic. Click here for more details.
- scrapely/__init__.py +1 -1
- scrapely/actor.py +43 -10
- scrapely/cdp_retry.py +226 -0
- scrapely/patch_browser_pool.py +14 -3
- scrapely/storage/api_client.py +109 -10
- {scrapely_python-0.1.4.dist-info → scrapely_python-0.1.6.dist-info}/METADATA +1 -1
- {scrapely_python-0.1.4.dist-info → scrapely_python-0.1.6.dist-info}/RECORD +9 -8
- {scrapely_python-0.1.4.dist-info → scrapely_python-0.1.6.dist-info}/WHEEL +0 -0
- {scrapely_python-0.1.4.dist-info → scrapely_python-0.1.6.dist-info}/top_level.txt +0 -0
scrapely/__init__.py
CHANGED
scrapely/actor.py
CHANGED
|
@@ -18,7 +18,7 @@ import warnings
|
|
|
18
18
|
from collections.abc import MutableMapping
|
|
19
19
|
from datetime import datetime, timedelta
|
|
20
20
|
from decimal import Decimal
|
|
21
|
-
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast
|
|
21
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union, cast
|
|
22
22
|
|
|
23
23
|
from crawlee.configuration import Configuration as CrawleeConfiguration
|
|
24
24
|
from crawlee.storage_clients._base import StorageClient
|
|
@@ -287,6 +287,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
287
287
|
cls._exit_code = self._instance_exit_code
|
|
288
288
|
if self._instance_status_message:
|
|
289
289
|
cls._status_message = self._instance_status_message
|
|
290
|
+
await cls._persist_terminal_status_message(self._instance_status_message)
|
|
290
291
|
await cls._shutdown(timeout_secs=int(self._cleanup_timeout.total_seconds()))
|
|
291
292
|
|
|
292
293
|
@classmethod
|
|
@@ -589,6 +590,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
589
590
|
cls._exit_code = opts.exit_code
|
|
590
591
|
cls._status_message = opts.status_message
|
|
591
592
|
|
|
593
|
+
await cls._persist_terminal_status_message(opts.status_message)
|
|
592
594
|
await cls._shutdown(timeout_secs=opts.timeout_secs)
|
|
593
595
|
|
|
594
596
|
if opts.exit:
|
|
@@ -1021,13 +1023,32 @@ class Actor(metaclass=_ActorMeta):
|
|
|
1021
1023
|
else:
|
|
1022
1024
|
logger.info(logged)
|
|
1023
1025
|
|
|
1026
|
+
result = await cls._put_run_status_message(message, is_terminal=is_terminal)
|
|
1027
|
+
if result is None:
|
|
1028
|
+
return ActorRun(id='', actor_id='', status='')
|
|
1029
|
+
run_id, data = result
|
|
1030
|
+
|
|
1031
|
+
return ActorRun(
|
|
1032
|
+
id=data.get('id', run_id),
|
|
1033
|
+
actor_id=data.get('actorId', ''),
|
|
1034
|
+
status=data.get('status', ''),
|
|
1035
|
+
started_at=data.get('startedAt'),
|
|
1036
|
+
finished_at=data.get('finishedAt'),
|
|
1037
|
+
status_message=data.get('statusMessage'),
|
|
1038
|
+
)
|
|
1039
|
+
|
|
1040
|
+
@classmethod
|
|
1041
|
+
async def _put_run_status_message(
|
|
1042
|
+
cls, message: str, *, is_terminal: bool
|
|
1043
|
+
) -> Optional[Tuple[str, Dict[str, Any]]]:
|
|
1044
|
+
"""PUT the status message to the current run. Returns ``None`` when not on the platform."""
|
|
1024
1045
|
config = cls.get_config()
|
|
1025
1046
|
api_url = config.get('api_base_url')
|
|
1026
1047
|
token = config.get('token')
|
|
1027
1048
|
run_id = config.get('actor_run_id')
|
|
1028
1049
|
|
|
1029
1050
|
if not api_url or not token or not run_id:
|
|
1030
|
-
return
|
|
1051
|
+
return None
|
|
1031
1052
|
|
|
1032
1053
|
client = cls.new_client(token=token, base_url=api_url)
|
|
1033
1054
|
response = await client.put(
|
|
@@ -1041,15 +1062,27 @@ class Actor(metaclass=_ActorMeta):
|
|
|
1041
1062
|
if response.content:
|
|
1042
1063
|
body = response.json()
|
|
1043
1064
|
data = body.get('data', body) if isinstance(body, dict) else {}
|
|
1065
|
+
return run_id, data if isinstance(data, dict) else {}
|
|
1044
1066
|
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1067
|
+
@classmethod
|
|
1068
|
+
async def _persist_terminal_status_message(cls, message: Optional[str], timeout_secs: float = 10.0) -> None:
|
|
1069
|
+
"""Store the exit/fail status message on the run as terminal (Apify parity).
|
|
1070
|
+
|
|
1071
|
+
Runs before teardown so it lands while the run still exists and the
|
|
1072
|
+
process is alive, on success and failure alike. Best effort: a failed
|
|
1073
|
+
update is logged and never blocks the exit.
|
|
1074
|
+
"""
|
|
1075
|
+
if not message or not cls._initialized or cls._config is None:
|
|
1076
|
+
return
|
|
1077
|
+
try:
|
|
1078
|
+
await asyncio.wait_for(
|
|
1079
|
+
cls._put_run_status_message(message, is_terminal=True),
|
|
1080
|
+
timeout=max(1.0, timeout_secs),
|
|
1081
|
+
)
|
|
1082
|
+
except asyncio.CancelledError:
|
|
1083
|
+
raise
|
|
1084
|
+
except BaseException as e:
|
|
1085
|
+
logger.warning('Failed to store the final status message on the run: %s', e)
|
|
1053
1086
|
|
|
1054
1087
|
@_ClassProperty
|
|
1055
1088
|
def log(cls) -> logging.Logger:
|
scrapely/cdp_retry.py
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Retry remote-browser (CDP) connects that fail for transient reasons.
|
|
2
|
+
|
|
3
|
+
The remote browser answers ``429`` when it is at capacity and ``503`` when it
|
|
4
|
+
cannot start a browser or validate the session right now, both with a
|
|
5
|
+
``Retry-After`` delay (also repeated in the response body, because Playwright
|
|
6
|
+
only puts the status line and body into its error message). Connection errors
|
|
7
|
+
and timeouts are transient too. Those are retried with exponential backoff,
|
|
8
|
+
waiting at least the ``Retry-After`` delay when the error carries one. Any
|
|
9
|
+
other status (``400``/``401``/``402``/``403``/``404``) fails at once.
|
|
10
|
+
|
|
11
|
+
Tuning (environment):
|
|
12
|
+
|
|
13
|
+
- ``SCRAPELY_CDP_CONNECT_ATTEMPTS``: total attempts, default ``5`` (``1`` disables retries).
|
|
14
|
+
- ``SCRAPELY_CDP_RETRY_BASE_SECS``: first backoff delay, default ``1``; doubles per attempt.
|
|
15
|
+
- ``SCRAPELY_CDP_RETRY_MAX_SECS``: cap for one backoff delay, default ``30``.
|
|
16
|
+
- ``SCRAPELY_CDP_RETRY_AFTER_MAX_SECS``: longest ``Retry-After`` the SDK waits
|
|
17
|
+
out, default ``120``. A longer ``Retry-After`` ends the retries (the SDK never
|
|
18
|
+
retries earlier than the server asked).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import asyncio
|
|
24
|
+
import functools
|
|
25
|
+
import logging
|
|
26
|
+
import os
|
|
27
|
+
import random
|
|
28
|
+
import re
|
|
29
|
+
import time
|
|
30
|
+
from typing import Any, Awaitable, Callable, Optional, Tuple, TypeVar
|
|
31
|
+
from urllib.parse import urlsplit
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger('scrapely.browser')
|
|
34
|
+
|
|
35
|
+
T = TypeVar('T')
|
|
36
|
+
|
|
37
|
+
_RETRY_WRAPPED_ATTR = '_scrapely_cdp_retry_wrapped'
|
|
38
|
+
|
|
39
|
+
_REASONS = (
|
|
40
|
+
'Too Many Requests|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Timeout|'
|
|
41
|
+
'Request Timeout|Bad Request|Unauthorized|Payment Required|Forbidden|Not Found'
|
|
42
|
+
)
|
|
43
|
+
_STATUS_RE = re.compile(rf'\b([1-5]\d\d) (?:{_REASONS})\b')
|
|
44
|
+
_WS_STATUS_RE = re.compile(r'Unexpected server response:?\s*(\d{3})', re.IGNORECASE)
|
|
45
|
+
_RETRY_AFTER_RE = re.compile(r'Retry-After:?\s*(\d+(?:\.\d+)?)', re.IGNORECASE)
|
|
46
|
+
_TRANSIENT_MARKERS = (
|
|
47
|
+
'econnrefused',
|
|
48
|
+
'econnreset',
|
|
49
|
+
'etimedout',
|
|
50
|
+
'epipe',
|
|
51
|
+
'eai_again',
|
|
52
|
+
'ehostunreach',
|
|
53
|
+
'enetunreach',
|
|
54
|
+
'socket hang up',
|
|
55
|
+
'timeout',
|
|
56
|
+
'timed out',
|
|
57
|
+
'connection closed',
|
|
58
|
+
'connection refused',
|
|
59
|
+
'connection reset',
|
|
60
|
+
'websocket error',
|
|
61
|
+
'net::err_',
|
|
62
|
+
'target closed',
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _env_float(name: str, default: float) -> float:
|
|
67
|
+
try:
|
|
68
|
+
value = float(os.environ.get(name, ''))
|
|
69
|
+
except ValueError:
|
|
70
|
+
return default
|
|
71
|
+
return value if value > 0 else default
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _max_attempts() -> int:
|
|
75
|
+
return max(1, int(_env_float('SCRAPELY_CDP_CONNECT_ATTEMPTS', 5)))
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def parse_status(message: str) -> Optional[int]:
|
|
79
|
+
"""HTTP status from a Playwright/Puppeteer connect error message, if any."""
|
|
80
|
+
match = _WS_STATUS_RE.search(message) or _STATUS_RE.search(message)
|
|
81
|
+
return int(match.group(1)) if match else None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def parse_retry_after(message: str) -> Optional[float]:
|
|
85
|
+
match = _RETRY_AFTER_RE.search(message)
|
|
86
|
+
if not match:
|
|
87
|
+
return None
|
|
88
|
+
return float(match.group(1))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def classify_connect_error(err: BaseException) -> Tuple[bool, Optional[int], Optional[float]]:
|
|
92
|
+
"""Return ``(retryable, status, retry_after_secs)`` for a connect error."""
|
|
93
|
+
message = f'{type(err).__name__}: {err}'
|
|
94
|
+
status = parse_status(message)
|
|
95
|
+
retry_after = parse_retry_after(message)
|
|
96
|
+
if status is not None:
|
|
97
|
+
return (status == 408 or status == 429 or status >= 500), status, retry_after
|
|
98
|
+
if isinstance(err, (asyncio.TimeoutError, TimeoutError, ConnectionError, OSError)):
|
|
99
|
+
return True, None, retry_after
|
|
100
|
+
lowered = message.lower()
|
|
101
|
+
return any(marker in lowered for marker in _TRANSIENT_MARKERS), None, retry_after
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def backoff_delay(attempt: int, retry_after: Optional[float]) -> float:
|
|
105
|
+
"""Delay before retry number ``attempt`` (1-based), honoring Retry-After."""
|
|
106
|
+
base = _env_float('SCRAPELY_CDP_RETRY_BASE_SECS', 1.0)
|
|
107
|
+
cap = _env_float('SCRAPELY_CDP_RETRY_MAX_SECS', 30.0)
|
|
108
|
+
delay = min(cap, base * (2 ** (attempt - 1)))
|
|
109
|
+
delay = delay * (0.75 + random.random() * 0.5)
|
|
110
|
+
if retry_after is not None:
|
|
111
|
+
delay = max(delay, retry_after)
|
|
112
|
+
return delay
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _retry_after_too_long(retry_after: Optional[float]) -> bool:
|
|
116
|
+
return retry_after is not None and retry_after > _env_float('SCRAPELY_CDP_RETRY_AFTER_MAX_SECS', 120.0)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _describe(err: BaseException, status: Optional[int]) -> str:
|
|
120
|
+
if status is not None:
|
|
121
|
+
return f'HTTP {status}'
|
|
122
|
+
# First line only: connect errors can carry the session URL with ids.
|
|
123
|
+
text = str(err).strip().splitlines()[0] if str(err).strip() else type(err).__name__
|
|
124
|
+
return re.sub(r'(wss?|https?)://\S+', '<cdp-url>', text)[:200]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _log_retry(err: BaseException, status: Optional[int], attempt: int, attempts: int, delay: float) -> None:
|
|
128
|
+
logger.warning(
|
|
129
|
+
'Remote browser connect failed (%s); retrying in %.1fs (attempt %d/%d)',
|
|
130
|
+
_describe(err, status),
|
|
131
|
+
delay,
|
|
132
|
+
attempt + 1,
|
|
133
|
+
attempts,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
async def connect_with_retry_async(connect: Callable[[], Awaitable[T]]) -> T:
|
|
138
|
+
"""Run an async CDP connect, retrying transient failures."""
|
|
139
|
+
attempts = _max_attempts()
|
|
140
|
+
attempt = 1
|
|
141
|
+
while True:
|
|
142
|
+
try:
|
|
143
|
+
return await connect()
|
|
144
|
+
except asyncio.CancelledError:
|
|
145
|
+
raise
|
|
146
|
+
except Exception as err:
|
|
147
|
+
retryable, status, retry_after = classify_connect_error(err)
|
|
148
|
+
# Never retry earlier than the server asked; give up instead of waiting that long.
|
|
149
|
+
if not retryable or attempt >= attempts or _retry_after_too_long(retry_after):
|
|
150
|
+
raise
|
|
151
|
+
delay = backoff_delay(attempt, retry_after)
|
|
152
|
+
_log_retry(err, status, attempt, attempts, delay)
|
|
153
|
+
await asyncio.sleep(delay)
|
|
154
|
+
attempt += 1
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def connect_with_retry_sync(connect: Callable[[], T]) -> T:
|
|
158
|
+
"""Run a blocking CDP connect, retrying transient failures."""
|
|
159
|
+
attempts = _max_attempts()
|
|
160
|
+
attempt = 1
|
|
161
|
+
while True:
|
|
162
|
+
try:
|
|
163
|
+
return connect()
|
|
164
|
+
except Exception as err:
|
|
165
|
+
retryable, status, retry_after = classify_connect_error(err)
|
|
166
|
+
# Never retry earlier than the server asked; give up instead of waiting that long.
|
|
167
|
+
if not retryable or attempt >= attempts or _retry_after_too_long(retry_after):
|
|
168
|
+
raise
|
|
169
|
+
delay = backoff_delay(attempt, retry_after)
|
|
170
|
+
_log_retry(err, status, attempt, attempts, delay)
|
|
171
|
+
time.sleep(delay)
|
|
172
|
+
attempt += 1
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def is_platform_cdp_url(url: Any, platform_cdp_url: Optional[str]) -> bool:
|
|
176
|
+
"""True when ``url`` points at the same host as the platform's CDP_URL."""
|
|
177
|
+
if not isinstance(url, str) or not platform_cdp_url:
|
|
178
|
+
return False
|
|
179
|
+
try:
|
|
180
|
+
target, platform = urlsplit(url), urlsplit(platform_cdp_url)
|
|
181
|
+
except ValueError:
|
|
182
|
+
return False
|
|
183
|
+
return bool(target.netloc) and target.netloc.lower() == platform.netloc.lower()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def wrap_connect_over_cdp(chromium: Any, is_async: bool, get_platform_cdp_url: Callable[[], Optional[str]]) -> None:
|
|
187
|
+
"""Make ``chromium.connect_over_cdp`` retry transient failures for the platform CDP host.
|
|
188
|
+
|
|
189
|
+
Covers actor code that connects to ``CDP_URL`` directly as well as the
|
|
190
|
+
SDK's own redirected launches. Connects to other hosts are left alone.
|
|
191
|
+
Idempotent.
|
|
192
|
+
"""
|
|
193
|
+
original = getattr(chromium, 'connect_over_cdp', None)
|
|
194
|
+
if original is None or getattr(original, _RETRY_WRAPPED_ATTR, False):
|
|
195
|
+
return
|
|
196
|
+
|
|
197
|
+
def _target(args: Tuple[Any, ...], kwargs: dict) -> Any:
|
|
198
|
+
if args:
|
|
199
|
+
return args[0]
|
|
200
|
+
return kwargs.get('endpoint_url') or kwargs.get('endpointURL')
|
|
201
|
+
|
|
202
|
+
if is_async:
|
|
203
|
+
|
|
204
|
+
@functools.wraps(original)
|
|
205
|
+
async def connect_over_cdp(*args: Any, **kwargs: Any) -> Any:
|
|
206
|
+
if not is_platform_cdp_url(_target(args, kwargs), get_platform_cdp_url()):
|
|
207
|
+
return await original(*args, **kwargs)
|
|
208
|
+
return await connect_with_retry_async(lambda: original(*args, **kwargs))
|
|
209
|
+
|
|
210
|
+
else:
|
|
211
|
+
|
|
212
|
+
@functools.wraps(original)
|
|
213
|
+
def connect_over_cdp(*args: Any, **kwargs: Any) -> Any: # type: ignore[misc]
|
|
214
|
+
if not is_platform_cdp_url(_target(args, kwargs), get_platform_cdp_url()):
|
|
215
|
+
return original(*args, **kwargs)
|
|
216
|
+
return connect_with_retry_sync(lambda: original(*args, **kwargs))
|
|
217
|
+
|
|
218
|
+
setattr(connect_over_cdp, _RETRY_WRAPPED_ATTR, True)
|
|
219
|
+
try:
|
|
220
|
+
chromium.connect_over_cdp = connect_over_cdp
|
|
221
|
+
except (AttributeError, TypeError):
|
|
222
|
+
pass
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def is_retry_wrapped(fn: Any) -> bool:
|
|
226
|
+
return bool(getattr(fn, _RETRY_WRAPPED_ATTR, False))
|
scrapely/patch_browser_pool.py
CHANGED
|
@@ -70,6 +70,7 @@ from typing import Any, List, Optional, Tuple
|
|
|
70
70
|
from urllib.parse import SplitResult, parse_qsl, unquote, urlencode, urlsplit, urlunsplit
|
|
71
71
|
|
|
72
72
|
from ._browser_locations import subdivision_to_state_name
|
|
73
|
+
from .cdp_retry import connect_with_retry_async, wrap_connect_over_cdp
|
|
73
74
|
|
|
74
75
|
logger = logging.getLogger(__name__)
|
|
75
76
|
|
|
@@ -427,7 +428,9 @@ def _apply_patch() -> None:
|
|
|
427
428
|
cdp_url = apply_proxy_info_to_cdp_url(cdp_url, proxy_info, browser_configuration)
|
|
428
429
|
|
|
429
430
|
# Remote browser infrastructure is Chromium CDP (same as Node connectOverCDP).
|
|
430
|
-
|
|
431
|
+
chromium = self._playwright.chromium
|
|
432
|
+
wrap_connect_over_cdp(chromium, True, get_cdp_url)
|
|
433
|
+
browser = await chromium.connect_over_cdp(cdp_url)
|
|
431
434
|
|
|
432
435
|
controller = PlaywrightBrowserController(
|
|
433
436
|
browser,
|
|
@@ -518,7 +521,9 @@ def _apply_camoufox_patch() -> None:
|
|
|
518
521
|
# Same as the PlaywrightBrowserPlugin patch above: the first page's
|
|
519
522
|
# proxy info / browser configuration pins the connection's location.
|
|
520
523
|
cdp_url = apply_proxy_info_to_cdp_url(cdp_url, _launch_proxy_info.get(), browser_configuration)
|
|
521
|
-
|
|
524
|
+
chromium = playwright.chromium
|
|
525
|
+
wrap_connect_over_cdp(chromium, True, get_cdp_url)
|
|
526
|
+
return await chromium.connect_over_cdp(cdp_url)
|
|
522
527
|
|
|
523
528
|
camoufox.AsyncNewBrowser = _patched_async_new_browser
|
|
524
529
|
camoufox._scrapely_cdp_patched = True
|
|
@@ -726,6 +731,10 @@ def _patch_playwright_object(playwright: Any, is_async: bool) -> Any:
|
|
|
726
731
|
chromium = getattr(playwright, 'chromium', None)
|
|
727
732
|
if chromium is None or getattr(chromium, 'connect_over_cdp', None) is None:
|
|
728
733
|
return playwright
|
|
734
|
+
# Retry transient connect failures (429/5xx/network) for the platform CDP
|
|
735
|
+
# host: covers direct chromium.connect_over_cdp(CDP_URL) calls in actor
|
|
736
|
+
# code and the redirected launches below, which call this same attribute.
|
|
737
|
+
wrap_connect_over_cdp(chromium, is_async, get_cdp_url)
|
|
729
738
|
for name in ('chromium', 'firefox', 'webkit'):
|
|
730
739
|
browser_type = getattr(playwright, name, None)
|
|
731
740
|
if browser_type is None:
|
|
@@ -845,7 +854,9 @@ def _apply_pyppeteer_patch() -> None:
|
|
|
845
854
|
'ignored: %s (the remote browser manages headless mode, binary and flags).',
|
|
846
855
|
', '.join(sorted(merged.keys())),
|
|
847
856
|
)
|
|
848
|
-
return await
|
|
857
|
+
return await connect_with_retry_async(
|
|
858
|
+
lambda: pyppeteer_launcher.connect(browserWSEndpoint=cdp_url)
|
|
859
|
+
)
|
|
849
860
|
|
|
850
861
|
pyppeteer_launcher.launch = _patched_launch
|
|
851
862
|
if hasattr(pyppeteer_mod, 'launch'):
|
scrapely/storage/api_client.py
CHANGED
|
@@ -24,6 +24,32 @@ class ApiClientError(Exception):
|
|
|
24
24
|
self.body = body
|
|
25
25
|
|
|
26
26
|
|
|
27
|
+
# PUT is left out: a KV record PUT is counted (and billed) as a write each time
|
|
28
|
+
# it is applied, so a lost response must not be re-sent blindly.
|
|
29
|
+
IDEMPOTENT_METHODS = frozenset({'GET', 'HEAD', 'DELETE', 'OPTIONS'})
|
|
30
|
+
# HAProxy's own 503 page: no backend was available, so nothing was forwarded.
|
|
31
|
+
NO_BACKEND_MARKER = 'No server is available to handle this request'
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def is_retryable_response(
|
|
35
|
+
status: int, idempotent: bool, error_type: Optional[str] = None, body_text: Optional[str] = None
|
|
36
|
+
) -> bool:
|
|
37
|
+
"""Whether a non-2xx answer is retried.
|
|
38
|
+
|
|
39
|
+
Idempotent methods: 500/502/503/504. Others (dataset push, KV PUT) only on
|
|
40
|
+
a 503 that certainly was not processed: the rate limiter refused it
|
|
41
|
+
("rate-limit-unavailable", before the handler), or HAProxy had no backend
|
|
42
|
+
(its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
|
|
43
|
+
(a direct dataset write whose commit was not confirmed), and re-sending
|
|
44
|
+
could store the items twice.
|
|
45
|
+
"""
|
|
46
|
+
if idempotent:
|
|
47
|
+
return status in (500, 502, 503, 504)
|
|
48
|
+
if status != 503:
|
|
49
|
+
return False
|
|
50
|
+
return error_type == 'rate-limit-unavailable' or NO_BACKEND_MARKER in (body_text or '')
|
|
51
|
+
|
|
52
|
+
|
|
27
53
|
def unwrap_data(value: Any) -> Any:
|
|
28
54
|
"""Unwrap Apify-style v2 response envelopes ({ data: ... })."""
|
|
29
55
|
if isinstance(value, list):
|
|
@@ -33,6 +59,35 @@ def unwrap_data(value: Any) -> Any:
|
|
|
33
59
|
return value
|
|
34
60
|
|
|
35
61
|
|
|
62
|
+
def _error_type(response: httpx.Response) -> Optional[str]:
|
|
63
|
+
try:
|
|
64
|
+
body = response.json()
|
|
65
|
+
except Exception:
|
|
66
|
+
return None
|
|
67
|
+
if isinstance(body, dict) and isinstance(body.get('error'), dict):
|
|
68
|
+
t = body['error'].get('type')
|
|
69
|
+
return t if isinstance(t, str) else None
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def server_retry_delay(server_attempt: int, retry_after: Optional[str], now: Optional[float] = None) -> float:
|
|
74
|
+
"""Exponential backoff with jitter, in seconds; honours a Retry-After (seconds or HTTP date) of up to 30 s."""
|
|
75
|
+
if retry_after is not None and retry_after.strip():
|
|
76
|
+
secs: Optional[float] = None
|
|
77
|
+
try:
|
|
78
|
+
secs = float(retry_after)
|
|
79
|
+
except ValueError:
|
|
80
|
+
try:
|
|
81
|
+
from email.utils import parsedate_to_datetime
|
|
82
|
+
|
|
83
|
+
secs = max(0.0, parsedate_to_datetime(retry_after).timestamp() - (time.time() if now is None else now))
|
|
84
|
+
except Exception:
|
|
85
|
+
secs = None
|
|
86
|
+
if secs is not None and secs >= 0:
|
|
87
|
+
return min(secs, 30.0) + random.random() * 0.1
|
|
88
|
+
return min(8.0, 0.5 * 2 ** server_attempt) + random.random() * 0.25
|
|
89
|
+
|
|
90
|
+
|
|
36
91
|
class AdaptiveRateLimiter:
|
|
37
92
|
"""Adaptive throttling based on X-RateLimit-* headers (TS AdaptiveRateLimiter)."""
|
|
38
93
|
|
|
@@ -116,6 +171,10 @@ class ApiClient:
|
|
|
116
171
|
|
|
117
172
|
_rate_limiter = AdaptiveRateLimiter()
|
|
118
173
|
_max_retries = 3
|
|
174
|
+
# Retries while the API is briefly unavailable (503/5xx, connection
|
|
175
|
+
# refused): backoff 0.5 s doubling to 8 s, about 45 s in total, which
|
|
176
|
+
# covers a main redeploy.
|
|
177
|
+
_max_server_retries = 8
|
|
119
178
|
|
|
120
179
|
def __init__(
|
|
121
180
|
self,
|
|
@@ -198,6 +257,7 @@ class ApiClient:
|
|
|
198
257
|
headers: Optional[Dict[str, str]] = None,
|
|
199
258
|
raise_for_status: bool = True,
|
|
200
259
|
do_not_retry_timeouts: bool = False,
|
|
260
|
+
idempotent: Optional[bool] = None,
|
|
201
261
|
) -> httpx.Response:
|
|
202
262
|
"""Make an HTTP request with adaptive rate limiting and 429 retries."""
|
|
203
263
|
client = await self._ensure_client()
|
|
@@ -208,8 +268,11 @@ class ApiClient:
|
|
|
208
268
|
if json_data is not None and 'Content-Type' not in request_headers:
|
|
209
269
|
request_headers['Content-Type'] = 'application/json'
|
|
210
270
|
|
|
211
|
-
|
|
212
|
-
|
|
271
|
+
if idempotent is None:
|
|
272
|
+
idempotent = method.upper() in IDEMPOTENT_METHODS
|
|
273
|
+
attempt = 0 # timeouts and 429s
|
|
274
|
+
server_attempt = 0 # 503/5xx and connection failures (API briefly unavailable)
|
|
275
|
+
while True:
|
|
213
276
|
await self._rate_limiter.wait_for_slot()
|
|
214
277
|
try:
|
|
215
278
|
response = await client.request(
|
|
@@ -220,27 +283,59 @@ class ApiClient:
|
|
|
220
283
|
content=content,
|
|
221
284
|
headers=request_headers or None,
|
|
222
285
|
)
|
|
223
|
-
except httpx.TimeoutException
|
|
224
|
-
last_error = exc
|
|
286
|
+
except httpx.TimeoutException:
|
|
225
287
|
if do_not_retry_timeouts or attempt >= self._max_retries:
|
|
226
288
|
raise
|
|
227
|
-
|
|
289
|
+
attempt += 1
|
|
290
|
+
await asyncio.sleep(0.25 * attempt)
|
|
228
291
|
continue
|
|
292
|
+
except httpx.TransportError as exc:
|
|
293
|
+
# No response. A refused/unresolvable connection never reached
|
|
294
|
+
# the API, so it is safe for every method; anything else (reset
|
|
295
|
+
# mid-request) only for idempotent ones.
|
|
296
|
+
if (idempotent or isinstance(exc, httpx.ConnectError)) and server_attempt < self._max_server_retries:
|
|
297
|
+
wait = server_retry_delay(server_attempt, None)
|
|
298
|
+
server_attempt += 1
|
|
299
|
+
logger.warning(
|
|
300
|
+
'%s %s failed (%s); retry %d/%d in %.1fs',
|
|
301
|
+
method, path, type(exc).__name__, server_attempt, self._max_server_retries, wait,
|
|
302
|
+
)
|
|
303
|
+
await asyncio.sleep(wait)
|
|
304
|
+
continue
|
|
305
|
+
raise
|
|
229
306
|
|
|
230
307
|
self._rate_limiter.update_from_headers(response.headers)
|
|
231
308
|
|
|
232
309
|
if response.status_code == 429 and attempt < self._max_retries:
|
|
310
|
+
attempt += 1
|
|
233
311
|
await self._rate_limiter.handle_rate_limited(response)
|
|
234
312
|
continue
|
|
235
313
|
|
|
314
|
+
# API briefly unavailable (main redeploy, limiter store blip); see
|
|
315
|
+
# is_retryable_response for which answers are safe to re-send.
|
|
316
|
+
if (
|
|
317
|
+
not response.is_success
|
|
318
|
+
and server_attempt < self._max_server_retries
|
|
319
|
+
and is_retryable_response(
|
|
320
|
+
response.status_code,
|
|
321
|
+
idempotent,
|
|
322
|
+
_error_type(response) if response.status_code == 503 else None,
|
|
323
|
+
response.text if response.status_code == 503 else None,
|
|
324
|
+
)
|
|
325
|
+
):
|
|
326
|
+
wait = server_retry_delay(server_attempt, response.headers.get('Retry-After'))
|
|
327
|
+
server_attempt += 1
|
|
328
|
+
logger.warning(
|
|
329
|
+
'%s %s answered %d; retry %d/%d in %.1fs',
|
|
330
|
+
method, path, response.status_code, server_attempt, self._max_server_retries, wait,
|
|
331
|
+
)
|
|
332
|
+
await asyncio.sleep(wait)
|
|
333
|
+
continue
|
|
334
|
+
|
|
236
335
|
if raise_for_status:
|
|
237
336
|
await self._raise_for_status(response)
|
|
238
337
|
return response
|
|
239
338
|
|
|
240
|
-
if last_error is not None:
|
|
241
|
-
raise last_error
|
|
242
|
-
raise ApiClientError('Request failed after retries', 429)
|
|
243
|
-
|
|
244
339
|
async def get(
|
|
245
340
|
self,
|
|
246
341
|
path: str,
|
|
@@ -268,6 +363,7 @@ class ApiClient:
|
|
|
268
363
|
params: Optional[Dict[str, Any]] = None,
|
|
269
364
|
*,
|
|
270
365
|
raise_for_status: bool = True,
|
|
366
|
+
idempotent: Optional[bool] = None,
|
|
271
367
|
) -> httpx.Response:
|
|
272
368
|
return await self.request(
|
|
273
369
|
'POST',
|
|
@@ -277,6 +373,7 @@ class ApiClient:
|
|
|
277
373
|
content=content,
|
|
278
374
|
headers=headers,
|
|
279
375
|
raise_for_status=raise_for_status,
|
|
376
|
+
idempotent=idempotent,
|
|
280
377
|
)
|
|
281
378
|
|
|
282
379
|
async def put(
|
|
@@ -325,8 +422,10 @@ class ApiClient:
|
|
|
325
422
|
data: Any = None,
|
|
326
423
|
params: Optional[Dict[str, Any]] = None,
|
|
327
424
|
headers: Optional[Dict[str, str]] = None,
|
|
425
|
+
*,
|
|
426
|
+
idempotent: Optional[bool] = None,
|
|
328
427
|
) -> Any:
|
|
329
|
-
response = await self.post(path, json_data=data, params=params, headers=headers)
|
|
428
|
+
response = await self.post(path, json_data=data, params=params, headers=headers, idempotent=idempotent)
|
|
330
429
|
if not response.content:
|
|
331
430
|
return {}
|
|
332
431
|
try:
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
scrapely/__init__.py,sha256=
|
|
1
|
+
scrapely/__init__.py,sha256=IcAS39wGSGYhV2mkHPkaTPFdZyoTjdVln3wBZaCAwfo,5745
|
|
2
2
|
scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
|
|
3
|
-
scrapely/actor.py,sha256=
|
|
3
|
+
scrapely/actor.py,sha256=j61rLc2JXxmIkRi1M_Gym_G4PVcK19NfOp0dVpEu_Bw,76660
|
|
4
4
|
scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
|
|
5
|
+
scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
|
|
5
6
|
scrapely/charging.py,sha256=MFz_Z4qJ-N3qLffleQb2BxjomygM3UMvHSBw1HabG80,56857
|
|
6
7
|
scrapely/charging_client.py,sha256=tDtlPwLSVFvoRa2NBgNjyfHkAUM77SSIyYT2ACgpDm8,3710
|
|
7
8
|
scrapely/configuration.py,sha256=YUnbft9JeWlVTgWalr-CgJCiyJ_e_rgoQItVPvjeWRs,15406
|
|
8
9
|
scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
|
|
9
10
|
scrapely/log.py,sha256=iVKoQdMLQXEWz1STzbSXrNUxsS_FNwiIdqoEg7CzeqU,1440
|
|
10
|
-
scrapely/patch_browser_pool.py,sha256=
|
|
11
|
+
scrapely/patch_browser_pool.py,sha256=qZstdQ1rFw8kGjzB9fm3GXco8Jt40eklTw1Czf_vh-8,38018
|
|
11
12
|
scrapely/pricing_types.py,sha256=J6wCddZd-U1MYeOtaaEdeHSsp1HBgiQLfvSInBOvhIo,5305
|
|
12
13
|
scrapely/proxy_configuration.py,sha256=rSCXJFuUEIK31chk05FVLRQKeVA34h_pBRbr8IKNXo0,36195
|
|
13
14
|
scrapely/py.typed,sha256=XtWlywqUjc6x_M2gAXlDtYXbAsYkCjKZEjFyJvaHvm4,105
|
|
@@ -22,7 +23,7 @@ scrapely/request_loaders/__init__.py,sha256=aNgr4pRt3MziUbU6UQ_bjTT3moMNvvyUsPeC
|
|
|
22
23
|
scrapely/request_loaders/_scrapely_request_list.py,sha256=cq7xx7Ai4qOhcqGV2TLqPR3o53FPCPKpVxWEMw5qKaM,7367
|
|
23
24
|
scrapely/storage/__init__.py,sha256=diAhxEvOXjc3YVhZvzkO3MttJUXQWJ5ogxstiApsh7U,1405
|
|
24
25
|
scrapely/storage/_metadata.py,sha256=WWawBnVN0b1iK2K-SlxZXUc6czb5wXWf8iC_0tzkgAQ,4764
|
|
25
|
-
scrapely/storage/api_client.py,sha256=
|
|
26
|
+
scrapely/storage/api_client.py,sha256=q48maCfgEnyWGD_gIhehiBvA30AlbqKkNjouNu6BIa4,15870
|
|
26
27
|
scrapely/storage/client.py,sha256=M3o_a_9azHNn6eL_UyEeVz-VQJvnT_th1HJPuCA4PkE,21746
|
|
27
28
|
scrapely/storage/patch_storage.py,sha256=o3Fv0BByeZ1T9HhkzhY6bjolOFsIQcnqBoHq1EdG8Fw,8758
|
|
28
29
|
scrapely/storage/scrapely_dataset_client.py,sha256=gRvlCDw9c0yO5GE5LQOa1xAyXc9pnb-E37Eq7fHqHCM,10338
|
|
@@ -37,7 +38,7 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
|
|
|
37
38
|
scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
|
|
38
39
|
scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
|
|
39
40
|
scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
|
|
40
|
-
scrapely_python-0.1.
|
|
41
|
-
scrapely_python-0.1.
|
|
42
|
-
scrapely_python-0.1.
|
|
43
|
-
scrapely_python-0.1.
|
|
41
|
+
scrapely_python-0.1.6.dist-info/METADATA,sha256=M7O94gZCIK2wmgvm4JCGYK6Z2pcaolNCjpRlkF_jsQI,3104
|
|
42
|
+
scrapely_python-0.1.6.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
43
|
+
scrapely_python-0.1.6.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
|
|
44
|
+
scrapely_python-0.1.6.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|