scrapely-python 0.1.4__tar.gz → 0.1.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of scrapely-python might be problematic. Click here for more details.

Files changed (62) hide show
  1. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/PKG-INFO +1 -1
  2. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/pyproject.toml +1 -1
  3. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/__init__.py +1 -1
  4. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/actor.py +43 -10
  5. scrapely_python-0.1.6/src/scrapely/cdp_retry.py +226 -0
  6. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/patch_browser_pool.py +14 -3
  7. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/api_client.py +109 -10
  8. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely_python.egg-info/PKG-INFO +1 -1
  9. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely_python.egg-info/SOURCES.txt +5 -1
  10. scrapely_python-0.1.6/tests/test_api_client_retries.py +170 -0
  11. scrapely_python-0.1.6/tests/test_cdp_retry.py +169 -0
  12. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/tests/test_storage.py +5 -1
  13. scrapely_python-0.1.6/tests/test_terminal_status_message.py +130 -0
  14. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/README.md +0 -0
  15. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/setup.cfg +0 -0
  16. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/_browser_locations.py +0 -0
  17. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/browser_configuration.py +0 -0
  18. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/charging.py +0 -0
  19. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/charging_client.py +0 -0
  20. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/configuration.py +0 -0
  21. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/crypto/__init__.py +0 -0
  22. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/crypto/_secrets.py +0 -0
  23. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/errors.py +0 -0
  24. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/events/__init__.py +0 -0
  25. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/events/_types.py +0 -0
  26. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/events/event_manager.py +0 -0
  27. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/log.py +0 -0
  28. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/pricing_types.py +0 -0
  29. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/proxy_configuration.py +0 -0
  30. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/py.typed +0 -0
  31. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/request_loaders/__init__.py +0 -0
  32. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/request_loaders/_scrapely_request_list.py +0 -0
  33. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/__init__.py +0 -0
  34. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/_file_system/__init__.py +0 -0
  35. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/_file_system/_key_value_store_client.py +0 -0
  36. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/_file_system/_storage_client.py +0 -0
  37. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/_metadata.py +0 -0
  38. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/client.py +0 -0
  39. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/patch_storage.py +0 -0
  40. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/scrapely_dataset_client.py +0 -0
  41. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/scrapely_kvs_client.py +0 -0
  42. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/scrapely_rq_client.py +0 -0
  43. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/scrapely_storage_client.py +0 -0
  44. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/smart_scrapely_storage_client.py +0 -0
  45. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/types.py +0 -0
  46. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage/utils.py +0 -0
  47. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storage_clients/__init__.py +0 -0
  48. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/storages/__init__.py +0 -0
  49. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/types.py +0 -0
  50. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely/webhook.py +0 -0
  51. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely_python.egg-info/dependency_links.txt +0 -0
  52. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely_python.egg-info/requires.txt +0 -0
  53. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/src/scrapely_python.egg-info/top_level.txt +0 -0
  54. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/tests/test_apify_parity.py +0 -0
  55. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/tests/test_browser_configuration.py +0 -0
  56. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/tests/test_charging_parity.py +0 -0
  57. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/tests/test_charging_pending.py +0 -0
  58. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/tests/test_parity.py +0 -0
  59. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/tests/test_patch_browser_pool.py +0 -0
  60. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/tests/test_proxy_configuration.py +0 -0
  61. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/tests/test_run_control.py +0 -0
  62. {scrapely_python-0.1.4 → scrapely_python-0.1.6}/tests/test_secrets.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-python
3
- Version: 0.1.4
3
+ Version: 0.1.6
4
4
  Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
5
5
  Author-email: Scrapely <support@scrape.ly>
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "scrapely-python"
7
- version = "0.1.4"
7
+ version = "0.1.6"
8
8
  description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -132,7 +132,7 @@ from .errors import (
132
132
  raise_for_status,
133
133
  )
134
134
 
135
- __version__ = '0.1.4'
135
+ __version__ = '0.1.6'
136
136
  __all__ = [
137
137
  # Core
138
138
  'Actor',
@@ -18,7 +18,7 @@ import warnings
18
18
  from collections.abc import MutableMapping
19
19
  from datetime import datetime, timedelta
20
20
  from decimal import Decimal
21
- from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast
21
+ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union, cast
22
22
 
23
23
  from crawlee.configuration import Configuration as CrawleeConfiguration
24
24
  from crawlee.storage_clients._base import StorageClient
@@ -287,6 +287,7 @@ class Actor(metaclass=_ActorMeta):
287
287
  cls._exit_code = self._instance_exit_code
288
288
  if self._instance_status_message:
289
289
  cls._status_message = self._instance_status_message
290
+ await cls._persist_terminal_status_message(self._instance_status_message)
290
291
  await cls._shutdown(timeout_secs=int(self._cleanup_timeout.total_seconds()))
291
292
 
292
293
  @classmethod
@@ -589,6 +590,7 @@ class Actor(metaclass=_ActorMeta):
589
590
  cls._exit_code = opts.exit_code
590
591
  cls._status_message = opts.status_message
591
592
 
593
+ await cls._persist_terminal_status_message(opts.status_message)
592
594
  await cls._shutdown(timeout_secs=opts.timeout_secs)
593
595
 
594
596
  if opts.exit:
@@ -1021,13 +1023,32 @@ class Actor(metaclass=_ActorMeta):
1021
1023
  else:
1022
1024
  logger.info(logged)
1023
1025
 
1026
+ result = await cls._put_run_status_message(message, is_terminal=is_terminal)
1027
+ if result is None:
1028
+ return ActorRun(id='', actor_id='', status='')
1029
+ run_id, data = result
1030
+
1031
+ return ActorRun(
1032
+ id=data.get('id', run_id),
1033
+ actor_id=data.get('actorId', ''),
1034
+ status=data.get('status', ''),
1035
+ started_at=data.get('startedAt'),
1036
+ finished_at=data.get('finishedAt'),
1037
+ status_message=data.get('statusMessage'),
1038
+ )
1039
+
1040
+ @classmethod
1041
+ async def _put_run_status_message(
1042
+ cls, message: str, *, is_terminal: bool
1043
+ ) -> Optional[Tuple[str, Dict[str, Any]]]:
1044
+ """PUT the status message to the current run. Returns ``None`` when not on the platform."""
1024
1045
  config = cls.get_config()
1025
1046
  api_url = config.get('api_base_url')
1026
1047
  token = config.get('token')
1027
1048
  run_id = config.get('actor_run_id')
1028
1049
 
1029
1050
  if not api_url or not token or not run_id:
1030
- return ActorRun(id='', actor_id='', status='')
1051
+ return None
1031
1052
 
1032
1053
  client = cls.new_client(token=token, base_url=api_url)
1033
1054
  response = await client.put(
@@ -1041,15 +1062,27 @@ class Actor(metaclass=_ActorMeta):
1041
1062
  if response.content:
1042
1063
  body = response.json()
1043
1064
  data = body.get('data', body) if isinstance(body, dict) else {}
1065
+ return run_id, data if isinstance(data, dict) else {}
1044
1066
 
1045
- return ActorRun(
1046
- id=data.get('id', run_id),
1047
- actor_id=data.get('actorId', ''),
1048
- status=data.get('status', ''),
1049
- started_at=data.get('startedAt'),
1050
- finished_at=data.get('finishedAt'),
1051
- status_message=data.get('statusMessage'),
1052
- )
1067
+ @classmethod
1068
+ async def _persist_terminal_status_message(cls, message: Optional[str], timeout_secs: float = 10.0) -> None:
1069
+ """Store the exit/fail status message on the run as terminal (Apify parity).
1070
+
1071
+ Runs before teardown so it lands while the run still exists and the
1072
+ process is alive, on success and failure alike. Best effort: a failed
1073
+ update is logged and never blocks the exit.
1074
+ """
1075
+ if not message or not cls._initialized or cls._config is None:
1076
+ return
1077
+ try:
1078
+ await asyncio.wait_for(
1079
+ cls._put_run_status_message(message, is_terminal=True),
1080
+ timeout=max(1.0, timeout_secs),
1081
+ )
1082
+ except asyncio.CancelledError:
1083
+ raise
1084
+ except BaseException as e:
1085
+ logger.warning('Failed to store the final status message on the run: %s', e)
1053
1086
 
1054
1087
  @_ClassProperty
1055
1088
  def log(cls) -> logging.Logger:
@@ -0,0 +1,226 @@
1
+ """Retry remote-browser (CDP) connects that fail for transient reasons.
2
+
3
+ The remote browser answers ``429`` when it is at capacity and ``503`` when it
4
+ cannot start a browser or validate the session right now, both with a
5
+ ``Retry-After`` delay (also repeated in the response body, because Playwright
6
+ only puts the status line and body into its error message). Connection errors
7
+ and timeouts are transient too. Those are retried with exponential backoff,
8
+ waiting at least the ``Retry-After`` delay when the error carries one. Any
9
+ other status (``400``/``401``/``402``/``403``/``404``) fails at once.
10
+
11
+ Tuning (environment):
12
+
13
+ - ``SCRAPELY_CDP_CONNECT_ATTEMPTS``: total attempts, default ``5`` (``1`` disables retries).
14
+ - ``SCRAPELY_CDP_RETRY_BASE_SECS``: first backoff delay, default ``1``; doubles per attempt.
15
+ - ``SCRAPELY_CDP_RETRY_MAX_SECS``: cap for one backoff delay, default ``30``.
16
+ - ``SCRAPELY_CDP_RETRY_AFTER_MAX_SECS``: longest ``Retry-After`` the SDK waits
17
+ out, default ``120``. A longer ``Retry-After`` ends the retries (the SDK never
18
+ retries earlier than the server asked).
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import asyncio
24
+ import functools
25
+ import logging
26
+ import os
27
+ import random
28
+ import re
29
+ import time
30
+ from typing import Any, Awaitable, Callable, Optional, Tuple, TypeVar
31
+ from urllib.parse import urlsplit
32
+
33
+ logger = logging.getLogger('scrapely.browser')
34
+
35
+ T = TypeVar('T')
36
+
37
+ _RETRY_WRAPPED_ATTR = '_scrapely_cdp_retry_wrapped'
38
+
39
+ _REASONS = (
40
+ 'Too Many Requests|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Timeout|'
41
+ 'Request Timeout|Bad Request|Unauthorized|Payment Required|Forbidden|Not Found'
42
+ )
43
+ _STATUS_RE = re.compile(rf'\b([1-5]\d\d) (?:{_REASONS})\b')
44
+ _WS_STATUS_RE = re.compile(r'Unexpected server response:?\s*(\d{3})', re.IGNORECASE)
45
+ _RETRY_AFTER_RE = re.compile(r'Retry-After:?\s*(\d+(?:\.\d+)?)', re.IGNORECASE)
46
+ _TRANSIENT_MARKERS = (
47
+ 'econnrefused',
48
+ 'econnreset',
49
+ 'etimedout',
50
+ 'epipe',
51
+ 'eai_again',
52
+ 'ehostunreach',
53
+ 'enetunreach',
54
+ 'socket hang up',
55
+ 'timeout',
56
+ 'timed out',
57
+ 'connection closed',
58
+ 'connection refused',
59
+ 'connection reset',
60
+ 'websocket error',
61
+ 'net::err_',
62
+ 'target closed',
63
+ )
64
+
65
+
66
+ def _env_float(name: str, default: float) -> float:
67
+ try:
68
+ value = float(os.environ.get(name, ''))
69
+ except ValueError:
70
+ return default
71
+ return value if value > 0 else default
72
+
73
+
74
+ def _max_attempts() -> int:
75
+ return max(1, int(_env_float('SCRAPELY_CDP_CONNECT_ATTEMPTS', 5)))
76
+
77
+
78
+ def parse_status(message: str) -> Optional[int]:
79
+ """HTTP status from a Playwright/Puppeteer connect error message, if any."""
80
+ match = _WS_STATUS_RE.search(message) or _STATUS_RE.search(message)
81
+ return int(match.group(1)) if match else None
82
+
83
+
84
+ def parse_retry_after(message: str) -> Optional[float]:
85
+ match = _RETRY_AFTER_RE.search(message)
86
+ if not match:
87
+ return None
88
+ return float(match.group(1))
89
+
90
+
91
+ def classify_connect_error(err: BaseException) -> Tuple[bool, Optional[int], Optional[float]]:
92
+ """Return ``(retryable, status, retry_after_secs)`` for a connect error."""
93
+ message = f'{type(err).__name__}: {err}'
94
+ status = parse_status(message)
95
+ retry_after = parse_retry_after(message)
96
+ if status is not None:
97
+ return (status == 408 or status == 429 or status >= 500), status, retry_after
98
+ if isinstance(err, (asyncio.TimeoutError, TimeoutError, ConnectionError, OSError)):
99
+ return True, None, retry_after
100
+ lowered = message.lower()
101
+ return any(marker in lowered for marker in _TRANSIENT_MARKERS), None, retry_after
102
+
103
+
104
+ def backoff_delay(attempt: int, retry_after: Optional[float]) -> float:
105
+ """Delay before retry number ``attempt`` (1-based), honoring Retry-After."""
106
+ base = _env_float('SCRAPELY_CDP_RETRY_BASE_SECS', 1.0)
107
+ cap = _env_float('SCRAPELY_CDP_RETRY_MAX_SECS', 30.0)
108
+ delay = min(cap, base * (2 ** (attempt - 1)))
109
+ delay = delay * (0.75 + random.random() * 0.5)
110
+ if retry_after is not None:
111
+ delay = max(delay, retry_after)
112
+ return delay
113
+
114
+
115
+ def _retry_after_too_long(retry_after: Optional[float]) -> bool:
116
+ return retry_after is not None and retry_after > _env_float('SCRAPELY_CDP_RETRY_AFTER_MAX_SECS', 120.0)
117
+
118
+
119
+ def _describe(err: BaseException, status: Optional[int]) -> str:
120
+ if status is not None:
121
+ return f'HTTP {status}'
122
+ # First line only: connect errors can carry the session URL with ids.
123
+ text = str(err).strip().splitlines()[0] if str(err).strip() else type(err).__name__
124
+ return re.sub(r'(wss?|https?)://\S+', '<cdp-url>', text)[:200]
125
+
126
+
127
+ def _log_retry(err: BaseException, status: Optional[int], attempt: int, attempts: int, delay: float) -> None:
128
+ logger.warning(
129
+ 'Remote browser connect failed (%s); retrying in %.1fs (attempt %d/%d)',
130
+ _describe(err, status),
131
+ delay,
132
+ attempt + 1,
133
+ attempts,
134
+ )
135
+
136
+
137
+ async def connect_with_retry_async(connect: Callable[[], Awaitable[T]]) -> T:
138
+ """Run an async CDP connect, retrying transient failures."""
139
+ attempts = _max_attempts()
140
+ attempt = 1
141
+ while True:
142
+ try:
143
+ return await connect()
144
+ except asyncio.CancelledError:
145
+ raise
146
+ except Exception as err:
147
+ retryable, status, retry_after = classify_connect_error(err)
148
+ # Never retry earlier than the server asked; give up instead of waiting that long.
149
+ if not retryable or attempt >= attempts or _retry_after_too_long(retry_after):
150
+ raise
151
+ delay = backoff_delay(attempt, retry_after)
152
+ _log_retry(err, status, attempt, attempts, delay)
153
+ await asyncio.sleep(delay)
154
+ attempt += 1
155
+
156
+
157
+ def connect_with_retry_sync(connect: Callable[[], T]) -> T:
158
+ """Run a blocking CDP connect, retrying transient failures."""
159
+ attempts = _max_attempts()
160
+ attempt = 1
161
+ while True:
162
+ try:
163
+ return connect()
164
+ except Exception as err:
165
+ retryable, status, retry_after = classify_connect_error(err)
166
+ # Never retry earlier than the server asked; give up instead of waiting that long.
167
+ if not retryable or attempt >= attempts or _retry_after_too_long(retry_after):
168
+ raise
169
+ delay = backoff_delay(attempt, retry_after)
170
+ _log_retry(err, status, attempt, attempts, delay)
171
+ time.sleep(delay)
172
+ attempt += 1
173
+
174
+
175
+ def is_platform_cdp_url(url: Any, platform_cdp_url: Optional[str]) -> bool:
176
+ """True when ``url`` points at the same host as the platform's CDP_URL."""
177
+ if not isinstance(url, str) or not platform_cdp_url:
178
+ return False
179
+ try:
180
+ target, platform = urlsplit(url), urlsplit(platform_cdp_url)
181
+ except ValueError:
182
+ return False
183
+ return bool(target.netloc) and target.netloc.lower() == platform.netloc.lower()
184
+
185
+
186
+ def wrap_connect_over_cdp(chromium: Any, is_async: bool, get_platform_cdp_url: Callable[[], Optional[str]]) -> None:
187
+ """Make ``chromium.connect_over_cdp`` retry transient failures for the platform CDP host.
188
+
189
+ Covers actor code that connects to ``CDP_URL`` directly as well as the
190
+ SDK's own redirected launches. Connects to other hosts are left alone.
191
+ Idempotent.
192
+ """
193
+ original = getattr(chromium, 'connect_over_cdp', None)
194
+ if original is None or getattr(original, _RETRY_WRAPPED_ATTR, False):
195
+ return
196
+
197
+ def _target(args: Tuple[Any, ...], kwargs: dict) -> Any:
198
+ if args:
199
+ return args[0]
200
+ return kwargs.get('endpoint_url') or kwargs.get('endpointURL')
201
+
202
+ if is_async:
203
+
204
+ @functools.wraps(original)
205
+ async def connect_over_cdp(*args: Any, **kwargs: Any) -> Any:
206
+ if not is_platform_cdp_url(_target(args, kwargs), get_platform_cdp_url()):
207
+ return await original(*args, **kwargs)
208
+ return await connect_with_retry_async(lambda: original(*args, **kwargs))
209
+
210
+ else:
211
+
212
+ @functools.wraps(original)
213
+ def connect_over_cdp(*args: Any, **kwargs: Any) -> Any: # type: ignore[misc]
214
+ if not is_platform_cdp_url(_target(args, kwargs), get_platform_cdp_url()):
215
+ return original(*args, **kwargs)
216
+ return connect_with_retry_sync(lambda: original(*args, **kwargs))
217
+
218
+ setattr(connect_over_cdp, _RETRY_WRAPPED_ATTR, True)
219
+ try:
220
+ chromium.connect_over_cdp = connect_over_cdp
221
+ except (AttributeError, TypeError):
222
+ pass
223
+
224
+
225
+ def is_retry_wrapped(fn: Any) -> bool:
226
+ return bool(getattr(fn, _RETRY_WRAPPED_ATTR, False))
@@ -70,6 +70,7 @@ from typing import Any, List, Optional, Tuple
70
70
  from urllib.parse import SplitResult, parse_qsl, unquote, urlencode, urlsplit, urlunsplit
71
71
 
72
72
  from ._browser_locations import subdivision_to_state_name
73
+ from .cdp_retry import connect_with_retry_async, wrap_connect_over_cdp
73
74
 
74
75
  logger = logging.getLogger(__name__)
75
76
 
@@ -427,7 +428,9 @@ def _apply_patch() -> None:
427
428
  cdp_url = apply_proxy_info_to_cdp_url(cdp_url, proxy_info, browser_configuration)
428
429
 
429
430
  # Remote browser infrastructure is Chromium CDP (same as Node connectOverCDP).
430
- browser = await self._playwright.chromium.connect_over_cdp(cdp_url)
431
+ chromium = self._playwright.chromium
432
+ wrap_connect_over_cdp(chromium, True, get_cdp_url)
433
+ browser = await chromium.connect_over_cdp(cdp_url)
431
434
 
432
435
  controller = PlaywrightBrowserController(
433
436
  browser,
@@ -518,7 +521,9 @@ def _apply_camoufox_patch() -> None:
518
521
  # Same as the PlaywrightBrowserPlugin patch above: the first page's
519
522
  # proxy info / browser configuration pins the connection's location.
520
523
  cdp_url = apply_proxy_info_to_cdp_url(cdp_url, _launch_proxy_info.get(), browser_configuration)
521
- return await playwright.chromium.connect_over_cdp(cdp_url)
524
+ chromium = playwright.chromium
525
+ wrap_connect_over_cdp(chromium, True, get_cdp_url)
526
+ return await chromium.connect_over_cdp(cdp_url)
522
527
 
523
528
  camoufox.AsyncNewBrowser = _patched_async_new_browser
524
529
  camoufox._scrapely_cdp_patched = True
@@ -726,6 +731,10 @@ def _patch_playwright_object(playwright: Any, is_async: bool) -> Any:
726
731
  chromium = getattr(playwright, 'chromium', None)
727
732
  if chromium is None or getattr(chromium, 'connect_over_cdp', None) is None:
728
733
  return playwright
734
+ # Retry transient connect failures (429/5xx/network) for the platform CDP
735
+ # host: covers direct chromium.connect_over_cdp(CDP_URL) calls in actor
736
+ # code and the redirected launches below, which call this same attribute.
737
+ wrap_connect_over_cdp(chromium, is_async, get_cdp_url)
729
738
  for name in ('chromium', 'firefox', 'webkit'):
730
739
  browser_type = getattr(playwright, name, None)
731
740
  if browser_type is None:
@@ -845,7 +854,9 @@ def _apply_pyppeteer_patch() -> None:
845
854
  'ignored: %s (the remote browser manages headless mode, binary and flags).',
846
855
  ', '.join(sorted(merged.keys())),
847
856
  )
848
- return await pyppeteer_launcher.connect(browserWSEndpoint=cdp_url)
857
+ return await connect_with_retry_async(
858
+ lambda: pyppeteer_launcher.connect(browserWSEndpoint=cdp_url)
859
+ )
849
860
 
850
861
  pyppeteer_launcher.launch = _patched_launch
851
862
  if hasattr(pyppeteer_mod, 'launch'):
@@ -24,6 +24,32 @@ class ApiClientError(Exception):
24
24
  self.body = body
25
25
 
26
26
 
27
+ # PUT is left out: a KV record PUT is counted (and billed) as a write each time
28
+ # it is applied, so a lost response must not be re-sent blindly.
29
+ IDEMPOTENT_METHODS = frozenset({'GET', 'HEAD', 'DELETE', 'OPTIONS'})
30
+ # HAProxy's own 503 page: no backend was available, so nothing was forwarded.
31
+ NO_BACKEND_MARKER = 'No server is available to handle this request'
32
+
33
+
34
+ def is_retryable_response(
35
+ status: int, idempotent: bool, error_type: Optional[str] = None, body_text: Optional[str] = None
36
+ ) -> bool:
37
+ """Whether a non-2xx answer is retried.
38
+
39
+ Idempotent methods: 500/502/503/504. Others (dataset push, KV PUT) only on
40
+ a 503 that certainly was not processed: the rate limiter refused it
41
+ ("rate-limit-unavailable", before the handler), or HAProxy had no backend
42
+ (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
43
+ (a direct dataset write whose commit was not confirmed), and re-sending
44
+ could store the items twice.
45
+ """
46
+ if idempotent:
47
+ return status in (500, 502, 503, 504)
48
+ if status != 503:
49
+ return False
50
+ return error_type == 'rate-limit-unavailable' or NO_BACKEND_MARKER in (body_text or '')
51
+
52
+
27
53
  def unwrap_data(value: Any) -> Any:
28
54
  """Unwrap Apify-style v2 response envelopes ({ data: ... })."""
29
55
  if isinstance(value, list):
@@ -33,6 +59,35 @@ def unwrap_data(value: Any) -> Any:
33
59
  return value
34
60
 
35
61
 
62
+ def _error_type(response: httpx.Response) -> Optional[str]:
63
+ try:
64
+ body = response.json()
65
+ except Exception:
66
+ return None
67
+ if isinstance(body, dict) and isinstance(body.get('error'), dict):
68
+ t = body['error'].get('type')
69
+ return t if isinstance(t, str) else None
70
+ return None
71
+
72
+
73
+ def server_retry_delay(server_attempt: int, retry_after: Optional[str], now: Optional[float] = None) -> float:
74
+ """Exponential backoff with jitter, in seconds; honours a Retry-After (seconds or HTTP date) of up to 30 s."""
75
+ if retry_after is not None and retry_after.strip():
76
+ secs: Optional[float] = None
77
+ try:
78
+ secs = float(retry_after)
79
+ except ValueError:
80
+ try:
81
+ from email.utils import parsedate_to_datetime
82
+
83
+ secs = max(0.0, parsedate_to_datetime(retry_after).timestamp() - (time.time() if now is None else now))
84
+ except Exception:
85
+ secs = None
86
+ if secs is not None and secs >= 0:
87
+ return min(secs, 30.0) + random.random() * 0.1
88
+ return min(8.0, 0.5 * 2 ** server_attempt) + random.random() * 0.25
89
+
90
+
36
91
  class AdaptiveRateLimiter:
37
92
  """Adaptive throttling based on X-RateLimit-* headers (TS AdaptiveRateLimiter)."""
38
93
 
@@ -116,6 +171,10 @@ class ApiClient:
116
171
 
117
172
  _rate_limiter = AdaptiveRateLimiter()
118
173
  _max_retries = 3
174
+ # Retries while the API is briefly unavailable (503/5xx, connection
175
+ # refused): backoff 0.5 s doubling to 8 s, about 45 s in total, which
176
+ # covers a main redeploy.
177
+ _max_server_retries = 8
119
178
 
120
179
  def __init__(
121
180
  self,
@@ -198,6 +257,7 @@ class ApiClient:
198
257
  headers: Optional[Dict[str, str]] = None,
199
258
  raise_for_status: bool = True,
200
259
  do_not_retry_timeouts: bool = False,
260
+ idempotent: Optional[bool] = None,
201
261
  ) -> httpx.Response:
202
262
  """Make an HTTP request with adaptive rate limiting and 429 retries."""
203
263
  client = await self._ensure_client()
@@ -208,8 +268,11 @@ class ApiClient:
208
268
  if json_data is not None and 'Content-Type' not in request_headers:
209
269
  request_headers['Content-Type'] = 'application/json'
210
270
 
211
- last_error: Optional[Exception] = None
212
- for attempt in range(self._max_retries + 1):
271
+ if idempotent is None:
272
+ idempotent = method.upper() in IDEMPOTENT_METHODS
273
+ attempt = 0 # timeouts and 429s
274
+ server_attempt = 0 # 503/5xx and connection failures (API briefly unavailable)
275
+ while True:
213
276
  await self._rate_limiter.wait_for_slot()
214
277
  try:
215
278
  response = await client.request(
@@ -220,27 +283,59 @@ class ApiClient:
220
283
  content=content,
221
284
  headers=request_headers or None,
222
285
  )
223
- except httpx.TimeoutException as exc:
224
- last_error = exc
286
+ except httpx.TimeoutException:
225
287
  if do_not_retry_timeouts or attempt >= self._max_retries:
226
288
  raise
227
- await asyncio.sleep(0.25 * (attempt + 1))
289
+ attempt += 1
290
+ await asyncio.sleep(0.25 * attempt)
228
291
  continue
292
+ except httpx.TransportError as exc:
293
+ # No response. A refused/unresolvable connection never reached
294
+ # the API, so it is safe for every method; anything else (reset
295
+ # mid-request) only for idempotent ones.
296
+ if (idempotent or isinstance(exc, httpx.ConnectError)) and server_attempt < self._max_server_retries:
297
+ wait = server_retry_delay(server_attempt, None)
298
+ server_attempt += 1
299
+ logger.warning(
300
+ '%s %s failed (%s); retry %d/%d in %.1fs',
301
+ method, path, type(exc).__name__, server_attempt, self._max_server_retries, wait,
302
+ )
303
+ await asyncio.sleep(wait)
304
+ continue
305
+ raise
229
306
 
230
307
  self._rate_limiter.update_from_headers(response.headers)
231
308
 
232
309
  if response.status_code == 429 and attempt < self._max_retries:
310
+ attempt += 1
233
311
  await self._rate_limiter.handle_rate_limited(response)
234
312
  continue
235
313
 
314
+ # API briefly unavailable (main redeploy, limiter store blip); see
315
+ # is_retryable_response for which answers are safe to re-send.
316
+ if (
317
+ not response.is_success
318
+ and server_attempt < self._max_server_retries
319
+ and is_retryable_response(
320
+ response.status_code,
321
+ idempotent,
322
+ _error_type(response) if response.status_code == 503 else None,
323
+ response.text if response.status_code == 503 else None,
324
+ )
325
+ ):
326
+ wait = server_retry_delay(server_attempt, response.headers.get('Retry-After'))
327
+ server_attempt += 1
328
+ logger.warning(
329
+ '%s %s answered %d; retry %d/%d in %.1fs',
330
+ method, path, response.status_code, server_attempt, self._max_server_retries, wait,
331
+ )
332
+ await asyncio.sleep(wait)
333
+ continue
334
+
236
335
  if raise_for_status:
237
336
  await self._raise_for_status(response)
238
337
  return response
239
338
 
240
- if last_error is not None:
241
- raise last_error
242
- raise ApiClientError('Request failed after retries', 429)
243
-
244
339
  async def get(
245
340
  self,
246
341
  path: str,
@@ -268,6 +363,7 @@ class ApiClient:
268
363
  params: Optional[Dict[str, Any]] = None,
269
364
  *,
270
365
  raise_for_status: bool = True,
366
+ idempotent: Optional[bool] = None,
271
367
  ) -> httpx.Response:
272
368
  return await self.request(
273
369
  'POST',
@@ -277,6 +373,7 @@ class ApiClient:
277
373
  content=content,
278
374
  headers=headers,
279
375
  raise_for_status=raise_for_status,
376
+ idempotent=idempotent,
280
377
  )
281
378
 
282
379
  async def put(
@@ -325,8 +422,10 @@ class ApiClient:
325
422
  data: Any = None,
326
423
  params: Optional[Dict[str, Any]] = None,
327
424
  headers: Optional[Dict[str, str]] = None,
425
+ *,
426
+ idempotent: Optional[bool] = None,
328
427
  ) -> Any:
329
- response = await self.post(path, json_data=data, params=params, headers=headers)
428
+ response = await self.post(path, json_data=data, params=params, headers=headers, idempotent=idempotent)
330
429
  if not response.content:
331
430
  return {}
332
431
  try:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-python
3
- Version: 0.1.4
3
+ Version: 0.1.6
4
4
  Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
5
5
  Author-email: Scrapely <support@scrape.ly>
6
6
  License-Expression: MIT
@@ -4,6 +4,7 @@ src/scrapely/__init__.py
4
4
  src/scrapely/_browser_locations.py
5
5
  src/scrapely/actor.py
6
6
  src/scrapely/browser_configuration.py
7
+ src/scrapely/cdp_retry.py
7
8
  src/scrapely/charging.py
8
9
  src/scrapely/charging_client.py
9
10
  src/scrapely/configuration.py
@@ -44,8 +45,10 @@ src/scrapely_python.egg-info/SOURCES.txt
44
45
  src/scrapely_python.egg-info/dependency_links.txt
45
46
  src/scrapely_python.egg-info/requires.txt
46
47
  src/scrapely_python.egg-info/top_level.txt
48
+ tests/test_api_client_retries.py
47
49
  tests/test_apify_parity.py
48
50
  tests/test_browser_configuration.py
51
+ tests/test_cdp_retry.py
49
52
  tests/test_charging_parity.py
50
53
  tests/test_charging_pending.py
51
54
  tests/test_parity.py
@@ -53,4 +56,5 @@ tests/test_patch_browser_pool.py
53
56
  tests/test_proxy_configuration.py
54
57
  tests/test_run_control.py
55
58
  tests/test_secrets.py
56
- tests/test_storage.py
59
+ tests/test_storage.py
60
+ tests/test_terminal_status_message.py