scrapely-client-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of scrapely-client-python might be problematic. Click here for more details.
- scrapely_client/__init__.py +30 -0
- scrapely_client/_charging.py +186 -0
- scrapely_client/_client_registry.py +136 -0
- scrapely_client/_consts.py +96 -0
- scrapely_client/_docs.py +37 -0
- scrapely_client/_literals.py +539 -0
- scrapely_client/_logging.py +180 -0
- scrapely_client/_models.py +4172 -0
- scrapely_client/_pagination.py +220 -0
- scrapely_client/_resource_clients/__init__.py +84 -0
- scrapely_client/_resource_clients/_resource_client.py +567 -0
- scrapely_client/_resource_clients/actor.py +1042 -0
- scrapely_client/_resource_clients/actor_collection.py +377 -0
- scrapely_client/_resource_clients/actor_env_var.py +154 -0
- scrapely_client/_resource_clients/actor_env_var_collection.py +165 -0
- scrapely_client/_resource_clients/actor_version.py +254 -0
- scrapely_client/_resource_clients/actor_version_collection.py +232 -0
- scrapely_client/_resource_clients/build.py +244 -0
- scrapely_client/_resource_clients/build_collection.py +164 -0
- scrapely_client/_resource_clients/dataset.py +1151 -0
- scrapely_client/_resource_clients/dataset_collection.py +244 -0
- scrapely_client/_resource_clients/key_value_store.py +906 -0
- scrapely_client/_resource_clients/key_value_store_collection.py +243 -0
- scrapely_client/_resource_clients/log.py +225 -0
- scrapely_client/_resource_clients/request_queue.py +1167 -0
- scrapely_client/_resource_clients/request_queue_collection.py +240 -0
- scrapely_client/_resource_clients/run.py +875 -0
- scrapely_client/_resource_clients/run_collection.py +226 -0
- scrapely_client/_resource_clients/schedule.py +243 -0
- scrapely_client/_resource_clients/schedule_collection.py +260 -0
- scrapely_client/_resource_clients/store_collection.py +242 -0
- scrapely_client/_resource_clients/task.py +793 -0
- scrapely_client/_resource_clients/task_collection.py +322 -0
- scrapely_client/_resource_clients/user.py +262 -0
- scrapely_client/_resource_clients/webhook.py +292 -0
- scrapely_client/_resource_clients/webhook_collection.py +281 -0
- scrapely_client/_resource_clients/webhook_dispatch.py +84 -0
- scrapely_client/_resource_clients/webhook_dispatch_collection.py +158 -0
- scrapely_client/_scrapely_client.py +837 -0
- scrapely_client/_statistics.py +29 -0
- scrapely_client/_status_message_watcher.py +197 -0
- scrapely_client/_streamed_log.py +250 -0
- scrapely_client/_typeddicts.py +408 -0
- scrapely_client/_utils/__init__.py +0 -0
- scrapely_client/_utils/crypto.py +87 -0
- scrapely_client/_utils/encoding.py +133 -0
- scrapely_client/_utils/errors.py +33 -0
- scrapely_client/_utils/http.py +137 -0
- scrapely_client/_utils/time.py +32 -0
- scrapely_client/_utils/try_import.py +61 -0
- scrapely_client/errors.py +190 -0
- scrapely_client/http_clients/__init__.py +10 -0
- scrapely_client/http_clients/_base.py +430 -0
- scrapely_client/http_clients/_impit.py +598 -0
- scrapely_client/http_compressors/__init__.py +18 -0
- scrapely_client/http_compressors/_base.py +25 -0
- scrapely_client/http_compressors/_brotli.py +38 -0
- scrapely_client/http_compressors/_gzip.py +38 -0
- scrapely_client/http_compressors/_resolve.py +38 -0
- scrapely_client/py.typed +0 -0
- scrapely_client/types.py +52 -0
- scrapely_client_python-1.0.0.dist-info/METADATA +244 -0
- scrapely_client_python-1.0.0.dist-info/RECORD +65 -0
- scrapely_client_python-1.0.0.dist-info/WHEEL +4 -0
- scrapely_client_python-1.0.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,598 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
import random
|
|
6
|
+
import time
|
|
7
|
+
from contextlib import suppress
|
|
8
|
+
from datetime import timedelta
|
|
9
|
+
from http import HTTPStatus
|
|
10
|
+
from typing import TYPE_CHECKING, Any, TypeVar
|
|
11
|
+
|
|
12
|
+
import impit
|
|
13
|
+
|
|
14
|
+
from scrapely_client._consts import (
|
|
15
|
+
DEFAULT_MAX_RETRIES,
|
|
16
|
+
DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
|
|
17
|
+
DEFAULT_TIMEOUT_LONG,
|
|
18
|
+
DEFAULT_TIMEOUT_MAX,
|
|
19
|
+
DEFAULT_TIMEOUT_MEDIUM,
|
|
20
|
+
DEFAULT_TIMEOUT_SHORT,
|
|
21
|
+
)
|
|
22
|
+
from scrapely_client._docs import docs_group
|
|
23
|
+
from scrapely_client._logging import log_context, logger_name
|
|
24
|
+
from scrapely_client._utils.time import to_seconds
|
|
25
|
+
from scrapely_client.errors import ApifyApiError
|
|
26
|
+
from scrapely_client.http_clients._base import HttpClient, HttpClientAsync
|
|
27
|
+
|
|
28
|
+
if TYPE_CHECKING:
|
|
29
|
+
from collections.abc import Awaitable, Callable
|
|
30
|
+
|
|
31
|
+
from scrapely_client._statistics import ClientStatistics
|
|
32
|
+
from scrapely_client.http_clients._base import HttpResponse
|
|
33
|
+
from scrapely_client.http_compressors._base import HttpCompressor
|
|
34
|
+
from scrapely_client.types import JsonSerializable, Timeout
|
|
35
|
+
|
|
36
|
+
T = TypeVar('T')
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger(logger_name)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
_PERMANENT_ERRORS = (
|
|
42
|
+
# A request Impit rejects before sending it, e.g. one carrying an invalid header value.
|
|
43
|
+
impit.LocalProtocolError,
|
|
44
|
+
# The class Impit declares for a scheme it refuses to speak, but does not raise today - it reports an unsupported
|
|
45
|
+
# scheme as `impit.InvalidURL`, which sits outside the `impit.HTTPError` tree and is non-retryable anyway. Listed
|
|
46
|
+
# so the classifier stays right if Impit switches over.
|
|
47
|
+
impit.UnsupportedProtocol,
|
|
48
|
+
# An over-long redirect chain is a routing loop, which repeating the request cannot break.
|
|
49
|
+
impit.TooManyRedirects,
|
|
50
|
+
# Only `Response.raise_for_status()` raises this, and the client never calls it - `_make_request` decides on
|
|
51
|
+
# status codes from the response itself.
|
|
52
|
+
impit.HTTPStatusError,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _is_retryable_error(exc: Exception) -> bool:
|
|
57
|
+
"""Check if an exception represents a transient transport failure that should be retried.
|
|
58
|
+
|
|
59
|
+
Every error from Impit's own hierarchy counts as transient except the permanently-failing types listed in
|
|
60
|
+
`_PERMANENT_ERRORS`. Retrying is the default because Impit also reports genuinely transient failures through
|
|
61
|
+
its generic base class, e.g. a bare `impit.HTTPError` wrapping a failure its internal HTTP library did not
|
|
62
|
+
classify. HTTP status code errors are handled separately in `_make_request` based on the response status code,
|
|
63
|
+
not here.
|
|
64
|
+
"""
|
|
65
|
+
return isinstance(exc, impit.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@docs_group('HTTP clients')
|
|
69
|
+
class ImpitHttpClient(HttpClient):
|
|
70
|
+
"""Synchronous HTTP client for the Scrapely API built on top of [Impit](https://github.com/apify/impit).
|
|
71
|
+
|
|
72
|
+
Impit is a high-performance HTTP client written in Rust that provides browser-like TLS fingerprints,
|
|
73
|
+
automatic header ordering, and HTTP/2 support. This client wraps `impit.Client` and adds automatic retries
|
|
74
|
+
with exponential backoff for rate-limited (HTTP 429) and server error (HTTP 5xx) responses.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
def __init__(
|
|
78
|
+
self,
|
|
79
|
+
*,
|
|
80
|
+
token: str | None = None,
|
|
81
|
+
timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT,
|
|
82
|
+
timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM,
|
|
83
|
+
timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
|
|
84
|
+
timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
|
|
85
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
86
|
+
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
|
|
87
|
+
statistics: ClientStatistics | None = None,
|
|
88
|
+
headers: dict[str, str] | None = None,
|
|
89
|
+
http_compressor: HttpCompressor | None = None,
|
|
90
|
+
) -> None:
|
|
91
|
+
"""Initialize the Impit-based synchronous HTTP client.
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
token: Scrapely API token for authentication.
|
|
95
|
+
timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
|
|
96
|
+
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
|
|
97
|
+
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
|
|
98
|
+
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
|
|
99
|
+
max_retries: Maximum number of retry attempts for failed requests.
|
|
100
|
+
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
|
|
101
|
+
statistics: Statistics tracker for API calls. Created automatically if not provided.
|
|
102
|
+
headers: Additional HTTP headers to include in all requests.
|
|
103
|
+
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
|
|
104
|
+
"""
|
|
105
|
+
super().__init__(
|
|
106
|
+
token=token,
|
|
107
|
+
timeout_short=timeout_short,
|
|
108
|
+
timeout_medium=timeout_medium,
|
|
109
|
+
timeout_long=timeout_long,
|
|
110
|
+
timeout_max=timeout_max,
|
|
111
|
+
max_retries=max_retries,
|
|
112
|
+
min_delay_between_retries=min_delay_between_retries,
|
|
113
|
+
statistics=statistics,
|
|
114
|
+
headers=headers,
|
|
115
|
+
http_compressor=http_compressor,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
self._impit_client = impit.Client(
|
|
119
|
+
follow_redirects=True,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
def call(
|
|
123
|
+
self,
|
|
124
|
+
*,
|
|
125
|
+
method: str,
|
|
126
|
+
url: str,
|
|
127
|
+
headers: dict[str, str] | None = None,
|
|
128
|
+
params: dict[str, Any] | None = None,
|
|
129
|
+
data: str | bytes | bytearray | None = None,
|
|
130
|
+
json: JsonSerializable | None = None,
|
|
131
|
+
stream: bool | None = None,
|
|
132
|
+
timeout: Timeout = 'medium',
|
|
133
|
+
) -> HttpResponse:
|
|
134
|
+
"""Make an HTTP request with automatic retry and exponential backoff.
|
|
135
|
+
|
|
136
|
+
Args:
|
|
137
|
+
method: HTTP method (GET, POST, PUT, DELETE, etc.).
|
|
138
|
+
url: Full URL to make the request to.
|
|
139
|
+
headers: Additional headers to include.
|
|
140
|
+
params: Query parameters to append to the URL.
|
|
141
|
+
data: Raw request body data. Cannot be used together with json.
|
|
142
|
+
json: JSON-serializable data for the request body. Cannot be used together with data.
|
|
143
|
+
stream: Whether to stream the response body.
|
|
144
|
+
timeout: Timeout for the API HTTP request. Use `short`, `medium`, or `long` tier literals for
|
|
145
|
+
preconfigured timeouts. A `timedelta` overrides it for this call (capped at `timeout_max`), and
|
|
146
|
+
`no_timeout` disables the timeout entirely.
|
|
147
|
+
|
|
148
|
+
Returns:
|
|
149
|
+
The HTTP response object.
|
|
150
|
+
|
|
151
|
+
Raises:
|
|
152
|
+
ApifyApiError: If the request fails after all retries or returns a non-retryable error status.
|
|
153
|
+
ValueError: If both json and data are provided.
|
|
154
|
+
"""
|
|
155
|
+
log_context.method.set(method)
|
|
156
|
+
log_context.url.set(url)
|
|
157
|
+
|
|
158
|
+
self._statistics.calls += 1
|
|
159
|
+
|
|
160
|
+
prepared_headers, prepared_params, content = self._prepare_request_call(
|
|
161
|
+
headers=headers,
|
|
162
|
+
params=params,
|
|
163
|
+
data=data,
|
|
164
|
+
json=json,
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
return self._retry_with_exp_backoff(
|
|
168
|
+
lambda stop_retrying, attempt: self._make_request(
|
|
169
|
+
stop_retrying=stop_retrying,
|
|
170
|
+
attempt=attempt,
|
|
171
|
+
method=method,
|
|
172
|
+
url=url,
|
|
173
|
+
headers=prepared_headers,
|
|
174
|
+
params=prepared_params,
|
|
175
|
+
content=content,
|
|
176
|
+
stream=stream,
|
|
177
|
+
timeout=timeout,
|
|
178
|
+
),
|
|
179
|
+
max_retries=self._max_retries,
|
|
180
|
+
backoff_base=self._min_delay_between_retries,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
def _make_request(
|
|
184
|
+
self,
|
|
185
|
+
*,
|
|
186
|
+
stop_retrying: Callable[[], None],
|
|
187
|
+
attempt: int,
|
|
188
|
+
method: str,
|
|
189
|
+
url: str,
|
|
190
|
+
headers: dict[str, str],
|
|
191
|
+
params: dict[str, Any] | None,
|
|
192
|
+
content: bytes | None,
|
|
193
|
+
stream: bool | None,
|
|
194
|
+
timeout: Timeout,
|
|
195
|
+
) -> impit.Response:
|
|
196
|
+
"""Execute a single HTTP request attempt.
|
|
197
|
+
|
|
198
|
+
Args:
|
|
199
|
+
stop_retrying: Callback to signal that retries should stop.
|
|
200
|
+
attempt: Current attempt number (1-indexed).
|
|
201
|
+
method: HTTP method.
|
|
202
|
+
url: Request URL.
|
|
203
|
+
headers: Request headers.
|
|
204
|
+
params: Query parameters.
|
|
205
|
+
content: Request body content.
|
|
206
|
+
stream: Whether to stream the response.
|
|
207
|
+
timeout: Timeout for this request.
|
|
208
|
+
|
|
209
|
+
Returns:
|
|
210
|
+
The HTTP response object.
|
|
211
|
+
|
|
212
|
+
Raises:
|
|
213
|
+
ApifyApiError: If the request fails with an error status.
|
|
214
|
+
"""
|
|
215
|
+
log_context.attempt.set(attempt)
|
|
216
|
+
logger.debug('Sending request')
|
|
217
|
+
|
|
218
|
+
self._statistics.requests += 1
|
|
219
|
+
|
|
220
|
+
try:
|
|
221
|
+
url_with_params = self._build_url_with_params(url, params=params)
|
|
222
|
+
|
|
223
|
+
# Impit treats timeout=None as "use client default (30s)", not "no timeout".
|
|
224
|
+
# Use a large value (24 hours) to effectively disable the timeout.
|
|
225
|
+
# This can be removed once impit updates its behaviour: https://github.com/apify/impit/issues/401
|
|
226
|
+
computed_timeout = self._compute_timeout(timeout, attempt=attempt)
|
|
227
|
+
impit_timeout = 86_400 if computed_timeout is None else computed_timeout
|
|
228
|
+
|
|
229
|
+
response = self._impit_client.request(
|
|
230
|
+
method=method,
|
|
231
|
+
url=url_with_params,
|
|
232
|
+
headers=headers,
|
|
233
|
+
content=content,
|
|
234
|
+
timeout=impit_timeout,
|
|
235
|
+
stream=stream or False,
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
if response.status_code < HTTPStatus.MULTIPLE_CHOICES:
|
|
239
|
+
logger.debug('Request successful', extra={'status_code': response.status_code})
|
|
240
|
+
return response
|
|
241
|
+
|
|
242
|
+
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
|
|
243
|
+
self._statistics.add_rate_limit_error(attempt)
|
|
244
|
+
|
|
245
|
+
except Exception as exc:
|
|
246
|
+
logger.debug('Request threw exception', exc_info=exc)
|
|
247
|
+
if not _is_retryable_error(exc):
|
|
248
|
+
logger.debug('Exception is not retryable', exc_info=exc)
|
|
249
|
+
stop_retrying()
|
|
250
|
+
raise
|
|
251
|
+
|
|
252
|
+
# Retry only server errors (5xx) and rate limits (429).
|
|
253
|
+
logger.debug('Request unsuccessful', extra={'status_code': response.status_code})
|
|
254
|
+
if (
|
|
255
|
+
response.status_code < HTTPStatus.INTERNAL_SERVER_ERROR
|
|
256
|
+
and response.status_code != HTTPStatus.TOO_MANY_REQUESTS
|
|
257
|
+
):
|
|
258
|
+
logger.debug('Status code is not retryable', extra={'status_code': response.status_code})
|
|
259
|
+
stop_retrying()
|
|
260
|
+
|
|
261
|
+
# Read the response in case it is a stream, so we can raise the error properly. A failed read goes through
|
|
262
|
+
# the same classification as a failed send.
|
|
263
|
+
try:
|
|
264
|
+
response.read()
|
|
265
|
+
except Exception as exc:
|
|
266
|
+
logger.debug('Reading the error response failed', exc_info=exc)
|
|
267
|
+
with suppress(Exception):
|
|
268
|
+
response.close()
|
|
269
|
+
if not _is_retryable_error(exc):
|
|
270
|
+
logger.debug('Exception is not retryable', exc_info=exc)
|
|
271
|
+
stop_retrying()
|
|
272
|
+
raise
|
|
273
|
+
|
|
274
|
+
raise ApifyApiError(response, attempt, method=method)
|
|
275
|
+
|
|
276
|
+
@staticmethod
|
|
277
|
+
def _retry_with_exp_backoff(
|
|
278
|
+
func: Callable[[Callable[[], None], int], T],
|
|
279
|
+
*,
|
|
280
|
+
max_retries: int = 8,
|
|
281
|
+
backoff_base: timedelta = timedelta(milliseconds=500),
|
|
282
|
+
backoff_factor: float = 2,
|
|
283
|
+
random_factor: float = 1,
|
|
284
|
+
) -> T:
|
|
285
|
+
"""Retry a function with exponential backoff and jitter.
|
|
286
|
+
|
|
287
|
+
Args:
|
|
288
|
+
func: Function to retry. Receives (stop_retrying callback, attempt number).
|
|
289
|
+
max_retries: Maximum retry attempts.
|
|
290
|
+
backoff_base: Base delay.
|
|
291
|
+
backoff_factor: Exponential multiplier (clamped to 1-10).
|
|
292
|
+
random_factor: Jitter factor (clamped to 0-1).
|
|
293
|
+
|
|
294
|
+
Returns:
|
|
295
|
+
The function's return value on success.
|
|
296
|
+
|
|
297
|
+
Raises:
|
|
298
|
+
Exception: Re-raises the last exception if all retries fail or stop_retrying is called.
|
|
299
|
+
"""
|
|
300
|
+
if max_retries < 1:
|
|
301
|
+
raise ValueError(f'max_retries must be at least 1, got {max_retries}')
|
|
302
|
+
|
|
303
|
+
random_factor = min(max(0, random_factor), 1)
|
|
304
|
+
backoff_factor = min(max(1, backoff_factor), 10)
|
|
305
|
+
swallow = True
|
|
306
|
+
|
|
307
|
+
def stop_retrying() -> None:
|
|
308
|
+
nonlocal swallow
|
|
309
|
+
swallow = False
|
|
310
|
+
|
|
311
|
+
for attempt in range(1, max_retries + 1):
|
|
312
|
+
try:
|
|
313
|
+
return func(stop_retrying, attempt)
|
|
314
|
+
except Exception:
|
|
315
|
+
if not swallow:
|
|
316
|
+
raise
|
|
317
|
+
|
|
318
|
+
random_sleep_factor = random.uniform(1, 1 + random_factor)
|
|
319
|
+
backoff_base_secs = to_seconds(backoff_base)
|
|
320
|
+
backoff_exp_factor = backoff_factor ** (attempt - 1)
|
|
321
|
+
|
|
322
|
+
sleep_time_secs = random_sleep_factor * backoff_base_secs * backoff_exp_factor
|
|
323
|
+
time.sleep(sleep_time_secs)
|
|
324
|
+
|
|
325
|
+
return func(stop_retrying, max_retries + 1)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
@docs_group('HTTP clients')
|
|
329
|
+
class ImpitHttpClientAsync(HttpClientAsync):
|
|
330
|
+
"""Asynchronous HTTP client for the Scrapely API built on top of [Impit](https://github.com/apify/impit).
|
|
331
|
+
|
|
332
|
+
Impit is a high-performance HTTP client written in Rust that provides browser-like TLS fingerprints,
|
|
333
|
+
automatic header ordering, and HTTP/2 support. This client wraps `impit.AsyncClient` and adds automatic retries
|
|
334
|
+
with exponential backoff for rate-limited (HTTP 429) and server error (HTTP 5xx) responses.
|
|
335
|
+
"""
|
|
336
|
+
|
|
337
|
+
def __init__(
|
|
338
|
+
self,
|
|
339
|
+
*,
|
|
340
|
+
token: str | None = None,
|
|
341
|
+
timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT,
|
|
342
|
+
timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM,
|
|
343
|
+
timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
|
|
344
|
+
timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
|
|
345
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
346
|
+
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
|
|
347
|
+
statistics: ClientStatistics | None = None,
|
|
348
|
+
headers: dict[str, str] | None = None,
|
|
349
|
+
http_compressor: HttpCompressor | None = None,
|
|
350
|
+
) -> None:
|
|
351
|
+
"""Initialize the Impit-based asynchronous HTTP client.
|
|
352
|
+
|
|
353
|
+
Args:
|
|
354
|
+
token: Scrapely API token for authentication.
|
|
355
|
+
timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
|
|
356
|
+
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
|
|
357
|
+
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
|
|
358
|
+
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
|
|
359
|
+
max_retries: Maximum number of retry attempts for failed requests.
|
|
360
|
+
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
|
|
361
|
+
statistics: Statistics tracker for API calls. Created automatically if not provided.
|
|
362
|
+
headers: Additional HTTP headers to include in all requests.
|
|
363
|
+
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
|
|
364
|
+
"""
|
|
365
|
+
super().__init__(
|
|
366
|
+
token=token,
|
|
367
|
+
timeout_short=timeout_short,
|
|
368
|
+
timeout_medium=timeout_medium,
|
|
369
|
+
timeout_long=timeout_long,
|
|
370
|
+
timeout_max=timeout_max,
|
|
371
|
+
max_retries=max_retries,
|
|
372
|
+
min_delay_between_retries=min_delay_between_retries,
|
|
373
|
+
statistics=statistics,
|
|
374
|
+
headers=headers,
|
|
375
|
+
http_compressor=http_compressor,
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
self._impit_async_client = impit.AsyncClient(
|
|
379
|
+
follow_redirects=True,
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
async def call(
|
|
383
|
+
self,
|
|
384
|
+
*,
|
|
385
|
+
method: str,
|
|
386
|
+
url: str,
|
|
387
|
+
headers: dict[str, str] | None = None,
|
|
388
|
+
params: dict[str, Any] | None = None,
|
|
389
|
+
data: str | bytes | bytearray | None = None,
|
|
390
|
+
json: JsonSerializable | None = None,
|
|
391
|
+
stream: bool | None = None,
|
|
392
|
+
timeout: Timeout = 'medium',
|
|
393
|
+
) -> HttpResponse:
|
|
394
|
+
"""Make an HTTP request with automatic retry and exponential backoff.
|
|
395
|
+
|
|
396
|
+
Args:
|
|
397
|
+
method: HTTP method (GET, POST, PUT, DELETE, etc.).
|
|
398
|
+
url: Full URL to make the request to.
|
|
399
|
+
headers: Additional headers to include.
|
|
400
|
+
params: Query parameters to append to the URL.
|
|
401
|
+
data: Raw request body data. Cannot be used together with json.
|
|
402
|
+
json: JSON-serializable data for the request body. Cannot be used together with data.
|
|
403
|
+
stream: Whether to stream the response body.
|
|
404
|
+
timeout: Timeout for the API HTTP request. Use `short`, `medium`, or `long` tier literals for
|
|
405
|
+
preconfigured timeouts. A `timedelta` overrides it for this call (capped at `timeout_max`), and
|
|
406
|
+
`no_timeout` disables the timeout entirely.
|
|
407
|
+
|
|
408
|
+
Returns:
|
|
409
|
+
The HTTP response object.
|
|
410
|
+
|
|
411
|
+
Raises:
|
|
412
|
+
ApifyApiError: If the request fails after all retries or returns a non-retryable error status.
|
|
413
|
+
ValueError: If both json and data are provided.
|
|
414
|
+
"""
|
|
415
|
+
log_context.method.set(method)
|
|
416
|
+
log_context.url.set(url)
|
|
417
|
+
|
|
418
|
+
self._statistics.calls += 1
|
|
419
|
+
|
|
420
|
+
# Serializing and compressing a request body is CPU-bound and would block the event loop, so
|
|
421
|
+
# offload preparation to a worker thread whenever there is something to compress. A body the
|
|
422
|
+
# client sends as it is costs less to prepare inline than the hop itself. A `json` body always
|
|
423
|
+
# hops, as its size is only known once serialized.
|
|
424
|
+
if json is not None or self._is_body_worth_compressing(data):
|
|
425
|
+
prepared_headers, prepared_params, content = await asyncio.to_thread(
|
|
426
|
+
self._prepare_request_call,
|
|
427
|
+
headers=headers,
|
|
428
|
+
params=params,
|
|
429
|
+
data=data,
|
|
430
|
+
json=json,
|
|
431
|
+
)
|
|
432
|
+
else:
|
|
433
|
+
prepared_headers, prepared_params, content = self._prepare_request_call(
|
|
434
|
+
headers=headers,
|
|
435
|
+
params=params,
|
|
436
|
+
data=data,
|
|
437
|
+
json=json,
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
return await self._retry_with_exp_backoff(
|
|
441
|
+
lambda stop_retrying, attempt: self._make_request(
|
|
442
|
+
stop_retrying=stop_retrying,
|
|
443
|
+
attempt=attempt,
|
|
444
|
+
method=method,
|
|
445
|
+
url=url,
|
|
446
|
+
headers=prepared_headers,
|
|
447
|
+
params=prepared_params,
|
|
448
|
+
content=content,
|
|
449
|
+
stream=stream,
|
|
450
|
+
timeout=timeout,
|
|
451
|
+
),
|
|
452
|
+
max_retries=self._max_retries,
|
|
453
|
+
backoff_base=self._min_delay_between_retries,
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
async def _make_request(
|
|
457
|
+
self,
|
|
458
|
+
*,
|
|
459
|
+
stop_retrying: Callable[[], None],
|
|
460
|
+
attempt: int,
|
|
461
|
+
method: str,
|
|
462
|
+
url: str,
|
|
463
|
+
headers: dict[str, str],
|
|
464
|
+
params: dict[str, Any] | None,
|
|
465
|
+
content: bytes | None,
|
|
466
|
+
stream: bool | None,
|
|
467
|
+
timeout: Timeout,
|
|
468
|
+
) -> impit.Response:
|
|
469
|
+
"""Execute a single HTTP request attempt.
|
|
470
|
+
|
|
471
|
+
Args:
|
|
472
|
+
stop_retrying: Callback to signal that retries should stop.
|
|
473
|
+
attempt: Current attempt number (1-indexed).
|
|
474
|
+
method: HTTP method.
|
|
475
|
+
url: Request URL.
|
|
476
|
+
headers: Request headers.
|
|
477
|
+
params: Query parameters.
|
|
478
|
+
content: Request body content.
|
|
479
|
+
stream: Whether to stream the response.
|
|
480
|
+
timeout: Timeout for this request.
|
|
481
|
+
|
|
482
|
+
Returns:
|
|
483
|
+
The HTTP response object.
|
|
484
|
+
|
|
485
|
+
Raises:
|
|
486
|
+
ApifyApiError: If the request fails with an error status.
|
|
487
|
+
"""
|
|
488
|
+
log_context.attempt.set(attempt)
|
|
489
|
+
logger.debug('Sending request')
|
|
490
|
+
|
|
491
|
+
self._statistics.requests += 1
|
|
492
|
+
|
|
493
|
+
try:
|
|
494
|
+
url_with_params = self._build_url_with_params(url, params=params)
|
|
495
|
+
|
|
496
|
+
# Impit treats timeout=None as "use client default (30s)", not "no timeout".
|
|
497
|
+
# Use a large value (24 hours) to effectively disable the timeout.
|
|
498
|
+
# This can be removed once impit updates its behaviour: https://github.com/apify/impit/issues/401
|
|
499
|
+
computed_timeout = self._compute_timeout(timeout, attempt=attempt)
|
|
500
|
+
impit_timeout = 86_400 if computed_timeout is None else computed_timeout
|
|
501
|
+
|
|
502
|
+
response = await self._impit_async_client.request(
|
|
503
|
+
method=method,
|
|
504
|
+
url=url_with_params,
|
|
505
|
+
headers=headers,
|
|
506
|
+
content=content,
|
|
507
|
+
timeout=impit_timeout,
|
|
508
|
+
stream=stream or False,
|
|
509
|
+
)
|
|
510
|
+
|
|
511
|
+
if response.status_code < HTTPStatus.MULTIPLE_CHOICES:
|
|
512
|
+
logger.debug('Request successful', extra={'status_code': response.status_code})
|
|
513
|
+
return response
|
|
514
|
+
|
|
515
|
+
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
|
|
516
|
+
self._statistics.add_rate_limit_error(attempt)
|
|
517
|
+
|
|
518
|
+
except Exception as exc:
|
|
519
|
+
logger.debug('Request threw exception', exc_info=exc)
|
|
520
|
+
if not _is_retryable_error(exc):
|
|
521
|
+
logger.debug('Exception is not retryable', exc_info=exc)
|
|
522
|
+
stop_retrying()
|
|
523
|
+
raise
|
|
524
|
+
|
|
525
|
+
# Retry only server errors (5xx) and rate limits (429).
|
|
526
|
+
logger.debug('Request unsuccessful', extra={'status_code': response.status_code})
|
|
527
|
+
if (
|
|
528
|
+
response.status_code < HTTPStatus.INTERNAL_SERVER_ERROR
|
|
529
|
+
and response.status_code != HTTPStatus.TOO_MANY_REQUESTS
|
|
530
|
+
):
|
|
531
|
+
logger.debug('Status code is not retryable', extra={'status_code': response.status_code})
|
|
532
|
+
stop_retrying()
|
|
533
|
+
|
|
534
|
+
# Read the response in case it is a stream, so we can raise the error properly. A failed read goes through
|
|
535
|
+
# the same classification as a failed send.
|
|
536
|
+
try:
|
|
537
|
+
await response.aread()
|
|
538
|
+
except Exception as exc:
|
|
539
|
+
logger.debug('Reading the error response failed', exc_info=exc)
|
|
540
|
+
with suppress(Exception):
|
|
541
|
+
await response.aclose()
|
|
542
|
+
if not _is_retryable_error(exc):
|
|
543
|
+
logger.debug('Exception is not retryable', exc_info=exc)
|
|
544
|
+
stop_retrying()
|
|
545
|
+
raise
|
|
546
|
+
|
|
547
|
+
raise ApifyApiError(response, attempt, method=method)
|
|
548
|
+
|
|
549
|
+
@staticmethod
|
|
550
|
+
async def _retry_with_exp_backoff(
|
|
551
|
+
func: Callable[[Callable[[], None], int], Awaitable[T]],
|
|
552
|
+
*,
|
|
553
|
+
max_retries: int = 8,
|
|
554
|
+
backoff_base: timedelta = timedelta(milliseconds=500),
|
|
555
|
+
backoff_factor: float = 2,
|
|
556
|
+
random_factor: float = 1,
|
|
557
|
+
) -> T:
|
|
558
|
+
"""Retry an async function with exponential backoff and jitter.
|
|
559
|
+
|
|
560
|
+
Args:
|
|
561
|
+
func: Async function to retry. Receives (stop_retrying callback, attempt number).
|
|
562
|
+
max_retries: Maximum retry attempts.
|
|
563
|
+
backoff_base: Base delay.
|
|
564
|
+
backoff_factor: Exponential multiplier (clamped to 1-10).
|
|
565
|
+
random_factor: Jitter factor (clamped to 0-1).
|
|
566
|
+
|
|
567
|
+
Returns:
|
|
568
|
+
The function's return value on success.
|
|
569
|
+
|
|
570
|
+
Raises:
|
|
571
|
+
Exception: Re-raises the last exception if all retries fail or stop_retrying is called.
|
|
572
|
+
"""
|
|
573
|
+
if max_retries < 1:
|
|
574
|
+
raise ValueError(f'max_retries must be at least 1, got {max_retries}')
|
|
575
|
+
|
|
576
|
+
random_factor = min(max(0, random_factor), 1)
|
|
577
|
+
backoff_factor = min(max(1, backoff_factor), 10)
|
|
578
|
+
swallow = True
|
|
579
|
+
|
|
580
|
+
def stop_retrying() -> None:
|
|
581
|
+
nonlocal swallow
|
|
582
|
+
swallow = False
|
|
583
|
+
|
|
584
|
+
for attempt in range(1, max_retries + 1):
|
|
585
|
+
try:
|
|
586
|
+
return await func(stop_retrying, attempt)
|
|
587
|
+
except Exception:
|
|
588
|
+
if not swallow:
|
|
589
|
+
raise
|
|
590
|
+
|
|
591
|
+
random_sleep_factor = random.uniform(1, 1 + random_factor)
|
|
592
|
+
backoff_base_secs = to_seconds(backoff_base)
|
|
593
|
+
backoff_exp_factor = backoff_factor ** (attempt - 1)
|
|
594
|
+
|
|
595
|
+
sleep_time_secs = random_sleep_factor * backoff_base_secs * backoff_exp_factor
|
|
596
|
+
await asyncio.sleep(sleep_time_secs)
|
|
597
|
+
|
|
598
|
+
return await func(stop_retrying, max_retries + 1)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from scrapely_client._utils.try_import import install_import_hook as _install_import_hook
|
|
2
|
+
from scrapely_client._utils.try_import import try_import as _try_import
|
|
3
|
+
|
|
4
|
+
# These imports have only mandatory dependencies, so they are imported directly.
|
|
5
|
+
from scrapely_client.http_compressors._base import HttpCompressor
|
|
6
|
+
from scrapely_client.http_compressors._gzip import GzipHttpCompressor
|
|
7
|
+
|
|
8
|
+
_install_import_hook(__name__)
|
|
9
|
+
|
|
10
|
+
# `brotli` is an optional extra, so it's wrapped in try_import. Accessing `BrotliHttpCompressor`
|
|
11
|
+
# without the extra installed raises a clear ImportError instead of failing at package import time.
|
|
12
|
+
with _try_import(__name__, 'BrotliHttpCompressor', dependency_name='brotli') as _brotli_import:
|
|
13
|
+
from scrapely_client.http_compressors._brotli import BrotliHttpCompressor
|
|
14
|
+
|
|
15
|
+
if _brotli_import.available:
|
|
16
|
+
__all__ = ['BrotliHttpCompressor', 'GzipHttpCompressor', 'HttpCompressor']
|
|
17
|
+
else:
|
|
18
|
+
__all__ = ['GzipHttpCompressor', 'HttpCompressor']
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class HttpCompressor(ABC):
|
|
7
|
+
"""Strategy for compressing HTTP request bodies.
|
|
8
|
+
|
|
9
|
+
Extend this class to create a custom compressor. Set `content_encoding` to the value
|
|
10
|
+
that should be sent in the `Content-Encoding` header and implement `compress`.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
content_encoding: str
|
|
14
|
+
"""Value sent in the `Content-Encoding` header, for example `gzip` or `br`."""
|
|
15
|
+
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def compress(self, data: bytes) -> bytes:
|
|
18
|
+
"""Compress a request body.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
data: The raw bytes to compress.
|
|
22
|
+
|
|
23
|
+
Returns:
|
|
24
|
+
The compressed bytes.
|
|
25
|
+
"""
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import brotli
|
|
4
|
+
|
|
5
|
+
from scrapely_client.http_compressors._base import HttpCompressor
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BrotliHttpCompressor(HttpCompressor):
|
|
9
|
+
"""Compresses request bodies using brotli.
|
|
10
|
+
|
|
11
|
+
Requires the `brotli` extra: `pip install "apify-client[brotli]"`.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
content_encoding = 'br'
|
|
15
|
+
|
|
16
|
+
_min_quality = 0
|
|
17
|
+
"""Lowest valid quality (fastest, least compression)."""
|
|
18
|
+
|
|
19
|
+
_max_quality = 11
|
|
20
|
+
"""Highest valid quality (slowest, best compression)."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, *, quality: int = 6) -> None:
|
|
23
|
+
"""Initialize the brotli compressor.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
quality: Compression level, from the fastest to the best compression.
|
|
27
|
+
|
|
28
|
+
Raises:
|
|
29
|
+
ValueError: If `quality` is out of the valid range.
|
|
30
|
+
"""
|
|
31
|
+
if not self._min_quality <= quality <= self._max_quality:
|
|
32
|
+
raise ValueError(
|
|
33
|
+
f'brotli quality must be between {self._min_quality} and {self._max_quality}, got {quality}.'
|
|
34
|
+
)
|
|
35
|
+
self._quality = quality
|
|
36
|
+
|
|
37
|
+
def compress(self, data: bytes) -> bytes:
|
|
38
|
+
return brotli.compress(data, quality=self._quality)
|