scrapely-client-python 3.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of scrapely-client-python might be problematic. Click here for more details.

Files changed (65) hide show
  1. scrapely_client/__init__.py +30 -0
  2. scrapely_client/_charging.py +186 -0
  3. scrapely_client/_client_registry.py +136 -0
  4. scrapely_client/_consts.py +96 -0
  5. scrapely_client/_docs.py +37 -0
  6. scrapely_client/_literals.py +539 -0
  7. scrapely_client/_logging.py +180 -0
  8. scrapely_client/_models.py +4172 -0
  9. scrapely_client/_pagination.py +220 -0
  10. scrapely_client/_resource_clients/__init__.py +84 -0
  11. scrapely_client/_resource_clients/_resource_client.py +567 -0
  12. scrapely_client/_resource_clients/actor.py +1042 -0
  13. scrapely_client/_resource_clients/actor_collection.py +377 -0
  14. scrapely_client/_resource_clients/actor_env_var.py +154 -0
  15. scrapely_client/_resource_clients/actor_env_var_collection.py +165 -0
  16. scrapely_client/_resource_clients/actor_version.py +254 -0
  17. scrapely_client/_resource_clients/actor_version_collection.py +232 -0
  18. scrapely_client/_resource_clients/build.py +244 -0
  19. scrapely_client/_resource_clients/build_collection.py +164 -0
  20. scrapely_client/_resource_clients/dataset.py +1151 -0
  21. scrapely_client/_resource_clients/dataset_collection.py +244 -0
  22. scrapely_client/_resource_clients/key_value_store.py +906 -0
  23. scrapely_client/_resource_clients/key_value_store_collection.py +243 -0
  24. scrapely_client/_resource_clients/log.py +225 -0
  25. scrapely_client/_resource_clients/request_queue.py +1167 -0
  26. scrapely_client/_resource_clients/request_queue_collection.py +240 -0
  27. scrapely_client/_resource_clients/run.py +875 -0
  28. scrapely_client/_resource_clients/run_collection.py +226 -0
  29. scrapely_client/_resource_clients/schedule.py +243 -0
  30. scrapely_client/_resource_clients/schedule_collection.py +260 -0
  31. scrapely_client/_resource_clients/store_collection.py +242 -0
  32. scrapely_client/_resource_clients/task.py +793 -0
  33. scrapely_client/_resource_clients/task_collection.py +322 -0
  34. scrapely_client/_resource_clients/user.py +262 -0
  35. scrapely_client/_resource_clients/webhook.py +292 -0
  36. scrapely_client/_resource_clients/webhook_collection.py +281 -0
  37. scrapely_client/_resource_clients/webhook_dispatch.py +84 -0
  38. scrapely_client/_resource_clients/webhook_dispatch_collection.py +158 -0
  39. scrapely_client/_scrapely_client.py +837 -0
  40. scrapely_client/_statistics.py +29 -0
  41. scrapely_client/_status_message_watcher.py +197 -0
  42. scrapely_client/_streamed_log.py +250 -0
  43. scrapely_client/_typeddicts.py +408 -0
  44. scrapely_client/_utils/__init__.py +0 -0
  45. scrapely_client/_utils/crypto.py +87 -0
  46. scrapely_client/_utils/encoding.py +133 -0
  47. scrapely_client/_utils/errors.py +33 -0
  48. scrapely_client/_utils/http.py +137 -0
  49. scrapely_client/_utils/time.py +32 -0
  50. scrapely_client/_utils/try_import.py +61 -0
  51. scrapely_client/errors.py +190 -0
  52. scrapely_client/http_clients/__init__.py +10 -0
  53. scrapely_client/http_clients/_base.py +430 -0
  54. scrapely_client/http_clients/_impit.py +598 -0
  55. scrapely_client/http_compressors/__init__.py +18 -0
  56. scrapely_client/http_compressors/_base.py +25 -0
  57. scrapely_client/http_compressors/_brotli.py +38 -0
  58. scrapely_client/http_compressors/_gzip.py +38 -0
  59. scrapely_client/http_compressors/_resolve.py +38 -0
  60. scrapely_client/py.typed +0 -0
  61. scrapely_client/types.py +52 -0
  62. scrapely_client_python-3.1.3.dist-info/METADATA +269 -0
  63. scrapely_client_python-3.1.3.dist-info/RECORD +65 -0
  64. scrapely_client_python-3.1.3.dist-info/WHEEL +4 -0
  65. scrapely_client_python-3.1.3.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,837 @@
1
+ from __future__ import annotations
2
+
3
+ from functools import cached_property
4
+ from typing import TYPE_CHECKING
5
+
6
+ from scrapely_client._charging import ChargingHooks, wrap_dataset_client_with_charging
7
+ from scrapely_client._client_registry import ClientRegistry, ClientRegistryAsync
8
+ from scrapely_client._consts import (
9
+ API_VERSION,
10
+ DEFAULT_API_URL,
11
+ DEFAULT_MAX_RETRIES,
12
+ DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
13
+ DEFAULT_TIMEOUT_LONG,
14
+ DEFAULT_TIMEOUT_MAX,
15
+ DEFAULT_TIMEOUT_MEDIUM,
16
+ DEFAULT_TIMEOUT_SHORT,
17
+ )
18
+ from scrapely_client._docs import docs_group
19
+ from scrapely_client._resource_clients import (
20
+ ActorClient,
21
+ ActorClientAsync,
22
+ ActorCollectionClient,
23
+ ActorCollectionClientAsync,
24
+ ActorEnvVarClient,
25
+ ActorEnvVarClientAsync,
26
+ ActorEnvVarCollectionClient,
27
+ ActorEnvVarCollectionClientAsync,
28
+ ActorVersionClient,
29
+ ActorVersionClientAsync,
30
+ ActorVersionCollectionClient,
31
+ ActorVersionCollectionClientAsync,
32
+ BuildClient,
33
+ BuildClientAsync,
34
+ BuildCollectionClient,
35
+ BuildCollectionClientAsync,
36
+ DatasetClient,
37
+ DatasetClientAsync,
38
+ DatasetCollectionClient,
39
+ DatasetCollectionClientAsync,
40
+ KeyValueStoreClient,
41
+ KeyValueStoreClientAsync,
42
+ KeyValueStoreCollectionClient,
43
+ KeyValueStoreCollectionClientAsync,
44
+ LogClient,
45
+ LogClientAsync,
46
+ RequestQueueClient,
47
+ RequestQueueClientAsync,
48
+ RequestQueueCollectionClient,
49
+ RequestQueueCollectionClientAsync,
50
+ RunClient,
51
+ RunClientAsync,
52
+ RunCollectionClient,
53
+ RunCollectionClientAsync,
54
+ ScheduleClient,
55
+ ScheduleClientAsync,
56
+ ScheduleCollectionClient,
57
+ ScheduleCollectionClientAsync,
58
+ StoreCollectionClient,
59
+ StoreCollectionClientAsync,
60
+ TaskClient,
61
+ TaskClientAsync,
62
+ TaskCollectionClient,
63
+ TaskCollectionClientAsync,
64
+ UserClient,
65
+ UserClientAsync,
66
+ WebhookClient,
67
+ WebhookClientAsync,
68
+ WebhookCollectionClient,
69
+ WebhookCollectionClientAsync,
70
+ WebhookDispatchClient,
71
+ WebhookDispatchClientAsync,
72
+ WebhookDispatchCollectionClient,
73
+ WebhookDispatchCollectionClientAsync,
74
+ )
75
+ from scrapely_client._statistics import ClientStatistics
76
+ from scrapely_client._utils.http import check_custom_headers
77
+ from scrapely_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync
78
+ from scrapely_client.http_compressors._resolve import resolve_compressor
79
+
80
+ if TYPE_CHECKING:
81
+ from datetime import timedelta
82
+
83
+ from scrapely_client.http_compressors._base import HttpCompressor
84
+ from scrapely_client.types import HttpCompressionAlgorithm
85
+
86
+
87
+ @docs_group('Scrapely API clients')
88
+ class ApifyClient:
89
+ """Synchronous client for the Scrapely API.
90
+
91
+ This is the main entry point for interacting with the Scrapely platform. It provides methods to access
92
+ resource-specific sub-clients for managing Actors, runs, datasets, key-value stores, request queues,
93
+ schedules, webhooks, and more.
94
+
95
+ The client automatically handles retries with exponential backoff for failed or rate-limited requests.
96
+
97
+ ### Usage
98
+
99
+ ```python
100
+ from scrapely_client import ApifyClient
101
+
102
+ client = ApifyClient(token='MY-APIFY-TOKEN')
103
+
104
+ # Start an Actor and wait for it to finish.
105
+ actor_client = client.actor('apify/python-example')
106
+ run = actor_client.call(run_input={'first_number': 1, 'second_number': 2})
107
+
108
+ # Fetch results from the run's default dataset.
109
+ if run is not None:
110
+ dataset_client = client.dataset(run.default_dataset_id)
111
+ items = dataset_client.list_items().items
112
+ for item in items:
113
+ print(item)
114
+ ```
115
+ """
116
+
117
+ def __init__(
118
+ self,
119
+ token: str | None = None,
120
+ *,
121
+ api_url: str = DEFAULT_API_URL,
122
+ api_public_url: str | None = DEFAULT_API_URL,
123
+ max_retries: int = DEFAULT_MAX_RETRIES,
124
+ min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
125
+ timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT,
126
+ timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM,
127
+ timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
128
+ timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
129
+ headers: dict[str, str] | None = None,
130
+ compression: HttpCompressionAlgorithm | HttpCompressor = 'gzip',
131
+ charging_hooks: ChargingHooks | None = None,
132
+ default_dataset_id: str | None = None,
133
+ ) -> None:
134
+ """Initialize the Scrapely API client.
135
+
136
+ To use a custom HTTP client, use the `with_custom_http_client` class method instead.
137
+
138
+ Args:
139
+ token: The Scrapely API token. You can find your token on the
140
+ Integrations page in the Scrapely Console.
141
+ api_url: The URL of the Scrapely API server to connect to. Defaults to https://api-staging.scrape.ly. It can
142
+ be an internal URL that is not globally accessible, in which case `api_public_url` should be set
143
+ as well.
144
+ api_public_url: The globally accessible URL of the Scrapely API server. Should be set only if `api_url`
145
+ is an internal URL that is not globally accessible. Defaults to https://api-staging.scrape.ly.
146
+ max_retries: How many times to retry a failed request at most.
147
+ min_delay_between_retries: How long will the client wait between retrying requests
148
+ (increases exponentially from this value).
149
+ timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
150
+ timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
151
+ timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
152
+ timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
153
+ headers: Additional HTTP headers to include in all API requests.
154
+ compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm,
155
+ or an `HttpCompressor` instance for finer-grained control.
156
+ charging_hooks: Charging hooks for pay-per-event dataset pushes (Scrapely extension). When set
157
+ together with `default_dataset_id`, pushes to the default dataset are routed through the hooks.
158
+ Without hooks the client behaves exactly like stock `apify-client`.
159
+ default_dataset_id: ID of the actor run's default dataset, used to gate charging interception.
160
+ """
161
+ # We need to do this because of mocking in tests and default mutable arguments.
162
+ api_url = DEFAULT_API_URL if api_url is None else api_url
163
+ api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
164
+
165
+ if headers:
166
+ check_custom_headers(self.__class__.__name__, headers)
167
+
168
+ self._charging_hooks = charging_hooks
169
+ self._default_dataset_id = default_dataset_id
170
+ self._token = token
171
+ """Scrapely API token for authentication."""
172
+
173
+ self._base_url = f'{api_url.rstrip("/")}/{API_VERSION}'
174
+ """Base URL of the Scrapely API."""
175
+
176
+ self._public_base_url = f'{api_public_url.rstrip("/")}/{API_VERSION}'
177
+ """Public base URL for CDN access."""
178
+
179
+ self._statistics = ClientStatistics()
180
+ """Collector for client request statistics."""
181
+
182
+ self._http_client: HttpClient | None = None
183
+ """HTTP client used to communicate with the Scrapely API. Lazily initialized on first access."""
184
+
185
+ self._client_registry = ClientRegistry(
186
+ actor_client=ActorClient,
187
+ actor_collection_client=ActorCollectionClient,
188
+ actor_env_var_client=ActorEnvVarClient,
189
+ actor_env_var_collection_client=ActorEnvVarCollectionClient,
190
+ actor_version_client=ActorVersionClient,
191
+ actor_version_collection_client=ActorVersionCollectionClient,
192
+ build_client=BuildClient,
193
+ build_collection_client=BuildCollectionClient,
194
+ dataset_client=DatasetClient,
195
+ dataset_collection_client=DatasetCollectionClient,
196
+ key_value_store_client=KeyValueStoreClient,
197
+ key_value_store_collection_client=KeyValueStoreCollectionClient,
198
+ log_client=LogClient,
199
+ request_queue_client=RequestQueueClient,
200
+ request_queue_collection_client=RequestQueueCollectionClient,
201
+ run_client=RunClient,
202
+ run_collection_client=RunCollectionClient,
203
+ schedule_client=ScheduleClient,
204
+ schedule_collection_client=ScheduleCollectionClient,
205
+ store_collection_client=StoreCollectionClient,
206
+ task_client=TaskClient,
207
+ task_collection_client=TaskCollectionClient,
208
+ user_client=UserClient,
209
+ webhook_client=WebhookClient,
210
+ webhook_collection_client=WebhookCollectionClient,
211
+ webhook_dispatch_client=WebhookDispatchClient,
212
+ webhook_dispatch_collection_client=WebhookDispatchCollectionClient,
213
+ )
214
+ """Registry of resource client classes used for dependency injection."""
215
+
216
+ # Configuration for the default HTTP client (used if a custom client is not provided).
217
+ self._max_retries = max_retries
218
+ self._min_delay_between_retries = min_delay_between_retries
219
+ self._timeout_short = timeout_short
220
+ self._timeout_medium = timeout_medium
221
+ self._timeout_long = timeout_long
222
+ self._timeout_max = timeout_max
223
+ self._headers = headers
224
+ self._http_compressor = resolve_compressor(compression)
225
+
226
+ @classmethod
227
+ def with_custom_http_client(
228
+ cls,
229
+ token: str | None = None,
230
+ *,
231
+ api_url: str = DEFAULT_API_URL,
232
+ api_public_url: str | None = DEFAULT_API_URL,
233
+ http_client: HttpClient,
234
+ ) -> ApifyClient:
235
+ """Create an `ApifyClient` instance with a custom HTTP client.
236
+
237
+ Use this alternative constructor when you want to provide your own HTTP client implementation
238
+ instead of the default one. The custom client is responsible for its own configuration
239
+ (retries, timeouts, etc.); only the token is applied to it, as described below.
240
+
241
+ ### Usage
242
+
243
+ ```python
244
+ from scrapely_client import ApifyClient
245
+ from scrapely_client.http_clients import HttpClient, HttpResponse
246
+
247
+ class MyHttpClient(HttpClient):
248
+ def call(self, *, method, url, **kwargs) -> HttpResponse:
249
+ ...
250
+
251
+ client = ApifyClient.with_custom_http_client(
252
+ token='MY-APIFY-TOKEN',
253
+ http_client=MyHttpClient(),
254
+ )
255
+ ```
256
+
257
+ Args:
258
+ token: The Scrapely API token. It is set as the `Authorization` header on the custom client,
259
+ unless the client already has one configured.
260
+ api_url: The URL of the Scrapely API server to connect to. Defaults to https://api-staging.scrape.ly.
261
+ api_public_url: The globally accessible URL of the Scrapely API server. Defaults to https://api-staging.scrape.ly.
262
+ http_client: A custom HTTP client instance extending `HttpClient`.
263
+ """
264
+ instance = cls(token=token, api_url=api_url, api_public_url=api_public_url)
265
+ if token is not None:
266
+ http_client.set_default_authorization(token)
267
+ instance._http_client = http_client
268
+ return instance
269
+
270
+ @property
271
+ def token(self) -> str | None:
272
+ """The Scrapely API token used by the client."""
273
+ return self._token
274
+
275
+ @property
276
+ def http_client(self) -> HttpClient:
277
+ """The HTTP client instance used for API communication.
278
+
279
+ Returns the custom HTTP client if one was provided via `with_custom_http_client`,
280
+ or the default `ImpitHttpClient` otherwise (lazily created on first access).
281
+ """
282
+ if self._http_client is None:
283
+ self._http_client = ImpitHttpClient(
284
+ token=self._token,
285
+ timeout_short=self._timeout_short,
286
+ timeout_medium=self._timeout_medium,
287
+ timeout_long=self._timeout_long,
288
+ timeout_max=self._timeout_max,
289
+ max_retries=self._max_retries,
290
+ min_delay_between_retries=self._min_delay_between_retries,
291
+ statistics=self._statistics,
292
+ headers=self._headers,
293
+ http_compressor=self._http_compressor,
294
+ )
295
+
296
+ return self._http_client
297
+
298
+ @cached_property
299
+ def _base_kwargs(self) -> dict:
300
+ """Base keyword arguments for resource client construction."""
301
+ return {
302
+ 'base_url': self._base_url,
303
+ 'public_base_url': self._public_base_url,
304
+ 'http_client': self.http_client,
305
+ 'client_registry': self._client_registry,
306
+ }
307
+
308
+ def actor(self, actor_id: str) -> ActorClient:
309
+ """Get the sub-client for a specific Actor.
310
+
311
+ Args:
312
+ actor_id: ID of the Actor to be manipulated.
313
+ """
314
+ return ActorClient(resource_id=actor_id, **self._base_kwargs)
315
+
316
+ def actors(self) -> ActorCollectionClient:
317
+ """Get the sub-client for the Actor collection, allowing to list and create Actors."""
318
+ return ActorCollectionClient(**self._base_kwargs)
319
+
320
+ def build(self, build_id: str) -> BuildClient:
321
+ """Get the sub-client for a specific Actor build.
322
+
323
+ Args:
324
+ build_id: ID of the Actor build to be manipulated.
325
+ """
326
+ return BuildClient(resource_id=build_id, **self._base_kwargs)
327
+
328
+ def builds(self) -> BuildCollectionClient:
329
+ """Get the sub-client for the build collection, allowing to list builds."""
330
+ return BuildCollectionClient(**self._base_kwargs)
331
+
332
+ def run(self, run_id: str) -> RunClient:
333
+ """Get the sub-client for a specific Actor run.
334
+
335
+ Args:
336
+ run_id: ID of the Actor run to be manipulated.
337
+ """
338
+ return RunClient(resource_id=run_id, **self._base_kwargs)
339
+
340
+ def runs(self) -> RunCollectionClient:
341
+ """Get the sub-client for the run collection, allowing to list Actor runs."""
342
+ return RunCollectionClient(**self._base_kwargs)
343
+
344
+ def dataset(self, dataset_id: str) -> DatasetClient:
345
+ """Get the sub-client for a specific dataset.
346
+
347
+ Args:
348
+ dataset_id: ID of the dataset to be manipulated.
349
+ """
350
+ dataset_client = DatasetClient(resource_id=dataset_id, **self._base_kwargs)
351
+ if (
352
+ self._charging_hooks is not None
353
+ and dataset_id == self._default_dataset_id
354
+ and self._charging_hooks.should_charge_dataset_pushes(dataset_id)
355
+ ):
356
+ return wrap_dataset_client_with_charging(dataset_client, self._charging_hooks, is_async=False)
357
+ return dataset_client
358
+
359
+ def datasets(self) -> DatasetCollectionClient:
360
+ """Get the sub-client for the dataset collection, allowing to list and create datasets."""
361
+ return DatasetCollectionClient(**self._base_kwargs)
362
+
363
+ def key_value_store(self, key_value_store_id: str) -> KeyValueStoreClient:
364
+ """Get the sub-client for a specific key-value store.
365
+
366
+ Args:
367
+ key_value_store_id: ID of the key-value store to be manipulated.
368
+ """
369
+ return KeyValueStoreClient(resource_id=key_value_store_id, **self._base_kwargs)
370
+
371
+ def key_value_stores(self) -> KeyValueStoreCollectionClient:
372
+ """Get the sub-client for the key-value store collection, allowing to list and create key-value stores."""
373
+ return KeyValueStoreCollectionClient(**self._base_kwargs)
374
+
375
+ def request_queue(self, request_queue_id: str, *, client_key: str | None = None) -> RequestQueueClient:
376
+ """Get the sub-client for a specific request queue.
377
+
378
+ Args:
379
+ request_queue_id: ID of the request queue to be manipulated.
380
+ client_key: A unique identifier of the client accessing the request queue.
381
+ """
382
+ return RequestQueueClient(resource_id=request_queue_id, client_key=client_key, **self._base_kwargs)
383
+
384
+ def request_queues(self) -> RequestQueueCollectionClient:
385
+ """Get the sub-client for the request queue collection, allowing to list and create request queues."""
386
+ return RequestQueueCollectionClient(**self._base_kwargs)
387
+
388
+ def webhook(self, webhook_id: str) -> WebhookClient:
389
+ """Get the sub-client for a specific webhook.
390
+
391
+ Args:
392
+ webhook_id: ID of the webhook to be manipulated.
393
+ """
394
+ return WebhookClient(resource_id=webhook_id, **self._base_kwargs)
395
+
396
+ def webhooks(self) -> WebhookCollectionClient:
397
+ """Get the sub-client for the webhook collection, allowing to list and create webhooks."""
398
+ return WebhookCollectionClient(**self._base_kwargs)
399
+
400
+ def webhook_dispatch(self, webhook_dispatch_id: str) -> WebhookDispatchClient:
401
+ """Get the sub-client for a specific webhook dispatch.
402
+
403
+ Args:
404
+ webhook_dispatch_id: ID of the webhook dispatch to access.
405
+ """
406
+ return WebhookDispatchClient(resource_id=webhook_dispatch_id, **self._base_kwargs)
407
+
408
+ def webhook_dispatches(self) -> WebhookDispatchCollectionClient:
409
+ """Get the sub-client for the webhook dispatch collection, allowing to list webhook dispatches."""
410
+ return WebhookDispatchCollectionClient(**self._base_kwargs)
411
+
412
+ def schedule(self, schedule_id: str) -> ScheduleClient:
413
+ """Get the sub-client for a specific schedule.
414
+
415
+ Args:
416
+ schedule_id: ID of the schedule to be manipulated.
417
+ """
418
+ return ScheduleClient(resource_id=schedule_id, **self._base_kwargs)
419
+
420
+ def schedules(self) -> ScheduleCollectionClient:
421
+ """Get the sub-client for the schedule collection, allowing to list and create schedules."""
422
+ return ScheduleCollectionClient(**self._base_kwargs)
423
+
424
+ def log(self, build_or_run_id: str) -> LogClient:
425
+ """Get the sub-client for retrieving logs of an Actor build or run.
426
+
427
+ Args:
428
+ build_or_run_id: ID of the Actor build or run for which to access the log.
429
+ """
430
+ return LogClient(resource_id=build_or_run_id, **self._base_kwargs)
431
+
432
+ def task(self, task_id: str) -> TaskClient:
433
+ """Get the sub-client for a specific Actor task.
434
+
435
+ Args:
436
+ task_id: ID of the task to be manipulated.
437
+ """
438
+ return TaskClient(resource_id=task_id, **self._base_kwargs)
439
+
440
+ def tasks(self) -> TaskCollectionClient:
441
+ """Get the sub-client for the task collection, allowing to list and create Actor tasks."""
442
+ return TaskCollectionClient(**self._base_kwargs)
443
+
444
+ def user(self, user_id: str | None = None) -> UserClient:
445
+ """Get the sub-client for querying user data.
446
+
447
+ Args:
448
+ user_id: ID of user to be queried. If None, queries the user belonging to the token supplied to the client.
449
+ """
450
+ return UserClient(resource_id=user_id, **self._base_kwargs)
451
+
452
+ def store(self) -> StoreCollectionClient:
453
+ """Get the sub-client for the Scrapely Store, allowing to list Actors published in the store."""
454
+ return StoreCollectionClient(**self._base_kwargs)
455
+
456
+
457
+ @docs_group('Scrapely API clients')
458
+ class ApifyClientAsync:
459
+ """Asynchronous client for the Scrapely API.
460
+
461
+ This is the main entry point for interacting with the Scrapely platform using async/await. It provides
462
+ methods to access resource-specific sub-clients for managing Actors, runs, datasets, key-value stores,
463
+ request queues, schedules, webhooks, and more.
464
+
465
+ The client automatically handles retries with exponential backoff for failed or rate-limited requests.
466
+
467
+ ### Usage
468
+
469
+ ```python
470
+ import asyncio
471
+
472
+ from scrapely_client import ApifyClientAsync
473
+
474
+
475
+ async def main() -> None:
476
+ client = ApifyClientAsync(token='MY-APIFY-TOKEN')
477
+
478
+ # Start an Actor and wait for it to finish.
479
+ actor_client = client.actor('apify/python-example')
480
+ run = await actor_client.call(run_input={'first_number': 1, 'second_number': 2})
481
+
482
+ # Fetch results from the run's default dataset.
483
+ if run is not None:
484
+ dataset_client = client.dataset(run.default_dataset_id)
485
+ items = (await dataset_client.list_items()).items
486
+ for item in items:
487
+ print(item)
488
+
489
+
490
+ asyncio.run(main())
491
+ ```
492
+ """
493
+
494
+ def __init__(
495
+ self,
496
+ token: str | None = None,
497
+ *,
498
+ api_url: str = DEFAULT_API_URL,
499
+ api_public_url: str | None = DEFAULT_API_URL,
500
+ max_retries: int = DEFAULT_MAX_RETRIES,
501
+ min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
502
+ timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT,
503
+ timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM,
504
+ timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
505
+ timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
506
+ headers: dict[str, str] | None = None,
507
+ compression: HttpCompressionAlgorithm | HttpCompressor = 'gzip',
508
+ charging_hooks: ChargingHooks | None = None,
509
+ default_dataset_id: str | None = None,
510
+ ) -> None:
511
+ """Initialize the Scrapely API client.
512
+
513
+ To use a custom HTTP client, use the `with_custom_http_client` class method instead.
514
+
515
+ Args:
516
+ token: The Scrapely API token. You can find your token on the
517
+ Integrations page in the Scrapely Console.
518
+ api_url: The URL of the Scrapely API server to connect to. Defaults to https://api-staging.scrape.ly. It can
519
+ be an internal URL that is not globally accessible, in which case `api_public_url` should be set
520
+ as well.
521
+ api_public_url: The globally accessible URL of the Scrapely API server. Should be set only if `api_url`
522
+ is an internal URL that is not globally accessible. Defaults to https://api-staging.scrape.ly.
523
+ max_retries: How many times to retry a failed request at most.
524
+ min_delay_between_retries: How long will the client wait between retrying requests
525
+ (increases exponentially from this value).
526
+ timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
527
+ timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
528
+ timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
529
+ timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
530
+ headers: Additional HTTP headers to include in all API requests.
531
+ compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm,
532
+ or an `HttpCompressor` instance for finer-grained control.
533
+ charging_hooks: Charging hooks for pay-per-event dataset pushes (Scrapely extension). When set
534
+ together with `default_dataset_id`, pushes to the default dataset are routed through the hooks.
535
+ Without hooks the client behaves exactly like stock `apify-client`.
536
+ default_dataset_id: ID of the actor run's default dataset, used to gate charging interception.
537
+ """
538
+ # We need to do this because of mocking in tests and default mutable arguments.
539
+ api_url = DEFAULT_API_URL if api_url is None else api_url
540
+ api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
541
+
542
+ if headers:
543
+ check_custom_headers(self.__class__.__name__, headers)
544
+
545
+ self._charging_hooks = charging_hooks
546
+ self._default_dataset_id = default_dataset_id
547
+ self._token = token
548
+ """Scrapely API token for authentication."""
549
+
550
+ self._base_url = f'{api_url.rstrip("/")}/{API_VERSION}'
551
+ """Base URL of the Scrapely API."""
552
+
553
+ self._public_base_url = f'{api_public_url.rstrip("/")}/{API_VERSION}'
554
+ """Public base URL for CDN access."""
555
+
556
+ self._statistics = ClientStatistics()
557
+ """Collector for client request statistics."""
558
+
559
+ self._http_client: HttpClientAsync | None = None
560
+ """HTTP client used to communicate with the Scrapely API. Lazily initialized on first access."""
561
+
562
+ self._client_registry = ClientRegistryAsync(
563
+ actor_client=ActorClientAsync,
564
+ actor_collection_client=ActorCollectionClientAsync,
565
+ actor_env_var_client=ActorEnvVarClientAsync,
566
+ actor_env_var_collection_client=ActorEnvVarCollectionClientAsync,
567
+ actor_version_client=ActorVersionClientAsync,
568
+ actor_version_collection_client=ActorVersionCollectionClientAsync,
569
+ build_client=BuildClientAsync,
570
+ build_collection_client=BuildCollectionClientAsync,
571
+ dataset_client=DatasetClientAsync,
572
+ dataset_collection_client=DatasetCollectionClientAsync,
573
+ key_value_store_client=KeyValueStoreClientAsync,
574
+ key_value_store_collection_client=KeyValueStoreCollectionClientAsync,
575
+ log_client=LogClientAsync,
576
+ request_queue_client=RequestQueueClientAsync,
577
+ request_queue_collection_client=RequestQueueCollectionClientAsync,
578
+ run_client=RunClientAsync,
579
+ run_collection_client=RunCollectionClientAsync,
580
+ schedule_client=ScheduleClientAsync,
581
+ schedule_collection_client=ScheduleCollectionClientAsync,
582
+ store_collection_client=StoreCollectionClientAsync,
583
+ task_client=TaskClientAsync,
584
+ task_collection_client=TaskCollectionClientAsync,
585
+ user_client=UserClientAsync,
586
+ webhook_client=WebhookClientAsync,
587
+ webhook_collection_client=WebhookCollectionClientAsync,
588
+ webhook_dispatch_client=WebhookDispatchClientAsync,
589
+ webhook_dispatch_collection_client=WebhookDispatchCollectionClientAsync,
590
+ )
591
+ """Registry of resource client classes used for dependency injection."""
592
+
593
+ # Configuration for the default HTTP client (used if a custom client is not provided).
594
+ self._max_retries = max_retries
595
+ self._min_delay_between_retries = min_delay_between_retries
596
+ self._timeout_short = timeout_short
597
+ self._timeout_medium = timeout_medium
598
+ self._timeout_long = timeout_long
599
+ self._timeout_max = timeout_max
600
+ self._headers = headers
601
+ self._http_compressor = resolve_compressor(compression)
602
+
603
+ @classmethod
604
+ def with_custom_http_client(
605
+ cls,
606
+ token: str | None = None,
607
+ *,
608
+ api_url: str = DEFAULT_API_URL,
609
+ api_public_url: str | None = DEFAULT_API_URL,
610
+ http_client: HttpClientAsync,
611
+ ) -> ApifyClientAsync:
612
+ """Create an `ApifyClientAsync` instance with a custom HTTP client.
613
+
614
+ Use this alternative constructor when you want to provide your own HTTP client implementation
615
+ instead of the default one. The custom client is responsible for its own configuration
616
+ (retries, timeouts, etc.); only the token is applied to it, as described below.
617
+
618
+ ### Usage
619
+
620
+ ```python
621
+ from scrapely_client import ApifyClientAsync
622
+ from scrapely_client.http_clients import HttpClientAsync, HttpResponse
623
+
624
+ class MyHttpClient(HttpClientAsync):
625
+ async def call(self, *, method, url, **kwargs) -> HttpResponse:
626
+ ...
627
+
628
+ client = ApifyClientAsync.with_custom_http_client(
629
+ token='MY-APIFY-TOKEN',
630
+ http_client=MyHttpClient(),
631
+ )
632
+ ```
633
+
634
+ Args:
635
+ token: The Scrapely API token. It is set as the `Authorization` header on the custom client,
636
+ unless the client already has one configured.
637
+ api_url: The URL of the Scrapely API server to connect to. Defaults to https://api-staging.scrape.ly.
638
+ api_public_url: The globally accessible URL of the Scrapely API server. Defaults to https://api-staging.scrape.ly.
639
+ http_client: A custom HTTP client instance extending `HttpClientAsync`.
640
+ """
641
+ instance = cls(token=token, api_url=api_url, api_public_url=api_public_url)
642
+ if token is not None:
643
+ http_client.set_default_authorization(token)
644
+ instance._http_client = http_client
645
+ return instance
646
+
647
+ @property
648
+ def token(self) -> str | None:
649
+ """The Scrapely API token used by the client."""
650
+ return self._token
651
+
652
+ @property
653
+ def http_client(self) -> HttpClientAsync:
654
+ """The HTTP client instance used for API communication.
655
+
656
+ Returns the custom HTTP client if one was provided via `with_custom_http_client`,
657
+ or the default `ImpitHttpClientAsync` otherwise (lazily created on first access).
658
+ """
659
+ if self._http_client is None:
660
+ self._http_client = ImpitHttpClientAsync(
661
+ token=self._token,
662
+ timeout_short=self._timeout_short,
663
+ timeout_medium=self._timeout_medium,
664
+ timeout_long=self._timeout_long,
665
+ timeout_max=self._timeout_max,
666
+ max_retries=self._max_retries,
667
+ min_delay_between_retries=self._min_delay_between_retries,
668
+ statistics=self._statistics,
669
+ headers=self._headers,
670
+ http_compressor=self._http_compressor,
671
+ )
672
+ return self._http_client
673
+
674
+ @cached_property
675
+ def _base_kwargs(self) -> dict:
676
+ """Base keyword arguments for resource client construction."""
677
+ return {
678
+ 'base_url': self._base_url,
679
+ 'public_base_url': self._public_base_url,
680
+ 'http_client': self.http_client,
681
+ 'client_registry': self._client_registry,
682
+ }
683
+
684
+ def actor(self, actor_id: str) -> ActorClientAsync:
685
+ """Get the sub-client for a specific Actor.
686
+
687
+ Args:
688
+ actor_id: ID of the Actor to be manipulated.
689
+ """
690
+ return ActorClientAsync(resource_id=actor_id, **self._base_kwargs)
691
+
692
+ def actors(self) -> ActorCollectionClientAsync:
693
+ """Get the sub-client for the Actor collection, allowing to list and create Actors."""
694
+ return ActorCollectionClientAsync(**self._base_kwargs)
695
+
696
+ def build(self, build_id: str) -> BuildClientAsync:
697
+ """Get the sub-client for a specific Actor build.
698
+
699
+ Args:
700
+ build_id: ID of the Actor build to be manipulated.
701
+ """
702
+ return BuildClientAsync(resource_id=build_id, **self._base_kwargs)
703
+
704
+ def builds(self) -> BuildCollectionClientAsync:
705
+ """Get the sub-client for the build collection, allowing to list builds."""
706
+ return BuildCollectionClientAsync(**self._base_kwargs)
707
+
708
+ def run(self, run_id: str) -> RunClientAsync:
709
+ """Get the sub-client for a specific Actor run.
710
+
711
+ Args:
712
+ run_id: ID of the Actor run to be manipulated.
713
+ """
714
+ return RunClientAsync(resource_id=run_id, **self._base_kwargs)
715
+
716
+ def runs(self) -> RunCollectionClientAsync:
717
+ """Get the sub-client for the run collection, allowing to list Actor runs."""
718
+ return RunCollectionClientAsync(**self._base_kwargs)
719
+
720
+ def dataset(self, dataset_id: str) -> DatasetClientAsync:
721
+ """Get the sub-client for a specific dataset.
722
+
723
+ Args:
724
+ dataset_id: ID of the dataset to be manipulated.
725
+ """
726
+ dataset_client = DatasetClientAsync(resource_id=dataset_id, **self._base_kwargs)
727
+ if (
728
+ self._charging_hooks is not None
729
+ and dataset_id == self._default_dataset_id
730
+ and self._charging_hooks.should_charge_dataset_pushes(dataset_id)
731
+ ):
732
+ return wrap_dataset_client_with_charging(dataset_client, self._charging_hooks, is_async=True)
733
+ return dataset_client
734
+
735
+ def datasets(self) -> DatasetCollectionClientAsync:
736
+ """Get the sub-client for the dataset collection, allowing to list and create datasets."""
737
+ return DatasetCollectionClientAsync(**self._base_kwargs)
738
+
739
+ def key_value_store(self, key_value_store_id: str) -> KeyValueStoreClientAsync:
740
+ """Get the sub-client for a specific key-value store.
741
+
742
+ Args:
743
+ key_value_store_id: ID of the key-value store to be manipulated.
744
+ """
745
+ return KeyValueStoreClientAsync(resource_id=key_value_store_id, **self._base_kwargs)
746
+
747
+ def key_value_stores(self) -> KeyValueStoreCollectionClientAsync:
748
+ """Get the sub-client for the key-value store collection, allowing to list and create key-value stores."""
749
+ return KeyValueStoreCollectionClientAsync(**self._base_kwargs)
750
+
751
+ def request_queue(self, request_queue_id: str, *, client_key: str | None = None) -> RequestQueueClientAsync:
752
+ """Get the sub-client for a specific request queue.
753
+
754
+ Args:
755
+ request_queue_id: ID of the request queue to be manipulated.
756
+ client_key: A unique identifier of the client accessing the request queue.
757
+ """
758
+ return RequestQueueClientAsync(resource_id=request_queue_id, client_key=client_key, **self._base_kwargs)
759
+
760
+ def request_queues(self) -> RequestQueueCollectionClientAsync:
761
+ """Get the sub-client for the request queue collection, allowing to list and create request queues."""
762
+ return RequestQueueCollectionClientAsync(**self._base_kwargs)
763
+
764
+ def webhook(self, webhook_id: str) -> WebhookClientAsync:
765
+ """Get the sub-client for a specific webhook.
766
+
767
+ Args:
768
+ webhook_id: ID of the webhook to be manipulated.
769
+ """
770
+ return WebhookClientAsync(resource_id=webhook_id, **self._base_kwargs)
771
+
772
+ def webhooks(self) -> WebhookCollectionClientAsync:
773
+ """Get the sub-client for the webhook collection, allowing to list and create webhooks."""
774
+ return WebhookCollectionClientAsync(**self._base_kwargs)
775
+
776
+ def webhook_dispatch(self, webhook_dispatch_id: str) -> WebhookDispatchClientAsync:
777
+ """Get the sub-client for a specific webhook dispatch.
778
+
779
+ Args:
780
+ webhook_dispatch_id: ID of the webhook dispatch to access.
781
+ """
782
+ return WebhookDispatchClientAsync(resource_id=webhook_dispatch_id, **self._base_kwargs)
783
+
784
+ def webhook_dispatches(self) -> WebhookDispatchCollectionClientAsync:
785
+ """Get the sub-client for the webhook dispatch collection, allowing to list webhook dispatches."""
786
+ return WebhookDispatchCollectionClientAsync(**self._base_kwargs)
787
+
788
+ def schedule(self, schedule_id: str) -> ScheduleClientAsync:
789
+ """Get the sub-client for a specific schedule.
790
+
791
+ Args:
792
+ schedule_id: ID of the schedule to be manipulated.
793
+ """
794
+ return ScheduleClientAsync(resource_id=schedule_id, **self._base_kwargs)
795
+
796
+ def schedules(self) -> ScheduleCollectionClientAsync:
797
+ """Get the sub-client for the schedule collection, allowing to list and create schedules."""
798
+ return ScheduleCollectionClientAsync(**self._base_kwargs)
799
+
800
+ def log(self, build_or_run_id: str) -> LogClientAsync:
801
+ """Get the sub-client for retrieving logs of an Actor build or run.
802
+
803
+ Args:
804
+ build_or_run_id: ID of the Actor build or run for which to access the log.
805
+ """
806
+ return LogClientAsync(resource_id=build_or_run_id, **self._base_kwargs)
807
+
808
+ def task(self, task_id: str) -> TaskClientAsync:
809
+ """Get the sub-client for a specific Actor task.
810
+
811
+ Args:
812
+ task_id: ID of the task to be manipulated.
813
+ """
814
+ return TaskClientAsync(resource_id=task_id, **self._base_kwargs)
815
+
816
+ def tasks(self) -> TaskCollectionClientAsync:
817
+ """Get the sub-client for the task collection, allowing to list and create Actor tasks."""
818
+ return TaskCollectionClientAsync(**self._base_kwargs)
819
+
820
+ def user(self, user_id: str | None = None) -> UserClientAsync:
821
+ """Get the sub-client for querying user data.
822
+
823
+ Args:
824
+ user_id: ID of user to be queried. If None, queries the user belonging to the token supplied to the client.
825
+ """
826
+ return UserClientAsync(resource_id=user_id, **self._base_kwargs)
827
+
828
+ def store(self) -> StoreCollectionClientAsync:
829
+ """Get the sub-client for the Scrapely Store, allowing to list Actors published in the store."""
830
+ return StoreCollectionClientAsync(**self._base_kwargs)
831
+
832
+
833
+ ScrapelyClient = ApifyClient
834
+ """Primary name of the client; ``ApifyClient`` is kept as a drop-in compatibility alias."""
835
+
836
+ ScrapelyClientAsync = ApifyClientAsync
837
+ """Primary name of the async client; ``ApifyClientAsync`` is kept as a drop-in compatibility alias."""