scrapely-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. scrapely/__init__.py +246 -0
  2. scrapely/_browser_locations.py +5470 -0
  3. scrapely/actor.py +1836 -0
  4. scrapely/browser_configuration.py +146 -0
  5. scrapely/charging.py +1143 -0
  6. scrapely/charging_client.py +107 -0
  7. scrapely/configuration.py +375 -0
  8. scrapely/crypto/__init__.py +21 -0
  9. scrapely/crypto/_secrets.py +210 -0
  10. scrapely/errors.py +153 -0
  11. scrapely/events/__init__.py +57 -0
  12. scrapely/events/_types.py +194 -0
  13. scrapely/events/event_manager.py +292 -0
  14. scrapely/log.py +39 -0
  15. scrapely/patch_browser_pool.py +861 -0
  16. scrapely/pricing_types.py +172 -0
  17. scrapely/proxy_configuration.py +741 -0
  18. scrapely/py.typed +2 -0
  19. scrapely/request_loaders/__init__.py +5 -0
  20. scrapely/request_loaders/_scrapely_request_list.py +169 -0
  21. scrapely/storage/__init__.py +48 -0
  22. scrapely/storage/_file_system/__init__.py +9 -0
  23. scrapely/storage/_file_system/_key_value_store_client.py +155 -0
  24. scrapely/storage/_file_system/_storage_client.py +50 -0
  25. scrapely/storage/_metadata.py +138 -0
  26. scrapely/storage/api_client.py +350 -0
  27. scrapely/storage/client.py +618 -0
  28. scrapely/storage/patch_storage.py +257 -0
  29. scrapely/storage/scrapely_dataset_client.py +305 -0
  30. scrapely/storage/scrapely_kvs_client.py +220 -0
  31. scrapely/storage/scrapely_rq_client.py +496 -0
  32. scrapely/storage/scrapely_storage_client.py +192 -0
  33. scrapely/storage/smart_scrapely_storage_client.py +109 -0
  34. scrapely/storage/types.py +601 -0
  35. scrapely/storage/utils.py +38 -0
  36. scrapely/storage_clients/__init__.py +25 -0
  37. scrapely/storages/__init__.py +5 -0
  38. scrapely/types.py +269 -0
  39. scrapely/webhook.py +82 -0
  40. scrapely_python-0.1.0.dist-info/METADATA +83 -0
  41. scrapely_python-0.1.0.dist-info/RECORD +43 -0
  42. scrapely_python-0.1.0.dist-info/WHEEL +5 -0
  43. scrapely_python-0.1.0.dist-info/top_level.txt +1 -0
scrapely/actor.py ADDED
@@ -0,0 +1,1836 @@
1
+ """
2
+ Main Actor class for the Scrapely Python SDK.
3
+
4
+ The core class for building Actors on the Scrapely platform.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import asyncio
10
+ import base64
11
+ import json
12
+ import logging
13
+ import os
14
+ import signal
15
+ import sys
16
+ import time
17
+ import warnings
18
+ from collections.abc import MutableMapping
19
+ from datetime import datetime, timedelta
20
+ from decimal import Decimal
21
+ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast
22
+
23
+ from crawlee.configuration import Configuration as CrawleeConfiguration
24
+ from crawlee.storage_clients._base import StorageClient
25
+ from crawlee.storages import Dataset, KeyValueStore, RequestQueue
26
+
27
+ from .charging import ChargingManager, is_default_dataset_item_event, push_data_and_charge
28
+ from .configuration import Configuration
29
+ from .crypto import decrypt_secret_inputs
30
+ from crawlee.events import EventManager
31
+
32
+ from .events import Event, LocalEventManager, ScrapelyEventManager
33
+ from .charging_client import create_charging_client
34
+ from .errors import ForbiddenError
35
+ from .storage.api_client import ApiClient, ApiClientError
36
+ from .storage.client import is_storage_uuid
37
+ from .storage._metadata import StorageType, resolve_alias_from_storages_json
38
+ from .storage.patch_storage import (
39
+ disconnect_storage,
40
+ get_api_storage_factory,
41
+ get_smart_storage_client,
42
+ set_api_storage,
43
+ set_storage_client,
44
+ )
45
+ from .storage.scrapely_storage_client import ScrapelyStorageClient
46
+ from .storage.smart_scrapely_storage_client import SmartScrapelyStorageClient
47
+ from .types import (
48
+ AbortOptions,
49
+ ActorRun,
50
+ ApifyEnv,
51
+ CallOptions,
52
+ ChargeResult,
53
+ ExitOptions,
54
+ InitOptions,
55
+ MetamorphOptions,
56
+ OpenStorageOptions,
57
+ RebootOptions,
58
+ SetStatusMessageOptions,
59
+ StartOptions,
60
+ WebhookOptions,
61
+ )
62
+ from .log import _configure_logging
63
+ from .webhook import Webhook
64
+
65
+ if TYPE_CHECKING:
66
+ from scrapely_client import ApifyClientAsync
67
+
68
+ from .browser_configuration import BrowserConfiguration
69
+ from .proxy_configuration import ProxyConfiguration
70
+
71
+ logger = logging.getLogger(__name__)
72
+
73
+ # Terminal run statuses used by the run-control polling loops (call/call_task).
74
+ _RUN_TERMINAL_STATUSES = frozenset({'SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'})
75
+
76
+
77
+ def _payload_dict(payload: Any) -> Dict[str, Any]:
78
+ """Normalize an unwrapped API envelope payload to a plain dict.
79
+
80
+ ``ApiClient.get_json``/``post_json``/``put_json`` return the unwrapped
81
+ ``data`` field, which may be a non-dict (``{}`` fallback, list, scalar)
82
+ when the endpoint responds with an empty or non-JSON body.
83
+ """
84
+ if isinstance(payload, dict):
85
+ return cast(Dict[str, Any], payload)
86
+ return {}
87
+
88
+
89
+ def _timeout_seconds(value: Union[int, str, None]) -> Optional[int]:
90
+ """Normalize a ``CallOptions.timeout`` value to poll deadline seconds.
91
+
92
+ Accepts a positive int; ``'inherit'``, ``None`` and non-positive values
93
+ mean "no client-side deadline" (the platform's own run timeout applies).
94
+ """
95
+ if isinstance(value, bool):
96
+ return None
97
+ if isinstance(value, int) and value > 0:
98
+ return value
99
+ return None
100
+
101
+
102
+ def _resolve_run_timeout(value: Union[int, str, None]) -> Optional[int]:
103
+ """Resolve a ``StartOptions``/``CallOptions`` timeout to wire seconds.
104
+
105
+ ``'inherit'`` resolves to the current run's remaining seconds (Node SDK
106
+ ``resolveCallTimeout`` parity, warning when no platform timeout is set);
107
+ positive ints pass through; everything else means "not sent" so the
108
+ platform default applies.
109
+ """
110
+ if isinstance(value, str) and value.lower() == 'inherit':
111
+ remaining_ms = Actor.get_remaining_time()
112
+ if remaining_ms is None:
113
+ logger.warning(
114
+ "Using 'inherit' for timeout is only possible when running on the platform "
115
+ 'and when the timeout for the Actor run is set (ACTOR_TIMEOUT_AT).'
116
+ )
117
+ return None
118
+ return max(0, -(-remaining_ms // 1000)) # ceil(ms → secs)
119
+ return _timeout_seconds(value)
120
+
121
+
122
+ def _webhook_body(webhook: Any) -> Dict[str, Any]:
123
+ """Serialize a Webhook/WebhookOptions-like object to the API body format."""
124
+ to_dict = getattr(webhook, 'to_dict', None)
125
+ if callable(to_dict):
126
+ return cast(Dict[str, Any], to_dict())
127
+
128
+ body: Dict[str, Any] = {
129
+ 'eventTypes': [getattr(e, 'value', e) for e in (webhook.event_types or [])],
130
+ 'requestUrl': webhook.request_url,
131
+ }
132
+ if getattr(webhook, 'payload_template', None):
133
+ body['payloadTemplate'] = webhook.payload_template
134
+ if getattr(webhook, 'headers_template', None):
135
+ body['headersTemplate'] = webhook.headers_template
136
+ if getattr(webhook, 'idempotency_key', None):
137
+ body['idempotencyKey'] = webhook.idempotency_key
138
+ description = getattr(webhook, 'description', None)
139
+ if description:
140
+ body['description'] = description
141
+ ignore_ssl_errors = getattr(webhook, 'ignore_ssl_errors', None)
142
+ if ignore_ssl_errors is not None:
143
+ body['ignoreSslErrors'] = bool(ignore_ssl_errors)
144
+ do_not_retry = getattr(webhook, 'do_not_retry', None)
145
+ if do_not_retry is not None:
146
+ body['doNotRetry'] = bool(do_not_retry)
147
+ return body
148
+
149
+
150
+ def _run_options_params(
151
+ *,
152
+ build: Optional[str] = None,
153
+ timeout: Union[int, str, None] = None,
154
+ memory_mbytes: Optional[int] = None,
155
+ max_total_charge_usd: Optional[float] = None,
156
+ restart_on_error: Optional[bool] = None,
157
+ force_permission_level: Optional[Any] = None,
158
+ webhooks: Optional[List[Any]] = None,
159
+ ) -> Dict[str, Any]:
160
+ """Build the Apify-style run-option query parameters for run-start requests.
161
+
162
+ Server wire contract: the POST body of a run-start request is the run
163
+ input itself; run options (build/timeout/memory/maxTotalChargeUsd/
164
+ restartOnError/forcePermissionLevel) are read from query parameters only.
165
+ Inline webhooks ride along base64-encoded in the ``webhooks`` query param
166
+ (the query-param transport the server supports alongside the body one).
167
+ """
168
+ params: Dict[str, Any] = {}
169
+ if build:
170
+ params['build'] = build
171
+ timeout_secs = _resolve_run_timeout(timeout)
172
+ if timeout_secs is not None:
173
+ params['timeout'] = timeout_secs
174
+ if memory_mbytes:
175
+ params['memory'] = memory_mbytes
176
+ if max_total_charge_usd is not None:
177
+ params['maxTotalChargeUsd'] = max_total_charge_usd
178
+ if restart_on_error is not None:
179
+ params['restartOnError'] = 'true' if restart_on_error else 'false'
180
+ if force_permission_level is not None:
181
+ level = getattr(force_permission_level, 'value', force_permission_level)
182
+ params['forcePermissionLevel'] = str(level)
183
+ if webhooks:
184
+ encoded = base64.b64encode(
185
+ json.dumps([_webhook_body(w) for w in webhooks]).encode('utf-8'),
186
+ ).decode('ascii')
187
+ params['webhooks'] = encoded
188
+ return params
189
+
190
+
191
+ def _is_api_storage_config(storage: Any) -> bool:
192
+ return (
193
+ isinstance(storage, dict)
194
+ and 'api_url' in storage
195
+ and 'token' in storage
196
+ and 'run_id' in storage
197
+ )
198
+
199
+
200
+ class _ClassProperty:
201
+ """Read-only class property descriptor."""
202
+
203
+ def __init__(self, func: Callable[..., Any]) -> None:
204
+ self.func = func
205
+
206
+ def __get__(self, obj: Any, owner: Any = None) -> Any:
207
+ return self.func(owner)
208
+
209
+
210
+ class _ActorMeta(type):
211
+ """Metaclass enabling async with Actor: by delegating to classmethods."""
212
+
213
+ # Declarations for the Actor classmethods delegated to below (typing only).
214
+ init: Callable[..., Any]
215
+ _shutdown: Callable[..., Any]
216
+ _exit_code: int
217
+
218
+ async def __aenter__(cls) -> Any:
219
+ await cls.init()
220
+ return cls
221
+
222
+ async def __aexit__(cls, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
223
+ if exc_type is not None:
224
+ cls._exit_code = 1
225
+ logger.error(f'Actor failed: {exc_val}')
226
+ await cls._shutdown()
227
+
228
+
229
+ class Actor(metaclass=_ActorMeta):
230
+ """Core class for building Actors on the Scrapely platform."""
231
+
232
+ _instance: Optional['Actor'] = None
233
+ _initialized: bool = False
234
+ _config: Optional[Configuration] = None
235
+ _client: Optional[ApiClient] = None
236
+ _apify_client_instance: Optional['ApifyClientAsync'] = None
237
+ _scrapely_storage: Optional[ScrapelyStorageClient] = None
238
+ _smart_storage: Optional[SmartScrapelyStorageClient] = None
239
+ _default_dataset: Optional[Dataset] = None
240
+ _default_kvs: Optional[KeyValueStore] = None
241
+ _default_rq: Optional[RequestQueue] = None
242
+ _event_manager: Optional[EventManager] = None
243
+ _cached_input: Optional[Dict[str, Any]] = None
244
+ _exit_code: int = 0
245
+ _status_message: Optional[str] = None
246
+ _charging_manager: Optional[ChargingManager] = None
247
+ _use_state_stores: set = set()
248
+ # Lifecycle bookkeeping set in init() / cleared in _shutdown().
249
+ _lifecycle_config_snapshot: Optional[Dict[str, Any]] = None
250
+ _signal_shutdown_started: bool = False
251
+
252
+ def __init__(
253
+ self,
254
+ options: Optional[Dict[str, Any]] = None,
255
+ *,
256
+ configure_logging: bool = True,
257
+ exit_code: int = 0,
258
+ status_message: Optional[str] = None,
259
+ cleanup_timeout: timedelta = timedelta(seconds=30),
260
+ exit_process: Optional[bool] = None,
261
+ event_listeners_timeout: Optional[timedelta] = None,
262
+ ):
263
+ """Apify parity: ``async with Actor(configure_logging=False):`` as well as ``async with Actor:``.
264
+
265
+ ``options`` is a configuration dict (Apify passes a Configuration here). The context
266
+ manager never calls ``sys.exit`` (``exit_process`` is accepted for signature parity),
267
+ and ``event_listeners_timeout`` is accepted but not applied.
268
+ """
269
+ self._options = options or {}
270
+ self._configure_logging = configure_logging
271
+ self._instance_exit_code = exit_code
272
+ self._instance_status_message = status_message
273
+ self._cleanup_timeout = cleanup_timeout
274
+
275
+ async def __aenter__(self) -> 'Actor':
276
+ await type(self).init(
277
+ InitOptions(configuration=self._options or None, configure_logging=self._configure_logging)
278
+ )
279
+ return self
280
+
281
+ async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
282
+ cls = type(self)
283
+ if exc_type is not None:
284
+ cls._exit_code = 1
285
+ logger.error(f'Actor failed: {exc_val}')
286
+ else:
287
+ cls._exit_code = self._instance_exit_code
288
+ if self._instance_status_message:
289
+ cls._status_message = self._instance_status_message
290
+ await cls._shutdown(timeout_secs=int(self._cleanup_timeout.total_seconds()))
291
+
292
+ @classmethod
293
+ def get_default_instance(cls) -> 'Actor':
294
+ if cls._instance is None:
295
+ cls._instance = cls()
296
+ return cls._instance
297
+
298
+ @classmethod
299
+ async def init(cls, options: Optional[InitOptions] = None) -> None:
300
+ if cls._initialized:
301
+ return
302
+
303
+ if options is None or options.configure_logging:
304
+ _configure_logging()
305
+
306
+ # Do NOT set _initialized until everything succeeds (failed init must be retryable).
307
+ cls._config = Configuration.get_global_config()
308
+ # Snapshot keys we may overwrite so shutdown can restore prior lifecycle state.
309
+ cls._lifecycle_config_snapshot = {
310
+ 'api_base_url': cls._config.get('api_base_url'),
311
+ 'token': cls._config.get('token'),
312
+ 'actor_run_id': cls._config.get('actor_run_id'),
313
+ }
314
+ if options and options.configuration:
315
+ for key, value in options.configuration.items():
316
+ cls._config.set(key, value)
317
+
318
+ crawlee_config = CrawleeConfiguration.get_global_configuration()
319
+ if cls._config.get('is_at_home'):
320
+ crawlee_config.purge_on_start = False
321
+
322
+ storage_opt = options.storage if options else None
323
+
324
+ try:
325
+ # Apply explicit storage credentials BEFORE event manager / charging so
326
+ # WebSocket + PPE bill the same run as storage.
327
+ if storage_opt is not None and _is_api_storage_config(storage_opt):
328
+ if storage_opt.get('api_url'):
329
+ cls._config.set('api_base_url', storage_opt['api_url'])
330
+ if storage_opt.get('token'):
331
+ cls._config.set('token', storage_opt['token'])
332
+ if storage_opt.get('run_id'):
333
+ cls._config.set('actor_run_id', storage_opt['run_id'])
334
+
335
+ if storage_opt is not None:
336
+ if _is_api_storage_config(storage_opt):
337
+ logger.info(
338
+ '[Scrapely] Initializing API storage for run %s',
339
+ storage_opt['run_id'],
340
+ )
341
+ smart = set_api_storage(storage_opt)
342
+ elif isinstance(storage_opt, StorageClient):
343
+ smart = set_storage_client(storage_opt)
344
+ else:
345
+ raise TypeError(
346
+ 'Actor.init() storage option must be ApiStorageConfig or StorageClient'
347
+ )
348
+ else:
349
+ api_url = cls._config.get('api_base_url')
350
+ token = cls._config.get('token')
351
+ run_id = cls._config.get('actor_run_id')
352
+
353
+ if api_url and token and run_id:
354
+ logger.info('[Scrapely] Initializing API storage for run %s', run_id)
355
+ smart = set_api_storage({
356
+ 'api_url': api_url,
357
+ 'token': token,
358
+ 'run_id': run_id,
359
+ })
360
+ elif not cls._config.get('is_at_home'):
361
+ # Local development without credentials: Smart → FS (Apify docs shape)
362
+ logger.info('[Scrapely] Initializing local file-system storage')
363
+ smart = set_api_storage()
364
+ else:
365
+ raise RuntimeError(
366
+ 'Missing required configuration for secure storage. '
367
+ 'Required: apiBaseUrl (SCRAPELY_API_URL), token (SCRAPELY_TOKEN/APIFY_TOKEN), '
368
+ 'actorRunId (ACTOR_RUN_ID). Containers must be started by the scrapely-server, '
369
+ 'or pass storage in Actor.init().'
370
+ )
371
+
372
+ cls._smart_storage = smart if isinstance(smart, SmartScrapelyStorageClient) else get_smart_storage_client()
373
+ factory = get_api_storage_factory()
374
+ cls._scrapely_storage = factory
375
+ if factory is not None and factory.has_credentials():
376
+ cls._client = factory.api_client
377
+ await cls._client._ensure_client()
378
+
379
+ # Choose event manager AFTER config has effective storage credentials.
380
+ from datetime import timedelta
381
+
382
+ ws_url = cls._config.get('actor_events_ws_url')
383
+ persist_interval_ms = cls._config.get('persist_state_interval_millis', 60000)
384
+ persist_interval = timedelta(milliseconds=persist_interval_ms)
385
+
386
+ event_manager: EventManager
387
+ if ws_url:
388
+ event_manager = ScrapelyEventManager(
389
+ configuration=cls._config,
390
+ persist_state_interval=persist_interval,
391
+ )
392
+ else:
393
+ event_manager = LocalEventManager(
394
+ system_info_interval=timedelta(seconds=1),
395
+ persist_state_interval=persist_interval,
396
+ )
397
+ cls._event_manager = event_manager
398
+
399
+ await event_manager.__aenter__()
400
+
401
+ def handle_signal(signum: int, frame: Any) -> None:
402
+ # Bound drain so uncertain PPE intents can flush before hard exit.
403
+ # Fire-and-forget create_task alone drops pending charges on SIGTERM.
404
+ logger.info('Received signal %s, shutting down...', signum)
405
+ try:
406
+ loop = asyncio.get_running_loop()
407
+ except RuntimeError:
408
+ os._exit(0) # NoReturn
409
+
410
+ if getattr(cls, '_signal_shutdown_started', False):
411
+ return
412
+ cls._signal_shutdown_started = True
413
+
414
+ def _run_shutdown() -> None:
415
+ async def _bounded() -> None:
416
+ # Independent hard-exit timer: fires even if teardown hangs on a
417
+ # stuck close() — the deadline cannot be swallowed by exceptions.
418
+ hard_timer = loop.call_later(6.0, os._exit, 0)
419
+ try:
420
+ await asyncio.wait_for(cls._shutdown(timeout_secs=5), timeout=5.5)
421
+ except BaseException as err:
422
+ logger.warning('Shutdown on signal failed: %s', err)
423
+ finally:
424
+ hard_timer.cancel()
425
+ os._exit(0)
426
+
427
+ loop.create_task(_bounded())
428
+
429
+ try:
430
+ loop.call_soon_threadsafe(_run_shutdown)
431
+ except Exception:
432
+ os._exit(0)
433
+
434
+ signal.signal(signal.SIGINT, handle_signal)
435
+ signal.signal(signal.SIGTERM, handle_signal)
436
+
437
+ cls._initialized = True
438
+
439
+ # Initialize PPE charging once so Actor.charge() has prices + budget.
440
+ # Must run AFTER `_initialized = True`: ChargingManager.init() goes
441
+ # through Actor.get_charging_manager(), which requires an initialized
442
+ # Actor (otherwise this eager init always fails with a warning and
443
+ # charging is only lazily recovered on the first Actor.charge()).
444
+ try:
445
+ charging_manager = cls.get_charging_manager()
446
+ await charging_manager.init()
447
+ except Exception as e:
448
+ logger.warning('Failed to initialize ChargingManager: %s', e)
449
+ except Exception:
450
+ # Roll back partial init so a corrected retry can succeed.
451
+ try:
452
+ await cls._shutdown()
453
+ except Exception as shutdown_err:
454
+ logger.warning('Rollback after failed Actor.init: %s', shutdown_err)
455
+ raise
456
+
457
+ @classmethod
458
+ async def _shutdown(cls, timeout_secs: int = 30) -> None:
459
+ # Resource cleanup may raise; config restore + lifecycle reset MUST still
460
+ # run so a failed teardown cannot leave tenant/run credentials active for
461
+ # the next Actor.init() retry.
462
+ cleanup_error: Optional[BaseException] = None
463
+ try:
464
+ if cls._event_manager is not None:
465
+ try:
466
+ await cls._event_manager.__aexit__(None, None, None)
467
+ except BaseException as e:
468
+ cleanup_error = cleanup_error or e
469
+ logger.warning('Event manager shutdown failed: %s', e)
470
+ cls._event_manager = None
471
+
472
+ # Best-effort: confirm any unresolved PPE charge intents before teardown.
473
+ if cls._charging_manager is not None:
474
+ # Reject NEW charges before the final flush so nothing can be
475
+ # admitted after it while teardown closes storages/credentials.
476
+ close_fn = getattr(cls._charging_manager, 'close', None)
477
+ if close_fn is not None:
478
+ try:
479
+ close_fn()
480
+ except Exception:
481
+ pass
482
+ try:
483
+ flush = getattr(cls._charging_manager, 'flush_pending', None)
484
+ if flush is not None:
485
+ deadline_ms = min(max(1, timeout_secs) * 1000, 30_000)
486
+ await asyncio.wait_for(
487
+ flush(max_intents=64, deadline_ms=deadline_ms),
488
+ timeout=max(1.0, deadline_ms / 1000.0),
489
+ )
490
+ except BaseException as e:
491
+ # Never swallow cancellation — but also never let it skip the
492
+ # reference-clearing teardown below. Record it and re-raise at
493
+ # the END of _shutdown: an immediate raise would leave the
494
+ # closed charging manager and storage/client references
495
+ # installed, so a subsequent Actor.init() would reuse the
496
+ # closed manager and every later charge would fail. The
497
+ # signal-path deadline is still bounded by the independent
498
+ # loop.call_later(6.0, os._exit, 0) hard timer.
499
+ cleanup_error = cleanup_error or e
500
+ logger.warning('ChargingManager flush_pending failed during shutdown: %s', e)
501
+
502
+ if cls._scrapely_storage is not None:
503
+ try:
504
+ await cls._scrapely_storage.close()
505
+ except BaseException as e:
506
+ cleanup_error = cleanup_error or e
507
+ logger.warning('Storage close failed during shutdown: %s', e)
508
+ cls._scrapely_storage = None
509
+ cls._smart_storage = None
510
+ cls._client = None
511
+ # Clear charging + patched client so a second Actor lifecycle cannot
512
+ # post charges to the previous run_id / credentials.
513
+ cls._charging_manager = None
514
+ cls._apify_client_instance = None
515
+
516
+ cls._default_dataset = None
517
+ cls._default_kvs = None
518
+ cls._default_rq = None
519
+ cls._cached_input = None
520
+ try:
521
+ disconnect_storage()
522
+ except BaseException as e:
523
+ cleanup_error = cleanup_error or e
524
+ logger.warning('disconnect_storage failed during shutdown: %s', e)
525
+ finally:
526
+ # ALWAYS restore config keys mutated for this lifecycle so a later
527
+ # init without explicit storage does not inherit previous credentials.
528
+ snap = getattr(cls, '_lifecycle_config_snapshot', None)
529
+ if snap is not None and cls._config is not None:
530
+ for key, value in snap.items():
531
+ try:
532
+ cls._config.set(key, value)
533
+ except Exception:
534
+ pass
535
+ cls._lifecycle_config_snapshot = None
536
+ cls._initialized = False
537
+ # Allow a subsequent Actor lifecycle to handle SIGINT/SIGTERM again.
538
+ cls._signal_shutdown_started = False
539
+
540
+ if cleanup_error is not None:
541
+ raise cleanup_error
542
+
543
+
544
+ @classmethod
545
+ async def main(cls, user_func: Callable[[], Any]) -> Any:
546
+ try:
547
+ async with cls:
548
+ return await user_func()
549
+ except Exception as e:
550
+ logger.exception(f'Actor failed: {e}')
551
+ cls._exit_code = 1
552
+ raise
553
+ finally:
554
+ sys.exit(cls._exit_code)
555
+
556
+ @classmethod
557
+ async def exit(
558
+ cls,
559
+ message_or_options: Optional[Union[str, ExitOptions]] = None,
560
+ options: Optional[ExitOptions] = None,
561
+ *,
562
+ status_message: Optional[str] = None,
563
+ exit_code: Optional[int] = None,
564
+ ) -> None:
565
+ """Exit the Actor with an optional terminal status message.
566
+
567
+ Apify-Python parity: supports keyword-only ``status_message`` and
568
+ ``exit_code`` arguments (``await Actor.exit(status_message="Done")``).
569
+ The pre-existing positional forms (``Actor.exit("message")`` /
570
+ ``Actor.exit(ExitOptions(...))``) remain supported; explicit keyword
571
+ arguments take precedence over the positional values when provided.
572
+ """
573
+ if isinstance(message_or_options, str):
574
+ opts = ExitOptions(status_message=message_or_options)
575
+ else:
576
+ opts = message_or_options or ExitOptions()
577
+
578
+ if options:
579
+ opts = ExitOptions(**{**opts.__dict__, **options.__dict__})
580
+
581
+ if status_message is not None:
582
+ opts.status_message = status_message
583
+ if exit_code is not None:
584
+ opts.exit_code = exit_code
585
+
586
+ if opts.status_message:
587
+ logger.info(f'[Scrapely] {opts.status_message}')
588
+
589
+ cls._exit_code = opts.exit_code
590
+ cls._status_message = opts.status_message
591
+
592
+ await cls._shutdown(timeout_secs=opts.timeout_secs)
593
+
594
+ if opts.exit:
595
+ sys.exit(cls._exit_code)
596
+
597
+ @classmethod
598
+ async def fail(
599
+ cls,
600
+ message_or_options: Optional[Union[str, ExitOptions]] = None,
601
+ options: Optional[ExitOptions] = None,
602
+ *,
603
+ status_message: Optional[str] = None,
604
+ exit_code: Optional[int] = None,
605
+ ) -> None:
606
+ """Fail the Actor run (exit code defaults to ``1``).
607
+
608
+ Apify-Python parity: supports keyword-only ``status_message`` and
609
+ ``exit_code`` arguments (``await Actor.fail(status_message="Boom")``).
610
+ The pre-existing positional forms (``Actor.fail("message")`` /
611
+ ``Actor.fail(ExitOptions(...))``) remain supported; explicit keyword
612
+ arguments take precedence over the positional values when provided.
613
+ """
614
+ if isinstance(message_or_options, str):
615
+ opts = ExitOptions(exit_code=1, status_message=message_or_options)
616
+ else:
617
+ # Options-object form: an explicit exit_code in the object wins over
618
+ # the fail() default of 1 (JS SDK semantics). Build the dict first —
619
+ # passing it alongside exit_code=1 as kwargs raises
620
+ # "got multiple values for keyword argument 'exit_code'".
621
+ base = dict(message_or_options.__dict__) if message_or_options else {}
622
+ base.setdefault('exit_code', 1)
623
+ opts = ExitOptions(**base)
624
+
625
+ if options:
626
+ opts = ExitOptions(**{**opts.__dict__, **options.__dict__})
627
+
628
+ if status_message is not None:
629
+ opts.status_message = status_message
630
+ if exit_code is not None:
631
+ opts.exit_code = exit_code
632
+
633
+ logger.error(f'[Scrapely] Actor failed: {opts.status_message or "Unknown error"}')
634
+ await cls.exit(opts)
635
+
636
+ @classmethod
637
+ def is_at_home(cls) -> bool:
638
+ return cls._config.get('is_at_home', False) if cls._config else False
639
+
640
+ @classmethod
641
+ def is_standby(cls) -> bool:
642
+ """Return True when the actor run was started in standby mode (Node SDK ``isStandby`` parity)."""
643
+ return (cls._config.get('meta_origin') == 'STANDBY') if cls._config else False
644
+
645
+ @_ClassProperty
646
+ def is_paying(cls) -> bool:
647
+ """Whether the user who started this Actor run is a paying user.
648
+
649
+ Mirrors the Node SDK's ``Actor.isPaying`` (``APIFY_USER_IS_PAYING`` /
650
+ ``user_is_paying``). Note the platform currently sets this from the
651
+ subscription plan only — PAYG balance alone does not flip it.
652
+ """
653
+ config = cls.get_config() if cls._config is not None else Configuration.get_global_config()
654
+ return bool(config.get('user_is_paying'))
655
+
656
+ @classmethod
657
+ def get_env(cls) -> ApifyEnv:
658
+ if cls._config is None:
659
+ cls._config = Configuration.get_global_config()
660
+ return cls._config.get_env()
661
+
662
+ @classmethod
663
+ def get_remaining_time(cls) -> Optional[int]:
664
+ timeout_at = os.environ.get('ACTOR_TIMEOUT_AT')
665
+ if not timeout_at:
666
+ return None
667
+ try:
668
+ timeout_date = datetime.fromisoformat(timeout_at.replace('Z', '+00:00'))
669
+ remaining = int((timeout_date - datetime.now(timeout_date.tzinfo)).total_seconds() * 1000)
670
+ return max(0, remaining)
671
+ except (ValueError, TypeError):
672
+ return None
673
+
674
+ @classmethod
675
+ async def get_input(cls) -> Optional[Dict[str, Any]]:
676
+ cls._ensure_initialized('get_input')
677
+
678
+ if cls._cached_input is not None:
679
+ return cls._cached_input
680
+
681
+ metamorph_count = os.environ.get('ACTOR_METAMORPH_COUNT')
682
+ if metamorph_count:
683
+ count = int(metamorph_count)
684
+ if count > 0:
685
+ kvs = await cls.open_key_value_store()
686
+ metamorph_input = await kvs.get_value(f'INPUT-METAMORPH-{count}')
687
+ if metamorph_input is not None:
688
+ cls._cached_input = metamorph_input
689
+ return cast(Dict[str, Any], metamorph_input)
690
+
691
+ raw_input = os.environ.get('ACTOR_INPUT')
692
+ if raw_input:
693
+ try:
694
+ decoded = base64.b64decode(raw_input).decode('utf-8')
695
+ parsed = json.loads(decoded)
696
+ except Exception:
697
+ parsed = json.loads(raw_input)
698
+ # Decrypt secret inputs after loading (JS parity: only the main input
699
+ # path is decrypted, mirroring Actor.getInput() in the Node SDK)
700
+ if isinstance(parsed, dict):
701
+ parsed = decrypt_secret_inputs(parsed)
702
+ cls._cached_input = parsed
703
+ return cast(Dict[str, Any], cls._cached_input)
704
+
705
+ kvs = await cls.open_key_value_store()
706
+ input_key = cls._config.get('input_key', 'INPUT') if cls._config else 'INPUT'
707
+ result = await kvs.get_value(input_key)
708
+ if isinstance(result, dict):
709
+ result = decrypt_secret_inputs(result)
710
+ cls._cached_input = result
711
+ return cast(Optional[Dict[str, Any]], result)
712
+
713
+ @classmethod
714
+ async def get_input_or_throw(cls) -> Dict[str, Any]:
715
+ result = await cls.get_input()
716
+ if result is None:
717
+ raise ValueError('Input does not exist')
718
+ return result
719
+
720
+ @_ClassProperty
721
+ def input(cls) -> Optional[Dict[str, Any]]:
722
+ if cls._cached_input is not None:
723
+ return cls._cached_input
724
+
725
+ raw = os.environ.get('ACTOR_INPUT')
726
+ if not raw:
727
+ return None
728
+
729
+ try:
730
+ decoded = base64.b64decode(raw).decode('utf-8')
731
+ cls._cached_input = json.loads(decoded)
732
+ except Exception:
733
+ try:
734
+ cls._cached_input = json.loads(raw)
735
+ except json.JSONDecodeError:
736
+ return None
737
+
738
+ # Decrypt secret inputs after loading (JS parity with Actor.input getter)
739
+ if isinstance(cls._cached_input, dict):
740
+ cls._cached_input = decrypt_secret_inputs(cls._cached_input)
741
+
742
+ return cls._cached_input
743
+
744
+ @classmethod
745
+ async def push_data(
746
+ cls,
747
+ data: Union[Dict[str, Any], List[Dict[str, Any]]],
748
+ charged_event_name: Optional[str] = None,
749
+ *,
750
+ event_name: Optional[str] = None,
751
+ ) -> ChargeResult:
752
+ """Push data to the default dataset, charging pay-per-event items.
753
+
754
+ Matches the Apify Python SDK's ``Actor.push_data(data, charged_event_name=...)``:
755
+ when the Actor uses pay-per-event pricing, each pushed item is charged for the
756
+ default-dataset-item event (if priced) and, when ``charged_event_name`` is
757
+ provided, for that explicit event as well. Actors without pay-per-event
758
+ pricing push without charging and receive an empty result.
759
+
760
+ Args:
761
+ data: Item or list of items to push to the default dataset.
762
+ charged_event_name: Optional pricing event to charge for each pushed item
763
+ (Apify Python SDK name; also accepted positionally).
764
+ event_name: Deprecated alias of ``charged_event_name`` (JS ``eventName``),
765
+ kept so existing callers keep working.
766
+
767
+ Returns:
768
+ ChargeResult describing how many items were charged and pushed.
769
+
770
+ Raises:
771
+ ValueError: If the event refers to a synthetic event, or both names are given
772
+ with different values.
773
+ """
774
+ cls._ensure_initialized('push_data')
775
+ if charged_event_name is not None and event_name is not None and charged_event_name != event_name:
776
+ raise ValueError('Pass only one of charged_event_name / event_name')
777
+ event_name = charged_event_name if charged_event_name is not None else event_name
778
+ # Exact synthetic names only (JS parity) — a prefix heuristic would block
779
+ # real priced events like 'apify-foo'.
780
+ if event_name and is_default_dataset_item_event(event_name):
781
+ raise ValueError(f"Cannot charge for synthetic event '{event_name}' manually")
782
+
783
+ if cls._default_dataset is None:
784
+ cls._default_dataset = await cls.open_dataset()
785
+
786
+ items = data if isinstance(data, list) else [data]
787
+ charging_manager = cls.get_charging_manager()
788
+ return await push_data_and_charge(
789
+ charging_manager,
790
+ items,
791
+ event_name,
792
+ True, # Actor.push_data always targets the default dataset
793
+ cls._default_dataset.push_data,
794
+ )
795
+
796
+ @classmethod
797
+ async def get_value(cls, key: str, default_value: Any = None) -> Any:
798
+ cls._ensure_initialized('get_value')
799
+ if cls._default_kvs is None:
800
+ cls._default_kvs = await cls.open_key_value_store()
801
+ return await cls._default_kvs.get_value(key, default_value=default_value)
802
+
803
+ @classmethod
804
+ async def set_value(cls, key: str, value: Any, content_type: Optional[str] = None) -> None:
805
+ cls._ensure_initialized('set_value')
806
+ if cls._default_kvs is None:
807
+ cls._default_kvs = await cls.open_key_value_store()
808
+ await cls._default_kvs.set_value(key, value, content_type=content_type)
809
+
810
+ @classmethod
811
+ def _resolve_open_args(
812
+ cls,
813
+ name_or_id: Optional[str] = None,
814
+ *,
815
+ id: Optional[str] = None,
816
+ name: Optional[str] = None,
817
+ alias: Optional[str] = None,
818
+ force_cloud: bool = False,
819
+ options: Optional[OpenStorageOptions] = None,
820
+ storage_type: StorageType = 'datasets',
821
+ ) -> Dict[str, Any]:
822
+ force = force_cloud or bool(options and options.force_cloud)
823
+
824
+ if name_or_id is not None:
825
+ if id is not None or name is not None or alias is not None:
826
+ raise ValueError('Pass either positional name_or_id or id/name/alias kwargs, not both')
827
+ if is_storage_uuid(name_or_id):
828
+ id = name_or_id
829
+ else:
830
+ name = name_or_id
831
+
832
+ if alias is not None:
833
+ storages_json = cls._config.get('actor_storages_json') if cls._config else None
834
+ resolved = resolve_alias_from_storages_json(alias, storage_type, storages_json)
835
+ if resolved is not None:
836
+ id = resolved
837
+ alias = None
838
+ else:
839
+ # Outside platform: alias behaves like a name (TS)
840
+ name = alias
841
+ alias = None
842
+
843
+ return {'id': id, 'name': name, 'alias': alias, 'force_cloud': force}
844
+
845
+ @classmethod
846
+ def _storage_client_for_open(cls, *, force_cloud: bool = False) -> Optional[StorageClient]:
847
+ smart = cls._smart_storage or get_smart_storage_client()
848
+ if smart is not None:
849
+ return smart.get_suitable_storage_client(force_cloud=force_cloud)
850
+
851
+ if force_cloud:
852
+ cloud = get_api_storage_factory()
853
+ if cloud is None:
854
+ # Build from config like TS _buildStorageOpenOptions
855
+ config = cls.get_config()
856
+ api_url = config.get('api_base_url')
857
+ token = config.get('token')
858
+ run_id = config.get('actor_run_id')
859
+ if api_url and token and run_id:
860
+ return ScrapelyStorageClient(api_url=api_url, token=token, run_id=run_id)
861
+ raise RuntimeError('Cloud storage client is not configured')
862
+ return cloud
863
+ return None
864
+
865
+ @classmethod
866
+ async def open_dataset(
867
+ cls,
868
+ name_or_id: Optional[str] = None,
869
+ options: Optional[OpenStorageOptions] = None,
870
+ *,
871
+ id: Optional[str] = None,
872
+ name: Optional[str] = None,
873
+ alias: Optional[str] = None,
874
+ force_cloud: bool = False,
875
+ ) -> Dataset:
876
+ cls._ensure_initialized('open_dataset')
877
+ open_args = cls._resolve_open_args(
878
+ name_or_id,
879
+ id=id,
880
+ name=name,
881
+ alias=alias,
882
+ force_cloud=force_cloud,
883
+ options=options,
884
+ storage_type='datasets',
885
+ )
886
+ storage_client = cls._storage_client_for_open(force_cloud=open_args['force_cloud'])
887
+
888
+ if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
889
+ if cls._default_dataset is None:
890
+ cls._default_dataset = await Dataset.open(storage_client=storage_client)
891
+ return cls._default_dataset
892
+
893
+ return await Dataset.open(
894
+ id=open_args['id'],
895
+ name=open_args['name'],
896
+ alias=open_args['alias'],
897
+ storage_client=storage_client,
898
+ )
899
+
900
+ @classmethod
901
+ async def open_key_value_store(
902
+ cls,
903
+ name_or_id: Optional[str] = None,
904
+ options: Optional[OpenStorageOptions] = None,
905
+ *,
906
+ id: Optional[str] = None,
907
+ name: Optional[str] = None,
908
+ alias: Optional[str] = None,
909
+ force_cloud: bool = False,
910
+ ) -> KeyValueStore:
911
+ cls._ensure_initialized('open_key_value_store')
912
+ open_args = cls._resolve_open_args(
913
+ name_or_id,
914
+ id=id,
915
+ name=name,
916
+ alias=alias,
917
+ force_cloud=force_cloud,
918
+ options=options,
919
+ storage_type='keyValueStores',
920
+ )
921
+ storage_client = cls._storage_client_for_open(force_cloud=open_args['force_cloud'])
922
+
923
+ if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
924
+ if cls._default_kvs is None:
925
+ cls._default_kvs = await KeyValueStore.open(storage_client=storage_client)
926
+ return cls._default_kvs
927
+
928
+ return await KeyValueStore.open(
929
+ id=open_args['id'],
930
+ name=open_args['name'],
931
+ alias=open_args['alias'],
932
+ storage_client=storage_client,
933
+ )
934
+
935
+ @classmethod
936
+ async def open_request_queue(
937
+ cls,
938
+ name_or_id: Optional[str] = None,
939
+ options: Optional[OpenStorageOptions] = None,
940
+ *,
941
+ id: Optional[str] = None,
942
+ name: Optional[str] = None,
943
+ alias: Optional[str] = None,
944
+ force_cloud: bool = False,
945
+ ) -> RequestQueue:
946
+ cls._ensure_initialized('open_request_queue')
947
+ open_args = cls._resolve_open_args(
948
+ name_or_id,
949
+ id=id,
950
+ name=name,
951
+ alias=alias,
952
+ force_cloud=force_cloud,
953
+ options=options,
954
+ storage_type='requestQueues',
955
+ )
956
+ storage_client = cls._storage_client_for_open(force_cloud=open_args['force_cloud'])
957
+
958
+ if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
959
+ if cls._default_rq is None:
960
+ cls._default_rq = await RequestQueue.open(storage_client=storage_client)
961
+ return cls._default_rq
962
+
963
+ return await RequestQueue.open(
964
+ id=open_args['id'],
965
+ name=open_args['name'],
966
+ alias=open_args['alias'],
967
+ storage_client=storage_client,
968
+ )
969
+
970
+ @classmethod
971
+ def on(cls, event: Event, listener: Callable) -> None:
972
+ cls._ensure_initialized('on')
973
+ if cls._event_manager is None:
974
+ raise RuntimeError('Actor event manager is not initialized.')
975
+ cls._event_manager.on(event=event, listener=listener)
976
+
977
+ @classmethod
978
+ def off(cls, event: Event, listener: Optional[Callable] = None) -> None:
979
+ cls._ensure_initialized('off')
980
+ if cls._event_manager is None:
981
+ raise RuntimeError('Actor event manager is not initialized.')
982
+ cls._event_manager.off(event=event, listener=listener)
983
+
984
+ @classmethod
985
+ async def set_status_message(
986
+ cls,
987
+ message: str,
988
+ options: Optional[SetStatusMessageOptions] = None,
989
+ *,
990
+ is_terminal: Optional[bool] = None,
991
+ ) -> ActorRun:
992
+ """Set the status message for the current Actor run (Node SDK parity).
993
+
994
+ Logs the message at the requested level, then updates the run through
995
+ the platform API when credentials (``api_base_url``, ``token``,
996
+ ``actor_run_id``) are available. Locally, logs only and returns an
997
+ empty run object, mirroring the Node SDK behavior.
998
+
999
+ Args:
1000
+ message: The status message to set.
1001
+ options: Optional settings (``is_status_message_terminal``, ``level``).
1002
+ is_terminal: Apify Python SDK-style terminal flag. When given, it
1003
+ overrides ``options.is_status_message_terminal``.
1004
+
1005
+ Returns:
1006
+ The updated ActorRun object, or an empty ActorRun when running locally.
1007
+ """
1008
+ cls._ensure_initialized('set_status_message')
1009
+ opts = options or SetStatusMessageOptions()
1010
+ level = (getattr(opts, 'level', None) or 'INFO').upper()
1011
+ if is_terminal is None:
1012
+ is_terminal = bool(getattr(opts, 'is_status_message_terminal', False))
1013
+
1014
+ logged = f'[Status message]: {message}'
1015
+ if level == 'DEBUG':
1016
+ logger.debug(logged)
1017
+ elif level == 'WARNING':
1018
+ logger.warning(logged)
1019
+ elif level == 'ERROR':
1020
+ logger.error(logged)
1021
+ else:
1022
+ logger.info(logged)
1023
+
1024
+ config = cls.get_config()
1025
+ api_url = config.get('api_base_url')
1026
+ token = config.get('token')
1027
+ run_id = config.get('actor_run_id')
1028
+
1029
+ if not api_url or not token or not run_id:
1030
+ return ActorRun(id='', actor_id='', status='')
1031
+
1032
+ client = cls.new_client(token=token, base_url=api_url)
1033
+ response = await client.put(
1034
+ f'/v2/actor-runs/{run_id}',
1035
+ json_data={
1036
+ 'statusMessage': message,
1037
+ 'isStatusMessageTerminal': is_terminal,
1038
+ },
1039
+ )
1040
+ body: Any = {}
1041
+ if response.content:
1042
+ body = response.json()
1043
+ data = body.get('data', body) if isinstance(body, dict) else {}
1044
+
1045
+ return ActorRun(
1046
+ id=data.get('id', run_id),
1047
+ actor_id=data.get('actorId', ''),
1048
+ status=data.get('status', ''),
1049
+ started_at=data.get('startedAt'),
1050
+ finished_at=data.get('finishedAt'),
1051
+ status_message=data.get('statusMessage'),
1052
+ )
1053
+
1054
+ @_ClassProperty
1055
+ def log(cls) -> logging.Logger:
1056
+ return logger
1057
+
1058
+ @_ClassProperty
1059
+ def exit_code(cls) -> int:
1060
+ return cls._exit_code
1061
+
1062
+ @_ClassProperty
1063
+ def status_message(cls) -> Optional[str]:
1064
+ return cls._status_message
1065
+
1066
+ @classmethod
1067
+ def _ensure_initialized(cls, method_name: str) -> None:
1068
+ if not cls._initialized:
1069
+ raise RuntimeError(
1070
+ f'Actor.{method_name}() requires Actor.init() to be called first. '
1071
+ 'Hint: Use async with Actor: which calls init() for you.'
1072
+ )
1073
+
1074
+ @classmethod
1075
+ def get_config(cls) -> Configuration:
1076
+ if cls._config is None:
1077
+ cls._config = Configuration.get_global_config()
1078
+ return cls._config
1079
+
1080
+ @classmethod
1081
+ def new_client(cls, token: Optional[str] = None, base_url: Optional[str] = None) -> ApiClient:
1082
+ config = cls.get_config()
1083
+ api_url = base_url or config.get('api_base_url')
1084
+ api_token = token or config.get('token')
1085
+ run_id = config.get('actor_run_id', 'unknown')
1086
+ return ApiClient(api_url=api_url, token=api_token, run_id=run_id)
1087
+
1088
+ # -------------------------------------------------------------------------
1089
+ # Properties for Apify SDK compatibility
1090
+ # -------------------------------------------------------------------------
1091
+
1092
+ @classmethod
1093
+ def _get_apify_client(cls) -> 'ApifyClientAsync':
1094
+ """Get or create a cached charging-configured ``ApifyClientAsync`` instance.
1095
+
1096
+ This client provides full ApifyClient compatibility including
1097
+ actor(), dataset(), key_value_store(), etc. methods.
1098
+ """
1099
+ if cls._apify_client_instance is None:
1100
+ config = cls.get_config()
1101
+ api_url = config.get('api_base_url')
1102
+ token = config.get('token')
1103
+ default_dataset_id = config.get('default_dataset_id')
1104
+
1105
+ # Get charging manager if available
1106
+ charging_manager = None
1107
+ try:
1108
+ charging_manager = cls.get_charging_manager()
1109
+ except RuntimeError:
1110
+ pass # Actor not initialized yet
1111
+
1112
+ cls._apify_client_instance = create_charging_client(
1113
+ token=token,
1114
+ api_url=api_url,
1115
+ # Public URLs (record/keys public URLs) must be minted against the
1116
+ # configured public host, not the client's built-in default.
1117
+ api_public_url=config.get('api_public_base_url'),
1118
+ charging_manager=charging_manager,
1119
+ default_dataset_id=default_dataset_id,
1120
+ )
1121
+
1122
+ return cls._apify_client_instance
1123
+
1124
+ @_ClassProperty
1125
+ def apify_client(cls) -> 'ApifyClientAsync':
1126
+ """Get a charging-configured ``ApifyClientAsync`` for Apify SDK compatibility.
1127
+
1128
+ This provides full ApifyClient compatibility including:
1129
+ - client.actor('id').call()
1130
+ - client.dataset('id').list_items()
1131
+ - client.key_value_store('id').get_record('key')
1132
+ - client.run('id').wait()
1133
+ - etc.
1134
+ """
1135
+ return cls._get_apify_client()
1136
+
1137
+ @_ClassProperty
1138
+ def configuration(cls) -> Configuration:
1139
+ """Return the Configuration instance."""
1140
+ return cls.get_config()
1141
+
1142
+ @_ClassProperty
1143
+ def event_manager(cls) -> Optional[EventManager]:
1144
+ """Return the EventManager instance."""
1145
+ return cls._event_manager
1146
+
1147
+ # -------------------------------------------------------------------------
1148
+ # Charging methods
1149
+ # -------------------------------------------------------------------------
1150
+
1151
+ @classmethod
1152
+ def get_charging_manager(cls) -> ChargingManager:
1153
+ """Get the ChargingManager instance for pay-per-event pricing.
1154
+
1155
+ Returns:
1156
+ ChargingManager instance
1157
+
1158
+ Raises:
1159
+ RuntimeError: If Actor is not initialized
1160
+ """
1161
+ cls._ensure_initialized('get_charging_manager')
1162
+ config = cls.get_config()
1163
+ effective_run_id = config.get('actor_run_id')
1164
+
1165
+ # Recreate when missing or bound to a stale run (e.g. sequential lifecycles
1166
+ # that forgot to clear, or storage credentials changed after init).
1167
+ if cls._charging_manager is not None:
1168
+ manager_run_id = getattr(cls._charging_manager, '_actor_run_id', None)
1169
+ if manager_run_id != effective_run_id:
1170
+ cls._charging_manager = None
1171
+ cls._apify_client_instance = None
1172
+
1173
+ if cls._charging_manager is None:
1174
+ # Prefer the storage factory client (same URL/token/run as API storage).
1175
+ client = cls._client if cls._client is not None else cls.new_client()
1176
+ cls._charging_manager = ChargingManager(config, client)
1177
+ return cls._charging_manager
1178
+
1179
+
1180
+ @classmethod
1181
+ async def charge(cls, event_name: str, count: int = 1) -> ChargeResult:
1182
+ """Charge for a specified number of events - sub-operations of the Actor.
1183
+
1184
+ This is relevant only for the pay-per-event pricing model.
1185
+
1186
+ Args:
1187
+ event_name: Name of the event to be charged for
1188
+ count: Number of events to charge for
1189
+
1190
+ Returns:
1191
+ ChargeResult indicating if the limit was reached
1192
+ """
1193
+ cls._ensure_initialized('charge')
1194
+ charging_manager = cls.get_charging_manager()
1195
+ if not getattr(charging_manager, '_initialized', False):
1196
+ await charging_manager.init()
1197
+ return await charging_manager.charge(event_name, count=count)
1198
+
1199
+ # -------------------------------------------------------------------------
1200
+ # Actor run control methods
1201
+ # -------------------------------------------------------------------------
1202
+
1203
+ @classmethod
1204
+ async def abort(
1205
+ cls,
1206
+ run_id: Optional[str] = None,
1207
+ options: Optional[AbortOptions] = None,
1208
+ ) -> ActorRun:
1209
+ """Abort an Actor run on the Scrapely platform.
1210
+
1211
+ Args:
1212
+ run_id: The ID of the run to abort. Defaults to the current run.
1213
+ options: Abort options including token, status_message, gracefully
1214
+
1215
+ Returns:
1216
+ ActorRun object with details about the aborted run
1217
+ """
1218
+ cls._ensure_initialized('abort')
1219
+ config = cls.get_config()
1220
+ target_run_id = run_id or config.get('actor_run_id')
1221
+ if not target_run_id:
1222
+ raise ValueError('Actor.abort() requires a run ID')
1223
+
1224
+ opts = options or AbortOptions()
1225
+ client = cls.new_client(token=opts.token) if opts.token else cls.new_client()
1226
+
1227
+ # Set status message if provided
1228
+ if opts.status_message:
1229
+ await client.put_json(
1230
+ f'/v2/actor-runs/{target_run_id}',
1231
+ data={'statusMessage': opts.status_message},
1232
+ )
1233
+
1234
+ # Abort the run; `gracefully` travels as a query parameter (the server
1235
+ # reads it only from the query string) and is omitted when unset so the
1236
+ # server default applies (Node SDK parity).
1237
+ abort_params: Dict[str, Any] = {}
1238
+ if opts.gracefully is not None:
1239
+ abort_params['gracefully'] = 'true' if opts.gracefully else 'false'
1240
+ data = _payload_dict(
1241
+ await client.post_json(
1242
+ f'/v2/actor-runs/{target_run_id}/abort', data=None, params=abort_params or None
1243
+ )
1244
+ )
1245
+
1246
+ return ActorRun(
1247
+ id=data.get('id', target_run_id),
1248
+ actor_id=data.get('actorId', ''),
1249
+ status=data.get('status', 'ABORTED'),
1250
+ started_at=data.get('startedAt'),
1251
+ finished_at=data.get('finishedAt'),
1252
+ status_message=data.get('statusMessage'),
1253
+ )
1254
+
1255
+ @classmethod
1256
+ async def start(
1257
+ cls,
1258
+ actor_id: str,
1259
+ run_input: Optional[Any] = None,
1260
+ options: Optional[StartOptions] = None,
1261
+ ) -> ActorRun:
1262
+ """Start an Actor on the Scrapely platform without waiting for finish.
1263
+
1264
+ Args:
1265
+ actor_id: The ID of the Actor to start
1266
+ run_input: The input to pass to the Actor run
1267
+ options: Start options including token, build, memory_mbytes, etc.
1268
+
1269
+ Returns:
1270
+ ActorRun object with details about the started run
1271
+ """
1272
+ cls._ensure_initialized('start')
1273
+ opts = options or StartOptions()
1274
+ client = cls.new_client(token=opts.token) if opts.token else cls.new_client()
1275
+
1276
+ # Wire format (server contract): the POST body is the run input itself;
1277
+ # run options are Apify-style query parameters. Inline webhooks ride
1278
+ # along base64-encoded in the `webhooks` query param.
1279
+ params = _run_options_params(
1280
+ build=opts.build,
1281
+ timeout=opts.timeout,
1282
+ memory_mbytes=opts.memory_mbytes,
1283
+ max_total_charge_usd=opts.max_total_charge_usd,
1284
+ restart_on_error=opts.restart_on_error,
1285
+ force_permission_level=opts.force_permission_level,
1286
+ webhooks=opts.webhooks,
1287
+ )
1288
+
1289
+ data = _payload_dict(
1290
+ await client.post_json(f'/v2/actors/{actor_id}/runs', data=run_input, params=params or None)
1291
+ )
1292
+
1293
+ return ActorRun(
1294
+ id=data.get('id', ''),
1295
+ actor_id=data.get('actorId', actor_id),
1296
+ status=data.get('status', 'RUNNING'),
1297
+ started_at=data.get('startedAt'),
1298
+ build_id=data.get('buildId'),
1299
+ default_dataset_id=data.get('defaultDatasetId'),
1300
+ default_key_value_store_id=data.get('defaultKeyValueStoreId'),
1301
+ default_request_queue_id=data.get('defaultRequestQueueId'),
1302
+ )
1303
+
1304
+ @classmethod
1305
+ async def call(
1306
+ cls,
1307
+ actor_id: str,
1308
+ run_input: Optional[Any] = None,
1309
+ options: Optional[CallOptions] = None,
1310
+ ) -> ActorRun:
1311
+ """Start an Actor and wait for it to finish.
1312
+
1313
+ Args:
1314
+ actor_id: The ID of the Actor to call
1315
+ run_input: The input to pass to the Actor run
1316
+ options: Call options including token, build, timeout, etc.
1317
+
1318
+ Returns:
1319
+ ActorRun object with details about the finished run
1320
+ """
1321
+ cls._ensure_initialized('call')
1322
+ opts = options or CallOptions()
1323
+
1324
+ # Resolve the timeout once: 'inherit' becomes the remaining seconds of
1325
+ # this run, forwarded to the server AND used as the poll deadline.
1326
+ timeout_secs = _resolve_run_timeout(opts.timeout)
1327
+
1328
+ # Start the actor (timeout/webhooks forwarded so the server enforces
1329
+ # the run timeout and registers inline webhooks).
1330
+ start_opts = StartOptions(
1331
+ token=opts.token,
1332
+ build=opts.build,
1333
+ memory_mbytes=opts.memory_mbytes,
1334
+ max_total_charge_usd=opts.max_total_charge_usd,
1335
+ restart_on_error=opts.restart_on_error,
1336
+ timeout=timeout_secs,
1337
+ force_permission_level=opts.force_permission_level,
1338
+ webhooks=opts.webhooks,
1339
+ )
1340
+ run = await cls.start(actor_id, run_input, start_opts)
1341
+
1342
+ # Wait for finish (honors opts.timeout; polls indefinitely when unset)
1343
+ client = cls.new_client(token=opts.token) if opts.token else cls.new_client()
1344
+ return await cls._wait_for_terminal_run(
1345
+ client,
1346
+ run.id,
1347
+ actor_id_fallback=actor_id,
1348
+ timeout_seconds=timeout_secs,
1349
+ )
1350
+
1351
+ @classmethod
1352
+ async def _wait_for_terminal_run(
1353
+ cls,
1354
+ client: ApiClient,
1355
+ run_id: str,
1356
+ actor_id_fallback: str = '',
1357
+ timeout_seconds: Optional[int] = None,
1358
+ ) -> ActorRun:
1359
+ """Poll ``GET /v2/actor-runs/:runId`` until the run reaches a terminal status.
1360
+
1361
+ When ``timeout_seconds`` elapses first, the run is aborted (best-effort)
1362
+ and ``TimeoutError`` is raised — Node SDK (apify-client) parity.
1363
+ """
1364
+ deadline = time.monotonic() + timeout_seconds if timeout_seconds is not None else None
1365
+
1366
+ while True:
1367
+ try:
1368
+ data = _payload_dict(await client.get_json(f'/v2/actor-runs/{run_id}'))
1369
+ except (ApiClientError, ForbiddenError) as exc:
1370
+ if getattr(exc, 'status_code', None) != 403:
1371
+ raise
1372
+ # The token cannot observe this run (e.g. a LIMITED_PERMISSIONS
1373
+ # run token polling a run it did not start). Fail fast with an
1374
+ # actionable message instead of a bare 403 so callers know the
1375
+ # child run may still be running unobserved.
1376
+ # NOTE: the real ApiClient raises ApiClientError for every
1377
+ # non-2xx response; ForbiddenError is kept for parity with
1378
+ # callers that surface the typed errors from scrapely.errors.
1379
+ raise RuntimeError(
1380
+ f'Actor: cannot observe run {run_id} with the current token (403 Forbidden). '
1381
+ 'A LIMITED_PERMISSIONS run token may only poll or abort runs it started itself; '
1382
+ 'the run may still be executing. Started from another actor? Pass a user API '
1383
+ 'token via options.token.'
1384
+ ) from exc
1385
+ status = data.get('status', '').upper()
1386
+
1387
+ if status in _RUN_TERMINAL_STATUSES:
1388
+ return ActorRun(
1389
+ id=data.get('id', run_id),
1390
+ actor_id=data.get('actorId', actor_id_fallback),
1391
+ status=status,
1392
+ started_at=data.get('startedAt'),
1393
+ finished_at=data.get('finishedAt'),
1394
+ duration_millis=data.get('durationMillis'),
1395
+ exit_code=data.get('exitCode'),
1396
+ status_message=data.get('statusMessage'),
1397
+ default_dataset_id=data.get('defaultDatasetId'),
1398
+ default_key_value_store_id=data.get('defaultKeyValueStoreId'),
1399
+ default_request_queue_id=data.get('defaultRequestQueueId'),
1400
+ )
1401
+
1402
+ if deadline is not None:
1403
+ remaining = deadline - time.monotonic()
1404
+ if remaining <= 0:
1405
+ try:
1406
+ await client.post_json(f'/v2/actor-runs/{run_id}/abort', data=None)
1407
+ except Exception:
1408
+ logger.warning('Actor: best-effort abort of run %s after timeout failed', run_id)
1409
+ raise TimeoutError(
1410
+ f'The actor run {run_id} did not finish within {timeout_seconds} seconds. '
1411
+ 'The run was aborted.'
1412
+ ) from None
1413
+ await asyncio.sleep(min(1.0, remaining))
1414
+ else:
1415
+ await asyncio.sleep(1)
1416
+
1417
+ @classmethod
1418
+ async def call_task(
1419
+ cls,
1420
+ task_id: str,
1421
+ task_input: Optional[Dict[str, Any]] = None,
1422
+ options: Optional[CallOptions] = None,
1423
+ ) -> ActorRun:
1424
+ """Start an Actor task and wait for it to finish.
1425
+
1426
+ Args:
1427
+ task_id: The ID of the task to call
1428
+ task_input: Overrides the input for the task
1429
+ options: Call options including token, build, timeout, etc.
1430
+
1431
+ Returns:
1432
+ ActorRun object with details about the finished run
1433
+ """
1434
+ cls._ensure_initialized('call_task')
1435
+ opts = options or CallOptions()
1436
+ client = cls.new_client(token=opts.token) if opts.token else cls.new_client()
1437
+
1438
+ # Wire format (server contract): the POST body is the task input
1439
+ # override (merged over the task's saved input by the server); run
1440
+ # options are Apify-style query parameters. The task-run endpoint is
1441
+ # /v2/actor-tasks/:taskId/runs (there is no /v2/tasks/:taskId/runs).
1442
+ params = _run_options_params(
1443
+ build=opts.build,
1444
+ timeout=opts.timeout,
1445
+ memory_mbytes=opts.memory_mbytes,
1446
+ max_total_charge_usd=opts.max_total_charge_usd,
1447
+ restart_on_error=opts.restart_on_error,
1448
+ force_permission_level=opts.force_permission_level,
1449
+ webhooks=opts.webhooks,
1450
+ )
1451
+
1452
+ data = _payload_dict(
1453
+ await client.post_json(f'/v2/actor-tasks/{task_id}/runs', data=task_input, params=params or None)
1454
+ )
1455
+ run_id = data.get('id', '')
1456
+ if not run_id:
1457
+ raise RuntimeError(f'Actor.call_task(): creating a run for task {task_id} returned no run ID')
1458
+
1459
+ # Wait for finish (honors opts.timeout, incl. 'inherit'; polls
1460
+ # indefinitely when unset)
1461
+ return await cls._wait_for_terminal_run(
1462
+ client,
1463
+ run_id,
1464
+ timeout_seconds=_resolve_run_timeout(opts.timeout),
1465
+ )
1466
+
1467
+ # -------------------------------------------------------------------------
1468
+ # Platform integration methods
1469
+ # -------------------------------------------------------------------------
1470
+
1471
+ @classmethod
1472
+ async def metamorph(
1473
+ cls,
1474
+ target_actor_id: str,
1475
+ run_input: Optional[Any] = None,
1476
+ options: Optional[MetamorphOptions] = None,
1477
+ ) -> None:
1478
+ """Transform this Actor run to an Actor run of a different Actor.
1479
+
1480
+ The platform stops the current Actor container and starts a new container
1481
+ with the new Actor instead. All default storages are preserved, and the
1482
+ new input is stored under the INPUT-METAMORPH-1 key in the same default
1483
+ key-value store.
1484
+
1485
+ Args:
1486
+ target_actor_id: ID of the target Actor to transform into
1487
+ run_input: The input to pass to the new run
1488
+ options: Metamorph options including build, content_type
1489
+ """
1490
+ cls._ensure_initialized('metamorph')
1491
+
1492
+ if not cls.is_at_home():
1493
+ logger.warning('Actor.metamorph() is only supported when running on the Scrapely platform.')
1494
+ return
1495
+
1496
+ config = cls.get_config()
1497
+ run_id = config.get('actor_run_id')
1498
+ if not run_id:
1499
+ raise RuntimeError('actor_run_id is not set - metamorph requires running on the platform')
1500
+
1501
+ opts = options or MetamorphOptions()
1502
+
1503
+ # Wire format (server contract): the server copies the current INPUT to
1504
+ # INPUT-METAMORPH-<n> itself and tracks the metamorph count in run
1505
+ # stats; the new input (if any) is read from the request body's
1506
+ # `input` field. targetActorId/build are accepted in the body.
1507
+ client = cls.new_client()
1508
+ body: Dict[str, Any] = {'targetActorId': target_actor_id}
1509
+ if opts.build:
1510
+ body['build'] = opts.build
1511
+ if run_input is not None:
1512
+ body['input'] = run_input
1513
+
1514
+ logger.info(f'[Scrapely] Metamorphosing to actor {target_actor_id}')
1515
+
1516
+ await client.post_json(f'/v2/actor-runs/{run_id}/metamorph', data=body)
1517
+
1518
+ # Flush pending PPE intents before hard exit.
1519
+ try:
1520
+ await cls._shutdown(timeout_secs=5)
1521
+ except Exception as err:
1522
+ logger.warning('flush_pending during metamorph failed: %s', err)
1523
+
1524
+ # Exit - the platform will restart with new actor
1525
+ logger.info('[Scrapely] Metamorph request accepted, exiting...')
1526
+ sys.exit(0)
1527
+
1528
+ @classmethod
1529
+ async def reboot(cls, options: Optional[RebootOptions] = None) -> None:
1530
+ """Internally reboot this Actor.
1531
+
1532
+ The system stops the current container and starts a new one with the
1533
+ same run ID and default storages.
1534
+
1535
+ Args:
1536
+ options: Reboot options including custom_after_sleep_millis
1537
+ """
1538
+ cls._ensure_initialized('reboot')
1539
+
1540
+ if not cls.is_at_home():
1541
+ logger.warning('Actor.reboot() is only supported when running on the Scrapely platform.')
1542
+ return
1543
+
1544
+ config = cls.get_config()
1545
+ run_id = config.get('actor_run_id')
1546
+ if not run_id:
1547
+ raise RuntimeError('actor_run_id is not set - reboot requires running on the platform')
1548
+
1549
+ opts = options or RebootOptions()
1550
+
1551
+ # Emit persist state event
1552
+ if cls._event_manager:
1553
+ from crawlee.events import EventPersistStateData
1554
+ cls._event_manager.emit(
1555
+ event=Event.PERSIST_STATE,
1556
+ event_data=EventPersistStateData(is_migrating=True),
1557
+ )
1558
+
1559
+ # Wait for state to be persisted
1560
+ await asyncio.sleep(1)
1561
+
1562
+ logger.info('[Scrapely] Rebooting...')
1563
+
1564
+ # Call reboot endpoint
1565
+ client = cls.new_client()
1566
+ await client.post(f'/v2/actor-runs/{run_id}/reboot')
1567
+
1568
+ # Wait before exiting
1569
+ sleep_ms = opts.custom_after_sleep_millis or config.get('metamorph_after_sleep_millis', 1000)
1570
+ await asyncio.sleep(sleep_ms / 1000)
1571
+
1572
+ # Flush pending PPE intents before hard exit.
1573
+ try:
1574
+ await cls._shutdown(timeout_secs=5)
1575
+ except Exception as err:
1576
+ logger.warning('flush_pending during reboot failed: %s', err)
1577
+
1578
+ logger.info('[Scrapely] Reboot request accepted, exiting...')
1579
+ sys.exit(0)
1580
+
1581
+ @classmethod
1582
+ async def add_webhook(cls, webhook: Webhook) -> Optional[Webhook]:
1583
+ """Create an ad-hoc webhook for the current Actor run.
1584
+
1585
+ This webhook lets you receive a notification when the Actor run
1586
+ finished or failed.
1587
+
1588
+ Note that webhooks are only supported for Actors running on the
1589
+ Scrapely platform. When running the Actor locally, the function
1590
+ will print a warning and return None.
1591
+
1592
+ Args:
1593
+ webhook: The webhook to add. It is automatically bound to the
1594
+ current Actor run.
1595
+
1596
+ Returns:
1597
+ The created webhook, or None if running locally
1598
+ """
1599
+ cls._ensure_initialized('add_webhook')
1600
+
1601
+ if not cls.is_at_home():
1602
+ logger.warning('Actor.add_webhook() is only supported on the Scrapely platform. Ignoring.')
1603
+ return None
1604
+
1605
+ config = cls.get_config()
1606
+ run_id = config.get('actor_run_id')
1607
+ if not run_id:
1608
+ raise RuntimeError('actor_run_id is not set - add_webhook requires running on the platform')
1609
+
1610
+ client = cls.new_client()
1611
+
1612
+ # Build webhook creation request
1613
+ body = {
1614
+ 'eventTypes': webhook.event_types,
1615
+ 'requestUrl': webhook.request_url,
1616
+ 'isAdHoc': True,
1617
+ 'condition': {'actorRunId': run_id},
1618
+ }
1619
+ if webhook.payload_template:
1620
+ body['payloadTemplate'] = webhook.payload_template
1621
+ if webhook.idempotency_key:
1622
+ body['idempotencyKey'] = webhook.idempotency_key
1623
+ if webhook.headers_template:
1624
+ body['headersTemplate'] = webhook.headers_template
1625
+ # `description` is not part of the Webhook dataclass — tolerate future additions.
1626
+ description = getattr(webhook, 'description', None)
1627
+ if description:
1628
+ body['description'] = description
1629
+ body['ignoreSslErrors'] = bool(webhook.ignore_ssl_errors)
1630
+ body['doNotRetry'] = bool(webhook.do_not_retry)
1631
+
1632
+ data = _payload_dict(await client.post_json('/v2/webhooks', data=body))
1633
+
1634
+ # Populate the server-assigned ID on the returned webhook so callers
1635
+ # can reference it later (Node SDK parity).
1636
+ webhook.id = data.get('id') or webhook.id
1637
+
1638
+ return webhook
1639
+
1640
+ @classmethod
1641
+ async def use_state(
1642
+ cls,
1643
+ default_value: Optional[Dict[str, Any]] = None,
1644
+ key: Optional[str] = None,
1645
+ kvs_name: Optional[str] = None,
1646
+ ) -> MutableMapping[str, Any]:
1647
+ """Easily create and manage state values. All state values are automatically persisted.
1648
+
1649
+ Values can be modified by simply using the assignment operator.
1650
+
1651
+ Args:
1652
+ default_value: The default value to initialize the state if not already set
1653
+ key: The key in the key-value store where state is stored. Defaults to 'APIFY_GLOBAL_STATE'
1654
+ kvs_name: The name of the key-value store where state is stored. Defaults to default KVS.
1655
+
1656
+ Returns:
1657
+ The state dictionary with automatic persistence
1658
+ """
1659
+ cls._ensure_initialized('use_state')
1660
+
1661
+ state_key = key or 'APIFY_GLOBAL_STATE'
1662
+ initial_value = default_value or {}
1663
+
1664
+ cls._use_state_stores.add(kvs_name)
1665
+ kvs = await cls.open_key_value_store(name=kvs_name)
1666
+
1667
+ # Use get_auto_saved_value if available (from patch_storage)
1668
+ if hasattr(kvs, 'get_auto_saved_value'):
1669
+ return await kvs.get_auto_saved_value(state_key, initial_value)
1670
+
1671
+ # Fallback: manual state management
1672
+ existing = await kvs.get_value(state_key)
1673
+ if existing is None:
1674
+ await kvs.set_value(state_key, initial_value)
1675
+ return initial_value
1676
+ return cast(MutableMapping[str, Any], existing)
1677
+
1678
+ # -------------------------------------------------------------------------
1679
+ # Remote browser configuration
1680
+ # -------------------------------------------------------------------------
1681
+
1682
+ @classmethod
1683
+ async def create_browser_configuration(
1684
+ cls,
1685
+ actor_browser_input: Optional[Dict[str, Any]] = None,
1686
+ *,
1687
+ country_code: Optional[str] = None,
1688
+ state: Optional[str] = None,
1689
+ city: Optional[str] = None,
1690
+ os: Optional[str] = None,
1691
+ timezone: Optional[str] = None,
1692
+ language: Optional[str] = None,
1693
+ adblock: Optional[bool] = None,
1694
+ solve_cloudflare: Optional[bool] = None,
1695
+ ) -> 'BrowserConfiguration':
1696
+ """Create a Scrapely remote browser configuration (location and fingerprint options).
1697
+
1698
+ Pass it to browser crawlers next to ``proxy_configuration``::
1699
+
1700
+ browser_configuration = await Actor.create_browser_configuration(country_code='US', os='windows')
1701
+ crawler = PlaywrightCrawler(browser_configuration=browser_configuration, request_handler=handler)
1702
+
1703
+ Ignored outside the Scrapely remote browser (local runs), so the same
1704
+ code runs locally and on the platform.
1705
+
1706
+ Args:
1707
+ actor_browser_input: Actor input fields ``browserCountry`` /
1708
+ ``browserState`` / ``browserCity``, used when the matching
1709
+ keyword argument is not given.
1710
+ country_code: Exit country (ISO 3166-1 alpha-2).
1711
+ state: Region name (e.g. ``California``).
1712
+ city: City name (e.g. ``San Jose``).
1713
+ os: Fingerprint OS: ``windows``, ``macos`` or ``android``.
1714
+ timezone: IANA timezone.
1715
+ language: Comma-separated language tags without q-values.
1716
+ adblock: Built-in ad/tracker blocking.
1717
+ solve_cloudflare: Automatic Cloudflare Turnstile solving.
1718
+ """
1719
+ from .browser_configuration import BrowserConfiguration
1720
+
1721
+ if actor_browser_input:
1722
+ country_code = country_code if country_code is not None else actor_browser_input.get('browserCountry')
1723
+ state = state if state is not None else actor_browser_input.get('browserState')
1724
+ city = city if city is not None else actor_browser_input.get('browserCity')
1725
+
1726
+ return BrowserConfiguration(
1727
+ country_code=country_code,
1728
+ state=state,
1729
+ city=city,
1730
+ os=os,
1731
+ timezone=timezone,
1732
+ language=language,
1733
+ adblock=adblock,
1734
+ solve_cloudflare=solve_cloudflare,
1735
+ )
1736
+
1737
+ # -------------------------------------------------------------------------
1738
+ # Proxy configuration
1739
+ # -------------------------------------------------------------------------
1740
+
1741
+ @classmethod
1742
+ async def create_proxy_configuration(
1743
+ cls,
1744
+ actor_proxy_input: Optional[Dict[str, Any]] = None,
1745
+ password: Optional[str] = None,
1746
+ groups: Optional[List[str]] = None,
1747
+ country_code: Optional[str] = None,
1748
+ subdivision_code: Optional[str] = None,
1749
+ proxy_urls: Optional[List[str]] = None,
1750
+ new_url_function: Optional[Callable] = None,
1751
+ tiered_proxy_urls: Optional[List[List[Optional[str]]]] = None,
1752
+ tiered_proxy_config: Optional[List[Dict[str, Any]]] = None,
1753
+ check_access: bool = True,
1754
+ ) -> Optional['ProxyConfiguration']:
1755
+ """Create a ProxyConfiguration object with the passed proxy configuration.
1756
+
1757
+ Configures connection to a proxy server with the provided options.
1758
+ Proxy servers are used to prevent target websites from blocking your
1759
+ crawlers based on IP address rate limits or blacklists.
1760
+
1761
+ Args:
1762
+ actor_proxy_input: Proxy configuration field from Actor input
1763
+ password: Password for the proxy
1764
+ groups: Proxy groups to use
1765
+ country_code: Country code for proxy
1766
+ subdivision_code: Subdivision code for proxy
1767
+ proxy_urls: Custom proxy server URLs to rotate through; may contain
1768
+ the 'auto' / 'http://auto' sentinel to mix in the platform proxy
1769
+ new_url_function: Function that returns a custom proxy URL
1770
+ tiered_proxy_urls: Proxy URL lists stratified into tiers
1771
+ tiered_proxy_config: Declarative tiers (proxy_urls / groups / ...)
1772
+ check_access: Whether to run the best-effort platform status check
1773
+
1774
+ Returns:
1775
+ ProxyConfiguration or None if no proxy should be used
1776
+ """
1777
+ cls._ensure_initialized('create_proxy_configuration')
1778
+
1779
+ from .proxy_configuration import ProxyConfiguration
1780
+
1781
+ # Handle actor_proxy_input (Node SDK normalizeProxyInput/createProxyConfiguration
1782
+ # parity): `useApifyProxy: false` explicitly disables the PLATFORM proxy only —
1783
+ # custom proxies (proxyUrls / tiered lists / new_url_function) from the same
1784
+ # input must still be used. The standard "Custom proxy URLs" input shape is
1785
+ # {useApifyProxy: false, proxyUrls: [...]}; returning None for it would
1786
+ # silently drop the user's own proxies and crawl on their real IPs. We only
1787
+ # bail when no custom proxies are given either. A MISSING key falls back to
1788
+ # platform mode when the server injected credentials — input like
1789
+ # {apifyProxyGroups: [...]} must not silently disable proxying.
1790
+ if actor_proxy_input is not None:
1791
+ if actor_proxy_input.get('useApifyProxy') is False and not (
1792
+ proxy_urls
1793
+ or new_url_function
1794
+ or tiered_proxy_urls
1795
+ or tiered_proxy_config
1796
+ or actor_proxy_input.get('proxyUrls')
1797
+ or actor_proxy_input.get('tieredProxyUrls')
1798
+ or actor_proxy_input.get('tieredProxyConfig')
1799
+ ):
1800
+ return None
1801
+ country_code = country_code or actor_proxy_input.get('apifyProxyCountry')
1802
+ subdivision_code = subdivision_code or actor_proxy_input.get('apifyProxySubdivision')
1803
+ groups = groups or actor_proxy_input.get('apifyProxyGroups')
1804
+ proxy_urls = proxy_urls or actor_proxy_input.get('proxyUrls')
1805
+ tiered_proxy_urls = tiered_proxy_urls or actor_proxy_input.get('tieredProxyUrls')
1806
+ tiered_proxy_config = tiered_proxy_config or actor_proxy_input.get('tieredProxyConfig')
1807
+
1808
+ # No custom proxies configured: use the platform proxy only when the
1809
+ # server injected credentials; locally, warn and run proxy-less (Node
1810
+ # SDK createProxyConfiguration parity).
1811
+ if not (proxy_urls or new_url_function or tiered_proxy_urls or tiered_proxy_config):
1812
+ from .proxy_configuration import _platform_env
1813
+ if not _platform_env():
1814
+ logger.warning(
1815
+ 'No proxy configured. Scrapely injects platform proxy credentials '
1816
+ '(SCRAPELY_PROXY_*/APIFY_PROXY_*) into runs when your account has '
1817
+ 'proxy access; locally, add proxy_urls to your actor input.'
1818
+ )
1819
+ return None
1820
+
1821
+ config = cls.get_config()
1822
+ proxy_config = ProxyConfiguration(
1823
+ password=password,
1824
+ groups=groups,
1825
+ country_code=country_code,
1826
+ subdivision_code=subdivision_code,
1827
+ proxy_urls=proxy_urls,
1828
+ new_url_function=new_url_function,
1829
+ tiered_proxy_urls=tiered_proxy_urls,
1830
+ tiered_proxy_config=tiered_proxy_config,
1831
+ _actor_config=config,
1832
+ _api_client=cls.new_client(),
1833
+ )
1834
+
1835
+ await proxy_config.initialize(check_access=check_access)
1836
+ return proxy_config