scrapely-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapely/__init__.py +246 -0
- scrapely/_browser_locations.py +5470 -0
- scrapely/actor.py +1836 -0
- scrapely/browser_configuration.py +146 -0
- scrapely/charging.py +1143 -0
- scrapely/charging_client.py +107 -0
- scrapely/configuration.py +375 -0
- scrapely/crypto/__init__.py +21 -0
- scrapely/crypto/_secrets.py +210 -0
- scrapely/errors.py +153 -0
- scrapely/events/__init__.py +57 -0
- scrapely/events/_types.py +194 -0
- scrapely/events/event_manager.py +292 -0
- scrapely/log.py +39 -0
- scrapely/patch_browser_pool.py +861 -0
- scrapely/pricing_types.py +172 -0
- scrapely/proxy_configuration.py +741 -0
- scrapely/py.typed +2 -0
- scrapely/request_loaders/__init__.py +5 -0
- scrapely/request_loaders/_scrapely_request_list.py +169 -0
- scrapely/storage/__init__.py +48 -0
- scrapely/storage/_file_system/__init__.py +9 -0
- scrapely/storage/_file_system/_key_value_store_client.py +155 -0
- scrapely/storage/_file_system/_storage_client.py +50 -0
- scrapely/storage/_metadata.py +138 -0
- scrapely/storage/api_client.py +350 -0
- scrapely/storage/client.py +618 -0
- scrapely/storage/patch_storage.py +257 -0
- scrapely/storage/scrapely_dataset_client.py +305 -0
- scrapely/storage/scrapely_kvs_client.py +220 -0
- scrapely/storage/scrapely_rq_client.py +496 -0
- scrapely/storage/scrapely_storage_client.py +192 -0
- scrapely/storage/smart_scrapely_storage_client.py +109 -0
- scrapely/storage/types.py +601 -0
- scrapely/storage/utils.py +38 -0
- scrapely/storage_clients/__init__.py +25 -0
- scrapely/storages/__init__.py +5 -0
- scrapely/types.py +269 -0
- scrapely/webhook.py +82 -0
- scrapely_python-0.1.0.dist-info/METADATA +83 -0
- scrapely_python-0.1.0.dist-info/RECORD +43 -0
- scrapely_python-0.1.0.dist-info/WHEEL +5 -0
- scrapely_python-0.1.0.dist-info/top_level.txt +1 -0
scrapely/actor.py
ADDED
|
@@ -0,0 +1,1836 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Main Actor class for the Scrapely Python SDK.
|
|
3
|
+
|
|
4
|
+
The core class for building Actors on the Scrapely platform.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
import base64
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
import signal
|
|
15
|
+
import sys
|
|
16
|
+
import time
|
|
17
|
+
import warnings
|
|
18
|
+
from collections.abc import MutableMapping
|
|
19
|
+
from datetime import datetime, timedelta
|
|
20
|
+
from decimal import Decimal
|
|
21
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast
|
|
22
|
+
|
|
23
|
+
from crawlee.configuration import Configuration as CrawleeConfiguration
|
|
24
|
+
from crawlee.storage_clients._base import StorageClient
|
|
25
|
+
from crawlee.storages import Dataset, KeyValueStore, RequestQueue
|
|
26
|
+
|
|
27
|
+
from .charging import ChargingManager, is_default_dataset_item_event, push_data_and_charge
|
|
28
|
+
from .configuration import Configuration
|
|
29
|
+
from .crypto import decrypt_secret_inputs
|
|
30
|
+
from crawlee.events import EventManager
|
|
31
|
+
|
|
32
|
+
from .events import Event, LocalEventManager, ScrapelyEventManager
|
|
33
|
+
from .charging_client import create_charging_client
|
|
34
|
+
from .errors import ForbiddenError
|
|
35
|
+
from .storage.api_client import ApiClient, ApiClientError
|
|
36
|
+
from .storage.client import is_storage_uuid
|
|
37
|
+
from .storage._metadata import StorageType, resolve_alias_from_storages_json
|
|
38
|
+
from .storage.patch_storage import (
|
|
39
|
+
disconnect_storage,
|
|
40
|
+
get_api_storage_factory,
|
|
41
|
+
get_smart_storage_client,
|
|
42
|
+
set_api_storage,
|
|
43
|
+
set_storage_client,
|
|
44
|
+
)
|
|
45
|
+
from .storage.scrapely_storage_client import ScrapelyStorageClient
|
|
46
|
+
from .storage.smart_scrapely_storage_client import SmartScrapelyStorageClient
|
|
47
|
+
from .types import (
|
|
48
|
+
AbortOptions,
|
|
49
|
+
ActorRun,
|
|
50
|
+
ApifyEnv,
|
|
51
|
+
CallOptions,
|
|
52
|
+
ChargeResult,
|
|
53
|
+
ExitOptions,
|
|
54
|
+
InitOptions,
|
|
55
|
+
MetamorphOptions,
|
|
56
|
+
OpenStorageOptions,
|
|
57
|
+
RebootOptions,
|
|
58
|
+
SetStatusMessageOptions,
|
|
59
|
+
StartOptions,
|
|
60
|
+
WebhookOptions,
|
|
61
|
+
)
|
|
62
|
+
from .log import _configure_logging
|
|
63
|
+
from .webhook import Webhook
|
|
64
|
+
|
|
65
|
+
if TYPE_CHECKING:
|
|
66
|
+
from scrapely_client import ApifyClientAsync
|
|
67
|
+
|
|
68
|
+
from .browser_configuration import BrowserConfiguration
|
|
69
|
+
from .proxy_configuration import ProxyConfiguration
|
|
70
|
+
|
|
71
|
+
logger = logging.getLogger(__name__)
|
|
72
|
+
|
|
73
|
+
# Terminal run statuses used by the run-control polling loops (call/call_task).
|
|
74
|
+
_RUN_TERMINAL_STATUSES = frozenset({'SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'})
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _payload_dict(payload: Any) -> Dict[str, Any]:
|
|
78
|
+
"""Normalize an unwrapped API envelope payload to a plain dict.
|
|
79
|
+
|
|
80
|
+
``ApiClient.get_json``/``post_json``/``put_json`` return the unwrapped
|
|
81
|
+
``data`` field, which may be a non-dict (``{}`` fallback, list, scalar)
|
|
82
|
+
when the endpoint responds with an empty or non-JSON body.
|
|
83
|
+
"""
|
|
84
|
+
if isinstance(payload, dict):
|
|
85
|
+
return cast(Dict[str, Any], payload)
|
|
86
|
+
return {}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _timeout_seconds(value: Union[int, str, None]) -> Optional[int]:
|
|
90
|
+
"""Normalize a ``CallOptions.timeout`` value to poll deadline seconds.
|
|
91
|
+
|
|
92
|
+
Accepts a positive int; ``'inherit'``, ``None`` and non-positive values
|
|
93
|
+
mean "no client-side deadline" (the platform's own run timeout applies).
|
|
94
|
+
"""
|
|
95
|
+
if isinstance(value, bool):
|
|
96
|
+
return None
|
|
97
|
+
if isinstance(value, int) and value > 0:
|
|
98
|
+
return value
|
|
99
|
+
return None
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _resolve_run_timeout(value: Union[int, str, None]) -> Optional[int]:
|
|
103
|
+
"""Resolve a ``StartOptions``/``CallOptions`` timeout to wire seconds.
|
|
104
|
+
|
|
105
|
+
``'inherit'`` resolves to the current run's remaining seconds (Node SDK
|
|
106
|
+
``resolveCallTimeout`` parity, warning when no platform timeout is set);
|
|
107
|
+
positive ints pass through; everything else means "not sent" so the
|
|
108
|
+
platform default applies.
|
|
109
|
+
"""
|
|
110
|
+
if isinstance(value, str) and value.lower() == 'inherit':
|
|
111
|
+
remaining_ms = Actor.get_remaining_time()
|
|
112
|
+
if remaining_ms is None:
|
|
113
|
+
logger.warning(
|
|
114
|
+
"Using 'inherit' for timeout is only possible when running on the platform "
|
|
115
|
+
'and when the timeout for the Actor run is set (ACTOR_TIMEOUT_AT).'
|
|
116
|
+
)
|
|
117
|
+
return None
|
|
118
|
+
return max(0, -(-remaining_ms // 1000)) # ceil(ms → secs)
|
|
119
|
+
return _timeout_seconds(value)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _webhook_body(webhook: Any) -> Dict[str, Any]:
|
|
123
|
+
"""Serialize a Webhook/WebhookOptions-like object to the API body format."""
|
|
124
|
+
to_dict = getattr(webhook, 'to_dict', None)
|
|
125
|
+
if callable(to_dict):
|
|
126
|
+
return cast(Dict[str, Any], to_dict())
|
|
127
|
+
|
|
128
|
+
body: Dict[str, Any] = {
|
|
129
|
+
'eventTypes': [getattr(e, 'value', e) for e in (webhook.event_types or [])],
|
|
130
|
+
'requestUrl': webhook.request_url,
|
|
131
|
+
}
|
|
132
|
+
if getattr(webhook, 'payload_template', None):
|
|
133
|
+
body['payloadTemplate'] = webhook.payload_template
|
|
134
|
+
if getattr(webhook, 'headers_template', None):
|
|
135
|
+
body['headersTemplate'] = webhook.headers_template
|
|
136
|
+
if getattr(webhook, 'idempotency_key', None):
|
|
137
|
+
body['idempotencyKey'] = webhook.idempotency_key
|
|
138
|
+
description = getattr(webhook, 'description', None)
|
|
139
|
+
if description:
|
|
140
|
+
body['description'] = description
|
|
141
|
+
ignore_ssl_errors = getattr(webhook, 'ignore_ssl_errors', None)
|
|
142
|
+
if ignore_ssl_errors is not None:
|
|
143
|
+
body['ignoreSslErrors'] = bool(ignore_ssl_errors)
|
|
144
|
+
do_not_retry = getattr(webhook, 'do_not_retry', None)
|
|
145
|
+
if do_not_retry is not None:
|
|
146
|
+
body['doNotRetry'] = bool(do_not_retry)
|
|
147
|
+
return body
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _run_options_params(
|
|
151
|
+
*,
|
|
152
|
+
build: Optional[str] = None,
|
|
153
|
+
timeout: Union[int, str, None] = None,
|
|
154
|
+
memory_mbytes: Optional[int] = None,
|
|
155
|
+
max_total_charge_usd: Optional[float] = None,
|
|
156
|
+
restart_on_error: Optional[bool] = None,
|
|
157
|
+
force_permission_level: Optional[Any] = None,
|
|
158
|
+
webhooks: Optional[List[Any]] = None,
|
|
159
|
+
) -> Dict[str, Any]:
|
|
160
|
+
"""Build the Apify-style run-option query parameters for run-start requests.
|
|
161
|
+
|
|
162
|
+
Server wire contract: the POST body of a run-start request is the run
|
|
163
|
+
input itself; run options (build/timeout/memory/maxTotalChargeUsd/
|
|
164
|
+
restartOnError/forcePermissionLevel) are read from query parameters only.
|
|
165
|
+
Inline webhooks ride along base64-encoded in the ``webhooks`` query param
|
|
166
|
+
(the query-param transport the server supports alongside the body one).
|
|
167
|
+
"""
|
|
168
|
+
params: Dict[str, Any] = {}
|
|
169
|
+
if build:
|
|
170
|
+
params['build'] = build
|
|
171
|
+
timeout_secs = _resolve_run_timeout(timeout)
|
|
172
|
+
if timeout_secs is not None:
|
|
173
|
+
params['timeout'] = timeout_secs
|
|
174
|
+
if memory_mbytes:
|
|
175
|
+
params['memory'] = memory_mbytes
|
|
176
|
+
if max_total_charge_usd is not None:
|
|
177
|
+
params['maxTotalChargeUsd'] = max_total_charge_usd
|
|
178
|
+
if restart_on_error is not None:
|
|
179
|
+
params['restartOnError'] = 'true' if restart_on_error else 'false'
|
|
180
|
+
if force_permission_level is not None:
|
|
181
|
+
level = getattr(force_permission_level, 'value', force_permission_level)
|
|
182
|
+
params['forcePermissionLevel'] = str(level)
|
|
183
|
+
if webhooks:
|
|
184
|
+
encoded = base64.b64encode(
|
|
185
|
+
json.dumps([_webhook_body(w) for w in webhooks]).encode('utf-8'),
|
|
186
|
+
).decode('ascii')
|
|
187
|
+
params['webhooks'] = encoded
|
|
188
|
+
return params
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _is_api_storage_config(storage: Any) -> bool:
|
|
192
|
+
return (
|
|
193
|
+
isinstance(storage, dict)
|
|
194
|
+
and 'api_url' in storage
|
|
195
|
+
and 'token' in storage
|
|
196
|
+
and 'run_id' in storage
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class _ClassProperty:
|
|
201
|
+
"""Read-only class property descriptor."""
|
|
202
|
+
|
|
203
|
+
def __init__(self, func: Callable[..., Any]) -> None:
|
|
204
|
+
self.func = func
|
|
205
|
+
|
|
206
|
+
def __get__(self, obj: Any, owner: Any = None) -> Any:
|
|
207
|
+
return self.func(owner)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
class _ActorMeta(type):
|
|
211
|
+
"""Metaclass enabling async with Actor: by delegating to classmethods."""
|
|
212
|
+
|
|
213
|
+
# Declarations for the Actor classmethods delegated to below (typing only).
|
|
214
|
+
init: Callable[..., Any]
|
|
215
|
+
_shutdown: Callable[..., Any]
|
|
216
|
+
_exit_code: int
|
|
217
|
+
|
|
218
|
+
async def __aenter__(cls) -> Any:
|
|
219
|
+
await cls.init()
|
|
220
|
+
return cls
|
|
221
|
+
|
|
222
|
+
async def __aexit__(cls, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
|
223
|
+
if exc_type is not None:
|
|
224
|
+
cls._exit_code = 1
|
|
225
|
+
logger.error(f'Actor failed: {exc_val}')
|
|
226
|
+
await cls._shutdown()
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class Actor(metaclass=_ActorMeta):
|
|
230
|
+
"""Core class for building Actors on the Scrapely platform."""
|
|
231
|
+
|
|
232
|
+
_instance: Optional['Actor'] = None
|
|
233
|
+
_initialized: bool = False
|
|
234
|
+
_config: Optional[Configuration] = None
|
|
235
|
+
_client: Optional[ApiClient] = None
|
|
236
|
+
_apify_client_instance: Optional['ApifyClientAsync'] = None
|
|
237
|
+
_scrapely_storage: Optional[ScrapelyStorageClient] = None
|
|
238
|
+
_smart_storage: Optional[SmartScrapelyStorageClient] = None
|
|
239
|
+
_default_dataset: Optional[Dataset] = None
|
|
240
|
+
_default_kvs: Optional[KeyValueStore] = None
|
|
241
|
+
_default_rq: Optional[RequestQueue] = None
|
|
242
|
+
_event_manager: Optional[EventManager] = None
|
|
243
|
+
_cached_input: Optional[Dict[str, Any]] = None
|
|
244
|
+
_exit_code: int = 0
|
|
245
|
+
_status_message: Optional[str] = None
|
|
246
|
+
_charging_manager: Optional[ChargingManager] = None
|
|
247
|
+
_use_state_stores: set = set()
|
|
248
|
+
# Lifecycle bookkeeping set in init() / cleared in _shutdown().
|
|
249
|
+
_lifecycle_config_snapshot: Optional[Dict[str, Any]] = None
|
|
250
|
+
_signal_shutdown_started: bool = False
|
|
251
|
+
|
|
252
|
+
def __init__(
|
|
253
|
+
self,
|
|
254
|
+
options: Optional[Dict[str, Any]] = None,
|
|
255
|
+
*,
|
|
256
|
+
configure_logging: bool = True,
|
|
257
|
+
exit_code: int = 0,
|
|
258
|
+
status_message: Optional[str] = None,
|
|
259
|
+
cleanup_timeout: timedelta = timedelta(seconds=30),
|
|
260
|
+
exit_process: Optional[bool] = None,
|
|
261
|
+
event_listeners_timeout: Optional[timedelta] = None,
|
|
262
|
+
):
|
|
263
|
+
"""Apify parity: ``async with Actor(configure_logging=False):`` as well as ``async with Actor:``.
|
|
264
|
+
|
|
265
|
+
``options`` is a configuration dict (Apify passes a Configuration here). The context
|
|
266
|
+
manager never calls ``sys.exit`` (``exit_process`` is accepted for signature parity),
|
|
267
|
+
and ``event_listeners_timeout`` is accepted but not applied.
|
|
268
|
+
"""
|
|
269
|
+
self._options = options or {}
|
|
270
|
+
self._configure_logging = configure_logging
|
|
271
|
+
self._instance_exit_code = exit_code
|
|
272
|
+
self._instance_status_message = status_message
|
|
273
|
+
self._cleanup_timeout = cleanup_timeout
|
|
274
|
+
|
|
275
|
+
async def __aenter__(self) -> 'Actor':
|
|
276
|
+
await type(self).init(
|
|
277
|
+
InitOptions(configuration=self._options or None, configure_logging=self._configure_logging)
|
|
278
|
+
)
|
|
279
|
+
return self
|
|
280
|
+
|
|
281
|
+
async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
|
282
|
+
cls = type(self)
|
|
283
|
+
if exc_type is not None:
|
|
284
|
+
cls._exit_code = 1
|
|
285
|
+
logger.error(f'Actor failed: {exc_val}')
|
|
286
|
+
else:
|
|
287
|
+
cls._exit_code = self._instance_exit_code
|
|
288
|
+
if self._instance_status_message:
|
|
289
|
+
cls._status_message = self._instance_status_message
|
|
290
|
+
await cls._shutdown(timeout_secs=int(self._cleanup_timeout.total_seconds()))
|
|
291
|
+
|
|
292
|
+
@classmethod
|
|
293
|
+
def get_default_instance(cls) -> 'Actor':
|
|
294
|
+
if cls._instance is None:
|
|
295
|
+
cls._instance = cls()
|
|
296
|
+
return cls._instance
|
|
297
|
+
|
|
298
|
+
@classmethod
|
|
299
|
+
async def init(cls, options: Optional[InitOptions] = None) -> None:
|
|
300
|
+
if cls._initialized:
|
|
301
|
+
return
|
|
302
|
+
|
|
303
|
+
if options is None or options.configure_logging:
|
|
304
|
+
_configure_logging()
|
|
305
|
+
|
|
306
|
+
# Do NOT set _initialized until everything succeeds (failed init must be retryable).
|
|
307
|
+
cls._config = Configuration.get_global_config()
|
|
308
|
+
# Snapshot keys we may overwrite so shutdown can restore prior lifecycle state.
|
|
309
|
+
cls._lifecycle_config_snapshot = {
|
|
310
|
+
'api_base_url': cls._config.get('api_base_url'),
|
|
311
|
+
'token': cls._config.get('token'),
|
|
312
|
+
'actor_run_id': cls._config.get('actor_run_id'),
|
|
313
|
+
}
|
|
314
|
+
if options and options.configuration:
|
|
315
|
+
for key, value in options.configuration.items():
|
|
316
|
+
cls._config.set(key, value)
|
|
317
|
+
|
|
318
|
+
crawlee_config = CrawleeConfiguration.get_global_configuration()
|
|
319
|
+
if cls._config.get('is_at_home'):
|
|
320
|
+
crawlee_config.purge_on_start = False
|
|
321
|
+
|
|
322
|
+
storage_opt = options.storage if options else None
|
|
323
|
+
|
|
324
|
+
try:
|
|
325
|
+
# Apply explicit storage credentials BEFORE event manager / charging so
|
|
326
|
+
# WebSocket + PPE bill the same run as storage.
|
|
327
|
+
if storage_opt is not None and _is_api_storage_config(storage_opt):
|
|
328
|
+
if storage_opt.get('api_url'):
|
|
329
|
+
cls._config.set('api_base_url', storage_opt['api_url'])
|
|
330
|
+
if storage_opt.get('token'):
|
|
331
|
+
cls._config.set('token', storage_opt['token'])
|
|
332
|
+
if storage_opt.get('run_id'):
|
|
333
|
+
cls._config.set('actor_run_id', storage_opt['run_id'])
|
|
334
|
+
|
|
335
|
+
if storage_opt is not None:
|
|
336
|
+
if _is_api_storage_config(storage_opt):
|
|
337
|
+
logger.info(
|
|
338
|
+
'[Scrapely] Initializing API storage for run %s',
|
|
339
|
+
storage_opt['run_id'],
|
|
340
|
+
)
|
|
341
|
+
smart = set_api_storage(storage_opt)
|
|
342
|
+
elif isinstance(storage_opt, StorageClient):
|
|
343
|
+
smart = set_storage_client(storage_opt)
|
|
344
|
+
else:
|
|
345
|
+
raise TypeError(
|
|
346
|
+
'Actor.init() storage option must be ApiStorageConfig or StorageClient'
|
|
347
|
+
)
|
|
348
|
+
else:
|
|
349
|
+
api_url = cls._config.get('api_base_url')
|
|
350
|
+
token = cls._config.get('token')
|
|
351
|
+
run_id = cls._config.get('actor_run_id')
|
|
352
|
+
|
|
353
|
+
if api_url and token and run_id:
|
|
354
|
+
logger.info('[Scrapely] Initializing API storage for run %s', run_id)
|
|
355
|
+
smart = set_api_storage({
|
|
356
|
+
'api_url': api_url,
|
|
357
|
+
'token': token,
|
|
358
|
+
'run_id': run_id,
|
|
359
|
+
})
|
|
360
|
+
elif not cls._config.get('is_at_home'):
|
|
361
|
+
# Local development without credentials: Smart → FS (Apify docs shape)
|
|
362
|
+
logger.info('[Scrapely] Initializing local file-system storage')
|
|
363
|
+
smart = set_api_storage()
|
|
364
|
+
else:
|
|
365
|
+
raise RuntimeError(
|
|
366
|
+
'Missing required configuration for secure storage. '
|
|
367
|
+
'Required: apiBaseUrl (SCRAPELY_API_URL), token (SCRAPELY_TOKEN/APIFY_TOKEN), '
|
|
368
|
+
'actorRunId (ACTOR_RUN_ID). Containers must be started by the scrapely-server, '
|
|
369
|
+
'or pass storage in Actor.init().'
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
cls._smart_storage = smart if isinstance(smart, SmartScrapelyStorageClient) else get_smart_storage_client()
|
|
373
|
+
factory = get_api_storage_factory()
|
|
374
|
+
cls._scrapely_storage = factory
|
|
375
|
+
if factory is not None and factory.has_credentials():
|
|
376
|
+
cls._client = factory.api_client
|
|
377
|
+
await cls._client._ensure_client()
|
|
378
|
+
|
|
379
|
+
# Choose event manager AFTER config has effective storage credentials.
|
|
380
|
+
from datetime import timedelta
|
|
381
|
+
|
|
382
|
+
ws_url = cls._config.get('actor_events_ws_url')
|
|
383
|
+
persist_interval_ms = cls._config.get('persist_state_interval_millis', 60000)
|
|
384
|
+
persist_interval = timedelta(milliseconds=persist_interval_ms)
|
|
385
|
+
|
|
386
|
+
event_manager: EventManager
|
|
387
|
+
if ws_url:
|
|
388
|
+
event_manager = ScrapelyEventManager(
|
|
389
|
+
configuration=cls._config,
|
|
390
|
+
persist_state_interval=persist_interval,
|
|
391
|
+
)
|
|
392
|
+
else:
|
|
393
|
+
event_manager = LocalEventManager(
|
|
394
|
+
system_info_interval=timedelta(seconds=1),
|
|
395
|
+
persist_state_interval=persist_interval,
|
|
396
|
+
)
|
|
397
|
+
cls._event_manager = event_manager
|
|
398
|
+
|
|
399
|
+
await event_manager.__aenter__()
|
|
400
|
+
|
|
401
|
+
def handle_signal(signum: int, frame: Any) -> None:
|
|
402
|
+
# Bound drain so uncertain PPE intents can flush before hard exit.
|
|
403
|
+
# Fire-and-forget create_task alone drops pending charges on SIGTERM.
|
|
404
|
+
logger.info('Received signal %s, shutting down...', signum)
|
|
405
|
+
try:
|
|
406
|
+
loop = asyncio.get_running_loop()
|
|
407
|
+
except RuntimeError:
|
|
408
|
+
os._exit(0) # NoReturn
|
|
409
|
+
|
|
410
|
+
if getattr(cls, '_signal_shutdown_started', False):
|
|
411
|
+
return
|
|
412
|
+
cls._signal_shutdown_started = True
|
|
413
|
+
|
|
414
|
+
def _run_shutdown() -> None:
|
|
415
|
+
async def _bounded() -> None:
|
|
416
|
+
# Independent hard-exit timer: fires even if teardown hangs on a
|
|
417
|
+
# stuck close() — the deadline cannot be swallowed by exceptions.
|
|
418
|
+
hard_timer = loop.call_later(6.0, os._exit, 0)
|
|
419
|
+
try:
|
|
420
|
+
await asyncio.wait_for(cls._shutdown(timeout_secs=5), timeout=5.5)
|
|
421
|
+
except BaseException as err:
|
|
422
|
+
logger.warning('Shutdown on signal failed: %s', err)
|
|
423
|
+
finally:
|
|
424
|
+
hard_timer.cancel()
|
|
425
|
+
os._exit(0)
|
|
426
|
+
|
|
427
|
+
loop.create_task(_bounded())
|
|
428
|
+
|
|
429
|
+
try:
|
|
430
|
+
loop.call_soon_threadsafe(_run_shutdown)
|
|
431
|
+
except Exception:
|
|
432
|
+
os._exit(0)
|
|
433
|
+
|
|
434
|
+
signal.signal(signal.SIGINT, handle_signal)
|
|
435
|
+
signal.signal(signal.SIGTERM, handle_signal)
|
|
436
|
+
|
|
437
|
+
cls._initialized = True
|
|
438
|
+
|
|
439
|
+
# Initialize PPE charging once so Actor.charge() has prices + budget.
|
|
440
|
+
# Must run AFTER `_initialized = True`: ChargingManager.init() goes
|
|
441
|
+
# through Actor.get_charging_manager(), which requires an initialized
|
|
442
|
+
# Actor (otherwise this eager init always fails with a warning and
|
|
443
|
+
# charging is only lazily recovered on the first Actor.charge()).
|
|
444
|
+
try:
|
|
445
|
+
charging_manager = cls.get_charging_manager()
|
|
446
|
+
await charging_manager.init()
|
|
447
|
+
except Exception as e:
|
|
448
|
+
logger.warning('Failed to initialize ChargingManager: %s', e)
|
|
449
|
+
except Exception:
|
|
450
|
+
# Roll back partial init so a corrected retry can succeed.
|
|
451
|
+
try:
|
|
452
|
+
await cls._shutdown()
|
|
453
|
+
except Exception as shutdown_err:
|
|
454
|
+
logger.warning('Rollback after failed Actor.init: %s', shutdown_err)
|
|
455
|
+
raise
|
|
456
|
+
|
|
457
|
+
@classmethod
|
|
458
|
+
async def _shutdown(cls, timeout_secs: int = 30) -> None:
|
|
459
|
+
# Resource cleanup may raise; config restore + lifecycle reset MUST still
|
|
460
|
+
# run so a failed teardown cannot leave tenant/run credentials active for
|
|
461
|
+
# the next Actor.init() retry.
|
|
462
|
+
cleanup_error: Optional[BaseException] = None
|
|
463
|
+
try:
|
|
464
|
+
if cls._event_manager is not None:
|
|
465
|
+
try:
|
|
466
|
+
await cls._event_manager.__aexit__(None, None, None)
|
|
467
|
+
except BaseException as e:
|
|
468
|
+
cleanup_error = cleanup_error or e
|
|
469
|
+
logger.warning('Event manager shutdown failed: %s', e)
|
|
470
|
+
cls._event_manager = None
|
|
471
|
+
|
|
472
|
+
# Best-effort: confirm any unresolved PPE charge intents before teardown.
|
|
473
|
+
if cls._charging_manager is not None:
|
|
474
|
+
# Reject NEW charges before the final flush so nothing can be
|
|
475
|
+
# admitted after it while teardown closes storages/credentials.
|
|
476
|
+
close_fn = getattr(cls._charging_manager, 'close', None)
|
|
477
|
+
if close_fn is not None:
|
|
478
|
+
try:
|
|
479
|
+
close_fn()
|
|
480
|
+
except Exception:
|
|
481
|
+
pass
|
|
482
|
+
try:
|
|
483
|
+
flush = getattr(cls._charging_manager, 'flush_pending', None)
|
|
484
|
+
if flush is not None:
|
|
485
|
+
deadline_ms = min(max(1, timeout_secs) * 1000, 30_000)
|
|
486
|
+
await asyncio.wait_for(
|
|
487
|
+
flush(max_intents=64, deadline_ms=deadline_ms),
|
|
488
|
+
timeout=max(1.0, deadline_ms / 1000.0),
|
|
489
|
+
)
|
|
490
|
+
except BaseException as e:
|
|
491
|
+
# Never swallow cancellation — but also never let it skip the
|
|
492
|
+
# reference-clearing teardown below. Record it and re-raise at
|
|
493
|
+
# the END of _shutdown: an immediate raise would leave the
|
|
494
|
+
# closed charging manager and storage/client references
|
|
495
|
+
# installed, so a subsequent Actor.init() would reuse the
|
|
496
|
+
# closed manager and every later charge would fail. The
|
|
497
|
+
# signal-path deadline is still bounded by the independent
|
|
498
|
+
# loop.call_later(6.0, os._exit, 0) hard timer.
|
|
499
|
+
cleanup_error = cleanup_error or e
|
|
500
|
+
logger.warning('ChargingManager flush_pending failed during shutdown: %s', e)
|
|
501
|
+
|
|
502
|
+
if cls._scrapely_storage is not None:
|
|
503
|
+
try:
|
|
504
|
+
await cls._scrapely_storage.close()
|
|
505
|
+
except BaseException as e:
|
|
506
|
+
cleanup_error = cleanup_error or e
|
|
507
|
+
logger.warning('Storage close failed during shutdown: %s', e)
|
|
508
|
+
cls._scrapely_storage = None
|
|
509
|
+
cls._smart_storage = None
|
|
510
|
+
cls._client = None
|
|
511
|
+
# Clear charging + patched client so a second Actor lifecycle cannot
|
|
512
|
+
# post charges to the previous run_id / credentials.
|
|
513
|
+
cls._charging_manager = None
|
|
514
|
+
cls._apify_client_instance = None
|
|
515
|
+
|
|
516
|
+
cls._default_dataset = None
|
|
517
|
+
cls._default_kvs = None
|
|
518
|
+
cls._default_rq = None
|
|
519
|
+
cls._cached_input = None
|
|
520
|
+
try:
|
|
521
|
+
disconnect_storage()
|
|
522
|
+
except BaseException as e:
|
|
523
|
+
cleanup_error = cleanup_error or e
|
|
524
|
+
logger.warning('disconnect_storage failed during shutdown: %s', e)
|
|
525
|
+
finally:
|
|
526
|
+
# ALWAYS restore config keys mutated for this lifecycle so a later
|
|
527
|
+
# init without explicit storage does not inherit previous credentials.
|
|
528
|
+
snap = getattr(cls, '_lifecycle_config_snapshot', None)
|
|
529
|
+
if snap is not None and cls._config is not None:
|
|
530
|
+
for key, value in snap.items():
|
|
531
|
+
try:
|
|
532
|
+
cls._config.set(key, value)
|
|
533
|
+
except Exception:
|
|
534
|
+
pass
|
|
535
|
+
cls._lifecycle_config_snapshot = None
|
|
536
|
+
cls._initialized = False
|
|
537
|
+
# Allow a subsequent Actor lifecycle to handle SIGINT/SIGTERM again.
|
|
538
|
+
cls._signal_shutdown_started = False
|
|
539
|
+
|
|
540
|
+
if cleanup_error is not None:
|
|
541
|
+
raise cleanup_error
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
@classmethod
|
|
545
|
+
async def main(cls, user_func: Callable[[], Any]) -> Any:
|
|
546
|
+
try:
|
|
547
|
+
async with cls:
|
|
548
|
+
return await user_func()
|
|
549
|
+
except Exception as e:
|
|
550
|
+
logger.exception(f'Actor failed: {e}')
|
|
551
|
+
cls._exit_code = 1
|
|
552
|
+
raise
|
|
553
|
+
finally:
|
|
554
|
+
sys.exit(cls._exit_code)
|
|
555
|
+
|
|
556
|
+
@classmethod
|
|
557
|
+
async def exit(
|
|
558
|
+
cls,
|
|
559
|
+
message_or_options: Optional[Union[str, ExitOptions]] = None,
|
|
560
|
+
options: Optional[ExitOptions] = None,
|
|
561
|
+
*,
|
|
562
|
+
status_message: Optional[str] = None,
|
|
563
|
+
exit_code: Optional[int] = None,
|
|
564
|
+
) -> None:
|
|
565
|
+
"""Exit the Actor with an optional terminal status message.
|
|
566
|
+
|
|
567
|
+
Apify-Python parity: supports keyword-only ``status_message`` and
|
|
568
|
+
``exit_code`` arguments (``await Actor.exit(status_message="Done")``).
|
|
569
|
+
The pre-existing positional forms (``Actor.exit("message")`` /
|
|
570
|
+
``Actor.exit(ExitOptions(...))``) remain supported; explicit keyword
|
|
571
|
+
arguments take precedence over the positional values when provided.
|
|
572
|
+
"""
|
|
573
|
+
if isinstance(message_or_options, str):
|
|
574
|
+
opts = ExitOptions(status_message=message_or_options)
|
|
575
|
+
else:
|
|
576
|
+
opts = message_or_options or ExitOptions()
|
|
577
|
+
|
|
578
|
+
if options:
|
|
579
|
+
opts = ExitOptions(**{**opts.__dict__, **options.__dict__})
|
|
580
|
+
|
|
581
|
+
if status_message is not None:
|
|
582
|
+
opts.status_message = status_message
|
|
583
|
+
if exit_code is not None:
|
|
584
|
+
opts.exit_code = exit_code
|
|
585
|
+
|
|
586
|
+
if opts.status_message:
|
|
587
|
+
logger.info(f'[Scrapely] {opts.status_message}')
|
|
588
|
+
|
|
589
|
+
cls._exit_code = opts.exit_code
|
|
590
|
+
cls._status_message = opts.status_message
|
|
591
|
+
|
|
592
|
+
await cls._shutdown(timeout_secs=opts.timeout_secs)
|
|
593
|
+
|
|
594
|
+
if opts.exit:
|
|
595
|
+
sys.exit(cls._exit_code)
|
|
596
|
+
|
|
597
|
+
@classmethod
|
|
598
|
+
async def fail(
|
|
599
|
+
cls,
|
|
600
|
+
message_or_options: Optional[Union[str, ExitOptions]] = None,
|
|
601
|
+
options: Optional[ExitOptions] = None,
|
|
602
|
+
*,
|
|
603
|
+
status_message: Optional[str] = None,
|
|
604
|
+
exit_code: Optional[int] = None,
|
|
605
|
+
) -> None:
|
|
606
|
+
"""Fail the Actor run (exit code defaults to ``1``).
|
|
607
|
+
|
|
608
|
+
Apify-Python parity: supports keyword-only ``status_message`` and
|
|
609
|
+
``exit_code`` arguments (``await Actor.fail(status_message="Boom")``).
|
|
610
|
+
The pre-existing positional forms (``Actor.fail("message")`` /
|
|
611
|
+
``Actor.fail(ExitOptions(...))``) remain supported; explicit keyword
|
|
612
|
+
arguments take precedence over the positional values when provided.
|
|
613
|
+
"""
|
|
614
|
+
if isinstance(message_or_options, str):
|
|
615
|
+
opts = ExitOptions(exit_code=1, status_message=message_or_options)
|
|
616
|
+
else:
|
|
617
|
+
# Options-object form: an explicit exit_code in the object wins over
|
|
618
|
+
# the fail() default of 1 (JS SDK semantics). Build the dict first —
|
|
619
|
+
# passing it alongside exit_code=1 as kwargs raises
|
|
620
|
+
# "got multiple values for keyword argument 'exit_code'".
|
|
621
|
+
base = dict(message_or_options.__dict__) if message_or_options else {}
|
|
622
|
+
base.setdefault('exit_code', 1)
|
|
623
|
+
opts = ExitOptions(**base)
|
|
624
|
+
|
|
625
|
+
if options:
|
|
626
|
+
opts = ExitOptions(**{**opts.__dict__, **options.__dict__})
|
|
627
|
+
|
|
628
|
+
if status_message is not None:
|
|
629
|
+
opts.status_message = status_message
|
|
630
|
+
if exit_code is not None:
|
|
631
|
+
opts.exit_code = exit_code
|
|
632
|
+
|
|
633
|
+
logger.error(f'[Scrapely] Actor failed: {opts.status_message or "Unknown error"}')
|
|
634
|
+
await cls.exit(opts)
|
|
635
|
+
|
|
636
|
+
@classmethod
|
|
637
|
+
def is_at_home(cls) -> bool:
|
|
638
|
+
return cls._config.get('is_at_home', False) if cls._config else False
|
|
639
|
+
|
|
640
|
+
@classmethod
|
|
641
|
+
def is_standby(cls) -> bool:
|
|
642
|
+
"""Return True when the actor run was started in standby mode (Node SDK ``isStandby`` parity)."""
|
|
643
|
+
return (cls._config.get('meta_origin') == 'STANDBY') if cls._config else False
|
|
644
|
+
|
|
645
|
+
@_ClassProperty
|
|
646
|
+
def is_paying(cls) -> bool:
|
|
647
|
+
"""Whether the user who started this Actor run is a paying user.
|
|
648
|
+
|
|
649
|
+
Mirrors the Node SDK's ``Actor.isPaying`` (``APIFY_USER_IS_PAYING`` /
|
|
650
|
+
``user_is_paying``). Note the platform currently sets this from the
|
|
651
|
+
subscription plan only — PAYG balance alone does not flip it.
|
|
652
|
+
"""
|
|
653
|
+
config = cls.get_config() if cls._config is not None else Configuration.get_global_config()
|
|
654
|
+
return bool(config.get('user_is_paying'))
|
|
655
|
+
|
|
656
|
+
@classmethod
|
|
657
|
+
def get_env(cls) -> ApifyEnv:
|
|
658
|
+
if cls._config is None:
|
|
659
|
+
cls._config = Configuration.get_global_config()
|
|
660
|
+
return cls._config.get_env()
|
|
661
|
+
|
|
662
|
+
@classmethod
|
|
663
|
+
def get_remaining_time(cls) -> Optional[int]:
|
|
664
|
+
timeout_at = os.environ.get('ACTOR_TIMEOUT_AT')
|
|
665
|
+
if not timeout_at:
|
|
666
|
+
return None
|
|
667
|
+
try:
|
|
668
|
+
timeout_date = datetime.fromisoformat(timeout_at.replace('Z', '+00:00'))
|
|
669
|
+
remaining = int((timeout_date - datetime.now(timeout_date.tzinfo)).total_seconds() * 1000)
|
|
670
|
+
return max(0, remaining)
|
|
671
|
+
except (ValueError, TypeError):
|
|
672
|
+
return None
|
|
673
|
+
|
|
674
|
+
@classmethod
|
|
675
|
+
async def get_input(cls) -> Optional[Dict[str, Any]]:
|
|
676
|
+
cls._ensure_initialized('get_input')
|
|
677
|
+
|
|
678
|
+
if cls._cached_input is not None:
|
|
679
|
+
return cls._cached_input
|
|
680
|
+
|
|
681
|
+
metamorph_count = os.environ.get('ACTOR_METAMORPH_COUNT')
|
|
682
|
+
if metamorph_count:
|
|
683
|
+
count = int(metamorph_count)
|
|
684
|
+
if count > 0:
|
|
685
|
+
kvs = await cls.open_key_value_store()
|
|
686
|
+
metamorph_input = await kvs.get_value(f'INPUT-METAMORPH-{count}')
|
|
687
|
+
if metamorph_input is not None:
|
|
688
|
+
cls._cached_input = metamorph_input
|
|
689
|
+
return cast(Dict[str, Any], metamorph_input)
|
|
690
|
+
|
|
691
|
+
raw_input = os.environ.get('ACTOR_INPUT')
|
|
692
|
+
if raw_input:
|
|
693
|
+
try:
|
|
694
|
+
decoded = base64.b64decode(raw_input).decode('utf-8')
|
|
695
|
+
parsed = json.loads(decoded)
|
|
696
|
+
except Exception:
|
|
697
|
+
parsed = json.loads(raw_input)
|
|
698
|
+
# Decrypt secret inputs after loading (JS parity: only the main input
|
|
699
|
+
# path is decrypted, mirroring Actor.getInput() in the Node SDK)
|
|
700
|
+
if isinstance(parsed, dict):
|
|
701
|
+
parsed = decrypt_secret_inputs(parsed)
|
|
702
|
+
cls._cached_input = parsed
|
|
703
|
+
return cast(Dict[str, Any], cls._cached_input)
|
|
704
|
+
|
|
705
|
+
kvs = await cls.open_key_value_store()
|
|
706
|
+
input_key = cls._config.get('input_key', 'INPUT') if cls._config else 'INPUT'
|
|
707
|
+
result = await kvs.get_value(input_key)
|
|
708
|
+
if isinstance(result, dict):
|
|
709
|
+
result = decrypt_secret_inputs(result)
|
|
710
|
+
cls._cached_input = result
|
|
711
|
+
return cast(Optional[Dict[str, Any]], result)
|
|
712
|
+
|
|
713
|
+
@classmethod
|
|
714
|
+
async def get_input_or_throw(cls) -> Dict[str, Any]:
|
|
715
|
+
result = await cls.get_input()
|
|
716
|
+
if result is None:
|
|
717
|
+
raise ValueError('Input does not exist')
|
|
718
|
+
return result
|
|
719
|
+
|
|
720
|
+
@_ClassProperty
|
|
721
|
+
def input(cls) -> Optional[Dict[str, Any]]:
|
|
722
|
+
if cls._cached_input is not None:
|
|
723
|
+
return cls._cached_input
|
|
724
|
+
|
|
725
|
+
raw = os.environ.get('ACTOR_INPUT')
|
|
726
|
+
if not raw:
|
|
727
|
+
return None
|
|
728
|
+
|
|
729
|
+
try:
|
|
730
|
+
decoded = base64.b64decode(raw).decode('utf-8')
|
|
731
|
+
cls._cached_input = json.loads(decoded)
|
|
732
|
+
except Exception:
|
|
733
|
+
try:
|
|
734
|
+
cls._cached_input = json.loads(raw)
|
|
735
|
+
except json.JSONDecodeError:
|
|
736
|
+
return None
|
|
737
|
+
|
|
738
|
+
# Decrypt secret inputs after loading (JS parity with Actor.input getter)
|
|
739
|
+
if isinstance(cls._cached_input, dict):
|
|
740
|
+
cls._cached_input = decrypt_secret_inputs(cls._cached_input)
|
|
741
|
+
|
|
742
|
+
return cls._cached_input
|
|
743
|
+
|
|
744
|
+
@classmethod
|
|
745
|
+
async def push_data(
|
|
746
|
+
cls,
|
|
747
|
+
data: Union[Dict[str, Any], List[Dict[str, Any]]],
|
|
748
|
+
charged_event_name: Optional[str] = None,
|
|
749
|
+
*,
|
|
750
|
+
event_name: Optional[str] = None,
|
|
751
|
+
) -> ChargeResult:
|
|
752
|
+
"""Push data to the default dataset, charging pay-per-event items.
|
|
753
|
+
|
|
754
|
+
Matches the Apify Python SDK's ``Actor.push_data(data, charged_event_name=...)``:
|
|
755
|
+
when the Actor uses pay-per-event pricing, each pushed item is charged for the
|
|
756
|
+
default-dataset-item event (if priced) and, when ``charged_event_name`` is
|
|
757
|
+
provided, for that explicit event as well. Actors without pay-per-event
|
|
758
|
+
pricing push without charging and receive an empty result.
|
|
759
|
+
|
|
760
|
+
Args:
|
|
761
|
+
data: Item or list of items to push to the default dataset.
|
|
762
|
+
charged_event_name: Optional pricing event to charge for each pushed item
|
|
763
|
+
(Apify Python SDK name; also accepted positionally).
|
|
764
|
+
event_name: Deprecated alias of ``charged_event_name`` (JS ``eventName``),
|
|
765
|
+
kept so existing callers keep working.
|
|
766
|
+
|
|
767
|
+
Returns:
|
|
768
|
+
ChargeResult describing how many items were charged and pushed.
|
|
769
|
+
|
|
770
|
+
Raises:
|
|
771
|
+
ValueError: If the event refers to a synthetic event, or both names are given
|
|
772
|
+
with different values.
|
|
773
|
+
"""
|
|
774
|
+
cls._ensure_initialized('push_data')
|
|
775
|
+
if charged_event_name is not None and event_name is not None and charged_event_name != event_name:
|
|
776
|
+
raise ValueError('Pass only one of charged_event_name / event_name')
|
|
777
|
+
event_name = charged_event_name if charged_event_name is not None else event_name
|
|
778
|
+
# Exact synthetic names only (JS parity) — a prefix heuristic would block
|
|
779
|
+
# real priced events like 'apify-foo'.
|
|
780
|
+
if event_name and is_default_dataset_item_event(event_name):
|
|
781
|
+
raise ValueError(f"Cannot charge for synthetic event '{event_name}' manually")
|
|
782
|
+
|
|
783
|
+
if cls._default_dataset is None:
|
|
784
|
+
cls._default_dataset = await cls.open_dataset()
|
|
785
|
+
|
|
786
|
+
items = data if isinstance(data, list) else [data]
|
|
787
|
+
charging_manager = cls.get_charging_manager()
|
|
788
|
+
return await push_data_and_charge(
|
|
789
|
+
charging_manager,
|
|
790
|
+
items,
|
|
791
|
+
event_name,
|
|
792
|
+
True, # Actor.push_data always targets the default dataset
|
|
793
|
+
cls._default_dataset.push_data,
|
|
794
|
+
)
|
|
795
|
+
|
|
796
|
+
@classmethod
|
|
797
|
+
async def get_value(cls, key: str, default_value: Any = None) -> Any:
|
|
798
|
+
cls._ensure_initialized('get_value')
|
|
799
|
+
if cls._default_kvs is None:
|
|
800
|
+
cls._default_kvs = await cls.open_key_value_store()
|
|
801
|
+
return await cls._default_kvs.get_value(key, default_value=default_value)
|
|
802
|
+
|
|
803
|
+
@classmethod
|
|
804
|
+
async def set_value(cls, key: str, value: Any, content_type: Optional[str] = None) -> None:
|
|
805
|
+
cls._ensure_initialized('set_value')
|
|
806
|
+
if cls._default_kvs is None:
|
|
807
|
+
cls._default_kvs = await cls.open_key_value_store()
|
|
808
|
+
await cls._default_kvs.set_value(key, value, content_type=content_type)
|
|
809
|
+
|
|
810
|
+
@classmethod
|
|
811
|
+
def _resolve_open_args(
|
|
812
|
+
cls,
|
|
813
|
+
name_or_id: Optional[str] = None,
|
|
814
|
+
*,
|
|
815
|
+
id: Optional[str] = None,
|
|
816
|
+
name: Optional[str] = None,
|
|
817
|
+
alias: Optional[str] = None,
|
|
818
|
+
force_cloud: bool = False,
|
|
819
|
+
options: Optional[OpenStorageOptions] = None,
|
|
820
|
+
storage_type: StorageType = 'datasets',
|
|
821
|
+
) -> Dict[str, Any]:
|
|
822
|
+
force = force_cloud or bool(options and options.force_cloud)
|
|
823
|
+
|
|
824
|
+
if name_or_id is not None:
|
|
825
|
+
if id is not None or name is not None or alias is not None:
|
|
826
|
+
raise ValueError('Pass either positional name_or_id or id/name/alias kwargs, not both')
|
|
827
|
+
if is_storage_uuid(name_or_id):
|
|
828
|
+
id = name_or_id
|
|
829
|
+
else:
|
|
830
|
+
name = name_or_id
|
|
831
|
+
|
|
832
|
+
if alias is not None:
|
|
833
|
+
storages_json = cls._config.get('actor_storages_json') if cls._config else None
|
|
834
|
+
resolved = resolve_alias_from_storages_json(alias, storage_type, storages_json)
|
|
835
|
+
if resolved is not None:
|
|
836
|
+
id = resolved
|
|
837
|
+
alias = None
|
|
838
|
+
else:
|
|
839
|
+
# Outside platform: alias behaves like a name (TS)
|
|
840
|
+
name = alias
|
|
841
|
+
alias = None
|
|
842
|
+
|
|
843
|
+
return {'id': id, 'name': name, 'alias': alias, 'force_cloud': force}
|
|
844
|
+
|
|
845
|
+
@classmethod
|
|
846
|
+
def _storage_client_for_open(cls, *, force_cloud: bool = False) -> Optional[StorageClient]:
|
|
847
|
+
smart = cls._smart_storage or get_smart_storage_client()
|
|
848
|
+
if smart is not None:
|
|
849
|
+
return smart.get_suitable_storage_client(force_cloud=force_cloud)
|
|
850
|
+
|
|
851
|
+
if force_cloud:
|
|
852
|
+
cloud = get_api_storage_factory()
|
|
853
|
+
if cloud is None:
|
|
854
|
+
# Build from config like TS _buildStorageOpenOptions
|
|
855
|
+
config = cls.get_config()
|
|
856
|
+
api_url = config.get('api_base_url')
|
|
857
|
+
token = config.get('token')
|
|
858
|
+
run_id = config.get('actor_run_id')
|
|
859
|
+
if api_url and token and run_id:
|
|
860
|
+
return ScrapelyStorageClient(api_url=api_url, token=token, run_id=run_id)
|
|
861
|
+
raise RuntimeError('Cloud storage client is not configured')
|
|
862
|
+
return cloud
|
|
863
|
+
return None
|
|
864
|
+
|
|
865
|
+
@classmethod
|
|
866
|
+
async def open_dataset(
|
|
867
|
+
cls,
|
|
868
|
+
name_or_id: Optional[str] = None,
|
|
869
|
+
options: Optional[OpenStorageOptions] = None,
|
|
870
|
+
*,
|
|
871
|
+
id: Optional[str] = None,
|
|
872
|
+
name: Optional[str] = None,
|
|
873
|
+
alias: Optional[str] = None,
|
|
874
|
+
force_cloud: bool = False,
|
|
875
|
+
) -> Dataset:
|
|
876
|
+
cls._ensure_initialized('open_dataset')
|
|
877
|
+
open_args = cls._resolve_open_args(
|
|
878
|
+
name_or_id,
|
|
879
|
+
id=id,
|
|
880
|
+
name=name,
|
|
881
|
+
alias=alias,
|
|
882
|
+
force_cloud=force_cloud,
|
|
883
|
+
options=options,
|
|
884
|
+
storage_type='datasets',
|
|
885
|
+
)
|
|
886
|
+
storage_client = cls._storage_client_for_open(force_cloud=open_args['force_cloud'])
|
|
887
|
+
|
|
888
|
+
if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
|
|
889
|
+
if cls._default_dataset is None:
|
|
890
|
+
cls._default_dataset = await Dataset.open(storage_client=storage_client)
|
|
891
|
+
return cls._default_dataset
|
|
892
|
+
|
|
893
|
+
return await Dataset.open(
|
|
894
|
+
id=open_args['id'],
|
|
895
|
+
name=open_args['name'],
|
|
896
|
+
alias=open_args['alias'],
|
|
897
|
+
storage_client=storage_client,
|
|
898
|
+
)
|
|
899
|
+
|
|
900
|
+
@classmethod
|
|
901
|
+
async def open_key_value_store(
|
|
902
|
+
cls,
|
|
903
|
+
name_or_id: Optional[str] = None,
|
|
904
|
+
options: Optional[OpenStorageOptions] = None,
|
|
905
|
+
*,
|
|
906
|
+
id: Optional[str] = None,
|
|
907
|
+
name: Optional[str] = None,
|
|
908
|
+
alias: Optional[str] = None,
|
|
909
|
+
force_cloud: bool = False,
|
|
910
|
+
) -> KeyValueStore:
|
|
911
|
+
cls._ensure_initialized('open_key_value_store')
|
|
912
|
+
open_args = cls._resolve_open_args(
|
|
913
|
+
name_or_id,
|
|
914
|
+
id=id,
|
|
915
|
+
name=name,
|
|
916
|
+
alias=alias,
|
|
917
|
+
force_cloud=force_cloud,
|
|
918
|
+
options=options,
|
|
919
|
+
storage_type='keyValueStores',
|
|
920
|
+
)
|
|
921
|
+
storage_client = cls._storage_client_for_open(force_cloud=open_args['force_cloud'])
|
|
922
|
+
|
|
923
|
+
if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
|
|
924
|
+
if cls._default_kvs is None:
|
|
925
|
+
cls._default_kvs = await KeyValueStore.open(storage_client=storage_client)
|
|
926
|
+
return cls._default_kvs
|
|
927
|
+
|
|
928
|
+
return await KeyValueStore.open(
|
|
929
|
+
id=open_args['id'],
|
|
930
|
+
name=open_args['name'],
|
|
931
|
+
alias=open_args['alias'],
|
|
932
|
+
storage_client=storage_client,
|
|
933
|
+
)
|
|
934
|
+
|
|
935
|
+
@classmethod
|
|
936
|
+
async def open_request_queue(
|
|
937
|
+
cls,
|
|
938
|
+
name_or_id: Optional[str] = None,
|
|
939
|
+
options: Optional[OpenStorageOptions] = None,
|
|
940
|
+
*,
|
|
941
|
+
id: Optional[str] = None,
|
|
942
|
+
name: Optional[str] = None,
|
|
943
|
+
alias: Optional[str] = None,
|
|
944
|
+
force_cloud: bool = False,
|
|
945
|
+
) -> RequestQueue:
|
|
946
|
+
cls._ensure_initialized('open_request_queue')
|
|
947
|
+
open_args = cls._resolve_open_args(
|
|
948
|
+
name_or_id,
|
|
949
|
+
id=id,
|
|
950
|
+
name=name,
|
|
951
|
+
alias=alias,
|
|
952
|
+
force_cloud=force_cloud,
|
|
953
|
+
options=options,
|
|
954
|
+
storage_type='requestQueues',
|
|
955
|
+
)
|
|
956
|
+
storage_client = cls._storage_client_for_open(force_cloud=open_args['force_cloud'])
|
|
957
|
+
|
|
958
|
+
if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
|
|
959
|
+
if cls._default_rq is None:
|
|
960
|
+
cls._default_rq = await RequestQueue.open(storage_client=storage_client)
|
|
961
|
+
return cls._default_rq
|
|
962
|
+
|
|
963
|
+
return await RequestQueue.open(
|
|
964
|
+
id=open_args['id'],
|
|
965
|
+
name=open_args['name'],
|
|
966
|
+
alias=open_args['alias'],
|
|
967
|
+
storage_client=storage_client,
|
|
968
|
+
)
|
|
969
|
+
|
|
970
|
+
@classmethod
|
|
971
|
+
def on(cls, event: Event, listener: Callable) -> None:
|
|
972
|
+
cls._ensure_initialized('on')
|
|
973
|
+
if cls._event_manager is None:
|
|
974
|
+
raise RuntimeError('Actor event manager is not initialized.')
|
|
975
|
+
cls._event_manager.on(event=event, listener=listener)
|
|
976
|
+
|
|
977
|
+
@classmethod
|
|
978
|
+
def off(cls, event: Event, listener: Optional[Callable] = None) -> None:
|
|
979
|
+
cls._ensure_initialized('off')
|
|
980
|
+
if cls._event_manager is None:
|
|
981
|
+
raise RuntimeError('Actor event manager is not initialized.')
|
|
982
|
+
cls._event_manager.off(event=event, listener=listener)
|
|
983
|
+
|
|
984
|
+
@classmethod
|
|
985
|
+
async def set_status_message(
|
|
986
|
+
cls,
|
|
987
|
+
message: str,
|
|
988
|
+
options: Optional[SetStatusMessageOptions] = None,
|
|
989
|
+
*,
|
|
990
|
+
is_terminal: Optional[bool] = None,
|
|
991
|
+
) -> ActorRun:
|
|
992
|
+
"""Set the status message for the current Actor run (Node SDK parity).
|
|
993
|
+
|
|
994
|
+
Logs the message at the requested level, then updates the run through
|
|
995
|
+
the platform API when credentials (``api_base_url``, ``token``,
|
|
996
|
+
``actor_run_id``) are available. Locally, logs only and returns an
|
|
997
|
+
empty run object, mirroring the Node SDK behavior.
|
|
998
|
+
|
|
999
|
+
Args:
|
|
1000
|
+
message: The status message to set.
|
|
1001
|
+
options: Optional settings (``is_status_message_terminal``, ``level``).
|
|
1002
|
+
is_terminal: Apify Python SDK-style terminal flag. When given, it
|
|
1003
|
+
overrides ``options.is_status_message_terminal``.
|
|
1004
|
+
|
|
1005
|
+
Returns:
|
|
1006
|
+
The updated ActorRun object, or an empty ActorRun when running locally.
|
|
1007
|
+
"""
|
|
1008
|
+
cls._ensure_initialized('set_status_message')
|
|
1009
|
+
opts = options or SetStatusMessageOptions()
|
|
1010
|
+
level = (getattr(opts, 'level', None) or 'INFO').upper()
|
|
1011
|
+
if is_terminal is None:
|
|
1012
|
+
is_terminal = bool(getattr(opts, 'is_status_message_terminal', False))
|
|
1013
|
+
|
|
1014
|
+
logged = f'[Status message]: {message}'
|
|
1015
|
+
if level == 'DEBUG':
|
|
1016
|
+
logger.debug(logged)
|
|
1017
|
+
elif level == 'WARNING':
|
|
1018
|
+
logger.warning(logged)
|
|
1019
|
+
elif level == 'ERROR':
|
|
1020
|
+
logger.error(logged)
|
|
1021
|
+
else:
|
|
1022
|
+
logger.info(logged)
|
|
1023
|
+
|
|
1024
|
+
config = cls.get_config()
|
|
1025
|
+
api_url = config.get('api_base_url')
|
|
1026
|
+
token = config.get('token')
|
|
1027
|
+
run_id = config.get('actor_run_id')
|
|
1028
|
+
|
|
1029
|
+
if not api_url or not token or not run_id:
|
|
1030
|
+
return ActorRun(id='', actor_id='', status='')
|
|
1031
|
+
|
|
1032
|
+
client = cls.new_client(token=token, base_url=api_url)
|
|
1033
|
+
response = await client.put(
|
|
1034
|
+
f'/v2/actor-runs/{run_id}',
|
|
1035
|
+
json_data={
|
|
1036
|
+
'statusMessage': message,
|
|
1037
|
+
'isStatusMessageTerminal': is_terminal,
|
|
1038
|
+
},
|
|
1039
|
+
)
|
|
1040
|
+
body: Any = {}
|
|
1041
|
+
if response.content:
|
|
1042
|
+
body = response.json()
|
|
1043
|
+
data = body.get('data', body) if isinstance(body, dict) else {}
|
|
1044
|
+
|
|
1045
|
+
return ActorRun(
|
|
1046
|
+
id=data.get('id', run_id),
|
|
1047
|
+
actor_id=data.get('actorId', ''),
|
|
1048
|
+
status=data.get('status', ''),
|
|
1049
|
+
started_at=data.get('startedAt'),
|
|
1050
|
+
finished_at=data.get('finishedAt'),
|
|
1051
|
+
status_message=data.get('statusMessage'),
|
|
1052
|
+
)
|
|
1053
|
+
|
|
1054
|
+
@_ClassProperty
|
|
1055
|
+
def log(cls) -> logging.Logger:
|
|
1056
|
+
return logger
|
|
1057
|
+
|
|
1058
|
+
@_ClassProperty
|
|
1059
|
+
def exit_code(cls) -> int:
|
|
1060
|
+
return cls._exit_code
|
|
1061
|
+
|
|
1062
|
+
@_ClassProperty
|
|
1063
|
+
def status_message(cls) -> Optional[str]:
|
|
1064
|
+
return cls._status_message
|
|
1065
|
+
|
|
1066
|
+
@classmethod
|
|
1067
|
+
def _ensure_initialized(cls, method_name: str) -> None:
|
|
1068
|
+
if not cls._initialized:
|
|
1069
|
+
raise RuntimeError(
|
|
1070
|
+
f'Actor.{method_name}() requires Actor.init() to be called first. '
|
|
1071
|
+
'Hint: Use async with Actor: which calls init() for you.'
|
|
1072
|
+
)
|
|
1073
|
+
|
|
1074
|
+
@classmethod
|
|
1075
|
+
def get_config(cls) -> Configuration:
|
|
1076
|
+
if cls._config is None:
|
|
1077
|
+
cls._config = Configuration.get_global_config()
|
|
1078
|
+
return cls._config
|
|
1079
|
+
|
|
1080
|
+
@classmethod
|
|
1081
|
+
def new_client(cls, token: Optional[str] = None, base_url: Optional[str] = None) -> ApiClient:
|
|
1082
|
+
config = cls.get_config()
|
|
1083
|
+
api_url = base_url or config.get('api_base_url')
|
|
1084
|
+
api_token = token or config.get('token')
|
|
1085
|
+
run_id = config.get('actor_run_id', 'unknown')
|
|
1086
|
+
return ApiClient(api_url=api_url, token=api_token, run_id=run_id)
|
|
1087
|
+
|
|
1088
|
+
# -------------------------------------------------------------------------
|
|
1089
|
+
# Properties for Apify SDK compatibility
|
|
1090
|
+
# -------------------------------------------------------------------------
|
|
1091
|
+
|
|
1092
|
+
@classmethod
|
|
1093
|
+
def _get_apify_client(cls) -> 'ApifyClientAsync':
|
|
1094
|
+
"""Get or create a cached charging-configured ``ApifyClientAsync`` instance.
|
|
1095
|
+
|
|
1096
|
+
This client provides full ApifyClient compatibility including
|
|
1097
|
+
actor(), dataset(), key_value_store(), etc. methods.
|
|
1098
|
+
"""
|
|
1099
|
+
if cls._apify_client_instance is None:
|
|
1100
|
+
config = cls.get_config()
|
|
1101
|
+
api_url = config.get('api_base_url')
|
|
1102
|
+
token = config.get('token')
|
|
1103
|
+
default_dataset_id = config.get('default_dataset_id')
|
|
1104
|
+
|
|
1105
|
+
# Get charging manager if available
|
|
1106
|
+
charging_manager = None
|
|
1107
|
+
try:
|
|
1108
|
+
charging_manager = cls.get_charging_manager()
|
|
1109
|
+
except RuntimeError:
|
|
1110
|
+
pass # Actor not initialized yet
|
|
1111
|
+
|
|
1112
|
+
cls._apify_client_instance = create_charging_client(
|
|
1113
|
+
token=token,
|
|
1114
|
+
api_url=api_url,
|
|
1115
|
+
# Public URLs (record/keys public URLs) must be minted against the
|
|
1116
|
+
# configured public host, not the client's built-in default.
|
|
1117
|
+
api_public_url=config.get('api_public_base_url'),
|
|
1118
|
+
charging_manager=charging_manager,
|
|
1119
|
+
default_dataset_id=default_dataset_id,
|
|
1120
|
+
)
|
|
1121
|
+
|
|
1122
|
+
return cls._apify_client_instance
|
|
1123
|
+
|
|
1124
|
+
@_ClassProperty
|
|
1125
|
+
def apify_client(cls) -> 'ApifyClientAsync':
|
|
1126
|
+
"""Get a charging-configured ``ApifyClientAsync`` for Apify SDK compatibility.
|
|
1127
|
+
|
|
1128
|
+
This provides full ApifyClient compatibility including:
|
|
1129
|
+
- client.actor('id').call()
|
|
1130
|
+
- client.dataset('id').list_items()
|
|
1131
|
+
- client.key_value_store('id').get_record('key')
|
|
1132
|
+
- client.run('id').wait()
|
|
1133
|
+
- etc.
|
|
1134
|
+
"""
|
|
1135
|
+
return cls._get_apify_client()
|
|
1136
|
+
|
|
1137
|
+
@_ClassProperty
|
|
1138
|
+
def configuration(cls) -> Configuration:
|
|
1139
|
+
"""Return the Configuration instance."""
|
|
1140
|
+
return cls.get_config()
|
|
1141
|
+
|
|
1142
|
+
@_ClassProperty
|
|
1143
|
+
def event_manager(cls) -> Optional[EventManager]:
|
|
1144
|
+
"""Return the EventManager instance."""
|
|
1145
|
+
return cls._event_manager
|
|
1146
|
+
|
|
1147
|
+
# -------------------------------------------------------------------------
|
|
1148
|
+
# Charging methods
|
|
1149
|
+
# -------------------------------------------------------------------------
|
|
1150
|
+
|
|
1151
|
+
@classmethod
|
|
1152
|
+
def get_charging_manager(cls) -> ChargingManager:
|
|
1153
|
+
"""Get the ChargingManager instance for pay-per-event pricing.
|
|
1154
|
+
|
|
1155
|
+
Returns:
|
|
1156
|
+
ChargingManager instance
|
|
1157
|
+
|
|
1158
|
+
Raises:
|
|
1159
|
+
RuntimeError: If Actor is not initialized
|
|
1160
|
+
"""
|
|
1161
|
+
cls._ensure_initialized('get_charging_manager')
|
|
1162
|
+
config = cls.get_config()
|
|
1163
|
+
effective_run_id = config.get('actor_run_id')
|
|
1164
|
+
|
|
1165
|
+
# Recreate when missing or bound to a stale run (e.g. sequential lifecycles
|
|
1166
|
+
# that forgot to clear, or storage credentials changed after init).
|
|
1167
|
+
if cls._charging_manager is not None:
|
|
1168
|
+
manager_run_id = getattr(cls._charging_manager, '_actor_run_id', None)
|
|
1169
|
+
if manager_run_id != effective_run_id:
|
|
1170
|
+
cls._charging_manager = None
|
|
1171
|
+
cls._apify_client_instance = None
|
|
1172
|
+
|
|
1173
|
+
if cls._charging_manager is None:
|
|
1174
|
+
# Prefer the storage factory client (same URL/token/run as API storage).
|
|
1175
|
+
client = cls._client if cls._client is not None else cls.new_client()
|
|
1176
|
+
cls._charging_manager = ChargingManager(config, client)
|
|
1177
|
+
return cls._charging_manager
|
|
1178
|
+
|
|
1179
|
+
|
|
1180
|
+
@classmethod
|
|
1181
|
+
async def charge(cls, event_name: str, count: int = 1) -> ChargeResult:
|
|
1182
|
+
"""Charge for a specified number of events - sub-operations of the Actor.
|
|
1183
|
+
|
|
1184
|
+
This is relevant only for the pay-per-event pricing model.
|
|
1185
|
+
|
|
1186
|
+
Args:
|
|
1187
|
+
event_name: Name of the event to be charged for
|
|
1188
|
+
count: Number of events to charge for
|
|
1189
|
+
|
|
1190
|
+
Returns:
|
|
1191
|
+
ChargeResult indicating if the limit was reached
|
|
1192
|
+
"""
|
|
1193
|
+
cls._ensure_initialized('charge')
|
|
1194
|
+
charging_manager = cls.get_charging_manager()
|
|
1195
|
+
if not getattr(charging_manager, '_initialized', False):
|
|
1196
|
+
await charging_manager.init()
|
|
1197
|
+
return await charging_manager.charge(event_name, count=count)
|
|
1198
|
+
|
|
1199
|
+
# -------------------------------------------------------------------------
|
|
1200
|
+
# Actor run control methods
|
|
1201
|
+
# -------------------------------------------------------------------------
|
|
1202
|
+
|
|
1203
|
+
@classmethod
|
|
1204
|
+
async def abort(
|
|
1205
|
+
cls,
|
|
1206
|
+
run_id: Optional[str] = None,
|
|
1207
|
+
options: Optional[AbortOptions] = None,
|
|
1208
|
+
) -> ActorRun:
|
|
1209
|
+
"""Abort an Actor run on the Scrapely platform.
|
|
1210
|
+
|
|
1211
|
+
Args:
|
|
1212
|
+
run_id: The ID of the run to abort. Defaults to the current run.
|
|
1213
|
+
options: Abort options including token, status_message, gracefully
|
|
1214
|
+
|
|
1215
|
+
Returns:
|
|
1216
|
+
ActorRun object with details about the aborted run
|
|
1217
|
+
"""
|
|
1218
|
+
cls._ensure_initialized('abort')
|
|
1219
|
+
config = cls.get_config()
|
|
1220
|
+
target_run_id = run_id or config.get('actor_run_id')
|
|
1221
|
+
if not target_run_id:
|
|
1222
|
+
raise ValueError('Actor.abort() requires a run ID')
|
|
1223
|
+
|
|
1224
|
+
opts = options or AbortOptions()
|
|
1225
|
+
client = cls.new_client(token=opts.token) if opts.token else cls.new_client()
|
|
1226
|
+
|
|
1227
|
+
# Set status message if provided
|
|
1228
|
+
if opts.status_message:
|
|
1229
|
+
await client.put_json(
|
|
1230
|
+
f'/v2/actor-runs/{target_run_id}',
|
|
1231
|
+
data={'statusMessage': opts.status_message},
|
|
1232
|
+
)
|
|
1233
|
+
|
|
1234
|
+
# Abort the run; `gracefully` travels as a query parameter (the server
|
|
1235
|
+
# reads it only from the query string) and is omitted when unset so the
|
|
1236
|
+
# server default applies (Node SDK parity).
|
|
1237
|
+
abort_params: Dict[str, Any] = {}
|
|
1238
|
+
if opts.gracefully is not None:
|
|
1239
|
+
abort_params['gracefully'] = 'true' if opts.gracefully else 'false'
|
|
1240
|
+
data = _payload_dict(
|
|
1241
|
+
await client.post_json(
|
|
1242
|
+
f'/v2/actor-runs/{target_run_id}/abort', data=None, params=abort_params or None
|
|
1243
|
+
)
|
|
1244
|
+
)
|
|
1245
|
+
|
|
1246
|
+
return ActorRun(
|
|
1247
|
+
id=data.get('id', target_run_id),
|
|
1248
|
+
actor_id=data.get('actorId', ''),
|
|
1249
|
+
status=data.get('status', 'ABORTED'),
|
|
1250
|
+
started_at=data.get('startedAt'),
|
|
1251
|
+
finished_at=data.get('finishedAt'),
|
|
1252
|
+
status_message=data.get('statusMessage'),
|
|
1253
|
+
)
|
|
1254
|
+
|
|
1255
|
+
@classmethod
|
|
1256
|
+
async def start(
|
|
1257
|
+
cls,
|
|
1258
|
+
actor_id: str,
|
|
1259
|
+
run_input: Optional[Any] = None,
|
|
1260
|
+
options: Optional[StartOptions] = None,
|
|
1261
|
+
) -> ActorRun:
|
|
1262
|
+
"""Start an Actor on the Scrapely platform without waiting for finish.
|
|
1263
|
+
|
|
1264
|
+
Args:
|
|
1265
|
+
actor_id: The ID of the Actor to start
|
|
1266
|
+
run_input: The input to pass to the Actor run
|
|
1267
|
+
options: Start options including token, build, memory_mbytes, etc.
|
|
1268
|
+
|
|
1269
|
+
Returns:
|
|
1270
|
+
ActorRun object with details about the started run
|
|
1271
|
+
"""
|
|
1272
|
+
cls._ensure_initialized('start')
|
|
1273
|
+
opts = options or StartOptions()
|
|
1274
|
+
client = cls.new_client(token=opts.token) if opts.token else cls.new_client()
|
|
1275
|
+
|
|
1276
|
+
# Wire format (server contract): the POST body is the run input itself;
|
|
1277
|
+
# run options are Apify-style query parameters. Inline webhooks ride
|
|
1278
|
+
# along base64-encoded in the `webhooks` query param.
|
|
1279
|
+
params = _run_options_params(
|
|
1280
|
+
build=opts.build,
|
|
1281
|
+
timeout=opts.timeout,
|
|
1282
|
+
memory_mbytes=opts.memory_mbytes,
|
|
1283
|
+
max_total_charge_usd=opts.max_total_charge_usd,
|
|
1284
|
+
restart_on_error=opts.restart_on_error,
|
|
1285
|
+
force_permission_level=opts.force_permission_level,
|
|
1286
|
+
webhooks=opts.webhooks,
|
|
1287
|
+
)
|
|
1288
|
+
|
|
1289
|
+
data = _payload_dict(
|
|
1290
|
+
await client.post_json(f'/v2/actors/{actor_id}/runs', data=run_input, params=params or None)
|
|
1291
|
+
)
|
|
1292
|
+
|
|
1293
|
+
return ActorRun(
|
|
1294
|
+
id=data.get('id', ''),
|
|
1295
|
+
actor_id=data.get('actorId', actor_id),
|
|
1296
|
+
status=data.get('status', 'RUNNING'),
|
|
1297
|
+
started_at=data.get('startedAt'),
|
|
1298
|
+
build_id=data.get('buildId'),
|
|
1299
|
+
default_dataset_id=data.get('defaultDatasetId'),
|
|
1300
|
+
default_key_value_store_id=data.get('defaultKeyValueStoreId'),
|
|
1301
|
+
default_request_queue_id=data.get('defaultRequestQueueId'),
|
|
1302
|
+
)
|
|
1303
|
+
|
|
1304
|
+
@classmethod
|
|
1305
|
+
async def call(
|
|
1306
|
+
cls,
|
|
1307
|
+
actor_id: str,
|
|
1308
|
+
run_input: Optional[Any] = None,
|
|
1309
|
+
options: Optional[CallOptions] = None,
|
|
1310
|
+
) -> ActorRun:
|
|
1311
|
+
"""Start an Actor and wait for it to finish.
|
|
1312
|
+
|
|
1313
|
+
Args:
|
|
1314
|
+
actor_id: The ID of the Actor to call
|
|
1315
|
+
run_input: The input to pass to the Actor run
|
|
1316
|
+
options: Call options including token, build, timeout, etc.
|
|
1317
|
+
|
|
1318
|
+
Returns:
|
|
1319
|
+
ActorRun object with details about the finished run
|
|
1320
|
+
"""
|
|
1321
|
+
cls._ensure_initialized('call')
|
|
1322
|
+
opts = options or CallOptions()
|
|
1323
|
+
|
|
1324
|
+
# Resolve the timeout once: 'inherit' becomes the remaining seconds of
|
|
1325
|
+
# this run, forwarded to the server AND used as the poll deadline.
|
|
1326
|
+
timeout_secs = _resolve_run_timeout(opts.timeout)
|
|
1327
|
+
|
|
1328
|
+
# Start the actor (timeout/webhooks forwarded so the server enforces
|
|
1329
|
+
# the run timeout and registers inline webhooks).
|
|
1330
|
+
start_opts = StartOptions(
|
|
1331
|
+
token=opts.token,
|
|
1332
|
+
build=opts.build,
|
|
1333
|
+
memory_mbytes=opts.memory_mbytes,
|
|
1334
|
+
max_total_charge_usd=opts.max_total_charge_usd,
|
|
1335
|
+
restart_on_error=opts.restart_on_error,
|
|
1336
|
+
timeout=timeout_secs,
|
|
1337
|
+
force_permission_level=opts.force_permission_level,
|
|
1338
|
+
webhooks=opts.webhooks,
|
|
1339
|
+
)
|
|
1340
|
+
run = await cls.start(actor_id, run_input, start_opts)
|
|
1341
|
+
|
|
1342
|
+
# Wait for finish (honors opts.timeout; polls indefinitely when unset)
|
|
1343
|
+
client = cls.new_client(token=opts.token) if opts.token else cls.new_client()
|
|
1344
|
+
return await cls._wait_for_terminal_run(
|
|
1345
|
+
client,
|
|
1346
|
+
run.id,
|
|
1347
|
+
actor_id_fallback=actor_id,
|
|
1348
|
+
timeout_seconds=timeout_secs,
|
|
1349
|
+
)
|
|
1350
|
+
|
|
1351
|
+
@classmethod
|
|
1352
|
+
async def _wait_for_terminal_run(
|
|
1353
|
+
cls,
|
|
1354
|
+
client: ApiClient,
|
|
1355
|
+
run_id: str,
|
|
1356
|
+
actor_id_fallback: str = '',
|
|
1357
|
+
timeout_seconds: Optional[int] = None,
|
|
1358
|
+
) -> ActorRun:
|
|
1359
|
+
"""Poll ``GET /v2/actor-runs/:runId`` until the run reaches a terminal status.
|
|
1360
|
+
|
|
1361
|
+
When ``timeout_seconds`` elapses first, the run is aborted (best-effort)
|
|
1362
|
+
and ``TimeoutError`` is raised — Node SDK (apify-client) parity.
|
|
1363
|
+
"""
|
|
1364
|
+
deadline = time.monotonic() + timeout_seconds if timeout_seconds is not None else None
|
|
1365
|
+
|
|
1366
|
+
while True:
|
|
1367
|
+
try:
|
|
1368
|
+
data = _payload_dict(await client.get_json(f'/v2/actor-runs/{run_id}'))
|
|
1369
|
+
except (ApiClientError, ForbiddenError) as exc:
|
|
1370
|
+
if getattr(exc, 'status_code', None) != 403:
|
|
1371
|
+
raise
|
|
1372
|
+
# The token cannot observe this run (e.g. a LIMITED_PERMISSIONS
|
|
1373
|
+
# run token polling a run it did not start). Fail fast with an
|
|
1374
|
+
# actionable message instead of a bare 403 so callers know the
|
|
1375
|
+
# child run may still be running unobserved.
|
|
1376
|
+
# NOTE: the real ApiClient raises ApiClientError for every
|
|
1377
|
+
# non-2xx response; ForbiddenError is kept for parity with
|
|
1378
|
+
# callers that surface the typed errors from scrapely.errors.
|
|
1379
|
+
raise RuntimeError(
|
|
1380
|
+
f'Actor: cannot observe run {run_id} with the current token (403 Forbidden). '
|
|
1381
|
+
'A LIMITED_PERMISSIONS run token may only poll or abort runs it started itself; '
|
|
1382
|
+
'the run may still be executing. Started from another actor? Pass a user API '
|
|
1383
|
+
'token via options.token.'
|
|
1384
|
+
) from exc
|
|
1385
|
+
status = data.get('status', '').upper()
|
|
1386
|
+
|
|
1387
|
+
if status in _RUN_TERMINAL_STATUSES:
|
|
1388
|
+
return ActorRun(
|
|
1389
|
+
id=data.get('id', run_id),
|
|
1390
|
+
actor_id=data.get('actorId', actor_id_fallback),
|
|
1391
|
+
status=status,
|
|
1392
|
+
started_at=data.get('startedAt'),
|
|
1393
|
+
finished_at=data.get('finishedAt'),
|
|
1394
|
+
duration_millis=data.get('durationMillis'),
|
|
1395
|
+
exit_code=data.get('exitCode'),
|
|
1396
|
+
status_message=data.get('statusMessage'),
|
|
1397
|
+
default_dataset_id=data.get('defaultDatasetId'),
|
|
1398
|
+
default_key_value_store_id=data.get('defaultKeyValueStoreId'),
|
|
1399
|
+
default_request_queue_id=data.get('defaultRequestQueueId'),
|
|
1400
|
+
)
|
|
1401
|
+
|
|
1402
|
+
if deadline is not None:
|
|
1403
|
+
remaining = deadline - time.monotonic()
|
|
1404
|
+
if remaining <= 0:
|
|
1405
|
+
try:
|
|
1406
|
+
await client.post_json(f'/v2/actor-runs/{run_id}/abort', data=None)
|
|
1407
|
+
except Exception:
|
|
1408
|
+
logger.warning('Actor: best-effort abort of run %s after timeout failed', run_id)
|
|
1409
|
+
raise TimeoutError(
|
|
1410
|
+
f'The actor run {run_id} did not finish within {timeout_seconds} seconds. '
|
|
1411
|
+
'The run was aborted.'
|
|
1412
|
+
) from None
|
|
1413
|
+
await asyncio.sleep(min(1.0, remaining))
|
|
1414
|
+
else:
|
|
1415
|
+
await asyncio.sleep(1)
|
|
1416
|
+
|
|
1417
|
+
@classmethod
|
|
1418
|
+
async def call_task(
|
|
1419
|
+
cls,
|
|
1420
|
+
task_id: str,
|
|
1421
|
+
task_input: Optional[Dict[str, Any]] = None,
|
|
1422
|
+
options: Optional[CallOptions] = None,
|
|
1423
|
+
) -> ActorRun:
|
|
1424
|
+
"""Start an Actor task and wait for it to finish.
|
|
1425
|
+
|
|
1426
|
+
Args:
|
|
1427
|
+
task_id: The ID of the task to call
|
|
1428
|
+
task_input: Overrides the input for the task
|
|
1429
|
+
options: Call options including token, build, timeout, etc.
|
|
1430
|
+
|
|
1431
|
+
Returns:
|
|
1432
|
+
ActorRun object with details about the finished run
|
|
1433
|
+
"""
|
|
1434
|
+
cls._ensure_initialized('call_task')
|
|
1435
|
+
opts = options or CallOptions()
|
|
1436
|
+
client = cls.new_client(token=opts.token) if opts.token else cls.new_client()
|
|
1437
|
+
|
|
1438
|
+
# Wire format (server contract): the POST body is the task input
|
|
1439
|
+
# override (merged over the task's saved input by the server); run
|
|
1440
|
+
# options are Apify-style query parameters. The task-run endpoint is
|
|
1441
|
+
# /v2/actor-tasks/:taskId/runs (there is no /v2/tasks/:taskId/runs).
|
|
1442
|
+
params = _run_options_params(
|
|
1443
|
+
build=opts.build,
|
|
1444
|
+
timeout=opts.timeout,
|
|
1445
|
+
memory_mbytes=opts.memory_mbytes,
|
|
1446
|
+
max_total_charge_usd=opts.max_total_charge_usd,
|
|
1447
|
+
restart_on_error=opts.restart_on_error,
|
|
1448
|
+
force_permission_level=opts.force_permission_level,
|
|
1449
|
+
webhooks=opts.webhooks,
|
|
1450
|
+
)
|
|
1451
|
+
|
|
1452
|
+
data = _payload_dict(
|
|
1453
|
+
await client.post_json(f'/v2/actor-tasks/{task_id}/runs', data=task_input, params=params or None)
|
|
1454
|
+
)
|
|
1455
|
+
run_id = data.get('id', '')
|
|
1456
|
+
if not run_id:
|
|
1457
|
+
raise RuntimeError(f'Actor.call_task(): creating a run for task {task_id} returned no run ID')
|
|
1458
|
+
|
|
1459
|
+
# Wait for finish (honors opts.timeout, incl. 'inherit'; polls
|
|
1460
|
+
# indefinitely when unset)
|
|
1461
|
+
return await cls._wait_for_terminal_run(
|
|
1462
|
+
client,
|
|
1463
|
+
run_id,
|
|
1464
|
+
timeout_seconds=_resolve_run_timeout(opts.timeout),
|
|
1465
|
+
)
|
|
1466
|
+
|
|
1467
|
+
# -------------------------------------------------------------------------
|
|
1468
|
+
# Platform integration methods
|
|
1469
|
+
# -------------------------------------------------------------------------
|
|
1470
|
+
|
|
1471
|
+
@classmethod
|
|
1472
|
+
async def metamorph(
|
|
1473
|
+
cls,
|
|
1474
|
+
target_actor_id: str,
|
|
1475
|
+
run_input: Optional[Any] = None,
|
|
1476
|
+
options: Optional[MetamorphOptions] = None,
|
|
1477
|
+
) -> None:
|
|
1478
|
+
"""Transform this Actor run to an Actor run of a different Actor.
|
|
1479
|
+
|
|
1480
|
+
The platform stops the current Actor container and starts a new container
|
|
1481
|
+
with the new Actor instead. All default storages are preserved, and the
|
|
1482
|
+
new input is stored under the INPUT-METAMORPH-1 key in the same default
|
|
1483
|
+
key-value store.
|
|
1484
|
+
|
|
1485
|
+
Args:
|
|
1486
|
+
target_actor_id: ID of the target Actor to transform into
|
|
1487
|
+
run_input: The input to pass to the new run
|
|
1488
|
+
options: Metamorph options including build, content_type
|
|
1489
|
+
"""
|
|
1490
|
+
cls._ensure_initialized('metamorph')
|
|
1491
|
+
|
|
1492
|
+
if not cls.is_at_home():
|
|
1493
|
+
logger.warning('Actor.metamorph() is only supported when running on the Scrapely platform.')
|
|
1494
|
+
return
|
|
1495
|
+
|
|
1496
|
+
config = cls.get_config()
|
|
1497
|
+
run_id = config.get('actor_run_id')
|
|
1498
|
+
if not run_id:
|
|
1499
|
+
raise RuntimeError('actor_run_id is not set - metamorph requires running on the platform')
|
|
1500
|
+
|
|
1501
|
+
opts = options or MetamorphOptions()
|
|
1502
|
+
|
|
1503
|
+
# Wire format (server contract): the server copies the current INPUT to
|
|
1504
|
+
# INPUT-METAMORPH-<n> itself and tracks the metamorph count in run
|
|
1505
|
+
# stats; the new input (if any) is read from the request body's
|
|
1506
|
+
# `input` field. targetActorId/build are accepted in the body.
|
|
1507
|
+
client = cls.new_client()
|
|
1508
|
+
body: Dict[str, Any] = {'targetActorId': target_actor_id}
|
|
1509
|
+
if opts.build:
|
|
1510
|
+
body['build'] = opts.build
|
|
1511
|
+
if run_input is not None:
|
|
1512
|
+
body['input'] = run_input
|
|
1513
|
+
|
|
1514
|
+
logger.info(f'[Scrapely] Metamorphosing to actor {target_actor_id}')
|
|
1515
|
+
|
|
1516
|
+
await client.post_json(f'/v2/actor-runs/{run_id}/metamorph', data=body)
|
|
1517
|
+
|
|
1518
|
+
# Flush pending PPE intents before hard exit.
|
|
1519
|
+
try:
|
|
1520
|
+
await cls._shutdown(timeout_secs=5)
|
|
1521
|
+
except Exception as err:
|
|
1522
|
+
logger.warning('flush_pending during metamorph failed: %s', err)
|
|
1523
|
+
|
|
1524
|
+
# Exit - the platform will restart with new actor
|
|
1525
|
+
logger.info('[Scrapely] Metamorph request accepted, exiting...')
|
|
1526
|
+
sys.exit(0)
|
|
1527
|
+
|
|
1528
|
+
@classmethod
|
|
1529
|
+
async def reboot(cls, options: Optional[RebootOptions] = None) -> None:
|
|
1530
|
+
"""Internally reboot this Actor.
|
|
1531
|
+
|
|
1532
|
+
The system stops the current container and starts a new one with the
|
|
1533
|
+
same run ID and default storages.
|
|
1534
|
+
|
|
1535
|
+
Args:
|
|
1536
|
+
options: Reboot options including custom_after_sleep_millis
|
|
1537
|
+
"""
|
|
1538
|
+
cls._ensure_initialized('reboot')
|
|
1539
|
+
|
|
1540
|
+
if not cls.is_at_home():
|
|
1541
|
+
logger.warning('Actor.reboot() is only supported when running on the Scrapely platform.')
|
|
1542
|
+
return
|
|
1543
|
+
|
|
1544
|
+
config = cls.get_config()
|
|
1545
|
+
run_id = config.get('actor_run_id')
|
|
1546
|
+
if not run_id:
|
|
1547
|
+
raise RuntimeError('actor_run_id is not set - reboot requires running on the platform')
|
|
1548
|
+
|
|
1549
|
+
opts = options or RebootOptions()
|
|
1550
|
+
|
|
1551
|
+
# Emit persist state event
|
|
1552
|
+
if cls._event_manager:
|
|
1553
|
+
from crawlee.events import EventPersistStateData
|
|
1554
|
+
cls._event_manager.emit(
|
|
1555
|
+
event=Event.PERSIST_STATE,
|
|
1556
|
+
event_data=EventPersistStateData(is_migrating=True),
|
|
1557
|
+
)
|
|
1558
|
+
|
|
1559
|
+
# Wait for state to be persisted
|
|
1560
|
+
await asyncio.sleep(1)
|
|
1561
|
+
|
|
1562
|
+
logger.info('[Scrapely] Rebooting...')
|
|
1563
|
+
|
|
1564
|
+
# Call reboot endpoint
|
|
1565
|
+
client = cls.new_client()
|
|
1566
|
+
await client.post(f'/v2/actor-runs/{run_id}/reboot')
|
|
1567
|
+
|
|
1568
|
+
# Wait before exiting
|
|
1569
|
+
sleep_ms = opts.custom_after_sleep_millis or config.get('metamorph_after_sleep_millis', 1000)
|
|
1570
|
+
await asyncio.sleep(sleep_ms / 1000)
|
|
1571
|
+
|
|
1572
|
+
# Flush pending PPE intents before hard exit.
|
|
1573
|
+
try:
|
|
1574
|
+
await cls._shutdown(timeout_secs=5)
|
|
1575
|
+
except Exception as err:
|
|
1576
|
+
logger.warning('flush_pending during reboot failed: %s', err)
|
|
1577
|
+
|
|
1578
|
+
logger.info('[Scrapely] Reboot request accepted, exiting...')
|
|
1579
|
+
sys.exit(0)
|
|
1580
|
+
|
|
1581
|
+
@classmethod
|
|
1582
|
+
async def add_webhook(cls, webhook: Webhook) -> Optional[Webhook]:
|
|
1583
|
+
"""Create an ad-hoc webhook for the current Actor run.
|
|
1584
|
+
|
|
1585
|
+
This webhook lets you receive a notification when the Actor run
|
|
1586
|
+
finished or failed.
|
|
1587
|
+
|
|
1588
|
+
Note that webhooks are only supported for Actors running on the
|
|
1589
|
+
Scrapely platform. When running the Actor locally, the function
|
|
1590
|
+
will print a warning and return None.
|
|
1591
|
+
|
|
1592
|
+
Args:
|
|
1593
|
+
webhook: The webhook to add. It is automatically bound to the
|
|
1594
|
+
current Actor run.
|
|
1595
|
+
|
|
1596
|
+
Returns:
|
|
1597
|
+
The created webhook, or None if running locally
|
|
1598
|
+
"""
|
|
1599
|
+
cls._ensure_initialized('add_webhook')
|
|
1600
|
+
|
|
1601
|
+
if not cls.is_at_home():
|
|
1602
|
+
logger.warning('Actor.add_webhook() is only supported on the Scrapely platform. Ignoring.')
|
|
1603
|
+
return None
|
|
1604
|
+
|
|
1605
|
+
config = cls.get_config()
|
|
1606
|
+
run_id = config.get('actor_run_id')
|
|
1607
|
+
if not run_id:
|
|
1608
|
+
raise RuntimeError('actor_run_id is not set - add_webhook requires running on the platform')
|
|
1609
|
+
|
|
1610
|
+
client = cls.new_client()
|
|
1611
|
+
|
|
1612
|
+
# Build webhook creation request
|
|
1613
|
+
body = {
|
|
1614
|
+
'eventTypes': webhook.event_types,
|
|
1615
|
+
'requestUrl': webhook.request_url,
|
|
1616
|
+
'isAdHoc': True,
|
|
1617
|
+
'condition': {'actorRunId': run_id},
|
|
1618
|
+
}
|
|
1619
|
+
if webhook.payload_template:
|
|
1620
|
+
body['payloadTemplate'] = webhook.payload_template
|
|
1621
|
+
if webhook.idempotency_key:
|
|
1622
|
+
body['idempotencyKey'] = webhook.idempotency_key
|
|
1623
|
+
if webhook.headers_template:
|
|
1624
|
+
body['headersTemplate'] = webhook.headers_template
|
|
1625
|
+
# `description` is not part of the Webhook dataclass — tolerate future additions.
|
|
1626
|
+
description = getattr(webhook, 'description', None)
|
|
1627
|
+
if description:
|
|
1628
|
+
body['description'] = description
|
|
1629
|
+
body['ignoreSslErrors'] = bool(webhook.ignore_ssl_errors)
|
|
1630
|
+
body['doNotRetry'] = bool(webhook.do_not_retry)
|
|
1631
|
+
|
|
1632
|
+
data = _payload_dict(await client.post_json('/v2/webhooks', data=body))
|
|
1633
|
+
|
|
1634
|
+
# Populate the server-assigned ID on the returned webhook so callers
|
|
1635
|
+
# can reference it later (Node SDK parity).
|
|
1636
|
+
webhook.id = data.get('id') or webhook.id
|
|
1637
|
+
|
|
1638
|
+
return webhook
|
|
1639
|
+
|
|
1640
|
+
@classmethod
|
|
1641
|
+
async def use_state(
|
|
1642
|
+
cls,
|
|
1643
|
+
default_value: Optional[Dict[str, Any]] = None,
|
|
1644
|
+
key: Optional[str] = None,
|
|
1645
|
+
kvs_name: Optional[str] = None,
|
|
1646
|
+
) -> MutableMapping[str, Any]:
|
|
1647
|
+
"""Easily create and manage state values. All state values are automatically persisted.
|
|
1648
|
+
|
|
1649
|
+
Values can be modified by simply using the assignment operator.
|
|
1650
|
+
|
|
1651
|
+
Args:
|
|
1652
|
+
default_value: The default value to initialize the state if not already set
|
|
1653
|
+
key: The key in the key-value store where state is stored. Defaults to 'APIFY_GLOBAL_STATE'
|
|
1654
|
+
kvs_name: The name of the key-value store where state is stored. Defaults to default KVS.
|
|
1655
|
+
|
|
1656
|
+
Returns:
|
|
1657
|
+
The state dictionary with automatic persistence
|
|
1658
|
+
"""
|
|
1659
|
+
cls._ensure_initialized('use_state')
|
|
1660
|
+
|
|
1661
|
+
state_key = key or 'APIFY_GLOBAL_STATE'
|
|
1662
|
+
initial_value = default_value or {}
|
|
1663
|
+
|
|
1664
|
+
cls._use_state_stores.add(kvs_name)
|
|
1665
|
+
kvs = await cls.open_key_value_store(name=kvs_name)
|
|
1666
|
+
|
|
1667
|
+
# Use get_auto_saved_value if available (from patch_storage)
|
|
1668
|
+
if hasattr(kvs, 'get_auto_saved_value'):
|
|
1669
|
+
return await kvs.get_auto_saved_value(state_key, initial_value)
|
|
1670
|
+
|
|
1671
|
+
# Fallback: manual state management
|
|
1672
|
+
existing = await kvs.get_value(state_key)
|
|
1673
|
+
if existing is None:
|
|
1674
|
+
await kvs.set_value(state_key, initial_value)
|
|
1675
|
+
return initial_value
|
|
1676
|
+
return cast(MutableMapping[str, Any], existing)
|
|
1677
|
+
|
|
1678
|
+
# -------------------------------------------------------------------------
|
|
1679
|
+
# Remote browser configuration
|
|
1680
|
+
# -------------------------------------------------------------------------
|
|
1681
|
+
|
|
1682
|
+
@classmethod
|
|
1683
|
+
async def create_browser_configuration(
|
|
1684
|
+
cls,
|
|
1685
|
+
actor_browser_input: Optional[Dict[str, Any]] = None,
|
|
1686
|
+
*,
|
|
1687
|
+
country_code: Optional[str] = None,
|
|
1688
|
+
state: Optional[str] = None,
|
|
1689
|
+
city: Optional[str] = None,
|
|
1690
|
+
os: Optional[str] = None,
|
|
1691
|
+
timezone: Optional[str] = None,
|
|
1692
|
+
language: Optional[str] = None,
|
|
1693
|
+
adblock: Optional[bool] = None,
|
|
1694
|
+
solve_cloudflare: Optional[bool] = None,
|
|
1695
|
+
) -> 'BrowserConfiguration':
|
|
1696
|
+
"""Create a Scrapely remote browser configuration (location and fingerprint options).
|
|
1697
|
+
|
|
1698
|
+
Pass it to browser crawlers next to ``proxy_configuration``::
|
|
1699
|
+
|
|
1700
|
+
browser_configuration = await Actor.create_browser_configuration(country_code='US', os='windows')
|
|
1701
|
+
crawler = PlaywrightCrawler(browser_configuration=browser_configuration, request_handler=handler)
|
|
1702
|
+
|
|
1703
|
+
Ignored outside the Scrapely remote browser (local runs), so the same
|
|
1704
|
+
code runs locally and on the platform.
|
|
1705
|
+
|
|
1706
|
+
Args:
|
|
1707
|
+
actor_browser_input: Actor input fields ``browserCountry`` /
|
|
1708
|
+
``browserState`` / ``browserCity``, used when the matching
|
|
1709
|
+
keyword argument is not given.
|
|
1710
|
+
country_code: Exit country (ISO 3166-1 alpha-2).
|
|
1711
|
+
state: Region name (e.g. ``California``).
|
|
1712
|
+
city: City name (e.g. ``San Jose``).
|
|
1713
|
+
os: Fingerprint OS: ``windows``, ``macos`` or ``android``.
|
|
1714
|
+
timezone: IANA timezone.
|
|
1715
|
+
language: Comma-separated language tags without q-values.
|
|
1716
|
+
adblock: Built-in ad/tracker blocking.
|
|
1717
|
+
solve_cloudflare: Automatic Cloudflare Turnstile solving.
|
|
1718
|
+
"""
|
|
1719
|
+
from .browser_configuration import BrowserConfiguration
|
|
1720
|
+
|
|
1721
|
+
if actor_browser_input:
|
|
1722
|
+
country_code = country_code if country_code is not None else actor_browser_input.get('browserCountry')
|
|
1723
|
+
state = state if state is not None else actor_browser_input.get('browserState')
|
|
1724
|
+
city = city if city is not None else actor_browser_input.get('browserCity')
|
|
1725
|
+
|
|
1726
|
+
return BrowserConfiguration(
|
|
1727
|
+
country_code=country_code,
|
|
1728
|
+
state=state,
|
|
1729
|
+
city=city,
|
|
1730
|
+
os=os,
|
|
1731
|
+
timezone=timezone,
|
|
1732
|
+
language=language,
|
|
1733
|
+
adblock=adblock,
|
|
1734
|
+
solve_cloudflare=solve_cloudflare,
|
|
1735
|
+
)
|
|
1736
|
+
|
|
1737
|
+
# -------------------------------------------------------------------------
|
|
1738
|
+
# Proxy configuration
|
|
1739
|
+
# -------------------------------------------------------------------------
|
|
1740
|
+
|
|
1741
|
+
@classmethod
|
|
1742
|
+
async def create_proxy_configuration(
|
|
1743
|
+
cls,
|
|
1744
|
+
actor_proxy_input: Optional[Dict[str, Any]] = None,
|
|
1745
|
+
password: Optional[str] = None,
|
|
1746
|
+
groups: Optional[List[str]] = None,
|
|
1747
|
+
country_code: Optional[str] = None,
|
|
1748
|
+
subdivision_code: Optional[str] = None,
|
|
1749
|
+
proxy_urls: Optional[List[str]] = None,
|
|
1750
|
+
new_url_function: Optional[Callable] = None,
|
|
1751
|
+
tiered_proxy_urls: Optional[List[List[Optional[str]]]] = None,
|
|
1752
|
+
tiered_proxy_config: Optional[List[Dict[str, Any]]] = None,
|
|
1753
|
+
check_access: bool = True,
|
|
1754
|
+
) -> Optional['ProxyConfiguration']:
|
|
1755
|
+
"""Create a ProxyConfiguration object with the passed proxy configuration.
|
|
1756
|
+
|
|
1757
|
+
Configures connection to a proxy server with the provided options.
|
|
1758
|
+
Proxy servers are used to prevent target websites from blocking your
|
|
1759
|
+
crawlers based on IP address rate limits or blacklists.
|
|
1760
|
+
|
|
1761
|
+
Args:
|
|
1762
|
+
actor_proxy_input: Proxy configuration field from Actor input
|
|
1763
|
+
password: Password for the proxy
|
|
1764
|
+
groups: Proxy groups to use
|
|
1765
|
+
country_code: Country code for proxy
|
|
1766
|
+
subdivision_code: Subdivision code for proxy
|
|
1767
|
+
proxy_urls: Custom proxy server URLs to rotate through; may contain
|
|
1768
|
+
the 'auto' / 'http://auto' sentinel to mix in the platform proxy
|
|
1769
|
+
new_url_function: Function that returns a custom proxy URL
|
|
1770
|
+
tiered_proxy_urls: Proxy URL lists stratified into tiers
|
|
1771
|
+
tiered_proxy_config: Declarative tiers (proxy_urls / groups / ...)
|
|
1772
|
+
check_access: Whether to run the best-effort platform status check
|
|
1773
|
+
|
|
1774
|
+
Returns:
|
|
1775
|
+
ProxyConfiguration or None if no proxy should be used
|
|
1776
|
+
"""
|
|
1777
|
+
cls._ensure_initialized('create_proxy_configuration')
|
|
1778
|
+
|
|
1779
|
+
from .proxy_configuration import ProxyConfiguration
|
|
1780
|
+
|
|
1781
|
+
# Handle actor_proxy_input (Node SDK normalizeProxyInput/createProxyConfiguration
|
|
1782
|
+
# parity): `useApifyProxy: false` explicitly disables the PLATFORM proxy only —
|
|
1783
|
+
# custom proxies (proxyUrls / tiered lists / new_url_function) from the same
|
|
1784
|
+
# input must still be used. The standard "Custom proxy URLs" input shape is
|
|
1785
|
+
# {useApifyProxy: false, proxyUrls: [...]}; returning None for it would
|
|
1786
|
+
# silently drop the user's own proxies and crawl on their real IPs. We only
|
|
1787
|
+
# bail when no custom proxies are given either. A MISSING key falls back to
|
|
1788
|
+
# platform mode when the server injected credentials — input like
|
|
1789
|
+
# {apifyProxyGroups: [...]} must not silently disable proxying.
|
|
1790
|
+
if actor_proxy_input is not None:
|
|
1791
|
+
if actor_proxy_input.get('useApifyProxy') is False and not (
|
|
1792
|
+
proxy_urls
|
|
1793
|
+
or new_url_function
|
|
1794
|
+
or tiered_proxy_urls
|
|
1795
|
+
or tiered_proxy_config
|
|
1796
|
+
or actor_proxy_input.get('proxyUrls')
|
|
1797
|
+
or actor_proxy_input.get('tieredProxyUrls')
|
|
1798
|
+
or actor_proxy_input.get('tieredProxyConfig')
|
|
1799
|
+
):
|
|
1800
|
+
return None
|
|
1801
|
+
country_code = country_code or actor_proxy_input.get('apifyProxyCountry')
|
|
1802
|
+
subdivision_code = subdivision_code or actor_proxy_input.get('apifyProxySubdivision')
|
|
1803
|
+
groups = groups or actor_proxy_input.get('apifyProxyGroups')
|
|
1804
|
+
proxy_urls = proxy_urls or actor_proxy_input.get('proxyUrls')
|
|
1805
|
+
tiered_proxy_urls = tiered_proxy_urls or actor_proxy_input.get('tieredProxyUrls')
|
|
1806
|
+
tiered_proxy_config = tiered_proxy_config or actor_proxy_input.get('tieredProxyConfig')
|
|
1807
|
+
|
|
1808
|
+
# No custom proxies configured: use the platform proxy only when the
|
|
1809
|
+
# server injected credentials; locally, warn and run proxy-less (Node
|
|
1810
|
+
# SDK createProxyConfiguration parity).
|
|
1811
|
+
if not (proxy_urls or new_url_function or tiered_proxy_urls or tiered_proxy_config):
|
|
1812
|
+
from .proxy_configuration import _platform_env
|
|
1813
|
+
if not _platform_env():
|
|
1814
|
+
logger.warning(
|
|
1815
|
+
'No proxy configured. Scrapely injects platform proxy credentials '
|
|
1816
|
+
'(SCRAPELY_PROXY_*/APIFY_PROXY_*) into runs when your account has '
|
|
1817
|
+
'proxy access; locally, add proxy_urls to your actor input.'
|
|
1818
|
+
)
|
|
1819
|
+
return None
|
|
1820
|
+
|
|
1821
|
+
config = cls.get_config()
|
|
1822
|
+
proxy_config = ProxyConfiguration(
|
|
1823
|
+
password=password,
|
|
1824
|
+
groups=groups,
|
|
1825
|
+
country_code=country_code,
|
|
1826
|
+
subdivision_code=subdivision_code,
|
|
1827
|
+
proxy_urls=proxy_urls,
|
|
1828
|
+
new_url_function=new_url_function,
|
|
1829
|
+
tiered_proxy_urls=tiered_proxy_urls,
|
|
1830
|
+
tiered_proxy_config=tiered_proxy_config,
|
|
1831
|
+
_actor_config=config,
|
|
1832
|
+
_api_client=cls.new_client(),
|
|
1833
|
+
)
|
|
1834
|
+
|
|
1835
|
+
await proxy_config.initialize(check_access=check_access)
|
|
1836
|
+
return proxy_config
|