scrapely-python 0.1.22__tar.gz → 0.1.23__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {scrapely_python-0.1.22/src/scrapely_python.egg-info → scrapely_python-0.1.23}/PKG-INFO +1 -1
  2. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/pyproject.toml +1 -1
  3. scrapely_python-0.1.23/src/scrapely/__init__.py +341 -0
  4. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/actor.py +86 -42
  5. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/patch_browser_pool.py +96 -3
  6. scrapely_python-0.1.23/src/scrapely/storage/__init__.py +113 -0
  7. {scrapely_python-0.1.22 → scrapely_python-0.1.23/src/scrapely_python.egg-info}/PKG-INFO +1 -1
  8. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely_python.egg-info/SOURCES.txt +1 -0
  9. scrapely_python-0.1.23/tests/test_lazy_crawlee.py +294 -0
  10. scrapely_python-0.1.22/src/scrapely/__init__.py +0 -246
  11. scrapely_python-0.1.22/src/scrapely/storage/__init__.py +0 -48
  12. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/LICENSE +0 -0
  13. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/LICENSE-APACHE +0 -0
  14. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/NOTICE +0 -0
  15. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/README.md +0 -0
  16. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/setup.cfg +0 -0
  17. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/_browser_locations.py +0 -0
  18. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/browser_configuration.py +0 -0
  19. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/cdp_retry.py +0 -0
  20. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/charging.py +0 -0
  21. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/charging_client.py +0 -0
  22. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/configuration.py +0 -0
  23. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/crypto/__init__.py +0 -0
  24. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/crypto/_secrets.py +0 -0
  25. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/errors.py +0 -0
  26. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/events/__init__.py +0 -0
  27. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/events/_types.py +0 -0
  28. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/events/event_manager.py +0 -0
  29. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/log.py +0 -0
  30. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/pricing_types.py +0 -0
  31. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/proxy_configuration.py +0 -0
  32. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/py.typed +0 -0
  33. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/request_loaders/__init__.py +0 -0
  34. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/request_loaders/_scrapely_request_list.py +0 -0
  35. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/solver.py +0 -0
  36. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/_file_system/__init__.py +0 -0
  37. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/_file_system/_key_value_store_client.py +0 -0
  38. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/_file_system/_storage_client.py +0 -0
  39. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/_metadata.py +0 -0
  40. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/api_client.py +0 -0
  41. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/client.py +0 -0
  42. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/patch_storage.py +0 -0
  43. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/scrapely_dataset_client.py +0 -0
  44. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/scrapely_kvs_client.py +0 -0
  45. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/scrapely_rq_client.py +0 -0
  46. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/scrapely_storage_client.py +0 -0
  47. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/smart_scrapely_storage_client.py +0 -0
  48. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/types.py +0 -0
  49. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/utils.py +0 -0
  50. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage_clients/__init__.py +0 -0
  51. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storages/__init__.py +0 -0
  52. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/types.py +0 -0
  53. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/webhook.py +0 -0
  54. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely_python.egg-info/dependency_links.txt +0 -0
  55. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely_python.egg-info/requires.txt +0 -0
  56. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely_python.egg-info/top_level.txt +0 -0
  57. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_api_client_retries.py +0 -0
  58. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_apify_parity.py +0 -0
  59. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_browser_configuration.py +0 -0
  60. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_cdp_retry.py +0 -0
  61. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_charge_limit_guard.py +0 -0
  62. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_charge_retries.py +0 -0
  63. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_charging_parity.py +0 -0
  64. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_charging_pending.py +0 -0
  65. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_input_proxy_aliases.py +0 -0
  66. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_parity.py +0 -0
  67. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_patch_browser_pool.py +0 -0
  68. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_proxy_configuration.py +0 -0
  69. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_push_idempotency.py +0 -0
  70. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_run_control.py +0 -0
  71. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_secrets.py +0 -0
  72. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_solver.py +0 -0
  73. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_storage.py +0 -0
  74. {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_terminal_status_message.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-python
3
- Version: 0.1.22
3
+ Version: 0.1.23
4
4
  Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
5
5
  Author-email: Scrapely <support@scrape.ly>
6
6
  License-Expression: MIT AND Apache-2.0
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "scrapely-python"
7
- version = "0.1.22"
7
+ version = "0.1.23"
8
8
  description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
9
9
  readme = "README.md"
10
10
  license = "MIT AND Apache-2.0"
@@ -0,0 +1,341 @@
1
+ """
2
+ Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform.
3
+
4
+ Usage:
5
+ from scrapely import Actor, Dataset
6
+
7
+ async def main():
8
+ async with Actor:
9
+ actor_input = await Actor.get_input()
10
+ await Actor.push_data({'url': actor_input['url']})
11
+ """
12
+
13
+ from typing import TYPE_CHECKING, Any
14
+
15
+ # Side-effect: patch browser launches (Crawlee BrowserPool, PlaywrightCrawler,
16
+ # raw Playwright, Camoufox, pyppeteer) to use the remote CDP browser (CDP_URL).
17
+ # Must run on package import so `from scrapely import Actor` enables the patch
18
+ # before any browser is launched (Node SDK parity). It does not import Crawlee:
19
+ # the Crawlee patches are applied when Crawlee's browser modules load.
20
+ from .patch_browser_pool import set_cdp_url # noqa: F401
21
+
22
+ # Crawlee is NOT imported here: it costs ~1 s per run under gVisor and many
23
+ # actors never touch it. Everything that needs Crawlee (the re-exported Crawlee
24
+ # classes, storage clients, events, request loaders) is resolved on first
25
+ # attribute access by ``__getattr__`` below; ``Actor`` imports what it needs on
26
+ # first use. Every ``from scrapely import X`` that worked before still works.
27
+ from .actor import Actor
28
+ from .configuration import Configuration
29
+ from .charging_client import (
30
+ USES_PUSH_DATA_INTERCEPTION,
31
+ create_charging_client,
32
+ push_data_charging_context,
33
+ )
34
+ from .proxy_configuration import ProxyConfiguration
35
+ from .browser_configuration import BrowserConfiguration
36
+ from .charging import (
37
+ ChargingManager,
38
+ ChargeUncertainError,
39
+ ChargeRejectedError,
40
+ has_default_dataset_item_event_in_pricing,
41
+ is_default_dataset_item_event,
42
+ merge_charge_results,
43
+ push_data_and_charge,
44
+ resolve_default_dataset_item_event_name,
45
+ APIFY_DEFAULT_DATASET_ITEM_EVENT,
46
+ SCRAPELY_DEFAULT_DATASET_ITEM_EVENT,
47
+ DEFAULT_DATASET_ITEM_EVENT,
48
+ )
49
+
50
+ from .types import (
51
+ ActorInput,
52
+ ActorPermissionLevel,
53
+ ActorRun,
54
+ ApifyEnv,
55
+ AbortOptions,
56
+ CallOptions,
57
+ ExitOptions,
58
+ InitOptions,
59
+ MetamorphOptions,
60
+ OpenStorageOptions,
61
+ ProxyInfo,
62
+ RebootOptions,
63
+ RecordOptions,
64
+ SetStatusMessageOptions,
65
+ StartOptions,
66
+ WebhookOptions,
67
+ )
68
+
69
+ from .webhook import (
70
+ Webhook,
71
+ WebhookEventType,
72
+ )
73
+ from .pricing_types import (
74
+ PricingModel,
75
+ FreeActorPricingInfo,
76
+ FlatPricePerMonthActorPricingInfo,
77
+ PricePerDatasetItemActorPricingInfo,
78
+ PayPerEventActorPricingInfo,
79
+ PricingPerEvent,
80
+ ActorPricingInfo,
81
+ ActorChargeEvent,
82
+ ChargeResult,
83
+ ActorPricingInfoResult,
84
+ )
85
+ from .errors import (
86
+ ScrapelyClientError,
87
+ ScrapelyApiError,
88
+ InvalidResponseBodyError,
89
+ InvalidRequestError,
90
+ UnauthorizedError,
91
+ ForbiddenError,
92
+ NotFoundError,
93
+ ConflictError,
94
+ RateLimitError,
95
+ ServerError,
96
+ raise_for_status,
97
+ )
98
+
99
+ # Public name -> (module, attribute) for names that need Crawlee. Loaded on
100
+ # first access and cached in the module namespace.
101
+ _LAZY_ATTRS = {
102
+ # Crawlee re-exports
103
+ 'Request': ('crawlee', 'Request'),
104
+ 'RequestLoader': ('crawlee.request_loaders', 'RequestLoader'),
105
+ 'RequestManager': ('crawlee.request_loaders', 'RequestManager'),
106
+ 'RequestManagerTandem': ('crawlee.request_loaders', 'RequestManagerTandem'),
107
+ 'SitemapRequestLoader': ('crawlee.request_loaders', 'SitemapRequestLoader'),
108
+ 'Dataset': ('crawlee.storages', 'Dataset'),
109
+ 'KeyValueStore': ('crawlee.storages', 'KeyValueStore'),
110
+ 'RequestQueue': ('crawlee.storages', 'RequestQueue'),
111
+ 'DatasetItemsListPage': ('crawlee.storage_clients.models', 'DatasetItemsListPage'),
112
+ 'EventListener': ('crawlee.events', 'EventListener'),
113
+ # Storage clients
114
+ 'ApiStorageClient': ('scrapely.storage', 'ApiStorageClient'),
115
+ 'FileSystemStorageClient': ('scrapely.storage', 'FileSystemStorageClient'),
116
+ 'MemoryStorageClient': ('scrapely.storage', 'MemoryStorageClient'),
117
+ 'ScrapelyStorageClient': ('scrapely.storage', 'ScrapelyStorageClient'),
118
+ 'SmartScrapelyStorageClient': ('scrapely.storage', 'SmartScrapelyStorageClient'),
119
+ 'StorageClient': ('scrapely.storage', 'StorageClient'),
120
+ 'set_api_storage': ('scrapely.storage', 'set_api_storage'),
121
+ # Request loaders
122
+ 'ScrapelyRequestList': ('scrapely.request_loaders', 'ScrapelyRequestList'),
123
+ # Events
124
+ **{
125
+ name: ('scrapely.events', name)
126
+ for name in (
127
+ 'AbortingEvent',
128
+ 'ActorEventTypes',
129
+ 'CrawleeEventManager',
130
+ 'DeprecatedEvent',
131
+ 'Event',
132
+ 'EventAbortingData',
133
+ 'EventExitData',
134
+ 'EventManager',
135
+ 'EventMessage',
136
+ 'EventMigratingData',
137
+ 'EventPersistStateData',
138
+ 'EventSystemInfoData',
139
+ 'EventWithoutData',
140
+ 'ExitEvent',
141
+ 'LocalEventManager',
142
+ 'MigratingEvent',
143
+ 'PersistStateEvent',
144
+ 'ScrapelyEventManager',
145
+ 'SystemInfoEvent',
146
+ 'SystemInfoEventData',
147
+ 'UnknownEvent',
148
+ )
149
+ },
150
+ }
151
+
152
+ # Submodules that used to be loaded (and so bound as attributes) by this
153
+ # package's eager imports; ``scrapely.events`` etc. keep working without an
154
+ # explicit ``import scrapely.events``.
155
+ _LAZY_SUBMODULES = frozenset({
156
+ 'events',
157
+ 'log',
158
+ 'request_loaders',
159
+ 'storage',
160
+ 'storage_clients',
161
+ 'storages',
162
+ })
163
+
164
+
165
+ def __getattr__(name: str) -> Any:
166
+ target = _LAZY_ATTRS.get(name)
167
+ if target is not None:
168
+ import importlib
169
+
170
+ value = getattr(importlib.import_module(target[0]), target[1])
171
+ globals()[name] = value
172
+ return value
173
+ if name in _LAZY_SUBMODULES:
174
+ import importlib
175
+
176
+ return importlib.import_module(f'{__name__}.{name}')
177
+ raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
178
+
179
+
180
+ def __dir__() -> list:
181
+ return sorted(set(globals()) | set(_LAZY_ATTRS) | _LAZY_SUBMODULES)
182
+
183
+
184
+ if TYPE_CHECKING: # Static analysers and IDEs see the lazy names as plain imports.
185
+ from crawlee import Request
186
+ from crawlee.events import EventListener
187
+ from crawlee.request_loaders import (
188
+ RequestLoader,
189
+ RequestManager,
190
+ RequestManagerTandem,
191
+ SitemapRequestLoader,
192
+ )
193
+ from crawlee.storage_clients.models import DatasetItemsListPage
194
+ from crawlee.storages import Dataset, KeyValueStore, RequestQueue
195
+
196
+ from .events import (
197
+ AbortingEvent,
198
+ ActorEventTypes,
199
+ CrawleeEventManager,
200
+ DeprecatedEvent,
201
+ Event,
202
+ EventAbortingData,
203
+ EventExitData,
204
+ EventManager,
205
+ EventMessage,
206
+ EventMigratingData,
207
+ EventPersistStateData,
208
+ EventSystemInfoData,
209
+ EventWithoutData,
210
+ ExitEvent,
211
+ LocalEventManager,
212
+ MigratingEvent,
213
+ PersistStateEvent,
214
+ ScrapelyEventManager,
215
+ SystemInfoEvent,
216
+ SystemInfoEventData,
217
+ UnknownEvent,
218
+ )
219
+ from .request_loaders import ScrapelyRequestList
220
+ from .storage import (
221
+ ApiStorageClient,
222
+ FileSystemStorageClient,
223
+ MemoryStorageClient,
224
+ ScrapelyStorageClient,
225
+ SmartScrapelyStorageClient,
226
+ StorageClient,
227
+ set_api_storage,
228
+ )
229
+
230
+ __version__ = '0.1.23'
231
+ __all__ = [
232
+ # Core
233
+ 'Actor',
234
+ 'Configuration',
235
+ 'set_cdp_url',
236
+ # ApifyClient
237
+ 'create_charging_client',
238
+ 'USES_PUSH_DATA_INTERCEPTION',
239
+ 'push_data_charging_context',
240
+
241
+ # Storage
242
+ 'Dataset',
243
+ 'KeyValueStore',
244
+ 'RequestQueue',
245
+ 'Request',
246
+ 'DatasetItemsListPage',
247
+ # Storage clients
248
+ 'ApiStorageClient',
249
+ 'ScrapelyStorageClient',
250
+ 'SmartScrapelyStorageClient',
251
+ 'FileSystemStorageClient',
252
+ 'MemoryStorageClient',
253
+ 'StorageClient',
254
+ 'set_api_storage',
255
+ # Request loaders
256
+ 'ScrapelyRequestList',
257
+ 'RequestLoader',
258
+ 'RequestManager',
259
+ 'RequestManagerTandem',
260
+ 'SitemapRequestLoader',
261
+ # Events
262
+ 'Event',
263
+ 'EventManager',
264
+ 'ScrapelyEventManager',
265
+ 'CrawleeEventManager',
266
+ 'LocalEventManager',
267
+ # Event data types
268
+ 'SystemInfoEventData',
269
+ 'EventAbortingData',
270
+ 'EventExitData',
271
+ 'EventListener',
272
+ 'EventMigratingData',
273
+ 'EventPersistStateData',
274
+ 'EventSystemInfoData',
275
+ # Event types
276
+ 'PersistStateEvent',
277
+ 'SystemInfoEvent',
278
+ 'MigratingEvent',
279
+ 'AbortingEvent',
280
+ 'ExitEvent',
281
+ 'EventWithoutData',
282
+ 'DeprecatedEvent',
283
+ 'UnknownEvent',
284
+ 'EventMessage',
285
+ 'ActorEventTypes',
286
+ # Proxy
287
+ 'ProxyConfiguration',
288
+ 'ProxyInfo',
289
+ # Remote browser
290
+ 'BrowserConfiguration',
291
+ # Charging
292
+ 'ChargingManager',
293
+ 'ChargeResult',
294
+ 'ChargeUncertainError',
295
+ 'ChargeRejectedError',
296
+ 'merge_charge_results',
297
+ 'push_data_and_charge',
298
+
299
+ # Types
300
+ 'ActorInput',
301
+ 'ActorPermissionLevel',
302
+ 'ActorRun',
303
+ 'ApifyEnv',
304
+ 'AbortOptions',
305
+ 'CallOptions',
306
+ 'ExitOptions',
307
+ 'InitOptions',
308
+ 'MetamorphOptions',
309
+ 'OpenStorageOptions',
310
+ 'RebootOptions',
311
+ 'RecordOptions',
312
+ 'SetStatusMessageOptions',
313
+ 'StartOptions',
314
+ 'WebhookOptions',
315
+
316
+ # Webhook
317
+ 'Webhook',
318
+ 'WebhookEventType',
319
+ # Pricing
320
+ 'PricingModel',
321
+ 'FreeActorPricingInfo',
322
+ 'FlatPricePerMonthActorPricingInfo',
323
+ 'PricePerDatasetItemActorPricingInfo',
324
+ 'PayPerEventActorPricingInfo',
325
+ 'PricingPerEvent',
326
+ 'ActorPricingInfo',
327
+ 'ActorChargeEvent',
328
+ 'ActorPricingInfoResult',
329
+ # Errors
330
+ 'ScrapelyClientError',
331
+ 'ScrapelyApiError',
332
+ 'InvalidResponseBodyError',
333
+ 'InvalidRequestError',
334
+ 'UnauthorizedError',
335
+ 'ForbiddenError',
336
+ 'NotFoundError',
337
+ 'ConflictError',
338
+ 'RateLimitError',
339
+ 'ServerError',
340
+ 'raise_for_status',
341
+ ]
@@ -20,10 +20,6 @@ from datetime import datetime, timedelta
20
20
  from decimal import Decimal
21
21
  from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union, cast
22
22
 
23
- from crawlee.configuration import Configuration as CrawleeConfiguration
24
- from crawlee.storage_clients._base import StorageClient
25
- from crawlee.storages import Dataset, KeyValueStore, RequestQueue
26
-
27
23
  from .charging import (
28
24
  ChargingManager,
29
25
  is_default_dataset_item_event,
@@ -34,23 +30,10 @@ from .charging import (
34
30
  from .configuration import Configuration
35
31
  from .crypto import decrypt_secret_inputs
36
32
  from .proxy_configuration import with_proxy_key_aliases
37
- from crawlee.events import EventManager
38
-
39
- from .events import Event, LocalEventManager, ScrapelyEventManager
40
33
  from .charging_client import create_charging_client
41
34
  from .errors import ForbiddenError
42
35
  from .storage.api_client import ApiClient, ApiClientError
43
36
  from .storage.client import is_storage_uuid
44
- from .storage._metadata import StorageType, resolve_alias_from_storages_json
45
- from .storage.patch_storage import (
46
- disconnect_storage,
47
- get_api_storage_factory,
48
- get_smart_storage_client,
49
- set_api_storage,
50
- set_storage_client,
51
- )
52
- from .storage.scrapely_storage_client import ScrapelyStorageClient
53
- from .storage.smart_scrapely_storage_client import SmartScrapelyStorageClient
54
37
  from .types import (
55
38
  AbortOptions,
56
39
  ActorRun,
@@ -66,17 +49,74 @@ from .types import (
66
49
  StartOptions,
67
50
  WebhookOptions,
68
51
  )
69
- from .log import _configure_logging
70
52
  from .webhook import Webhook
71
53
 
72
54
  if TYPE_CHECKING:
55
+ from crawlee.events import EventManager
56
+ from crawlee.storage_clients._base import StorageClient
57
+ from crawlee.storages import Dataset, KeyValueStore, RequestQueue
73
58
  from scrapely_client import ApifyClientAsync
74
59
 
60
+ from .events import Event
61
+ from .storage._metadata import StorageType
62
+ from .storage.scrapely_storage_client import ScrapelyStorageClient
63
+ from .storage.smart_scrapely_storage_client import SmartScrapelyStorageClient
64
+
75
65
  from .browser_configuration import BrowserConfiguration
76
66
  from .proxy_configuration import ProxyConfiguration
77
67
 
78
68
  logger = logging.getLogger(__name__)
79
69
 
70
+ # Names that need Crawlee, resolved on first use instead of at import: importing
71
+ # Crawlee costs ~1 s per run under gVisor. They stay module attributes
72
+ # (``scrapely.actor.Dataset`` works and can be monkeypatched): ``_lazy`` returns
73
+ # the module-global value when one is set, else imports and caches it.
74
+ _LAZY_ATTRS: Dict[str, Tuple[str, str]] = {
75
+ 'CrawleeConfiguration': ('crawlee.configuration', 'Configuration'),
76
+ 'StorageClient': ('crawlee.storage_clients._base', 'StorageClient'),
77
+ 'Dataset': ('crawlee.storages', 'Dataset'),
78
+ 'KeyValueStore': ('crawlee.storages', 'KeyValueStore'),
79
+ 'RequestQueue': ('crawlee.storages', 'RequestQueue'),
80
+ 'EventManager': ('crawlee.events', 'EventManager'),
81
+ 'Event': ('scrapely.events', 'Event'),
82
+ 'LocalEventManager': ('scrapely.events', 'LocalEventManager'),
83
+ 'ScrapelyEventManager': ('scrapely.events', 'ScrapelyEventManager'),
84
+ 'StorageType': ('scrapely.storage._metadata', 'StorageType'),
85
+ 'resolve_alias_from_storages_json': ('scrapely.storage._metadata', 'resolve_alias_from_storages_json'),
86
+ 'disconnect_storage': ('scrapely.storage.patch_storage', 'disconnect_storage'),
87
+ 'get_api_storage_factory': ('scrapely.storage.patch_storage', 'get_api_storage_factory'),
88
+ 'get_smart_storage_client': ('scrapely.storage.patch_storage', 'get_smart_storage_client'),
89
+ 'set_api_storage': ('scrapely.storage.patch_storage', 'set_api_storage'),
90
+ 'set_storage_client': ('scrapely.storage.patch_storage', 'set_storage_client'),
91
+ 'ScrapelyStorageClient': ('scrapely.storage.scrapely_storage_client', 'ScrapelyStorageClient'),
92
+ 'SmartScrapelyStorageClient': (
93
+ 'scrapely.storage.smart_scrapely_storage_client',
94
+ 'SmartScrapelyStorageClient',
95
+ ),
96
+ '_configure_logging': ('scrapely.log', '_configure_logging'),
97
+ }
98
+
99
+
100
+ def _lazy(name: str) -> Any:
101
+ """Return a Crawlee-dependent module attribute, importing it on first use."""
102
+ value = globals().get(name, _MISSING)
103
+ if value is _MISSING:
104
+ import importlib
105
+
106
+ module_name, attr = _LAZY_ATTRS[name]
107
+ value = getattr(importlib.import_module(module_name), attr)
108
+ globals()[name] = value
109
+ return value
110
+
111
+
112
+ _MISSING = object()
113
+
114
+
115
+ def __getattr__(name: str) -> Any:
116
+ if name in _LAZY_ATTRS:
117
+ return _lazy(name)
118
+ raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
119
+
80
120
  # Terminal run statuses used by the run-control polling loops (call/call_task).
81
121
  _RUN_TERMINAL_STATUSES = frozenset({'SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'})
82
122
 
@@ -324,7 +364,7 @@ class Actor(metaclass=_ActorMeta):
324
364
  return
325
365
 
326
366
  if options is None or options.configure_logging:
327
- _configure_logging()
367
+ _lazy('_configure_logging')()
328
368
 
329
369
  # Do NOT set _initialized until everything succeeds (failed init must be retryable).
330
370
  cls._config = Configuration.get_global_config()
@@ -338,7 +378,7 @@ class Actor(metaclass=_ActorMeta):
338
378
  for key, value in options.configuration.items():
339
379
  cls._config.set(key, value)
340
380
 
341
- crawlee_config = CrawleeConfiguration.get_global_configuration()
381
+ crawlee_config = _lazy('CrawleeConfiguration').get_global_configuration()
342
382
  if cls._config.get('is_at_home'):
343
383
  crawlee_config.purge_on_start = False
344
384
 
@@ -361,9 +401,9 @@ class Actor(metaclass=_ActorMeta):
361
401
  '[Scrapely] Initializing API storage for run %s',
362
402
  storage_opt['run_id'],
363
403
  )
364
- smart = set_api_storage(storage_opt)
365
- elif isinstance(storage_opt, StorageClient):
366
- smart = set_storage_client(storage_opt)
404
+ smart = _lazy('set_api_storage')(storage_opt)
405
+ elif isinstance(storage_opt, _lazy('StorageClient')):
406
+ smart = _lazy('set_storage_client')(storage_opt)
367
407
  else:
368
408
  raise TypeError(
369
409
  'Actor.init() storage option must be ApiStorageConfig or StorageClient'
@@ -375,7 +415,7 @@ class Actor(metaclass=_ActorMeta):
375
415
 
376
416
  if api_url and token and run_id:
377
417
  logger.info('[Scrapely] Initializing API storage for run %s', run_id)
378
- smart = set_api_storage({
418
+ smart = _lazy('set_api_storage')({
379
419
  'api_url': api_url,
380
420
  'token': token,
381
421
  'run_id': run_id,
@@ -383,7 +423,7 @@ class Actor(metaclass=_ActorMeta):
383
423
  elif not cls._config.get('is_at_home'):
384
424
  # Local development without credentials: Smart → FS (Apify docs shape)
385
425
  logger.info('[Scrapely] Initializing local file-system storage')
386
- smart = set_api_storage()
426
+ smart = _lazy('set_api_storage')()
387
427
  else:
388
428
  raise RuntimeError(
389
429
  'Missing required configuration for secure storage. '
@@ -392,8 +432,12 @@ class Actor(metaclass=_ActorMeta):
392
432
  'or pass storage in Actor.init().'
393
433
  )
394
434
 
395
- cls._smart_storage = smart if isinstance(smart, SmartScrapelyStorageClient) else get_smart_storage_client()
396
- factory = get_api_storage_factory()
435
+ cls._smart_storage = (
436
+ smart
437
+ if isinstance(smart, _lazy('SmartScrapelyStorageClient'))
438
+ else _lazy('get_smart_storage_client')()
439
+ )
440
+ factory = _lazy('get_api_storage_factory')()
397
441
  cls._scrapely_storage = factory
398
442
  if factory is not None and factory.has_credentials():
399
443
  cls._client = factory.api_client
@@ -408,12 +452,12 @@ class Actor(metaclass=_ActorMeta):
408
452
 
409
453
  event_manager: EventManager
410
454
  if ws_url:
411
- event_manager = ScrapelyEventManager(
455
+ event_manager = _lazy('ScrapelyEventManager')(
412
456
  configuration=cls._config,
413
457
  persist_state_interval=persist_interval,
414
458
  )
415
459
  else:
416
- event_manager = LocalEventManager(
460
+ event_manager = _lazy('LocalEventManager')(
417
461
  system_info_interval=timedelta(seconds=1),
418
462
  persist_state_interval=persist_interval,
419
463
  )
@@ -544,7 +588,7 @@ class Actor(metaclass=_ActorMeta):
544
588
  cls._default_rq = None
545
589
  cls._cached_input = None
546
590
  try:
547
- disconnect_storage()
591
+ _lazy('disconnect_storage')()
548
592
  except BaseException as e:
549
593
  cleanup_error = cleanup_error or e
550
594
  logger.warning('disconnect_storage failed during shutdown: %s', e)
@@ -836,7 +880,7 @@ class Actor(metaclass=_ActorMeta):
836
880
 
837
881
  target = cls._default_dataset
838
882
  if dataset is not None:
839
- target = dataset if isinstance(dataset, Dataset) else await cls.open_dataset(name=dataset)
883
+ target = dataset if isinstance(dataset, _lazy('Dataset')) else await cls.open_dataset(name=dataset)
840
884
  is_default = target is cls._default_dataset or (
841
885
  getattr(target, 'id', None) is not None and target.id == getattr(cls._default_dataset, 'id', None)
842
886
  )
@@ -902,7 +946,7 @@ class Actor(metaclass=_ActorMeta):
902
946
 
903
947
  if alias is not None:
904
948
  storages_json = cls._env_config().get('actor_storages_json')
905
- resolved = resolve_alias_from_storages_json(alias, storage_type, storages_json)
949
+ resolved = _lazy('resolve_alias_from_storages_json')(alias, storage_type, storages_json)
906
950
  if resolved is not None:
907
951
  id = resolved
908
952
  alias = None
@@ -915,12 +959,12 @@ class Actor(metaclass=_ActorMeta):
915
959
 
916
960
  @classmethod
917
961
  def _storage_client_for_open(cls, *, force_cloud: bool = False) -> Optional[StorageClient]:
918
- smart = cls._smart_storage or get_smart_storage_client()
962
+ smart = cls._smart_storage or _lazy('get_smart_storage_client')()
919
963
  if smart is not None:
920
964
  return smart.get_suitable_storage_client(force_cloud=force_cloud)
921
965
 
922
966
  if force_cloud:
923
- cloud = get_api_storage_factory()
967
+ cloud = _lazy('get_api_storage_factory')()
924
968
  if cloud is None:
925
969
  # Build from config like TS _buildStorageOpenOptions
926
970
  config = cls.get_config()
@@ -928,7 +972,7 @@ class Actor(metaclass=_ActorMeta):
928
972
  token = config.get('token')
929
973
  run_id = config.get('actor_run_id')
930
974
  if api_url and token and run_id:
931
- return ScrapelyStorageClient(api_url=api_url, token=token, run_id=run_id)
975
+ return _lazy('ScrapelyStorageClient')(api_url=api_url, token=token, run_id=run_id)
932
976
  raise RuntimeError('Cloud storage client is not configured')
933
977
  return cloud
934
978
  return None
@@ -958,10 +1002,10 @@ class Actor(metaclass=_ActorMeta):
958
1002
 
959
1003
  if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
960
1004
  if cls._default_dataset is None:
961
- cls._default_dataset = await Dataset.open(storage_client=storage_client)
1005
+ cls._default_dataset = await _lazy('Dataset').open(storage_client=storage_client)
962
1006
  return cls._default_dataset
963
1007
 
964
- return await Dataset.open(
1008
+ return await _lazy('Dataset').open(
965
1009
  id=open_args['id'],
966
1010
  name=open_args['name'],
967
1011
  alias=open_args['alias'],
@@ -993,10 +1037,10 @@ class Actor(metaclass=_ActorMeta):
993
1037
 
994
1038
  if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
995
1039
  if cls._default_kvs is None:
996
- cls._default_kvs = await KeyValueStore.open(storage_client=storage_client)
1040
+ cls._default_kvs = await _lazy('KeyValueStore').open(storage_client=storage_client)
997
1041
  return cls._default_kvs
998
1042
 
999
- return await KeyValueStore.open(
1043
+ return await _lazy('KeyValueStore').open(
1000
1044
  id=open_args['id'],
1001
1045
  name=open_args['name'],
1002
1046
  alias=open_args['alias'],
@@ -1028,10 +1072,10 @@ class Actor(metaclass=_ActorMeta):
1028
1072
 
1029
1073
  if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
1030
1074
  if cls._default_rq is None:
1031
- cls._default_rq = await RequestQueue.open(storage_client=storage_client)
1075
+ cls._default_rq = await _lazy('RequestQueue').open(storage_client=storage_client)
1032
1076
  return cls._default_rq
1033
1077
 
1034
- return await RequestQueue.open(
1078
+ return await _lazy('RequestQueue').open(
1035
1079
  id=open_args['id'],
1036
1080
  name=open_args['name'],
1037
1081
  alias=open_args['alias'],
@@ -1710,7 +1754,7 @@ class Actor(metaclass=_ActorMeta):
1710
1754
  if cls._event_manager:
1711
1755
  from crawlee.events import EventPersistStateData
1712
1756
  cls._event_manager.emit(
1713
- event=Event.PERSIST_STATE,
1757
+ event=_lazy('Event').PERSIST_STATE,
1714
1758
  event_data=EventPersistStateData(is_migrating=True),
1715
1759
  )
1716
1760