scrapely-python 0.1.22__tar.gz → 0.1.23__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scrapely_python-0.1.22/src/scrapely_python.egg-info → scrapely_python-0.1.23}/PKG-INFO +1 -1
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/pyproject.toml +1 -1
- scrapely_python-0.1.23/src/scrapely/__init__.py +341 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/actor.py +86 -42
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/patch_browser_pool.py +96 -3
- scrapely_python-0.1.23/src/scrapely/storage/__init__.py +113 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23/src/scrapely_python.egg-info}/PKG-INFO +1 -1
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely_python.egg-info/SOURCES.txt +1 -0
- scrapely_python-0.1.23/tests/test_lazy_crawlee.py +294 -0
- scrapely_python-0.1.22/src/scrapely/__init__.py +0 -246
- scrapely_python-0.1.22/src/scrapely/storage/__init__.py +0 -48
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/LICENSE +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/LICENSE-APACHE +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/NOTICE +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/README.md +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/setup.cfg +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/_browser_locations.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/browser_configuration.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/cdp_retry.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/charging.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/charging_client.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/configuration.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/crypto/__init__.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/crypto/_secrets.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/errors.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/events/__init__.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/events/_types.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/events/event_manager.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/log.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/pricing_types.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/proxy_configuration.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/py.typed +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/request_loaders/__init__.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/request_loaders/_scrapely_request_list.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/solver.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/_file_system/__init__.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/_file_system/_key_value_store_client.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/_file_system/_storage_client.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/_metadata.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/api_client.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/client.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/patch_storage.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/scrapely_dataset_client.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/scrapely_kvs_client.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/scrapely_rq_client.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/scrapely_storage_client.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/smart_scrapely_storage_client.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/types.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage/utils.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storage_clients/__init__.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/storages/__init__.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/types.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely/webhook.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely_python.egg-info/dependency_links.txt +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely_python.egg-info/requires.txt +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/src/scrapely_python.egg-info/top_level.txt +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_api_client_retries.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_apify_parity.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_browser_configuration.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_cdp_retry.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_charge_limit_guard.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_charge_retries.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_charging_parity.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_charging_pending.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_input_proxy_aliases.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_parity.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_patch_browser_pool.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_proxy_configuration.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_push_idempotency.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_run_control.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_secrets.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_solver.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_storage.py +0 -0
- {scrapely_python-0.1.22 → scrapely_python-0.1.23}/tests/test_terminal_status_message.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scrapely-python"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.23"
|
|
8
8
|
description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT AND Apache-2.0"
|
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
from scrapely import Actor, Dataset
|
|
6
|
+
|
|
7
|
+
async def main():
|
|
8
|
+
async with Actor:
|
|
9
|
+
actor_input = await Actor.get_input()
|
|
10
|
+
await Actor.push_data({'url': actor_input['url']})
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from typing import TYPE_CHECKING, Any
|
|
14
|
+
|
|
15
|
+
# Side-effect: patch browser launches (Crawlee BrowserPool, PlaywrightCrawler,
|
|
16
|
+
# raw Playwright, Camoufox, pyppeteer) to use the remote CDP browser (CDP_URL).
|
|
17
|
+
# Must run on package import so `from scrapely import Actor` enables the patch
|
|
18
|
+
# before any browser is launched (Node SDK parity). It does not import Crawlee:
|
|
19
|
+
# the Crawlee patches are applied when Crawlee's browser modules load.
|
|
20
|
+
from .patch_browser_pool import set_cdp_url # noqa: F401
|
|
21
|
+
|
|
22
|
+
# Crawlee is NOT imported here: it costs ~1 s per run under gVisor and many
|
|
23
|
+
# actors never touch it. Everything that needs Crawlee (the re-exported Crawlee
|
|
24
|
+
# classes, storage clients, events, request loaders) is resolved on first
|
|
25
|
+
# attribute access by ``__getattr__`` below; ``Actor`` imports what it needs on
|
|
26
|
+
# first use. Every ``from scrapely import X`` that worked before still works.
|
|
27
|
+
from .actor import Actor
|
|
28
|
+
from .configuration import Configuration
|
|
29
|
+
from .charging_client import (
|
|
30
|
+
USES_PUSH_DATA_INTERCEPTION,
|
|
31
|
+
create_charging_client,
|
|
32
|
+
push_data_charging_context,
|
|
33
|
+
)
|
|
34
|
+
from .proxy_configuration import ProxyConfiguration
|
|
35
|
+
from .browser_configuration import BrowserConfiguration
|
|
36
|
+
from .charging import (
|
|
37
|
+
ChargingManager,
|
|
38
|
+
ChargeUncertainError,
|
|
39
|
+
ChargeRejectedError,
|
|
40
|
+
has_default_dataset_item_event_in_pricing,
|
|
41
|
+
is_default_dataset_item_event,
|
|
42
|
+
merge_charge_results,
|
|
43
|
+
push_data_and_charge,
|
|
44
|
+
resolve_default_dataset_item_event_name,
|
|
45
|
+
APIFY_DEFAULT_DATASET_ITEM_EVENT,
|
|
46
|
+
SCRAPELY_DEFAULT_DATASET_ITEM_EVENT,
|
|
47
|
+
DEFAULT_DATASET_ITEM_EVENT,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
from .types import (
|
|
51
|
+
ActorInput,
|
|
52
|
+
ActorPermissionLevel,
|
|
53
|
+
ActorRun,
|
|
54
|
+
ApifyEnv,
|
|
55
|
+
AbortOptions,
|
|
56
|
+
CallOptions,
|
|
57
|
+
ExitOptions,
|
|
58
|
+
InitOptions,
|
|
59
|
+
MetamorphOptions,
|
|
60
|
+
OpenStorageOptions,
|
|
61
|
+
ProxyInfo,
|
|
62
|
+
RebootOptions,
|
|
63
|
+
RecordOptions,
|
|
64
|
+
SetStatusMessageOptions,
|
|
65
|
+
StartOptions,
|
|
66
|
+
WebhookOptions,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
from .webhook import (
|
|
70
|
+
Webhook,
|
|
71
|
+
WebhookEventType,
|
|
72
|
+
)
|
|
73
|
+
from .pricing_types import (
|
|
74
|
+
PricingModel,
|
|
75
|
+
FreeActorPricingInfo,
|
|
76
|
+
FlatPricePerMonthActorPricingInfo,
|
|
77
|
+
PricePerDatasetItemActorPricingInfo,
|
|
78
|
+
PayPerEventActorPricingInfo,
|
|
79
|
+
PricingPerEvent,
|
|
80
|
+
ActorPricingInfo,
|
|
81
|
+
ActorChargeEvent,
|
|
82
|
+
ChargeResult,
|
|
83
|
+
ActorPricingInfoResult,
|
|
84
|
+
)
|
|
85
|
+
from .errors import (
|
|
86
|
+
ScrapelyClientError,
|
|
87
|
+
ScrapelyApiError,
|
|
88
|
+
InvalidResponseBodyError,
|
|
89
|
+
InvalidRequestError,
|
|
90
|
+
UnauthorizedError,
|
|
91
|
+
ForbiddenError,
|
|
92
|
+
NotFoundError,
|
|
93
|
+
ConflictError,
|
|
94
|
+
RateLimitError,
|
|
95
|
+
ServerError,
|
|
96
|
+
raise_for_status,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# Public name -> (module, attribute) for names that need Crawlee. Loaded on
|
|
100
|
+
# first access and cached in the module namespace.
|
|
101
|
+
_LAZY_ATTRS = {
|
|
102
|
+
# Crawlee re-exports
|
|
103
|
+
'Request': ('crawlee', 'Request'),
|
|
104
|
+
'RequestLoader': ('crawlee.request_loaders', 'RequestLoader'),
|
|
105
|
+
'RequestManager': ('crawlee.request_loaders', 'RequestManager'),
|
|
106
|
+
'RequestManagerTandem': ('crawlee.request_loaders', 'RequestManagerTandem'),
|
|
107
|
+
'SitemapRequestLoader': ('crawlee.request_loaders', 'SitemapRequestLoader'),
|
|
108
|
+
'Dataset': ('crawlee.storages', 'Dataset'),
|
|
109
|
+
'KeyValueStore': ('crawlee.storages', 'KeyValueStore'),
|
|
110
|
+
'RequestQueue': ('crawlee.storages', 'RequestQueue'),
|
|
111
|
+
'DatasetItemsListPage': ('crawlee.storage_clients.models', 'DatasetItemsListPage'),
|
|
112
|
+
'EventListener': ('crawlee.events', 'EventListener'),
|
|
113
|
+
# Storage clients
|
|
114
|
+
'ApiStorageClient': ('scrapely.storage', 'ApiStorageClient'),
|
|
115
|
+
'FileSystemStorageClient': ('scrapely.storage', 'FileSystemStorageClient'),
|
|
116
|
+
'MemoryStorageClient': ('scrapely.storage', 'MemoryStorageClient'),
|
|
117
|
+
'ScrapelyStorageClient': ('scrapely.storage', 'ScrapelyStorageClient'),
|
|
118
|
+
'SmartScrapelyStorageClient': ('scrapely.storage', 'SmartScrapelyStorageClient'),
|
|
119
|
+
'StorageClient': ('scrapely.storage', 'StorageClient'),
|
|
120
|
+
'set_api_storage': ('scrapely.storage', 'set_api_storage'),
|
|
121
|
+
# Request loaders
|
|
122
|
+
'ScrapelyRequestList': ('scrapely.request_loaders', 'ScrapelyRequestList'),
|
|
123
|
+
# Events
|
|
124
|
+
**{
|
|
125
|
+
name: ('scrapely.events', name)
|
|
126
|
+
for name in (
|
|
127
|
+
'AbortingEvent',
|
|
128
|
+
'ActorEventTypes',
|
|
129
|
+
'CrawleeEventManager',
|
|
130
|
+
'DeprecatedEvent',
|
|
131
|
+
'Event',
|
|
132
|
+
'EventAbortingData',
|
|
133
|
+
'EventExitData',
|
|
134
|
+
'EventManager',
|
|
135
|
+
'EventMessage',
|
|
136
|
+
'EventMigratingData',
|
|
137
|
+
'EventPersistStateData',
|
|
138
|
+
'EventSystemInfoData',
|
|
139
|
+
'EventWithoutData',
|
|
140
|
+
'ExitEvent',
|
|
141
|
+
'LocalEventManager',
|
|
142
|
+
'MigratingEvent',
|
|
143
|
+
'PersistStateEvent',
|
|
144
|
+
'ScrapelyEventManager',
|
|
145
|
+
'SystemInfoEvent',
|
|
146
|
+
'SystemInfoEventData',
|
|
147
|
+
'UnknownEvent',
|
|
148
|
+
)
|
|
149
|
+
},
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
# Submodules that used to be loaded (and so bound as attributes) by this
|
|
153
|
+
# package's eager imports; ``scrapely.events`` etc. keep working without an
|
|
154
|
+
# explicit ``import scrapely.events``.
|
|
155
|
+
_LAZY_SUBMODULES = frozenset({
|
|
156
|
+
'events',
|
|
157
|
+
'log',
|
|
158
|
+
'request_loaders',
|
|
159
|
+
'storage',
|
|
160
|
+
'storage_clients',
|
|
161
|
+
'storages',
|
|
162
|
+
})
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def __getattr__(name: str) -> Any:
|
|
166
|
+
target = _LAZY_ATTRS.get(name)
|
|
167
|
+
if target is not None:
|
|
168
|
+
import importlib
|
|
169
|
+
|
|
170
|
+
value = getattr(importlib.import_module(target[0]), target[1])
|
|
171
|
+
globals()[name] = value
|
|
172
|
+
return value
|
|
173
|
+
if name in _LAZY_SUBMODULES:
|
|
174
|
+
import importlib
|
|
175
|
+
|
|
176
|
+
return importlib.import_module(f'{__name__}.{name}')
|
|
177
|
+
raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def __dir__() -> list:
|
|
181
|
+
return sorted(set(globals()) | set(_LAZY_ATTRS) | _LAZY_SUBMODULES)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
if TYPE_CHECKING: # Static analysers and IDEs see the lazy names as plain imports.
|
|
185
|
+
from crawlee import Request
|
|
186
|
+
from crawlee.events import EventListener
|
|
187
|
+
from crawlee.request_loaders import (
|
|
188
|
+
RequestLoader,
|
|
189
|
+
RequestManager,
|
|
190
|
+
RequestManagerTandem,
|
|
191
|
+
SitemapRequestLoader,
|
|
192
|
+
)
|
|
193
|
+
from crawlee.storage_clients.models import DatasetItemsListPage
|
|
194
|
+
from crawlee.storages import Dataset, KeyValueStore, RequestQueue
|
|
195
|
+
|
|
196
|
+
from .events import (
|
|
197
|
+
AbortingEvent,
|
|
198
|
+
ActorEventTypes,
|
|
199
|
+
CrawleeEventManager,
|
|
200
|
+
DeprecatedEvent,
|
|
201
|
+
Event,
|
|
202
|
+
EventAbortingData,
|
|
203
|
+
EventExitData,
|
|
204
|
+
EventManager,
|
|
205
|
+
EventMessage,
|
|
206
|
+
EventMigratingData,
|
|
207
|
+
EventPersistStateData,
|
|
208
|
+
EventSystemInfoData,
|
|
209
|
+
EventWithoutData,
|
|
210
|
+
ExitEvent,
|
|
211
|
+
LocalEventManager,
|
|
212
|
+
MigratingEvent,
|
|
213
|
+
PersistStateEvent,
|
|
214
|
+
ScrapelyEventManager,
|
|
215
|
+
SystemInfoEvent,
|
|
216
|
+
SystemInfoEventData,
|
|
217
|
+
UnknownEvent,
|
|
218
|
+
)
|
|
219
|
+
from .request_loaders import ScrapelyRequestList
|
|
220
|
+
from .storage import (
|
|
221
|
+
ApiStorageClient,
|
|
222
|
+
FileSystemStorageClient,
|
|
223
|
+
MemoryStorageClient,
|
|
224
|
+
ScrapelyStorageClient,
|
|
225
|
+
SmartScrapelyStorageClient,
|
|
226
|
+
StorageClient,
|
|
227
|
+
set_api_storage,
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
__version__ = '0.1.23'
|
|
231
|
+
__all__ = [
|
|
232
|
+
# Core
|
|
233
|
+
'Actor',
|
|
234
|
+
'Configuration',
|
|
235
|
+
'set_cdp_url',
|
|
236
|
+
# ApifyClient
|
|
237
|
+
'create_charging_client',
|
|
238
|
+
'USES_PUSH_DATA_INTERCEPTION',
|
|
239
|
+
'push_data_charging_context',
|
|
240
|
+
|
|
241
|
+
# Storage
|
|
242
|
+
'Dataset',
|
|
243
|
+
'KeyValueStore',
|
|
244
|
+
'RequestQueue',
|
|
245
|
+
'Request',
|
|
246
|
+
'DatasetItemsListPage',
|
|
247
|
+
# Storage clients
|
|
248
|
+
'ApiStorageClient',
|
|
249
|
+
'ScrapelyStorageClient',
|
|
250
|
+
'SmartScrapelyStorageClient',
|
|
251
|
+
'FileSystemStorageClient',
|
|
252
|
+
'MemoryStorageClient',
|
|
253
|
+
'StorageClient',
|
|
254
|
+
'set_api_storage',
|
|
255
|
+
# Request loaders
|
|
256
|
+
'ScrapelyRequestList',
|
|
257
|
+
'RequestLoader',
|
|
258
|
+
'RequestManager',
|
|
259
|
+
'RequestManagerTandem',
|
|
260
|
+
'SitemapRequestLoader',
|
|
261
|
+
# Events
|
|
262
|
+
'Event',
|
|
263
|
+
'EventManager',
|
|
264
|
+
'ScrapelyEventManager',
|
|
265
|
+
'CrawleeEventManager',
|
|
266
|
+
'LocalEventManager',
|
|
267
|
+
# Event data types
|
|
268
|
+
'SystemInfoEventData',
|
|
269
|
+
'EventAbortingData',
|
|
270
|
+
'EventExitData',
|
|
271
|
+
'EventListener',
|
|
272
|
+
'EventMigratingData',
|
|
273
|
+
'EventPersistStateData',
|
|
274
|
+
'EventSystemInfoData',
|
|
275
|
+
# Event types
|
|
276
|
+
'PersistStateEvent',
|
|
277
|
+
'SystemInfoEvent',
|
|
278
|
+
'MigratingEvent',
|
|
279
|
+
'AbortingEvent',
|
|
280
|
+
'ExitEvent',
|
|
281
|
+
'EventWithoutData',
|
|
282
|
+
'DeprecatedEvent',
|
|
283
|
+
'UnknownEvent',
|
|
284
|
+
'EventMessage',
|
|
285
|
+
'ActorEventTypes',
|
|
286
|
+
# Proxy
|
|
287
|
+
'ProxyConfiguration',
|
|
288
|
+
'ProxyInfo',
|
|
289
|
+
# Remote browser
|
|
290
|
+
'BrowserConfiguration',
|
|
291
|
+
# Charging
|
|
292
|
+
'ChargingManager',
|
|
293
|
+
'ChargeResult',
|
|
294
|
+
'ChargeUncertainError',
|
|
295
|
+
'ChargeRejectedError',
|
|
296
|
+
'merge_charge_results',
|
|
297
|
+
'push_data_and_charge',
|
|
298
|
+
|
|
299
|
+
# Types
|
|
300
|
+
'ActorInput',
|
|
301
|
+
'ActorPermissionLevel',
|
|
302
|
+
'ActorRun',
|
|
303
|
+
'ApifyEnv',
|
|
304
|
+
'AbortOptions',
|
|
305
|
+
'CallOptions',
|
|
306
|
+
'ExitOptions',
|
|
307
|
+
'InitOptions',
|
|
308
|
+
'MetamorphOptions',
|
|
309
|
+
'OpenStorageOptions',
|
|
310
|
+
'RebootOptions',
|
|
311
|
+
'RecordOptions',
|
|
312
|
+
'SetStatusMessageOptions',
|
|
313
|
+
'StartOptions',
|
|
314
|
+
'WebhookOptions',
|
|
315
|
+
|
|
316
|
+
# Webhook
|
|
317
|
+
'Webhook',
|
|
318
|
+
'WebhookEventType',
|
|
319
|
+
# Pricing
|
|
320
|
+
'PricingModel',
|
|
321
|
+
'FreeActorPricingInfo',
|
|
322
|
+
'FlatPricePerMonthActorPricingInfo',
|
|
323
|
+
'PricePerDatasetItemActorPricingInfo',
|
|
324
|
+
'PayPerEventActorPricingInfo',
|
|
325
|
+
'PricingPerEvent',
|
|
326
|
+
'ActorPricingInfo',
|
|
327
|
+
'ActorChargeEvent',
|
|
328
|
+
'ActorPricingInfoResult',
|
|
329
|
+
# Errors
|
|
330
|
+
'ScrapelyClientError',
|
|
331
|
+
'ScrapelyApiError',
|
|
332
|
+
'InvalidResponseBodyError',
|
|
333
|
+
'InvalidRequestError',
|
|
334
|
+
'UnauthorizedError',
|
|
335
|
+
'ForbiddenError',
|
|
336
|
+
'NotFoundError',
|
|
337
|
+
'ConflictError',
|
|
338
|
+
'RateLimitError',
|
|
339
|
+
'ServerError',
|
|
340
|
+
'raise_for_status',
|
|
341
|
+
]
|
|
@@ -20,10 +20,6 @@ from datetime import datetime, timedelta
|
|
|
20
20
|
from decimal import Decimal
|
|
21
21
|
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union, cast
|
|
22
22
|
|
|
23
|
-
from crawlee.configuration import Configuration as CrawleeConfiguration
|
|
24
|
-
from crawlee.storage_clients._base import StorageClient
|
|
25
|
-
from crawlee.storages import Dataset, KeyValueStore, RequestQueue
|
|
26
|
-
|
|
27
23
|
from .charging import (
|
|
28
24
|
ChargingManager,
|
|
29
25
|
is_default_dataset_item_event,
|
|
@@ -34,23 +30,10 @@ from .charging import (
|
|
|
34
30
|
from .configuration import Configuration
|
|
35
31
|
from .crypto import decrypt_secret_inputs
|
|
36
32
|
from .proxy_configuration import with_proxy_key_aliases
|
|
37
|
-
from crawlee.events import EventManager
|
|
38
|
-
|
|
39
|
-
from .events import Event, LocalEventManager, ScrapelyEventManager
|
|
40
33
|
from .charging_client import create_charging_client
|
|
41
34
|
from .errors import ForbiddenError
|
|
42
35
|
from .storage.api_client import ApiClient, ApiClientError
|
|
43
36
|
from .storage.client import is_storage_uuid
|
|
44
|
-
from .storage._metadata import StorageType, resolve_alias_from_storages_json
|
|
45
|
-
from .storage.patch_storage import (
|
|
46
|
-
disconnect_storage,
|
|
47
|
-
get_api_storage_factory,
|
|
48
|
-
get_smart_storage_client,
|
|
49
|
-
set_api_storage,
|
|
50
|
-
set_storage_client,
|
|
51
|
-
)
|
|
52
|
-
from .storage.scrapely_storage_client import ScrapelyStorageClient
|
|
53
|
-
from .storage.smart_scrapely_storage_client import SmartScrapelyStorageClient
|
|
54
37
|
from .types import (
|
|
55
38
|
AbortOptions,
|
|
56
39
|
ActorRun,
|
|
@@ -66,17 +49,74 @@ from .types import (
|
|
|
66
49
|
StartOptions,
|
|
67
50
|
WebhookOptions,
|
|
68
51
|
)
|
|
69
|
-
from .log import _configure_logging
|
|
70
52
|
from .webhook import Webhook
|
|
71
53
|
|
|
72
54
|
if TYPE_CHECKING:
|
|
55
|
+
from crawlee.events import EventManager
|
|
56
|
+
from crawlee.storage_clients._base import StorageClient
|
|
57
|
+
from crawlee.storages import Dataset, KeyValueStore, RequestQueue
|
|
73
58
|
from scrapely_client import ApifyClientAsync
|
|
74
59
|
|
|
60
|
+
from .events import Event
|
|
61
|
+
from .storage._metadata import StorageType
|
|
62
|
+
from .storage.scrapely_storage_client import ScrapelyStorageClient
|
|
63
|
+
from .storage.smart_scrapely_storage_client import SmartScrapelyStorageClient
|
|
64
|
+
|
|
75
65
|
from .browser_configuration import BrowserConfiguration
|
|
76
66
|
from .proxy_configuration import ProxyConfiguration
|
|
77
67
|
|
|
78
68
|
logger = logging.getLogger(__name__)
|
|
79
69
|
|
|
70
|
+
# Names that need Crawlee, resolved on first use instead of at import: importing
|
|
71
|
+
# Crawlee costs ~1 s per run under gVisor. They stay module attributes
|
|
72
|
+
# (``scrapely.actor.Dataset`` works and can be monkeypatched): ``_lazy`` returns
|
|
73
|
+
# the module-global value when one is set, else imports and caches it.
|
|
74
|
+
_LAZY_ATTRS: Dict[str, Tuple[str, str]] = {
|
|
75
|
+
'CrawleeConfiguration': ('crawlee.configuration', 'Configuration'),
|
|
76
|
+
'StorageClient': ('crawlee.storage_clients._base', 'StorageClient'),
|
|
77
|
+
'Dataset': ('crawlee.storages', 'Dataset'),
|
|
78
|
+
'KeyValueStore': ('crawlee.storages', 'KeyValueStore'),
|
|
79
|
+
'RequestQueue': ('crawlee.storages', 'RequestQueue'),
|
|
80
|
+
'EventManager': ('crawlee.events', 'EventManager'),
|
|
81
|
+
'Event': ('scrapely.events', 'Event'),
|
|
82
|
+
'LocalEventManager': ('scrapely.events', 'LocalEventManager'),
|
|
83
|
+
'ScrapelyEventManager': ('scrapely.events', 'ScrapelyEventManager'),
|
|
84
|
+
'StorageType': ('scrapely.storage._metadata', 'StorageType'),
|
|
85
|
+
'resolve_alias_from_storages_json': ('scrapely.storage._metadata', 'resolve_alias_from_storages_json'),
|
|
86
|
+
'disconnect_storage': ('scrapely.storage.patch_storage', 'disconnect_storage'),
|
|
87
|
+
'get_api_storage_factory': ('scrapely.storage.patch_storage', 'get_api_storage_factory'),
|
|
88
|
+
'get_smart_storage_client': ('scrapely.storage.patch_storage', 'get_smart_storage_client'),
|
|
89
|
+
'set_api_storage': ('scrapely.storage.patch_storage', 'set_api_storage'),
|
|
90
|
+
'set_storage_client': ('scrapely.storage.patch_storage', 'set_storage_client'),
|
|
91
|
+
'ScrapelyStorageClient': ('scrapely.storage.scrapely_storage_client', 'ScrapelyStorageClient'),
|
|
92
|
+
'SmartScrapelyStorageClient': (
|
|
93
|
+
'scrapely.storage.smart_scrapely_storage_client',
|
|
94
|
+
'SmartScrapelyStorageClient',
|
|
95
|
+
),
|
|
96
|
+
'_configure_logging': ('scrapely.log', '_configure_logging'),
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _lazy(name: str) -> Any:
|
|
101
|
+
"""Return a Crawlee-dependent module attribute, importing it on first use."""
|
|
102
|
+
value = globals().get(name, _MISSING)
|
|
103
|
+
if value is _MISSING:
|
|
104
|
+
import importlib
|
|
105
|
+
|
|
106
|
+
module_name, attr = _LAZY_ATTRS[name]
|
|
107
|
+
value = getattr(importlib.import_module(module_name), attr)
|
|
108
|
+
globals()[name] = value
|
|
109
|
+
return value
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
_MISSING = object()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def __getattr__(name: str) -> Any:
|
|
116
|
+
if name in _LAZY_ATTRS:
|
|
117
|
+
return _lazy(name)
|
|
118
|
+
raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
|
|
119
|
+
|
|
80
120
|
# Terminal run statuses used by the run-control polling loops (call/call_task).
|
|
81
121
|
_RUN_TERMINAL_STATUSES = frozenset({'SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'})
|
|
82
122
|
|
|
@@ -324,7 +364,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
324
364
|
return
|
|
325
365
|
|
|
326
366
|
if options is None or options.configure_logging:
|
|
327
|
-
_configure_logging()
|
|
367
|
+
_lazy('_configure_logging')()
|
|
328
368
|
|
|
329
369
|
# Do NOT set _initialized until everything succeeds (failed init must be retryable).
|
|
330
370
|
cls._config = Configuration.get_global_config()
|
|
@@ -338,7 +378,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
338
378
|
for key, value in options.configuration.items():
|
|
339
379
|
cls._config.set(key, value)
|
|
340
380
|
|
|
341
|
-
crawlee_config = CrawleeConfiguration.get_global_configuration()
|
|
381
|
+
crawlee_config = _lazy('CrawleeConfiguration').get_global_configuration()
|
|
342
382
|
if cls._config.get('is_at_home'):
|
|
343
383
|
crawlee_config.purge_on_start = False
|
|
344
384
|
|
|
@@ -361,9 +401,9 @@ class Actor(metaclass=_ActorMeta):
|
|
|
361
401
|
'[Scrapely] Initializing API storage for run %s',
|
|
362
402
|
storage_opt['run_id'],
|
|
363
403
|
)
|
|
364
|
-
smart = set_api_storage(storage_opt)
|
|
365
|
-
elif isinstance(storage_opt, StorageClient):
|
|
366
|
-
smart = set_storage_client(storage_opt)
|
|
404
|
+
smart = _lazy('set_api_storage')(storage_opt)
|
|
405
|
+
elif isinstance(storage_opt, _lazy('StorageClient')):
|
|
406
|
+
smart = _lazy('set_storage_client')(storage_opt)
|
|
367
407
|
else:
|
|
368
408
|
raise TypeError(
|
|
369
409
|
'Actor.init() storage option must be ApiStorageConfig or StorageClient'
|
|
@@ -375,7 +415,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
375
415
|
|
|
376
416
|
if api_url and token and run_id:
|
|
377
417
|
logger.info('[Scrapely] Initializing API storage for run %s', run_id)
|
|
378
|
-
smart = set_api_storage({
|
|
418
|
+
smart = _lazy('set_api_storage')({
|
|
379
419
|
'api_url': api_url,
|
|
380
420
|
'token': token,
|
|
381
421
|
'run_id': run_id,
|
|
@@ -383,7 +423,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
383
423
|
elif not cls._config.get('is_at_home'):
|
|
384
424
|
# Local development without credentials: Smart → FS (Apify docs shape)
|
|
385
425
|
logger.info('[Scrapely] Initializing local file-system storage')
|
|
386
|
-
smart = set_api_storage()
|
|
426
|
+
smart = _lazy('set_api_storage')()
|
|
387
427
|
else:
|
|
388
428
|
raise RuntimeError(
|
|
389
429
|
'Missing required configuration for secure storage. '
|
|
@@ -392,8 +432,12 @@ class Actor(metaclass=_ActorMeta):
|
|
|
392
432
|
'or pass storage in Actor.init().'
|
|
393
433
|
)
|
|
394
434
|
|
|
395
|
-
cls._smart_storage =
|
|
396
|
-
|
|
435
|
+
cls._smart_storage = (
|
|
436
|
+
smart
|
|
437
|
+
if isinstance(smart, _lazy('SmartScrapelyStorageClient'))
|
|
438
|
+
else _lazy('get_smart_storage_client')()
|
|
439
|
+
)
|
|
440
|
+
factory = _lazy('get_api_storage_factory')()
|
|
397
441
|
cls._scrapely_storage = factory
|
|
398
442
|
if factory is not None and factory.has_credentials():
|
|
399
443
|
cls._client = factory.api_client
|
|
@@ -408,12 +452,12 @@ class Actor(metaclass=_ActorMeta):
|
|
|
408
452
|
|
|
409
453
|
event_manager: EventManager
|
|
410
454
|
if ws_url:
|
|
411
|
-
event_manager = ScrapelyEventManager(
|
|
455
|
+
event_manager = _lazy('ScrapelyEventManager')(
|
|
412
456
|
configuration=cls._config,
|
|
413
457
|
persist_state_interval=persist_interval,
|
|
414
458
|
)
|
|
415
459
|
else:
|
|
416
|
-
event_manager = LocalEventManager(
|
|
460
|
+
event_manager = _lazy('LocalEventManager')(
|
|
417
461
|
system_info_interval=timedelta(seconds=1),
|
|
418
462
|
persist_state_interval=persist_interval,
|
|
419
463
|
)
|
|
@@ -544,7 +588,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
544
588
|
cls._default_rq = None
|
|
545
589
|
cls._cached_input = None
|
|
546
590
|
try:
|
|
547
|
-
disconnect_storage()
|
|
591
|
+
_lazy('disconnect_storage')()
|
|
548
592
|
except BaseException as e:
|
|
549
593
|
cleanup_error = cleanup_error or e
|
|
550
594
|
logger.warning('disconnect_storage failed during shutdown: %s', e)
|
|
@@ -836,7 +880,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
836
880
|
|
|
837
881
|
target = cls._default_dataset
|
|
838
882
|
if dataset is not None:
|
|
839
|
-
target = dataset if isinstance(dataset, Dataset) else await cls.open_dataset(name=dataset)
|
|
883
|
+
target = dataset if isinstance(dataset, _lazy('Dataset')) else await cls.open_dataset(name=dataset)
|
|
840
884
|
is_default = target is cls._default_dataset or (
|
|
841
885
|
getattr(target, 'id', None) is not None and target.id == getattr(cls._default_dataset, 'id', None)
|
|
842
886
|
)
|
|
@@ -902,7 +946,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
902
946
|
|
|
903
947
|
if alias is not None:
|
|
904
948
|
storages_json = cls._env_config().get('actor_storages_json')
|
|
905
|
-
resolved = resolve_alias_from_storages_json(alias, storage_type, storages_json)
|
|
949
|
+
resolved = _lazy('resolve_alias_from_storages_json')(alias, storage_type, storages_json)
|
|
906
950
|
if resolved is not None:
|
|
907
951
|
id = resolved
|
|
908
952
|
alias = None
|
|
@@ -915,12 +959,12 @@ class Actor(metaclass=_ActorMeta):
|
|
|
915
959
|
|
|
916
960
|
@classmethod
|
|
917
961
|
def _storage_client_for_open(cls, *, force_cloud: bool = False) -> Optional[StorageClient]:
|
|
918
|
-
smart = cls._smart_storage or get_smart_storage_client()
|
|
962
|
+
smart = cls._smart_storage or _lazy('get_smart_storage_client')()
|
|
919
963
|
if smart is not None:
|
|
920
964
|
return smart.get_suitable_storage_client(force_cloud=force_cloud)
|
|
921
965
|
|
|
922
966
|
if force_cloud:
|
|
923
|
-
cloud = get_api_storage_factory()
|
|
967
|
+
cloud = _lazy('get_api_storage_factory')()
|
|
924
968
|
if cloud is None:
|
|
925
969
|
# Build from config like TS _buildStorageOpenOptions
|
|
926
970
|
config = cls.get_config()
|
|
@@ -928,7 +972,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
928
972
|
token = config.get('token')
|
|
929
973
|
run_id = config.get('actor_run_id')
|
|
930
974
|
if api_url and token and run_id:
|
|
931
|
-
return ScrapelyStorageClient(api_url=api_url, token=token, run_id=run_id)
|
|
975
|
+
return _lazy('ScrapelyStorageClient')(api_url=api_url, token=token, run_id=run_id)
|
|
932
976
|
raise RuntimeError('Cloud storage client is not configured')
|
|
933
977
|
return cloud
|
|
934
978
|
return None
|
|
@@ -958,10 +1002,10 @@ class Actor(metaclass=_ActorMeta):
|
|
|
958
1002
|
|
|
959
1003
|
if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
|
|
960
1004
|
if cls._default_dataset is None:
|
|
961
|
-
cls._default_dataset = await Dataset.open(storage_client=storage_client)
|
|
1005
|
+
cls._default_dataset = await _lazy('Dataset').open(storage_client=storage_client)
|
|
962
1006
|
return cls._default_dataset
|
|
963
1007
|
|
|
964
|
-
return await Dataset.open(
|
|
1008
|
+
return await _lazy('Dataset').open(
|
|
965
1009
|
id=open_args['id'],
|
|
966
1010
|
name=open_args['name'],
|
|
967
1011
|
alias=open_args['alias'],
|
|
@@ -993,10 +1037,10 @@ class Actor(metaclass=_ActorMeta):
|
|
|
993
1037
|
|
|
994
1038
|
if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
|
|
995
1039
|
if cls._default_kvs is None:
|
|
996
|
-
cls._default_kvs = await KeyValueStore.open(storage_client=storage_client)
|
|
1040
|
+
cls._default_kvs = await _lazy('KeyValueStore').open(storage_client=storage_client)
|
|
997
1041
|
return cls._default_kvs
|
|
998
1042
|
|
|
999
|
-
return await KeyValueStore.open(
|
|
1043
|
+
return await _lazy('KeyValueStore').open(
|
|
1000
1044
|
id=open_args['id'],
|
|
1001
1045
|
name=open_args['name'],
|
|
1002
1046
|
alias=open_args['alias'],
|
|
@@ -1028,10 +1072,10 @@ class Actor(metaclass=_ActorMeta):
|
|
|
1028
1072
|
|
|
1029
1073
|
if open_args['id'] is None and open_args['name'] is None and open_args['alias'] is None:
|
|
1030
1074
|
if cls._default_rq is None:
|
|
1031
|
-
cls._default_rq = await RequestQueue.open(storage_client=storage_client)
|
|
1075
|
+
cls._default_rq = await _lazy('RequestQueue').open(storage_client=storage_client)
|
|
1032
1076
|
return cls._default_rq
|
|
1033
1077
|
|
|
1034
|
-
return await RequestQueue.open(
|
|
1078
|
+
return await _lazy('RequestQueue').open(
|
|
1035
1079
|
id=open_args['id'],
|
|
1036
1080
|
name=open_args['name'],
|
|
1037
1081
|
alias=open_args['alias'],
|
|
@@ -1710,7 +1754,7 @@ class Actor(metaclass=_ActorMeta):
|
|
|
1710
1754
|
if cls._event_manager:
|
|
1711
1755
|
from crawlee.events import EventPersistStateData
|
|
1712
1756
|
cls._event_manager.emit(
|
|
1713
|
-
event=Event.PERSIST_STATE,
|
|
1757
|
+
event=_lazy('Event').PERSIST_STATE,
|
|
1714
1758
|
event_data=EventPersistStateData(is_migrating=True),
|
|
1715
1759
|
)
|
|
1716
1760
|
|