apify 2.7.3__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apify might be problematic. Click here for more details.
- apify/_actor.py +194 -126
- apify/_charging.py +34 -9
- apify/_configuration.py +70 -6
- apify/_crypto.py +0 -6
- apify/_models.py +7 -7
- apify/_proxy_configuration.py +10 -10
- apify/_utils.py +25 -2
- apify/events/__init__.py +5 -0
- apify/events/_apify_event_manager.py +140 -0
- apify/events/_types.py +102 -0
- apify/log.py +0 -9
- apify/request_loaders/__init__.py +18 -0
- apify/{storages/_request_list.py → request_loaders/_apify_request_list.py} +25 -18
- apify/request_loaders/py.typed +0 -0
- apify/scrapy/_logging_config.py +1 -4
- apify/scrapy/extensions/_httpcache.py +9 -5
- apify/scrapy/requests.py +3 -3
- apify/scrapy/scheduler.py +8 -5
- apify/storage_clients/__init__.py +12 -0
- apify/storage_clients/_apify/__init__.py +11 -0
- apify/storage_clients/_apify/_dataset_client.py +328 -0
- apify/storage_clients/_apify/_key_value_store_client.py +265 -0
- apify/storage_clients/_apify/_models.py +131 -0
- apify/storage_clients/_apify/_request_queue_client.py +327 -0
- apify/storage_clients/_apify/_request_queue_shared_client.py +527 -0
- apify/storage_clients/_apify/_request_queue_single_client.py +399 -0
- apify/storage_clients/_apify/_storage_client.py +106 -0
- apify/storage_clients/_apify/_utils.py +194 -0
- apify/storage_clients/_apify/py.typed +0 -0
- apify/storage_clients/_file_system/__init__.py +2 -0
- apify/storage_clients/_file_system/_key_value_store_client.py +57 -0
- apify/storage_clients/_file_system/_storage_client.py +41 -0
- apify/storage_clients/_smart_apify/__init__.py +1 -0
- apify/storage_clients/_smart_apify/_storage_client.py +117 -0
- apify/storage_clients/py.typed +0 -0
- apify/storages/__init__.py +1 -3
- {apify-2.7.3.dist-info → apify-3.0.0.dist-info}/METADATA +25 -9
- apify-3.0.0.dist-info/RECORD +57 -0
- apify/_platform_event_manager.py +0 -231
- apify/apify_storage_client/__init__.py +0 -3
- apify/apify_storage_client/_apify_storage_client.py +0 -72
- apify/apify_storage_client/_dataset_client.py +0 -190
- apify/apify_storage_client/_dataset_collection_client.py +0 -51
- apify/apify_storage_client/_key_value_store_client.py +0 -109
- apify/apify_storage_client/_key_value_store_collection_client.py +0 -51
- apify/apify_storage_client/_request_queue_client.py +0 -176
- apify/apify_storage_client/_request_queue_collection_client.py +0 -51
- apify-2.7.3.dist-info/RECORD +0 -44
- /apify/{apify_storage_client → events}/py.typed +0 -0
- {apify-2.7.3.dist-info → apify-3.0.0.dist-info}/WHEEL +0 -0
- {apify-2.7.3.dist-info → apify-3.0.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
4
|
-
|
|
5
|
-
from typing_extensions import override
|
|
6
|
-
|
|
7
|
-
from crawlee.storage_clients._base import KeyValueStoreCollectionClient as BaseKeyValueStoreCollectionClient
|
|
8
|
-
from crawlee.storage_clients.models import KeyValueStoreListPage, KeyValueStoreMetadata
|
|
9
|
-
|
|
10
|
-
if TYPE_CHECKING:
|
|
11
|
-
from apify_client.clients import KeyValueStoreCollectionClientAsync
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class KeyValueStoreCollectionClient(BaseKeyValueStoreCollectionClient):
|
|
15
|
-
"""Key-value store collection resource client implementation based on the Apify platform storage."""
|
|
16
|
-
|
|
17
|
-
def __init__(self, apify_dataset_collection_client: KeyValueStoreCollectionClientAsync) -> None:
|
|
18
|
-
self._client = apify_dataset_collection_client
|
|
19
|
-
|
|
20
|
-
@override
|
|
21
|
-
async def get_or_create(
|
|
22
|
-
self,
|
|
23
|
-
*,
|
|
24
|
-
id: str | None = None,
|
|
25
|
-
name: str | None = None,
|
|
26
|
-
schema: dict | None = None,
|
|
27
|
-
) -> KeyValueStoreMetadata:
|
|
28
|
-
return KeyValueStoreMetadata.model_validate(
|
|
29
|
-
await self._client.get_or_create(
|
|
30
|
-
name=id if id is not None else name,
|
|
31
|
-
schema=schema,
|
|
32
|
-
)
|
|
33
|
-
)
|
|
34
|
-
|
|
35
|
-
@override
|
|
36
|
-
async def list(
|
|
37
|
-
self,
|
|
38
|
-
*,
|
|
39
|
-
unnamed: bool = False,
|
|
40
|
-
limit: int | None = None,
|
|
41
|
-
offset: int | None = None,
|
|
42
|
-
desc: bool = False,
|
|
43
|
-
) -> KeyValueStoreListPage:
|
|
44
|
-
return KeyValueStoreListPage.model_validate(
|
|
45
|
-
await self._client.list(
|
|
46
|
-
unnamed=unnamed,
|
|
47
|
-
limit=limit,
|
|
48
|
-
offset=offset,
|
|
49
|
-
desc=desc,
|
|
50
|
-
)
|
|
51
|
-
)
|
|
@@ -1,176 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
4
|
-
|
|
5
|
-
from typing_extensions import override
|
|
6
|
-
|
|
7
|
-
from crawlee import Request
|
|
8
|
-
from crawlee.storage_clients._base import RequestQueueClient as BaseRequestQueueClient
|
|
9
|
-
from crawlee.storage_clients.models import (
|
|
10
|
-
BatchRequestsOperationResponse,
|
|
11
|
-
ProcessedRequest,
|
|
12
|
-
ProlongRequestLockResponse,
|
|
13
|
-
RequestQueueHead,
|
|
14
|
-
RequestQueueHeadWithLocks,
|
|
15
|
-
RequestQueueMetadata,
|
|
16
|
-
)
|
|
17
|
-
|
|
18
|
-
if TYPE_CHECKING:
|
|
19
|
-
from collections.abc import Sequence
|
|
20
|
-
|
|
21
|
-
from apify_client.clients import RequestQueueClientAsync
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
class RequestQueueClient(BaseRequestQueueClient):
|
|
25
|
-
"""Request queue resource client implementation based on the Apify platform storage."""
|
|
26
|
-
|
|
27
|
-
def __init__(self, apify_request_queue_client: RequestQueueClientAsync) -> None:
|
|
28
|
-
self._client = apify_request_queue_client
|
|
29
|
-
|
|
30
|
-
@override
|
|
31
|
-
async def get(self) -> RequestQueueMetadata | None:
|
|
32
|
-
result = await self._client.get()
|
|
33
|
-
return RequestQueueMetadata.model_validate({'resourceDirectory': ''} | result) if result else None
|
|
34
|
-
|
|
35
|
-
@override
|
|
36
|
-
async def update(
|
|
37
|
-
self,
|
|
38
|
-
*,
|
|
39
|
-
name: str | None = None,
|
|
40
|
-
) -> RequestQueueMetadata:
|
|
41
|
-
return RequestQueueMetadata.model_validate(
|
|
42
|
-
{'resourceDirectory': ''}
|
|
43
|
-
| await self._client.update(
|
|
44
|
-
name=name,
|
|
45
|
-
)
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
@override
|
|
49
|
-
async def delete(self) -> None:
|
|
50
|
-
await self._client.delete()
|
|
51
|
-
|
|
52
|
-
@override
|
|
53
|
-
async def list_head(self, *, limit: int | None = None) -> RequestQueueHead:
|
|
54
|
-
return RequestQueueHead.model_validate(
|
|
55
|
-
await self._client.list_head(
|
|
56
|
-
limit=limit,
|
|
57
|
-
),
|
|
58
|
-
)
|
|
59
|
-
|
|
60
|
-
@override
|
|
61
|
-
async def list_and_lock_head(self, *, lock_secs: int, limit: int | None = None) -> RequestQueueHeadWithLocks:
|
|
62
|
-
return RequestQueueHeadWithLocks.model_validate(
|
|
63
|
-
await self._client.list_and_lock_head(
|
|
64
|
-
lock_secs=lock_secs,
|
|
65
|
-
limit=limit,
|
|
66
|
-
)
|
|
67
|
-
)
|
|
68
|
-
|
|
69
|
-
@override
|
|
70
|
-
async def add_request(
|
|
71
|
-
self,
|
|
72
|
-
request: Request,
|
|
73
|
-
*,
|
|
74
|
-
forefront: bool = False,
|
|
75
|
-
) -> ProcessedRequest:
|
|
76
|
-
return ProcessedRequest.model_validate(
|
|
77
|
-
{'id': request.id, 'uniqueKey': request.unique_key}
|
|
78
|
-
| await self._client.add_request(
|
|
79
|
-
request=request.model_dump(
|
|
80
|
-
by_alias=True,
|
|
81
|
-
exclude={
|
|
82
|
-
'id',
|
|
83
|
-
},
|
|
84
|
-
),
|
|
85
|
-
forefront=forefront,
|
|
86
|
-
)
|
|
87
|
-
)
|
|
88
|
-
|
|
89
|
-
@override
|
|
90
|
-
async def get_request(self, request_id: str) -> Request | None:
|
|
91
|
-
result = await self._client.get_request(request_id)
|
|
92
|
-
return Request.model_validate(result) if result else None
|
|
93
|
-
|
|
94
|
-
@override
|
|
95
|
-
async def update_request(
|
|
96
|
-
self,
|
|
97
|
-
request: Request,
|
|
98
|
-
*,
|
|
99
|
-
forefront: bool = False,
|
|
100
|
-
) -> ProcessedRequest:
|
|
101
|
-
return ProcessedRequest.model_validate(
|
|
102
|
-
{'id': request.id, 'uniqueKey': request.unique_key}
|
|
103
|
-
| await self._client.update_request(
|
|
104
|
-
request=request.model_dump(
|
|
105
|
-
by_alias=True,
|
|
106
|
-
),
|
|
107
|
-
forefront=forefront,
|
|
108
|
-
)
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
@override
|
|
112
|
-
async def delete_request(self, request_id: str) -> None:
|
|
113
|
-
await self._client.delete_request(request_id)
|
|
114
|
-
|
|
115
|
-
@override
|
|
116
|
-
async def prolong_request_lock(
|
|
117
|
-
self,
|
|
118
|
-
request_id: str,
|
|
119
|
-
*,
|
|
120
|
-
forefront: bool = False,
|
|
121
|
-
lock_secs: int,
|
|
122
|
-
) -> ProlongRequestLockResponse:
|
|
123
|
-
return ProlongRequestLockResponse.model_validate(
|
|
124
|
-
await self._client.prolong_request_lock(
|
|
125
|
-
request_id=request_id,
|
|
126
|
-
forefront=forefront,
|
|
127
|
-
lock_secs=lock_secs,
|
|
128
|
-
)
|
|
129
|
-
)
|
|
130
|
-
|
|
131
|
-
@override
|
|
132
|
-
async def delete_request_lock(
|
|
133
|
-
self,
|
|
134
|
-
request_id: str,
|
|
135
|
-
*,
|
|
136
|
-
forefront: bool = False,
|
|
137
|
-
) -> None:
|
|
138
|
-
await self._client.delete_request_lock(
|
|
139
|
-
request_id=request_id,
|
|
140
|
-
forefront=forefront,
|
|
141
|
-
)
|
|
142
|
-
|
|
143
|
-
@override
|
|
144
|
-
async def batch_add_requests(
|
|
145
|
-
self,
|
|
146
|
-
requests: Sequence[Request],
|
|
147
|
-
*,
|
|
148
|
-
forefront: bool = False,
|
|
149
|
-
) -> BatchRequestsOperationResponse:
|
|
150
|
-
return BatchRequestsOperationResponse.model_validate(
|
|
151
|
-
await self._client.batch_add_requests(
|
|
152
|
-
requests=[
|
|
153
|
-
r.model_dump(
|
|
154
|
-
by_alias=True,
|
|
155
|
-
exclude={
|
|
156
|
-
'id',
|
|
157
|
-
},
|
|
158
|
-
)
|
|
159
|
-
for r in requests
|
|
160
|
-
],
|
|
161
|
-
forefront=forefront,
|
|
162
|
-
)
|
|
163
|
-
)
|
|
164
|
-
|
|
165
|
-
@override
|
|
166
|
-
async def batch_delete_requests(self, requests: list[Request]) -> BatchRequestsOperationResponse:
|
|
167
|
-
return BatchRequestsOperationResponse.model_validate(
|
|
168
|
-
await self._client.batch_delete_requests(
|
|
169
|
-
requests=[
|
|
170
|
-
r.model_dump(
|
|
171
|
-
by_alias=True,
|
|
172
|
-
)
|
|
173
|
-
for r in requests
|
|
174
|
-
],
|
|
175
|
-
)
|
|
176
|
-
)
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
4
|
-
|
|
5
|
-
from typing_extensions import override
|
|
6
|
-
|
|
7
|
-
from crawlee.storage_clients._base import RequestQueueCollectionClient as BaseRequestQueueCollectionClient
|
|
8
|
-
from crawlee.storage_clients.models import RequestQueueListPage, RequestQueueMetadata
|
|
9
|
-
|
|
10
|
-
if TYPE_CHECKING:
|
|
11
|
-
from apify_client.clients import RequestQueueCollectionClientAsync
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class RequestQueueCollectionClient(BaseRequestQueueCollectionClient):
|
|
15
|
-
"""Request queue collection resource client implementation based on the Apify platform storage."""
|
|
16
|
-
|
|
17
|
-
def __init__(self, apify_request_queue_collection_client: RequestQueueCollectionClientAsync) -> None:
|
|
18
|
-
self._client = apify_request_queue_collection_client
|
|
19
|
-
|
|
20
|
-
@override
|
|
21
|
-
async def get_or_create(
|
|
22
|
-
self,
|
|
23
|
-
*,
|
|
24
|
-
id: str | None = None,
|
|
25
|
-
name: str | None = None,
|
|
26
|
-
schema: dict | None = None,
|
|
27
|
-
) -> RequestQueueMetadata:
|
|
28
|
-
return RequestQueueMetadata.model_validate(
|
|
29
|
-
{'resourceDirectory': ''}
|
|
30
|
-
| await self._client.get_or_create(
|
|
31
|
-
name=id if id is not None else name,
|
|
32
|
-
)
|
|
33
|
-
)
|
|
34
|
-
|
|
35
|
-
@override
|
|
36
|
-
async def list(
|
|
37
|
-
self,
|
|
38
|
-
*,
|
|
39
|
-
unnamed: bool = False,
|
|
40
|
-
limit: int | None = None,
|
|
41
|
-
offset: int | None = None,
|
|
42
|
-
desc: bool = False,
|
|
43
|
-
) -> RequestQueueListPage:
|
|
44
|
-
return RequestQueueListPage.model_validate(
|
|
45
|
-
await self._client.list(
|
|
46
|
-
unnamed=unnamed,
|
|
47
|
-
limit=limit,
|
|
48
|
-
offset=offset,
|
|
49
|
-
desc=desc,
|
|
50
|
-
)
|
|
51
|
-
)
|
apify-2.7.3.dist-info/RECORD
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
apify/__init__.py,sha256=HpgKg2FZWJuSPfDygzJ62psylhw4NN4tKFnoYUIhcd4,838
|
|
2
|
-
apify/_actor.py,sha256=nEuSeGjBIV_IkdFGCVxGKItzVasM2BzyVP2SYvzavvI,51882
|
|
3
|
-
apify/_charging.py,sha256=fFCwg2NL6qtGvnyddp8a-eyhsiaO8-vPEGQMXZ-ZUcU,12225
|
|
4
|
-
apify/_configuration.py,sha256=qICQYEn3yxPy0rRgDfXcdVFqNm27m-a-JqpnNWO2g64,12179
|
|
5
|
-
apify/_consts.py,sha256=CjhyEJ4Mi0lcIrzfqz8dN7nPJWGjCeBrrXQy1PZ6zRI,440
|
|
6
|
-
apify/_crypto.py,sha256=UICTbzjhIKV74biisdtr3kx4c9AO-cKGJnP-XlSTe4E,6960
|
|
7
|
-
apify/_models.py,sha256=-Y0rljBJWxMMCp8iDCTG4UV3bEvNZzp-kx2SYbPfeIY,7919
|
|
8
|
-
apify/_platform_event_manager.py,sha256=igi9dRTfB7t0mRBM1bCfzMh7RBbr5adrJ0iRymUQ8S8,7990
|
|
9
|
-
apify/_proxy_configuration.py,sha256=eMeYr9mr2ITURBmgZw4Cel9kfkiLhdaxE47EDh-3okM,13091
|
|
10
|
-
apify/_utils.py,sha256=7YL7VhmNND0XARsXtPQWWQCyK4825n-ZVB8n9cgWm0A,1838
|
|
11
|
-
apify/log.py,sha256=j-E4t-WeA93bc1NCQRG8sTntehQCiiN8ia-MdQe3_Ts,1291
|
|
12
|
-
apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
-
apify/apify_storage_client/__init__.py,sha256=-UbR68bFsDR6ln8OFs4t50eqcnY36hujO-SeOt-KmcA,114
|
|
14
|
-
apify/apify_storage_client/_apify_storage_client.py,sha256=qeWYsEQGeyyhJzS9TZTQFNqdSl8JzHz_4_HDKGY4I_Y,2736
|
|
15
|
-
apify/apify_storage_client/_dataset_client.py,sha256=9RxxhrJMic5QRJn2Vl4J-FnSlEigIpYW5Z_2B1dcRzM,5597
|
|
16
|
-
apify/apify_storage_client/_dataset_collection_client.py,sha256=gf5skMTkfpGhEscRy5bgo13vznxGZrSd7w9Ivh3Usyc,1516
|
|
17
|
-
apify/apify_storage_client/_key_value_store_client.py,sha256=OCFUAW0o-8KQvUpL8zmlZrpU3yRmDKdsO2529H2v40I,4002
|
|
18
|
-
apify/apify_storage_client/_key_value_store_collection_client.py,sha256=zjsbRW4zjme6dIzxxlHyCW3voBA5489MUhdjl5YMaro,1596
|
|
19
|
-
apify/apify_storage_client/_request_queue_client.py,sha256=cNMhXz85s1ZtjLpVqkduYl1y6o9QyNdcIGoy6ccD-h0,5178
|
|
20
|
-
apify/apify_storage_client/_request_queue_collection_client.py,sha256=MTLM2cG0txAe3cSjkGbXyq2Ek0R7wlsMbGGULmQGD3I,1603
|
|
21
|
-
apify/apify_storage_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
|
-
apify/scrapy/__init__.py,sha256=m2a0ts_JY9xJkBy4JU5mV8PJqjA3GGKLXBFu4nl-n-A,1048
|
|
23
|
-
apify/scrapy/_actor_runner.py,sha256=rXWSnlQWGskDUH8PtLCv5SkOIx4AiVa4QbCYeCett5c,938
|
|
24
|
-
apify/scrapy/_async_thread.py,sha256=8xif_fWce7vaMLuDc-XuDzZlHbCI-NY61YXdP2P27QY,4753
|
|
25
|
-
apify/scrapy/_logging_config.py,sha256=nhlZxS3rdg0j__2AeTrQTjdtUVkd22URnes8sWasI-M,1934
|
|
26
|
-
apify/scrapy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
27
|
-
apify/scrapy/requests.py,sha256=vZEU1IwNotCkqZ-b-LfM15iAyr1LnZ_fF8oWyMFVVFI,6553
|
|
28
|
-
apify/scrapy/scheduler.py,sha256=-r1wZjMmeRDPxZKGHO-EYDYpGdDgSPAdNgMFViqUK8E,6019
|
|
29
|
-
apify/scrapy/utils.py,sha256=Ssfa-P9-g9XYP1suDce6dQ8ta7PfijiPoMl2iplE6Ow,2126
|
|
30
|
-
apify/scrapy/extensions/__init__.py,sha256=cVQ8CCtOsJsRP28YKZWSUsi4FBwxI-yPJRNSXPFSa_o,98
|
|
31
|
-
apify/scrapy/extensions/_httpcache.py,sha256=OOz3eia6LRHakLcfS3GGSlVEOV83BDbAUhCg9GApXvU,8771
|
|
32
|
-
apify/scrapy/middlewares/__init__.py,sha256=tfW-d3WFWLeNEjL8fTmon6NwgD-OXx1Bw2fBdU-wPy4,114
|
|
33
|
-
apify/scrapy/middlewares/apify_proxy.py,sha256=CDAOXS3bcVDZHM3B0GvhXbxEikMIadLF_0P73WL_nI4,5550
|
|
34
|
-
apify/scrapy/middlewares/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
35
|
-
apify/scrapy/pipelines/__init__.py,sha256=GWPeLN_Zwj8vRBWtXW6DaxdB7mvyQ7Jw5Tz1ccgWlZI,119
|
|
36
|
-
apify/scrapy/pipelines/actor_dataset_push.py,sha256=XUUyznQTD-E3wYUUFt2WAOnWhbnRrY0WuedlfYfYhDI,846
|
|
37
|
-
apify/scrapy/pipelines/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
38
|
-
apify/storages/__init__.py,sha256=FW-z6ubuPnHGM-Wp15T8mR5q6lnpDGrCW-IkgZd5L30,177
|
|
39
|
-
apify/storages/_request_list.py,sha256=FCC4X2MX2V8vLZBCUi5Q1qg9w62y9UkF4ptOqyPVhG8,6052
|
|
40
|
-
apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
|
-
apify-2.7.3.dist-info/METADATA,sha256=4z_Jhxs3dbwcl24FvTuk_vtMof4pnjCPjlvMxf-Zp0M,21768
|
|
42
|
-
apify-2.7.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
43
|
-
apify-2.7.3.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
|
|
44
|
-
apify-2.7.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|