apify 2.7.2__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of apify might be problematic. Click here for more details.

Files changed (51) hide show
  1. apify/_actor.py +194 -126
  2. apify/_charging.py +34 -9
  3. apify/_configuration.py +79 -6
  4. apify/_crypto.py +0 -6
  5. apify/_models.py +7 -7
  6. apify/_proxy_configuration.py +10 -10
  7. apify/_utils.py +25 -2
  8. apify/events/__init__.py +5 -0
  9. apify/events/_apify_event_manager.py +140 -0
  10. apify/events/_types.py +102 -0
  11. apify/log.py +0 -9
  12. apify/request_loaders/__init__.py +18 -0
  13. apify/{storages/_request_list.py → request_loaders/_apify_request_list.py} +25 -18
  14. apify/request_loaders/py.typed +0 -0
  15. apify/scrapy/_logging_config.py +1 -4
  16. apify/scrapy/extensions/_httpcache.py +9 -5
  17. apify/scrapy/requests.py +3 -3
  18. apify/scrapy/scheduler.py +8 -5
  19. apify/storage_clients/__init__.py +12 -0
  20. apify/storage_clients/_apify/__init__.py +11 -0
  21. apify/storage_clients/_apify/_dataset_client.py +328 -0
  22. apify/storage_clients/_apify/_key_value_store_client.py +265 -0
  23. apify/storage_clients/_apify/_models.py +131 -0
  24. apify/storage_clients/_apify/_request_queue_client.py +327 -0
  25. apify/storage_clients/_apify/_request_queue_shared_client.py +527 -0
  26. apify/storage_clients/_apify/_request_queue_single_client.py +399 -0
  27. apify/storage_clients/_apify/_storage_client.py +106 -0
  28. apify/storage_clients/_apify/_utils.py +194 -0
  29. apify/storage_clients/_apify/py.typed +0 -0
  30. apify/storage_clients/_file_system/__init__.py +2 -0
  31. apify/storage_clients/_file_system/_key_value_store_client.py +57 -0
  32. apify/storage_clients/_file_system/_storage_client.py +41 -0
  33. apify/storage_clients/_smart_apify/__init__.py +1 -0
  34. apify/storage_clients/_smart_apify/_storage_client.py +117 -0
  35. apify/storage_clients/py.typed +0 -0
  36. apify/storages/__init__.py +1 -3
  37. {apify-2.7.2.dist-info → apify-3.0.0.dist-info}/METADATA +25 -9
  38. apify-3.0.0.dist-info/RECORD +57 -0
  39. apify/_platform_event_manager.py +0 -231
  40. apify/apify_storage_client/__init__.py +0 -3
  41. apify/apify_storage_client/_apify_storage_client.py +0 -72
  42. apify/apify_storage_client/_dataset_client.py +0 -190
  43. apify/apify_storage_client/_dataset_collection_client.py +0 -51
  44. apify/apify_storage_client/_key_value_store_client.py +0 -109
  45. apify/apify_storage_client/_key_value_store_collection_client.py +0 -51
  46. apify/apify_storage_client/_request_queue_client.py +0 -176
  47. apify/apify_storage_client/_request_queue_collection_client.py +0 -51
  48. apify-2.7.2.dist-info/RECORD +0 -44
  49. /apify/{apify_storage_client → events}/py.typed +0 -0
  50. {apify-2.7.2.dist-info → apify-3.0.0.dist-info}/WHEEL +0 -0
  51. {apify-2.7.2.dist-info → apify-3.0.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,51 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from typing import TYPE_CHECKING
4
-
5
- from typing_extensions import override
6
-
7
- from crawlee.storage_clients._base import KeyValueStoreCollectionClient as BaseKeyValueStoreCollectionClient
8
- from crawlee.storage_clients.models import KeyValueStoreListPage, KeyValueStoreMetadata
9
-
10
- if TYPE_CHECKING:
11
- from apify_client.clients import KeyValueStoreCollectionClientAsync
12
-
13
-
14
- class KeyValueStoreCollectionClient(BaseKeyValueStoreCollectionClient):
15
- """Key-value store collection resource client implementation based on the Apify platform storage."""
16
-
17
- def __init__(self, apify_dataset_collection_client: KeyValueStoreCollectionClientAsync) -> None:
18
- self._client = apify_dataset_collection_client
19
-
20
- @override
21
- async def get_or_create(
22
- self,
23
- *,
24
- id: str | None = None,
25
- name: str | None = None,
26
- schema: dict | None = None,
27
- ) -> KeyValueStoreMetadata:
28
- return KeyValueStoreMetadata.model_validate(
29
- await self._client.get_or_create(
30
- name=id if id is not None else name,
31
- schema=schema,
32
- )
33
- )
34
-
35
- @override
36
- async def list(
37
- self,
38
- *,
39
- unnamed: bool = False,
40
- limit: int | None = None,
41
- offset: int | None = None,
42
- desc: bool = False,
43
- ) -> KeyValueStoreListPage:
44
- return KeyValueStoreListPage.model_validate(
45
- await self._client.list(
46
- unnamed=unnamed,
47
- limit=limit,
48
- offset=offset,
49
- desc=desc,
50
- )
51
- )
@@ -1,176 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from typing import TYPE_CHECKING
4
-
5
- from typing_extensions import override
6
-
7
- from crawlee import Request
8
- from crawlee.storage_clients._base import RequestQueueClient as BaseRequestQueueClient
9
- from crawlee.storage_clients.models import (
10
- BatchRequestsOperationResponse,
11
- ProcessedRequest,
12
- ProlongRequestLockResponse,
13
- RequestQueueHead,
14
- RequestQueueHeadWithLocks,
15
- RequestQueueMetadata,
16
- )
17
-
18
- if TYPE_CHECKING:
19
- from collections.abc import Sequence
20
-
21
- from apify_client.clients import RequestQueueClientAsync
22
-
23
-
24
- class RequestQueueClient(BaseRequestQueueClient):
25
- """Request queue resource client implementation based on the Apify platform storage."""
26
-
27
- def __init__(self, apify_request_queue_client: RequestQueueClientAsync) -> None:
28
- self._client = apify_request_queue_client
29
-
30
- @override
31
- async def get(self) -> RequestQueueMetadata | None:
32
- result = await self._client.get()
33
- return RequestQueueMetadata.model_validate({'resourceDirectory': ''} | result) if result else None
34
-
35
- @override
36
- async def update(
37
- self,
38
- *,
39
- name: str | None = None,
40
- ) -> RequestQueueMetadata:
41
- return RequestQueueMetadata.model_validate(
42
- {'resourceDirectory': ''}
43
- | await self._client.update(
44
- name=name,
45
- )
46
- )
47
-
48
- @override
49
- async def delete(self) -> None:
50
- await self._client.delete()
51
-
52
- @override
53
- async def list_head(self, *, limit: int | None = None) -> RequestQueueHead:
54
- return RequestQueueHead.model_validate(
55
- await self._client.list_head(
56
- limit=limit,
57
- ),
58
- )
59
-
60
- @override
61
- async def list_and_lock_head(self, *, lock_secs: int, limit: int | None = None) -> RequestQueueHeadWithLocks:
62
- return RequestQueueHeadWithLocks.model_validate(
63
- await self._client.list_and_lock_head(
64
- lock_secs=lock_secs,
65
- limit=limit,
66
- )
67
- )
68
-
69
- @override
70
- async def add_request(
71
- self,
72
- request: Request,
73
- *,
74
- forefront: bool = False,
75
- ) -> ProcessedRequest:
76
- return ProcessedRequest.model_validate(
77
- {'id': request.id, 'uniqueKey': request.unique_key}
78
- | await self._client.add_request(
79
- request=request.model_dump(
80
- by_alias=True,
81
- exclude={
82
- 'id',
83
- },
84
- ),
85
- forefront=forefront,
86
- )
87
- )
88
-
89
- @override
90
- async def get_request(self, request_id: str) -> Request | None:
91
- result = await self._client.get_request(request_id)
92
- return Request.model_validate(result) if result else None
93
-
94
- @override
95
- async def update_request(
96
- self,
97
- request: Request,
98
- *,
99
- forefront: bool = False,
100
- ) -> ProcessedRequest:
101
- return ProcessedRequest.model_validate(
102
- {'id': request.id, 'uniqueKey': request.unique_key}
103
- | await self._client.update_request(
104
- request=request.model_dump(
105
- by_alias=True,
106
- ),
107
- forefront=forefront,
108
- )
109
- )
110
-
111
- @override
112
- async def delete_request(self, request_id: str) -> None:
113
- await self._client.delete_request(request_id)
114
-
115
- @override
116
- async def prolong_request_lock(
117
- self,
118
- request_id: str,
119
- *,
120
- forefront: bool = False,
121
- lock_secs: int,
122
- ) -> ProlongRequestLockResponse:
123
- return ProlongRequestLockResponse.model_validate(
124
- await self._client.prolong_request_lock(
125
- request_id=request_id,
126
- forefront=forefront,
127
- lock_secs=lock_secs,
128
- )
129
- )
130
-
131
- @override
132
- async def delete_request_lock(
133
- self,
134
- request_id: str,
135
- *,
136
- forefront: bool = False,
137
- ) -> None:
138
- await self._client.delete_request_lock(
139
- request_id=request_id,
140
- forefront=forefront,
141
- )
142
-
143
- @override
144
- async def batch_add_requests(
145
- self,
146
- requests: Sequence[Request],
147
- *,
148
- forefront: bool = False,
149
- ) -> BatchRequestsOperationResponse:
150
- return BatchRequestsOperationResponse.model_validate(
151
- await self._client.batch_add_requests(
152
- requests=[
153
- r.model_dump(
154
- by_alias=True,
155
- exclude={
156
- 'id',
157
- },
158
- )
159
- for r in requests
160
- ],
161
- forefront=forefront,
162
- )
163
- )
164
-
165
- @override
166
- async def batch_delete_requests(self, requests: list[Request]) -> BatchRequestsOperationResponse:
167
- return BatchRequestsOperationResponse.model_validate(
168
- await self._client.batch_delete_requests(
169
- requests=[
170
- r.model_dump(
171
- by_alias=True,
172
- )
173
- for r in requests
174
- ],
175
- )
176
- )
@@ -1,51 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from typing import TYPE_CHECKING
4
-
5
- from typing_extensions import override
6
-
7
- from crawlee.storage_clients._base import RequestQueueCollectionClient as BaseRequestQueueCollectionClient
8
- from crawlee.storage_clients.models import RequestQueueListPage, RequestQueueMetadata
9
-
10
- if TYPE_CHECKING:
11
- from apify_client.clients import RequestQueueCollectionClientAsync
12
-
13
-
14
- class RequestQueueCollectionClient(BaseRequestQueueCollectionClient):
15
- """Request queue collection resource client implementation based on the Apify platform storage."""
16
-
17
- def __init__(self, apify_request_queue_collection_client: RequestQueueCollectionClientAsync) -> None:
18
- self._client = apify_request_queue_collection_client
19
-
20
- @override
21
- async def get_or_create(
22
- self,
23
- *,
24
- id: str | None = None,
25
- name: str | None = None,
26
- schema: dict | None = None,
27
- ) -> RequestQueueMetadata:
28
- return RequestQueueMetadata.model_validate(
29
- {'resourceDirectory': ''}
30
- | await self._client.get_or_create(
31
- name=id if id is not None else name,
32
- )
33
- )
34
-
35
- @override
36
- async def list(
37
- self,
38
- *,
39
- unnamed: bool = False,
40
- limit: int | None = None,
41
- offset: int | None = None,
42
- desc: bool = False,
43
- ) -> RequestQueueListPage:
44
- return RequestQueueListPage.model_validate(
45
- await self._client.list(
46
- unnamed=unnamed,
47
- limit=limit,
48
- offset=offset,
49
- desc=desc,
50
- )
51
- )
@@ -1,44 +0,0 @@
1
- apify/__init__.py,sha256=HpgKg2FZWJuSPfDygzJ62psylhw4NN4tKFnoYUIhcd4,838
2
- apify/_actor.py,sha256=nEuSeGjBIV_IkdFGCVxGKItzVasM2BzyVP2SYvzavvI,51882
3
- apify/_charging.py,sha256=fFCwg2NL6qtGvnyddp8a-eyhsiaO8-vPEGQMXZ-ZUcU,12225
4
- apify/_configuration.py,sha256=AVztnlaBkHxBs0VkLIUhFHWwvlgHY-koMNUc0aqw9ZI,11908
5
- apify/_consts.py,sha256=CjhyEJ4Mi0lcIrzfqz8dN7nPJWGjCeBrrXQy1PZ6zRI,440
6
- apify/_crypto.py,sha256=UICTbzjhIKV74biisdtr3kx4c9AO-cKGJnP-XlSTe4E,6960
7
- apify/_models.py,sha256=-Y0rljBJWxMMCp8iDCTG4UV3bEvNZzp-kx2SYbPfeIY,7919
8
- apify/_platform_event_manager.py,sha256=igi9dRTfB7t0mRBM1bCfzMh7RBbr5adrJ0iRymUQ8S8,7990
9
- apify/_proxy_configuration.py,sha256=eMeYr9mr2ITURBmgZw4Cel9kfkiLhdaxE47EDh-3okM,13091
10
- apify/_utils.py,sha256=7YL7VhmNND0XARsXtPQWWQCyK4825n-ZVB8n9cgWm0A,1838
11
- apify/log.py,sha256=j-E4t-WeA93bc1NCQRG8sTntehQCiiN8ia-MdQe3_Ts,1291
12
- apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
13
- apify/apify_storage_client/__init__.py,sha256=-UbR68bFsDR6ln8OFs4t50eqcnY36hujO-SeOt-KmcA,114
14
- apify/apify_storage_client/_apify_storage_client.py,sha256=qeWYsEQGeyyhJzS9TZTQFNqdSl8JzHz_4_HDKGY4I_Y,2736
15
- apify/apify_storage_client/_dataset_client.py,sha256=9RxxhrJMic5QRJn2Vl4J-FnSlEigIpYW5Z_2B1dcRzM,5597
16
- apify/apify_storage_client/_dataset_collection_client.py,sha256=gf5skMTkfpGhEscRy5bgo13vznxGZrSd7w9Ivh3Usyc,1516
17
- apify/apify_storage_client/_key_value_store_client.py,sha256=OCFUAW0o-8KQvUpL8zmlZrpU3yRmDKdsO2529H2v40I,4002
18
- apify/apify_storage_client/_key_value_store_collection_client.py,sha256=zjsbRW4zjme6dIzxxlHyCW3voBA5489MUhdjl5YMaro,1596
19
- apify/apify_storage_client/_request_queue_client.py,sha256=cNMhXz85s1ZtjLpVqkduYl1y6o9QyNdcIGoy6ccD-h0,5178
20
- apify/apify_storage_client/_request_queue_collection_client.py,sha256=MTLM2cG0txAe3cSjkGbXyq2Ek0R7wlsMbGGULmQGD3I,1603
21
- apify/apify_storage_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
22
- apify/scrapy/__init__.py,sha256=m2a0ts_JY9xJkBy4JU5mV8PJqjA3GGKLXBFu4nl-n-A,1048
23
- apify/scrapy/_actor_runner.py,sha256=rXWSnlQWGskDUH8PtLCv5SkOIx4AiVa4QbCYeCett5c,938
24
- apify/scrapy/_async_thread.py,sha256=8xif_fWce7vaMLuDc-XuDzZlHbCI-NY61YXdP2P27QY,4753
25
- apify/scrapy/_logging_config.py,sha256=nhlZxS3rdg0j__2AeTrQTjdtUVkd22URnes8sWasI-M,1934
26
- apify/scrapy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
27
- apify/scrapy/requests.py,sha256=vZEU1IwNotCkqZ-b-LfM15iAyr1LnZ_fF8oWyMFVVFI,6553
28
- apify/scrapy/scheduler.py,sha256=-r1wZjMmeRDPxZKGHO-EYDYpGdDgSPAdNgMFViqUK8E,6019
29
- apify/scrapy/utils.py,sha256=Ssfa-P9-g9XYP1suDce6dQ8ta7PfijiPoMl2iplE6Ow,2126
30
- apify/scrapy/extensions/__init__.py,sha256=cVQ8CCtOsJsRP28YKZWSUsi4FBwxI-yPJRNSXPFSa_o,98
31
- apify/scrapy/extensions/_httpcache.py,sha256=OOz3eia6LRHakLcfS3GGSlVEOV83BDbAUhCg9GApXvU,8771
32
- apify/scrapy/middlewares/__init__.py,sha256=tfW-d3WFWLeNEjL8fTmon6NwgD-OXx1Bw2fBdU-wPy4,114
33
- apify/scrapy/middlewares/apify_proxy.py,sha256=CDAOXS3bcVDZHM3B0GvhXbxEikMIadLF_0P73WL_nI4,5550
34
- apify/scrapy/middlewares/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
35
- apify/scrapy/pipelines/__init__.py,sha256=GWPeLN_Zwj8vRBWtXW6DaxdB7mvyQ7Jw5Tz1ccgWlZI,119
36
- apify/scrapy/pipelines/actor_dataset_push.py,sha256=XUUyznQTD-E3wYUUFt2WAOnWhbnRrY0WuedlfYfYhDI,846
37
- apify/scrapy/pipelines/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
38
- apify/storages/__init__.py,sha256=FW-z6ubuPnHGM-Wp15T8mR5q6lnpDGrCW-IkgZd5L30,177
39
- apify/storages/_request_list.py,sha256=FCC4X2MX2V8vLZBCUi5Q1qg9w62y9UkF4ptOqyPVhG8,6052
40
- apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
41
- apify-2.7.2.dist-info/METADATA,sha256=DQzkuDhEVEc5ju3CENFV-wsP_U5iXB7KEU8TTs5k57g,21768
42
- apify-2.7.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
43
- apify-2.7.2.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
44
- apify-2.7.2.dist-info/RECORD,,
File without changes
File without changes