scrapely-client-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of scrapely-client-python might be problematic. Click here for more details.
- scrapely_client/__init__.py +30 -0
- scrapely_client/_charging.py +186 -0
- scrapely_client/_client_registry.py +136 -0
- scrapely_client/_consts.py +96 -0
- scrapely_client/_docs.py +37 -0
- scrapely_client/_literals.py +539 -0
- scrapely_client/_logging.py +180 -0
- scrapely_client/_models.py +4172 -0
- scrapely_client/_pagination.py +220 -0
- scrapely_client/_resource_clients/__init__.py +84 -0
- scrapely_client/_resource_clients/_resource_client.py +567 -0
- scrapely_client/_resource_clients/actor.py +1042 -0
- scrapely_client/_resource_clients/actor_collection.py +377 -0
- scrapely_client/_resource_clients/actor_env_var.py +154 -0
- scrapely_client/_resource_clients/actor_env_var_collection.py +165 -0
- scrapely_client/_resource_clients/actor_version.py +254 -0
- scrapely_client/_resource_clients/actor_version_collection.py +232 -0
- scrapely_client/_resource_clients/build.py +244 -0
- scrapely_client/_resource_clients/build_collection.py +164 -0
- scrapely_client/_resource_clients/dataset.py +1151 -0
- scrapely_client/_resource_clients/dataset_collection.py +244 -0
- scrapely_client/_resource_clients/key_value_store.py +906 -0
- scrapely_client/_resource_clients/key_value_store_collection.py +243 -0
- scrapely_client/_resource_clients/log.py +225 -0
- scrapely_client/_resource_clients/request_queue.py +1167 -0
- scrapely_client/_resource_clients/request_queue_collection.py +240 -0
- scrapely_client/_resource_clients/run.py +875 -0
- scrapely_client/_resource_clients/run_collection.py +226 -0
- scrapely_client/_resource_clients/schedule.py +243 -0
- scrapely_client/_resource_clients/schedule_collection.py +260 -0
- scrapely_client/_resource_clients/store_collection.py +242 -0
- scrapely_client/_resource_clients/task.py +793 -0
- scrapely_client/_resource_clients/task_collection.py +322 -0
- scrapely_client/_resource_clients/user.py +262 -0
- scrapely_client/_resource_clients/webhook.py +292 -0
- scrapely_client/_resource_clients/webhook_collection.py +281 -0
- scrapely_client/_resource_clients/webhook_dispatch.py +84 -0
- scrapely_client/_resource_clients/webhook_dispatch_collection.py +158 -0
- scrapely_client/_scrapely_client.py +837 -0
- scrapely_client/_statistics.py +29 -0
- scrapely_client/_status_message_watcher.py +197 -0
- scrapely_client/_streamed_log.py +250 -0
- scrapely_client/_typeddicts.py +408 -0
- scrapely_client/_utils/__init__.py +0 -0
- scrapely_client/_utils/crypto.py +87 -0
- scrapely_client/_utils/encoding.py +133 -0
- scrapely_client/_utils/errors.py +33 -0
- scrapely_client/_utils/http.py +137 -0
- scrapely_client/_utils/time.py +32 -0
- scrapely_client/_utils/try_import.py +61 -0
- scrapely_client/errors.py +190 -0
- scrapely_client/http_clients/__init__.py +10 -0
- scrapely_client/http_clients/_base.py +430 -0
- scrapely_client/http_clients/_impit.py +598 -0
- scrapely_client/http_compressors/__init__.py +18 -0
- scrapely_client/http_compressors/_base.py +25 -0
- scrapely_client/http_compressors/_brotli.py +38 -0
- scrapely_client/http_compressors/_gzip.py +38 -0
- scrapely_client/http_compressors/_resolve.py +38 -0
- scrapely_client/py.typed +0 -0
- scrapely_client/types.py +52 -0
- scrapely_client_python-1.0.0.dist-info/METADATA +244 -0
- scrapely_client_python-1.0.0.dist-info/RECORD +65 -0
- scrapely_client_python-1.0.0.dist-info/WHEEL +4 -0
- scrapely_client_python-1.0.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
|
+
|
|
5
|
+
from scrapely_client._docs import docs_group
|
|
6
|
+
from scrapely_client._models import Build, BuildResponse
|
|
7
|
+
from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
|
|
8
|
+
from scrapely_client._utils.http import response_to_dict
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from datetime import timedelta
|
|
12
|
+
|
|
13
|
+
from scrapely_client._resource_clients import LogClient, LogClientAsync
|
|
14
|
+
from scrapely_client.types import Timeout
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@docs_group('Resource clients')
|
|
18
|
+
class BuildClient(ResourceClient):
|
|
19
|
+
"""Sub-client for managing a specific Actor build.
|
|
20
|
+
|
|
21
|
+
Provides methods to manage a specific Actor build, e.g. get it, abort it, or wait for it to finish. Obtain an
|
|
22
|
+
instance via an appropriate method on the `ApifyClient` class.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
*,
|
|
28
|
+
resource_id: str,
|
|
29
|
+
resource_path: str = 'actor-builds',
|
|
30
|
+
**kwargs: Any,
|
|
31
|
+
) -> None:
|
|
32
|
+
super().__init__(
|
|
33
|
+
resource_id=resource_id,
|
|
34
|
+
resource_path=resource_path,
|
|
35
|
+
**kwargs,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
def get(self, *, timeout: Timeout = 'short') -> Build | None:
|
|
39
|
+
"""Return information about the Actor build.
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
timeout: Timeout for the API HTTP request.
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
The retrieved Actor build data.
|
|
47
|
+
"""
|
|
48
|
+
result = self._get(timeout=timeout)
|
|
49
|
+
if result is None:
|
|
50
|
+
return None
|
|
51
|
+
return BuildResponse.model_validate(result).data
|
|
52
|
+
|
|
53
|
+
def delete(self, *, timeout: Timeout = 'short') -> None:
|
|
54
|
+
"""Delete the build.
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
timeout: Timeout for the API HTTP request.
|
|
59
|
+
"""
|
|
60
|
+
self._delete(timeout=timeout)
|
|
61
|
+
|
|
62
|
+
def abort(self, *, timeout: Timeout = 'short') -> Build:
|
|
63
|
+
"""Abort the Actor build which is starting or currently running and return its details.
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
timeout: Timeout for the API HTTP request.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
The data of the aborted Actor build.
|
|
71
|
+
"""
|
|
72
|
+
response = self._http_client.call(
|
|
73
|
+
url=self._build_url('abort'),
|
|
74
|
+
method='POST',
|
|
75
|
+
params=self._build_params(),
|
|
76
|
+
timeout=timeout,
|
|
77
|
+
)
|
|
78
|
+
result = response_to_dict(response)
|
|
79
|
+
return BuildResponse.model_validate(result).data
|
|
80
|
+
|
|
81
|
+
def get_open_api_definition(self, *, timeout: Timeout = 'medium') -> dict:
|
|
82
|
+
"""Return OpenAPI definition of the Actor's build.
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
timeout: Timeout for the API HTTP request.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
OpenAPI definition of the Actor's build.
|
|
90
|
+
"""
|
|
91
|
+
response = self._http_client.call(
|
|
92
|
+
url=self._build_url('openapi.json'),
|
|
93
|
+
method='GET',
|
|
94
|
+
timeout=timeout,
|
|
95
|
+
)
|
|
96
|
+
return response_to_dict(response)
|
|
97
|
+
|
|
98
|
+
def wait_for_finish(
|
|
99
|
+
self, *, wait_duration: timedelta | None = None, timeout: Timeout = 'no_timeout'
|
|
100
|
+
) -> Build | None:
|
|
101
|
+
"""Wait synchronously until the build finishes or the server times out.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
wait_duration: How long does the client wait for build to finish. None for indefinite.
|
|
105
|
+
timeout: Timeout for the API HTTP request.
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
The Actor build data. If the status on the object is not one of the terminal statuses (SUCCEEDED, FAILED,
|
|
109
|
+
TIMED-OUT, ABORTED), then the build has not yet finished.
|
|
110
|
+
"""
|
|
111
|
+
result = self._wait_for_finish(
|
|
112
|
+
url=self._build_url(),
|
|
113
|
+
params=self._build_params(),
|
|
114
|
+
wait_duration=wait_duration,
|
|
115
|
+
timeout=timeout,
|
|
116
|
+
)
|
|
117
|
+
return Build.model_validate(result) if result is not None else None
|
|
118
|
+
|
|
119
|
+
def log(self) -> LogClient:
|
|
120
|
+
"""Get the client for the log of the Actor build.
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
Returns:
|
|
124
|
+
A client allowing access to the log of this Actor build.
|
|
125
|
+
"""
|
|
126
|
+
return self._client_registry.log_client(
|
|
127
|
+
resource_path='log',
|
|
128
|
+
**self._base_client_kwargs,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@docs_group('Resource clients')
|
|
133
|
+
class BuildClientAsync(ResourceClientAsync):
|
|
134
|
+
"""Sub-client for managing a specific Actor build.
|
|
135
|
+
|
|
136
|
+
Provides methods to manage a specific Actor build, e.g. get it, abort it, or wait for it to finish. Obtain an
|
|
137
|
+
instance via an appropriate method on the `ApifyClientAsync` class.
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
def __init__(
|
|
141
|
+
self,
|
|
142
|
+
*,
|
|
143
|
+
resource_id: str,
|
|
144
|
+
resource_path: str = 'actor-builds',
|
|
145
|
+
**kwargs: Any,
|
|
146
|
+
) -> None:
|
|
147
|
+
super().__init__(
|
|
148
|
+
resource_id=resource_id,
|
|
149
|
+
resource_path=resource_path,
|
|
150
|
+
**kwargs,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
async def get(self, *, timeout: Timeout = 'short') -> Build | None:
|
|
154
|
+
"""Return information about the Actor build.
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
Args:
|
|
158
|
+
timeout: Timeout for the API HTTP request.
|
|
159
|
+
|
|
160
|
+
Returns:
|
|
161
|
+
The retrieved Actor build data.
|
|
162
|
+
"""
|
|
163
|
+
result = await self._get(timeout=timeout)
|
|
164
|
+
if result is None:
|
|
165
|
+
return None
|
|
166
|
+
return BuildResponse.model_validate(result).data
|
|
167
|
+
|
|
168
|
+
async def abort(self, *, timeout: Timeout = 'short') -> Build:
|
|
169
|
+
"""Abort the Actor build which is starting or currently running and return its details.
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
Args:
|
|
173
|
+
timeout: Timeout for the API HTTP request.
|
|
174
|
+
|
|
175
|
+
Returns:
|
|
176
|
+
The data of the aborted Actor build.
|
|
177
|
+
"""
|
|
178
|
+
response = await self._http_client.call(
|
|
179
|
+
url=self._build_url('abort'),
|
|
180
|
+
method='POST',
|
|
181
|
+
params=self._build_params(),
|
|
182
|
+
timeout=timeout,
|
|
183
|
+
)
|
|
184
|
+
result = response_to_dict(response)
|
|
185
|
+
return BuildResponse.model_validate(result).data
|
|
186
|
+
|
|
187
|
+
async def delete(self, *, timeout: Timeout = 'short') -> None:
|
|
188
|
+
"""Delete the build.
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
timeout: Timeout for the API HTTP request.
|
|
193
|
+
"""
|
|
194
|
+
await self._delete(timeout=timeout)
|
|
195
|
+
|
|
196
|
+
async def get_open_api_definition(self, *, timeout: Timeout = 'medium') -> dict:
|
|
197
|
+
"""Return OpenAPI definition of the Actor's build.
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
Args:
|
|
201
|
+
timeout: Timeout for the API HTTP request.
|
|
202
|
+
|
|
203
|
+
Returns:
|
|
204
|
+
OpenAPI definition of the Actor's build.
|
|
205
|
+
"""
|
|
206
|
+
response = await self._http_client.call(
|
|
207
|
+
url=self._build_url('openapi.json'),
|
|
208
|
+
method='GET',
|
|
209
|
+
timeout=timeout,
|
|
210
|
+
)
|
|
211
|
+
return response_to_dict(response)
|
|
212
|
+
|
|
213
|
+
async def wait_for_finish(
|
|
214
|
+
self, *, wait_duration: timedelta | None = None, timeout: Timeout = 'no_timeout'
|
|
215
|
+
) -> Build | None:
|
|
216
|
+
"""Wait asynchronously until the build finishes or the server times out.
|
|
217
|
+
|
|
218
|
+
Args:
|
|
219
|
+
wait_duration: How long does the client wait for build to finish. None for indefinite.
|
|
220
|
+
timeout: Timeout for the API HTTP request.
|
|
221
|
+
|
|
222
|
+
Returns:
|
|
223
|
+
The Actor build data. If the status on the object is not one of the terminal statuses (SUCCEEDED, FAILED,
|
|
224
|
+
TIMED-OUT, ABORTED), then the build has not yet finished.
|
|
225
|
+
"""
|
|
226
|
+
result = await self._wait_for_finish(
|
|
227
|
+
url=self._build_url(),
|
|
228
|
+
params=self._build_params(),
|
|
229
|
+
wait_duration=wait_duration,
|
|
230
|
+
timeout=timeout,
|
|
231
|
+
)
|
|
232
|
+
return Build.model_validate(result) if result is not None else None
|
|
233
|
+
|
|
234
|
+
def log(self) -> LogClientAsync:
|
|
235
|
+
"""Get the client for the log of the Actor build.
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
Returns:
|
|
239
|
+
A client allowing access to the log of this Actor build.
|
|
240
|
+
"""
|
|
241
|
+
return self._client_registry.log_client(
|
|
242
|
+
resource_path='log',
|
|
243
|
+
**self._base_client_kwargs,
|
|
244
|
+
)
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
|
+
|
|
5
|
+
from scrapely_client._docs import docs_group
|
|
6
|
+
from scrapely_client._models import ListOfBuilds, ListOfBuildsResponse
|
|
7
|
+
from scrapely_client._pagination import get_items_iterator, get_items_iterator_async
|
|
8
|
+
from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from collections.abc import AsyncIterator, Iterator
|
|
12
|
+
|
|
13
|
+
from scrapely_client._models import BuildShort
|
|
14
|
+
from scrapely_client.types import Timeout
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@docs_group('Resource clients')
|
|
18
|
+
class BuildCollectionClient(ResourceClient):
|
|
19
|
+
"""Sub-client for the Actor build collection.
|
|
20
|
+
|
|
21
|
+
Provides methods to manage Actor builds, e.g. list them. Obtain an instance via an appropriate method on the
|
|
22
|
+
`ApifyClient` class.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
*,
|
|
28
|
+
resource_path: str = 'actor-builds',
|
|
29
|
+
**kwargs: Any,
|
|
30
|
+
) -> None:
|
|
31
|
+
super().__init__(
|
|
32
|
+
resource_path=resource_path,
|
|
33
|
+
**kwargs,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
def list(
|
|
37
|
+
self,
|
|
38
|
+
*,
|
|
39
|
+
limit: int | None = None,
|
|
40
|
+
offset: int | None = None,
|
|
41
|
+
desc: bool | None = None,
|
|
42
|
+
timeout: Timeout = 'medium',
|
|
43
|
+
) -> ListOfBuilds:
|
|
44
|
+
"""List all Actor builds.
|
|
45
|
+
|
|
46
|
+
List all Actor builds, either of a single Actor, or all user's Actors, depending on where this client
|
|
47
|
+
was initialized from.
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
limit: How many builds to retrieve.
|
|
52
|
+
offset: What build to include as first when retrieving the list.
|
|
53
|
+
desc: Whether to sort the builds in descending order based on their start date.
|
|
54
|
+
timeout: Timeout for the API HTTP request.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
The retrieved Actor builds.
|
|
58
|
+
"""
|
|
59
|
+
result = self._list(timeout=timeout, limit=limit, offset=offset, desc=desc)
|
|
60
|
+
return ListOfBuildsResponse.model_validate(result).data
|
|
61
|
+
|
|
62
|
+
def iterate(
|
|
63
|
+
self,
|
|
64
|
+
*,
|
|
65
|
+
limit: int | None = None,
|
|
66
|
+
offset: int | None = None,
|
|
67
|
+
desc: bool | None = None,
|
|
68
|
+
timeout: Timeout = 'medium',
|
|
69
|
+
) -> Iterator[BuildShort]:
|
|
70
|
+
"""Iterate over all Actor builds.
|
|
71
|
+
|
|
72
|
+
Simple `list` does only one API call, possibly not listing all items matching the criteria. This method
|
|
73
|
+
returns an iterator that is capable of making multiple API calls to retrieve all items matching the criteria.
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
limit: How many builds to retrieve.
|
|
78
|
+
offset: What build to include as first when retrieving the list.
|
|
79
|
+
desc: Whether to sort the builds in descending order based on their start date.
|
|
80
|
+
timeout: Timeout for the API HTTP request.
|
|
81
|
+
|
|
82
|
+
Yields:
|
|
83
|
+
The Actor builds matching the specified filters.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfBuilds:
|
|
87
|
+
return self.list(limit=limit, offset=offset, desc=desc, timeout=timeout)
|
|
88
|
+
|
|
89
|
+
return get_items_iterator(_callback, limit=limit, offset=offset)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@docs_group('Resource clients')
|
|
93
|
+
class BuildCollectionClientAsync(ResourceClientAsync):
|
|
94
|
+
"""Sub-client for the Actor build collection.
|
|
95
|
+
|
|
96
|
+
Provides methods to manage Actor builds, e.g. list them. Obtain an instance via an appropriate method on the
|
|
97
|
+
`ApifyClientAsync` class.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
def __init__(
|
|
101
|
+
self,
|
|
102
|
+
*,
|
|
103
|
+
resource_path: str = 'actor-builds',
|
|
104
|
+
**kwargs: Any,
|
|
105
|
+
) -> None:
|
|
106
|
+
super().__init__(
|
|
107
|
+
resource_path=resource_path,
|
|
108
|
+
**kwargs,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
async def list(
|
|
112
|
+
self,
|
|
113
|
+
*,
|
|
114
|
+
limit: int | None = None,
|
|
115
|
+
offset: int | None = None,
|
|
116
|
+
desc: bool | None = None,
|
|
117
|
+
timeout: Timeout = 'medium',
|
|
118
|
+
) -> ListOfBuilds:
|
|
119
|
+
"""List all Actor builds.
|
|
120
|
+
|
|
121
|
+
List all Actor builds, either of a single Actor, or all user's Actors, depending on where this client
|
|
122
|
+
was initialized from.
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
limit: How many builds to retrieve.
|
|
127
|
+
offset: What build to include as first when retrieving the list.
|
|
128
|
+
desc: Whether to sort the builds in descending order based on their start date.
|
|
129
|
+
timeout: Timeout for the API HTTP request.
|
|
130
|
+
|
|
131
|
+
Returns:
|
|
132
|
+
The retrieved Actor builds.
|
|
133
|
+
"""
|
|
134
|
+
result = await self._list(timeout=timeout, limit=limit, offset=offset, desc=desc)
|
|
135
|
+
return ListOfBuildsResponse.model_validate(result).data
|
|
136
|
+
|
|
137
|
+
def iterate(
|
|
138
|
+
self,
|
|
139
|
+
*,
|
|
140
|
+
limit: int | None = None,
|
|
141
|
+
offset: int | None = None,
|
|
142
|
+
desc: bool | None = None,
|
|
143
|
+
timeout: Timeout = 'medium',
|
|
144
|
+
) -> AsyncIterator[BuildShort]:
|
|
145
|
+
"""Iterate over all Actor builds.
|
|
146
|
+
|
|
147
|
+
Simple `list` does only one API call, possibly not listing all items matching the criteria. This method
|
|
148
|
+
returns an iterator that is capable of making multiple API calls to retrieve all items matching the criteria.
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
Args:
|
|
152
|
+
limit: How many builds to retrieve.
|
|
153
|
+
offset: What build to include as first when retrieving the list.
|
|
154
|
+
desc: Whether to sort the builds in descending order based on their start date.
|
|
155
|
+
timeout: Timeout for the API HTTP request.
|
|
156
|
+
|
|
157
|
+
Yields:
|
|
158
|
+
The Actor builds matching the specified filters.
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
async def _callback(*, limit: int | None = None, offset: int | None = None) -> ListOfBuilds:
|
|
162
|
+
return await self.list(limit=limit, offset=offset, desc=desc, timeout=timeout)
|
|
163
|
+
|
|
164
|
+
return get_items_iterator_async(_callback, limit=limit, offset=offset)
|