scrapely-client-python 3.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of scrapely-client-python might be problematic. Click here for more details.
- scrapely_client/__init__.py +30 -0
- scrapely_client/_charging.py +186 -0
- scrapely_client/_client_registry.py +136 -0
- scrapely_client/_consts.py +96 -0
- scrapely_client/_docs.py +37 -0
- scrapely_client/_literals.py +539 -0
- scrapely_client/_logging.py +180 -0
- scrapely_client/_models.py +4172 -0
- scrapely_client/_pagination.py +220 -0
- scrapely_client/_resource_clients/__init__.py +84 -0
- scrapely_client/_resource_clients/_resource_client.py +567 -0
- scrapely_client/_resource_clients/actor.py +1042 -0
- scrapely_client/_resource_clients/actor_collection.py +377 -0
- scrapely_client/_resource_clients/actor_env_var.py +154 -0
- scrapely_client/_resource_clients/actor_env_var_collection.py +165 -0
- scrapely_client/_resource_clients/actor_version.py +254 -0
- scrapely_client/_resource_clients/actor_version_collection.py +232 -0
- scrapely_client/_resource_clients/build.py +244 -0
- scrapely_client/_resource_clients/build_collection.py +164 -0
- scrapely_client/_resource_clients/dataset.py +1151 -0
- scrapely_client/_resource_clients/dataset_collection.py +244 -0
- scrapely_client/_resource_clients/key_value_store.py +906 -0
- scrapely_client/_resource_clients/key_value_store_collection.py +243 -0
- scrapely_client/_resource_clients/log.py +225 -0
- scrapely_client/_resource_clients/request_queue.py +1167 -0
- scrapely_client/_resource_clients/request_queue_collection.py +240 -0
- scrapely_client/_resource_clients/run.py +875 -0
- scrapely_client/_resource_clients/run_collection.py +226 -0
- scrapely_client/_resource_clients/schedule.py +243 -0
- scrapely_client/_resource_clients/schedule_collection.py +260 -0
- scrapely_client/_resource_clients/store_collection.py +242 -0
- scrapely_client/_resource_clients/task.py +793 -0
- scrapely_client/_resource_clients/task_collection.py +322 -0
- scrapely_client/_resource_clients/user.py +262 -0
- scrapely_client/_resource_clients/webhook.py +292 -0
- scrapely_client/_resource_clients/webhook_collection.py +281 -0
- scrapely_client/_resource_clients/webhook_dispatch.py +84 -0
- scrapely_client/_resource_clients/webhook_dispatch_collection.py +158 -0
- scrapely_client/_scrapely_client.py +837 -0
- scrapely_client/_statistics.py +29 -0
- scrapely_client/_status_message_watcher.py +197 -0
- scrapely_client/_streamed_log.py +250 -0
- scrapely_client/_typeddicts.py +408 -0
- scrapely_client/_utils/__init__.py +0 -0
- scrapely_client/_utils/crypto.py +87 -0
- scrapely_client/_utils/encoding.py +133 -0
- scrapely_client/_utils/errors.py +33 -0
- scrapely_client/_utils/http.py +137 -0
- scrapely_client/_utils/time.py +32 -0
- scrapely_client/_utils/try_import.py +61 -0
- scrapely_client/errors.py +190 -0
- scrapely_client/http_clients/__init__.py +10 -0
- scrapely_client/http_clients/_base.py +430 -0
- scrapely_client/http_clients/_impit.py +598 -0
- scrapely_client/http_compressors/__init__.py +18 -0
- scrapely_client/http_compressors/_base.py +25 -0
- scrapely_client/http_compressors/_brotli.py +38 -0
- scrapely_client/http_compressors/_gzip.py +38 -0
- scrapely_client/http_compressors/_resolve.py +38 -0
- scrapely_client/py.typed +0 -0
- scrapely_client/types.py +52 -0
- scrapely_client_python-3.1.3.dist-info/METADATA +269 -0
- scrapely_client_python-3.1.3.dist-info/RECORD +65 -0
- scrapely_client_python-3.1.3.dist-info/WHEEL +4 -0
- scrapely_client_python-3.1.3.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,793 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
|
+
|
|
5
|
+
from scrapely_client._docs import docs_group
|
|
6
|
+
from scrapely_client._models import (
|
|
7
|
+
ActorStandby,
|
|
8
|
+
Run,
|
|
9
|
+
RunResponse,
|
|
10
|
+
Task,
|
|
11
|
+
TaskInput,
|
|
12
|
+
TaskOptions,
|
|
13
|
+
TaskResponse,
|
|
14
|
+
UpdateTaskRequest,
|
|
15
|
+
)
|
|
16
|
+
from scrapely_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
|
|
17
|
+
from scrapely_client._utils.encoding import encode_webhooks_to_base64
|
|
18
|
+
from scrapely_client._utils.http import response_to_dict
|
|
19
|
+
from scrapely_client._utils.time import to_seconds
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from datetime import timedelta
|
|
23
|
+
|
|
24
|
+
from scrapely_client._literals import ActorJobStatus, RunOrigin
|
|
25
|
+
from scrapely_client._resource_clients import (
|
|
26
|
+
RunClient,
|
|
27
|
+
RunClientAsync,
|
|
28
|
+
RunCollectionClient,
|
|
29
|
+
RunCollectionClientAsync,
|
|
30
|
+
WebhookCollectionClient,
|
|
31
|
+
WebhookCollectionClientAsync,
|
|
32
|
+
)
|
|
33
|
+
from scrapely_client._typeddicts import TaskInputDict
|
|
34
|
+
from scrapely_client.types import Timeout, WebhooksList
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@docs_group('Resource clients')
|
|
38
|
+
class TaskClient(ResourceClient):
|
|
39
|
+
"""Sub-client for managing a specific task.
|
|
40
|
+
|
|
41
|
+
Provides methods to manage a specific task, e.g. update it, delete it, or start runs. Obtain an instance via an
|
|
42
|
+
appropriate method on the `ApifyClient` class.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
*,
|
|
48
|
+
resource_id: str,
|
|
49
|
+
resource_path: str = 'actor-tasks',
|
|
50
|
+
**kwargs: Any,
|
|
51
|
+
) -> None:
|
|
52
|
+
super().__init__(resource_id=resource_id, resource_path=resource_path, **kwargs)
|
|
53
|
+
|
|
54
|
+
def get(self, *, timeout: Timeout = 'short') -> Task | None:
|
|
55
|
+
"""Retrieve the task.
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
timeout: Timeout for the API HTTP request.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
The retrieved task.
|
|
63
|
+
"""
|
|
64
|
+
result = self._get(timeout=timeout)
|
|
65
|
+
if result is None:
|
|
66
|
+
return None
|
|
67
|
+
return TaskResponse.model_validate(result).data
|
|
68
|
+
|
|
69
|
+
def update(
|
|
70
|
+
self,
|
|
71
|
+
*,
|
|
72
|
+
name: str | None = None,
|
|
73
|
+
task_input: TaskInputDict | TaskInput | None = None,
|
|
74
|
+
build: str | None = None,
|
|
75
|
+
max_items: int | None = None,
|
|
76
|
+
memory_mbytes: int | None = None,
|
|
77
|
+
run_timeout: timedelta | None = None,
|
|
78
|
+
restart_on_error: bool | None = None,
|
|
79
|
+
title: str | None = None,
|
|
80
|
+
actor_standby_desired_requests_per_actor_run: int | None = None,
|
|
81
|
+
actor_standby_max_requests_per_actor_run: int | None = None,
|
|
82
|
+
actor_standby_idle_timeout: timedelta | None = None,
|
|
83
|
+
actor_standby_build: str | None = None,
|
|
84
|
+
actor_standby_memory_mbytes: int | None = None,
|
|
85
|
+
timeout: Timeout = 'short',
|
|
86
|
+
) -> Task:
|
|
87
|
+
"""Update the task with specified fields.
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
name: Name of the task.
|
|
92
|
+
build: Actor build to run. It can be either a build tag or build number. By default, the run uses
|
|
93
|
+
the build specified in the task settings (typically latest).
|
|
94
|
+
max_items: Maximum number of results that will be returned by this run. If the Actor is charged per result,
|
|
95
|
+
you will not be charged for more results than the given limit.
|
|
96
|
+
memory_mbytes: Memory limit for the run, in megabytes. By default, the run uses a memory limit specified
|
|
97
|
+
in the task settings.
|
|
98
|
+
run_timeout: Optional timeout for the run. By default, the run uses timeout specified
|
|
99
|
+
in the task settings.
|
|
100
|
+
restart_on_error: If true, the Task run process will be restarted whenever it exits with
|
|
101
|
+
a non-zero status code.
|
|
102
|
+
task_input: Task input dictionary.
|
|
103
|
+
title: A human-friendly equivalent of the name.
|
|
104
|
+
actor_standby_desired_requests_per_actor_run: The desired number of concurrent HTTP requests for
|
|
105
|
+
a single Actor Standby run.
|
|
106
|
+
actor_standby_max_requests_per_actor_run: The maximum number of concurrent HTTP requests for
|
|
107
|
+
a single Actor Standby run.
|
|
108
|
+
actor_standby_idle_timeout: If the Actor run does not receive any requests for this time,
|
|
109
|
+
it will be shut down.
|
|
110
|
+
actor_standby_build: The build tag or number to run when the Actor is in Standby mode.
|
|
111
|
+
actor_standby_memory_mbytes: The memory in megabytes to use when the Actor is in Standby mode.
|
|
112
|
+
timeout: Timeout for the API HTTP request.
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
The updated task.
|
|
116
|
+
"""
|
|
117
|
+
if task_input is not None and not isinstance(task_input, TaskInput):
|
|
118
|
+
task_input = TaskInput.model_validate(task_input)
|
|
119
|
+
|
|
120
|
+
task_fields = UpdateTaskRequest(
|
|
121
|
+
name=name,
|
|
122
|
+
title=title,
|
|
123
|
+
input=task_input,
|
|
124
|
+
options=TaskOptions(
|
|
125
|
+
build=build,
|
|
126
|
+
max_items=max_items,
|
|
127
|
+
memory_mbytes=memory_mbytes,
|
|
128
|
+
timeout_secs=to_seconds(run_timeout, as_int=True),
|
|
129
|
+
restart_on_error=restart_on_error,
|
|
130
|
+
),
|
|
131
|
+
actor_standby=ActorStandby(
|
|
132
|
+
desired_requests_per_actor_run=actor_standby_desired_requests_per_actor_run,
|
|
133
|
+
max_requests_per_actor_run=actor_standby_max_requests_per_actor_run,
|
|
134
|
+
idle_timeout_secs=to_seconds(actor_standby_idle_timeout, as_int=True),
|
|
135
|
+
build=actor_standby_build,
|
|
136
|
+
memory_mbytes=actor_standby_memory_mbytes,
|
|
137
|
+
),
|
|
138
|
+
)
|
|
139
|
+
result = self._update(timeout=timeout, **task_fields.model_dump(by_alias=True, exclude_none=True))
|
|
140
|
+
return TaskResponse.model_validate(result).data
|
|
141
|
+
|
|
142
|
+
def publish(
|
|
143
|
+
self,
|
|
144
|
+
*,
|
|
145
|
+
slug: str | None = None,
|
|
146
|
+
seo_title: str | None = None,
|
|
147
|
+
seo_description: str | None = None,
|
|
148
|
+
use_case: str | None = None,
|
|
149
|
+
timeout: Timeout = 'short',
|
|
150
|
+
) -> Task:
|
|
151
|
+
"""Publish the task on its public landing page (Scrapely-specific endpoint).
|
|
152
|
+
|
|
153
|
+
Scrapely exposes a dedicated publication endpoint (`POST /v2/actor-tasks/{task_id}/publish`) rather than
|
|
154
|
+
Upstream-compatible task update with `isPublic`.
|
|
155
|
+
|
|
156
|
+
Args:
|
|
157
|
+
slug: Custom slug for the public landing page URL.
|
|
158
|
+
seo_title: SEO title for the landing page (max 255 characters).
|
|
159
|
+
seo_description: SEO description for the landing page (max 255 characters).
|
|
160
|
+
use_case: Short description of what the task is used for.
|
|
161
|
+
timeout: Timeout for the API HTTP request.
|
|
162
|
+
|
|
163
|
+
Returns:
|
|
164
|
+
The published task.
|
|
165
|
+
"""
|
|
166
|
+
body = {
|
|
167
|
+
key: value
|
|
168
|
+
for key, value in {
|
|
169
|
+
'slug': slug,
|
|
170
|
+
'seoTitle': seo_title,
|
|
171
|
+
'seoDescription': seo_description,
|
|
172
|
+
'useCase': use_case,
|
|
173
|
+
}.items()
|
|
174
|
+
if value is not None
|
|
175
|
+
}
|
|
176
|
+
response = self._http_client.call(
|
|
177
|
+
url=self._build_url('publish'),
|
|
178
|
+
method='POST',
|
|
179
|
+
params=self._build_params(),
|
|
180
|
+
json=body,
|
|
181
|
+
timeout=timeout,
|
|
182
|
+
)
|
|
183
|
+
return TaskResponse.model_validate(response.json()).data
|
|
184
|
+
|
|
185
|
+
def unpublish(self, *, timeout: Timeout = 'short') -> Task:
|
|
186
|
+
"""Unpublish the task from its public landing page (Scrapely-specific endpoint).
|
|
187
|
+
|
|
188
|
+
Scrapely exposes a dedicated publication endpoint (`POST /v2/actor-tasks/{task_id}/unpublish`) rather than
|
|
189
|
+
Upstream-compatible task update with `isPublic`. The public display configuration is preserved, so the task can be
|
|
190
|
+
published again without re-entering it.
|
|
191
|
+
|
|
192
|
+
Args:
|
|
193
|
+
timeout: Timeout for the API HTTP request.
|
|
194
|
+
|
|
195
|
+
Returns:
|
|
196
|
+
The unpublished task.
|
|
197
|
+
"""
|
|
198
|
+
response = self._http_client.call(
|
|
199
|
+
url=self._build_url('unpublish'),
|
|
200
|
+
method='POST',
|
|
201
|
+
params=self._build_params(),
|
|
202
|
+
timeout=timeout,
|
|
203
|
+
)
|
|
204
|
+
return TaskResponse.model_validate(response.json()).data
|
|
205
|
+
|
|
206
|
+
def delete(self, *, timeout: Timeout = 'short') -> None:
|
|
207
|
+
"""Delete the task.
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
Args:
|
|
211
|
+
timeout: Timeout for the API HTTP request.
|
|
212
|
+
"""
|
|
213
|
+
self._delete(timeout=timeout)
|
|
214
|
+
|
|
215
|
+
def start(
|
|
216
|
+
self,
|
|
217
|
+
*,
|
|
218
|
+
task_input: TaskInputDict | TaskInput | None = None,
|
|
219
|
+
build: str | None = None,
|
|
220
|
+
max_items: int | None = None,
|
|
221
|
+
memory_mbytes: int | None = None,
|
|
222
|
+
run_timeout: timedelta | None = None,
|
|
223
|
+
restart_on_error: bool | None = None,
|
|
224
|
+
wait_for_finish: int | None = None,
|
|
225
|
+
webhooks: WebhooksList | None = None,
|
|
226
|
+
timeout: Timeout = 'medium',
|
|
227
|
+
) -> Run:
|
|
228
|
+
"""Start the task and immediately return the Run object.
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
Args:
|
|
232
|
+
task_input: Task input dictionary.
|
|
233
|
+
build: Specifies the Actor build to run. It can be either a build tag or build number. By default,
|
|
234
|
+
the run uses the build specified in the task settings (typically latest).
|
|
235
|
+
max_items: Maximum number of results that will be returned by this run. If the Actor is charged
|
|
236
|
+
per result, you will not be charged for more results than the given limit.
|
|
237
|
+
memory_mbytes: Memory limit for the run, in megabytes. By default, the run uses a memory limit specified
|
|
238
|
+
in the task settings.
|
|
239
|
+
run_timeout: Optional timeout for the run. By default, the run uses timeout specified
|
|
240
|
+
in the task settings.
|
|
241
|
+
restart_on_error: If true, the Task run process will be restarted whenever it exits with
|
|
242
|
+
a non-zero status code.
|
|
243
|
+
wait_for_finish: The maximum number of seconds the server waits for the run to finish. By default,
|
|
244
|
+
it is 0, the maximum value is 60.
|
|
245
|
+
webhooks: Optional ad-hoc webhooks associated with
|
|
246
|
+
the Actor run which can be used to receive a notification, e.g. when the Actor finished or failed.
|
|
247
|
+
If you already have a webhook set up for the Actor or task, you do not have to add it again here.
|
|
248
|
+
Each webhook is represented by a dictionary containing these items:
|
|
249
|
+
* `event_types`: List of `WebhookEventType` values which trigger the webhook.
|
|
250
|
+
* `request_url`: URL to which to send the webhook HTTP request.
|
|
251
|
+
* `payload_template`: Optional template for the request payload.
|
|
252
|
+
timeout: Timeout for the API HTTP request.
|
|
253
|
+
|
|
254
|
+
Returns:
|
|
255
|
+
The run object.
|
|
256
|
+
"""
|
|
257
|
+
if task_input is not None and not isinstance(task_input, TaskInput):
|
|
258
|
+
task_input = TaskInput.model_validate(task_input)
|
|
259
|
+
|
|
260
|
+
request_params = self._build_params(
|
|
261
|
+
build=build,
|
|
262
|
+
maxItems=max_items,
|
|
263
|
+
memory=memory_mbytes,
|
|
264
|
+
timeout=to_seconds(run_timeout, as_int=True),
|
|
265
|
+
restartOnError=restart_on_error,
|
|
266
|
+
waitForFinish=wait_for_finish,
|
|
267
|
+
webhooks=encode_webhooks_to_base64(webhooks),
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
response = self._http_client.call(
|
|
271
|
+
url=self._build_url('runs'),
|
|
272
|
+
method='POST',
|
|
273
|
+
headers={'content-type': 'application/json; charset=utf-8'},
|
|
274
|
+
json=task_input.model_dump() if task_input is not None else None,
|
|
275
|
+
params=request_params,
|
|
276
|
+
timeout=timeout,
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
result = response_to_dict(response)
|
|
280
|
+
return RunResponse.model_validate(result).data
|
|
281
|
+
|
|
282
|
+
def call(
|
|
283
|
+
self,
|
|
284
|
+
*,
|
|
285
|
+
task_input: TaskInputDict | TaskInput | None = None,
|
|
286
|
+
build: str | None = None,
|
|
287
|
+
max_items: int | None = None,
|
|
288
|
+
memory_mbytes: int | None = None,
|
|
289
|
+
run_timeout: timedelta | None = None,
|
|
290
|
+
restart_on_error: bool | None = None,
|
|
291
|
+
webhooks: WebhooksList | None = None,
|
|
292
|
+
wait_duration: timedelta | None = None,
|
|
293
|
+
timeout: Timeout = 'no_timeout',
|
|
294
|
+
) -> Run | None:
|
|
295
|
+
"""Start a task and wait for it to finish before returning the Run object.
|
|
296
|
+
|
|
297
|
+
It waits indefinitely, unless the wait_duration argument is provided.
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
Args:
|
|
301
|
+
task_input: Task input dictionary.
|
|
302
|
+
build: Specifies the Actor build to run. It can be either a build tag or build number. By default,
|
|
303
|
+
the run uses the build specified in the task settings (typically latest).
|
|
304
|
+
max_items: Maximum number of results that will be returned by this run. If the Actor is charged per result,
|
|
305
|
+
you will not be charged for more results than the given limit.
|
|
306
|
+
memory_mbytes: Memory limit for the run, in megabytes. By default, the run uses a memory limit specified
|
|
307
|
+
in the task settings.
|
|
308
|
+
run_timeout: Optional timeout for the run. By default, the run uses timeout specified
|
|
309
|
+
in the task settings.
|
|
310
|
+
restart_on_error: If true, the Task run process will be restarted whenever it exits with
|
|
311
|
+
a non-zero status code.
|
|
312
|
+
webhooks: Specifies optional webhooks associated with the Actor run, which can be used to receive
|
|
313
|
+
a notification e.g. when the Actor finished or failed. Note: if you already have a webhook set up for
|
|
314
|
+
the Actor or task, you do not have to add it again here.
|
|
315
|
+
wait_duration: The maximum time the server waits for the task run to finish. If not provided,
|
|
316
|
+
waits indefinitely.
|
|
317
|
+
timeout: Timeout for the API HTTP request.
|
|
318
|
+
|
|
319
|
+
Returns:
|
|
320
|
+
The run object.
|
|
321
|
+
"""
|
|
322
|
+
started_run = self.start(
|
|
323
|
+
task_input=task_input,
|
|
324
|
+
build=build,
|
|
325
|
+
max_items=max_items,
|
|
326
|
+
memory_mbytes=memory_mbytes,
|
|
327
|
+
run_timeout=run_timeout,
|
|
328
|
+
restart_on_error=restart_on_error,
|
|
329
|
+
webhooks=webhooks,
|
|
330
|
+
timeout=timeout,
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
run_client = self._client_registry.run_client(
|
|
334
|
+
resource_id=started_run.id,
|
|
335
|
+
base_url=self._base_url,
|
|
336
|
+
public_base_url=self._public_base_url,
|
|
337
|
+
http_client=self._http_client,
|
|
338
|
+
client_registry=self._client_registry,
|
|
339
|
+
)
|
|
340
|
+
return run_client.wait_for_finish(wait_duration=wait_duration)
|
|
341
|
+
|
|
342
|
+
def get_input(self, *, timeout: Timeout = 'short') -> dict:
|
|
343
|
+
"""Retrieve the default input for this task.
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
Args:
|
|
347
|
+
timeout: Timeout for the API HTTP request.
|
|
348
|
+
|
|
349
|
+
Returns:
|
|
350
|
+
Retrieved task input.
|
|
351
|
+
|
|
352
|
+
Raises:
|
|
353
|
+
NotFoundError: If the task does not exist.
|
|
354
|
+
"""
|
|
355
|
+
response = self._http_client.call(
|
|
356
|
+
url=self._build_url('input'),
|
|
357
|
+
method='GET',
|
|
358
|
+
params=self._build_params(),
|
|
359
|
+
timeout=timeout,
|
|
360
|
+
)
|
|
361
|
+
return response_to_dict(response)
|
|
362
|
+
|
|
363
|
+
def update_input(self, *, task_input: TaskInputDict | TaskInput, timeout: Timeout = 'short') -> dict:
|
|
364
|
+
"""Update the default input for this task.
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
Args:
|
|
368
|
+
task_input: The new default input for this task.
|
|
369
|
+
timeout: Timeout for the API HTTP request.
|
|
370
|
+
|
|
371
|
+
Returns:
|
|
372
|
+
The updated task input.
|
|
373
|
+
"""
|
|
374
|
+
if task_input is not None and not isinstance(task_input, TaskInput):
|
|
375
|
+
task_input = TaskInput.model_validate(task_input)
|
|
376
|
+
|
|
377
|
+
response = self._http_client.call(
|
|
378
|
+
url=self._build_url('input'),
|
|
379
|
+
method='PUT',
|
|
380
|
+
params=self._build_params(),
|
|
381
|
+
json=task_input.model_dump(),
|
|
382
|
+
timeout=timeout,
|
|
383
|
+
)
|
|
384
|
+
return response_to_dict(response)
|
|
385
|
+
|
|
386
|
+
def runs(self) -> RunCollectionClient:
|
|
387
|
+
"""Retrieve a client for the runs of this task."""
|
|
388
|
+
return self._client_registry.run_collection_client(
|
|
389
|
+
resource_path='runs',
|
|
390
|
+
**self._base_client_kwargs,
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
def last_run(self, *, status: ActorJobStatus | None = None, origin: RunOrigin | None = None) -> RunClient:
|
|
394
|
+
"""Retrieve the client for the last run of this task.
|
|
395
|
+
|
|
396
|
+
Last run is retrieved based on the start time of the runs.
|
|
397
|
+
|
|
398
|
+
Args:
|
|
399
|
+
status: Consider only runs with this status.
|
|
400
|
+
origin: Consider only runs started with this origin.
|
|
401
|
+
|
|
402
|
+
Returns:
|
|
403
|
+
The resource client for the last run of this task.
|
|
404
|
+
"""
|
|
405
|
+
return self._client_registry.run_client(
|
|
406
|
+
resource_id='last',
|
|
407
|
+
resource_path='runs',
|
|
408
|
+
params=self._build_params(status=status, origin=origin),
|
|
409
|
+
**self._base_client_kwargs,
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
def webhooks(self) -> WebhookCollectionClient:
|
|
413
|
+
"""Retrieve a client for webhooks associated with this task."""
|
|
414
|
+
return self._client_registry.webhook_collection_client(**self._base_client_kwargs)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
@docs_group('Resource clients')
|
|
418
|
+
class TaskClientAsync(ResourceClientAsync):
|
|
419
|
+
"""Sub-client for managing a specific task.
|
|
420
|
+
|
|
421
|
+
Provides methods to manage a specific task, e.g. update it, delete it, or start runs. Obtain an instance via an
|
|
422
|
+
appropriate method on the `ApifyClientAsync` class.
|
|
423
|
+
"""
|
|
424
|
+
|
|
425
|
+
def __init__(
|
|
426
|
+
self,
|
|
427
|
+
*,
|
|
428
|
+
resource_id: str,
|
|
429
|
+
resource_path: str = 'actor-tasks',
|
|
430
|
+
**kwargs: Any,
|
|
431
|
+
) -> None:
|
|
432
|
+
super().__init__(resource_id=resource_id, resource_path=resource_path, **kwargs)
|
|
433
|
+
|
|
434
|
+
async def get(self, *, timeout: Timeout = 'short') -> Task | None:
|
|
435
|
+
"""Retrieve the task.
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
Args:
|
|
439
|
+
timeout: Timeout for the API HTTP request.
|
|
440
|
+
|
|
441
|
+
Returns:
|
|
442
|
+
The retrieved task.
|
|
443
|
+
"""
|
|
444
|
+
result = await self._get(timeout=timeout)
|
|
445
|
+
if result is None:
|
|
446
|
+
return None
|
|
447
|
+
return TaskResponse.model_validate(result).data
|
|
448
|
+
|
|
449
|
+
async def update(
|
|
450
|
+
self,
|
|
451
|
+
*,
|
|
452
|
+
name: str | None = None,
|
|
453
|
+
task_input: TaskInputDict | TaskInput | None = None,
|
|
454
|
+
build: str | None = None,
|
|
455
|
+
max_items: int | None = None,
|
|
456
|
+
memory_mbytes: int | None = None,
|
|
457
|
+
run_timeout: timedelta | None = None,
|
|
458
|
+
restart_on_error: bool | None = None,
|
|
459
|
+
title: str | None = None,
|
|
460
|
+
actor_standby_desired_requests_per_actor_run: int | None = None,
|
|
461
|
+
actor_standby_max_requests_per_actor_run: int | None = None,
|
|
462
|
+
actor_standby_idle_timeout: timedelta | None = None,
|
|
463
|
+
actor_standby_build: str | None = None,
|
|
464
|
+
actor_standby_memory_mbytes: int | None = None,
|
|
465
|
+
timeout: Timeout = 'short',
|
|
466
|
+
) -> Task:
|
|
467
|
+
"""Update the task with specified fields.
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
Args:
|
|
471
|
+
name: Name of the task.
|
|
472
|
+
build: Actor build to run. It can be either a build tag or build number. By default, the run uses
|
|
473
|
+
the build specified in the task settings (typically latest).
|
|
474
|
+
max_items: Maximum number of results that will be returned by this run. If the Actor is charged per result,
|
|
475
|
+
you will not be charged for more results than the given limit.
|
|
476
|
+
memory_mbytes: Memory limit for the run, in megabytes. By default, the run uses a memory limit specified
|
|
477
|
+
in the task settings.
|
|
478
|
+
run_timeout: Optional timeout for the run. By default, the run uses timeout specified
|
|
479
|
+
in the task settings.
|
|
480
|
+
restart_on_error: If true, the Task run process will be restarted whenever it exits with
|
|
481
|
+
a non-zero status code.
|
|
482
|
+
task_input: Task input dictionary.
|
|
483
|
+
title: A human-friendly equivalent of the name.
|
|
484
|
+
actor_standby_desired_requests_per_actor_run: The desired number of concurrent HTTP requests for
|
|
485
|
+
a single Actor Standby run.
|
|
486
|
+
actor_standby_max_requests_per_actor_run: The maximum number of concurrent HTTP requests for
|
|
487
|
+
a single Actor Standby run.
|
|
488
|
+
actor_standby_idle_timeout: If the Actor run does not receive any requests for this time,
|
|
489
|
+
it will be shut down.
|
|
490
|
+
actor_standby_build: The build tag or number to run when the Actor is in Standby mode.
|
|
491
|
+
actor_standby_memory_mbytes: The memory in megabytes to use when the Actor is in Standby mode.
|
|
492
|
+
timeout: Timeout for the API HTTP request.
|
|
493
|
+
|
|
494
|
+
Returns:
|
|
495
|
+
The updated task.
|
|
496
|
+
"""
|
|
497
|
+
if task_input is not None and not isinstance(task_input, TaskInput):
|
|
498
|
+
task_input = TaskInput.model_validate(task_input)
|
|
499
|
+
|
|
500
|
+
task_fields = UpdateTaskRequest(
|
|
501
|
+
name=name,
|
|
502
|
+
title=title,
|
|
503
|
+
input=task_input,
|
|
504
|
+
options=TaskOptions(
|
|
505
|
+
build=build,
|
|
506
|
+
max_items=max_items,
|
|
507
|
+
memory_mbytes=memory_mbytes,
|
|
508
|
+
timeout_secs=to_seconds(run_timeout, as_int=True),
|
|
509
|
+
restart_on_error=restart_on_error,
|
|
510
|
+
),
|
|
511
|
+
actor_standby=ActorStandby(
|
|
512
|
+
desired_requests_per_actor_run=actor_standby_desired_requests_per_actor_run,
|
|
513
|
+
max_requests_per_actor_run=actor_standby_max_requests_per_actor_run,
|
|
514
|
+
idle_timeout_secs=to_seconds(actor_standby_idle_timeout, as_int=True),
|
|
515
|
+
build=actor_standby_build,
|
|
516
|
+
memory_mbytes=actor_standby_memory_mbytes,
|
|
517
|
+
),
|
|
518
|
+
)
|
|
519
|
+
result = await self._update(timeout=timeout, **task_fields.model_dump(by_alias=True, exclude_none=True))
|
|
520
|
+
return TaskResponse.model_validate(result).data
|
|
521
|
+
|
|
522
|
+
async def publish(
|
|
523
|
+
self,
|
|
524
|
+
*,
|
|
525
|
+
slug: str | None = None,
|
|
526
|
+
seo_title: str | None = None,
|
|
527
|
+
seo_description: str | None = None,
|
|
528
|
+
use_case: str | None = None,
|
|
529
|
+
timeout: Timeout = 'short',
|
|
530
|
+
) -> Task:
|
|
531
|
+
"""Publish the task on its public landing page (Scrapely-specific endpoint).
|
|
532
|
+
|
|
533
|
+
Scrapely exposes a dedicated publication endpoint (`POST /v2/actor-tasks/{task_id}/publish`) rather than
|
|
534
|
+
Upstream-compatible task update with `isPublic`.
|
|
535
|
+
|
|
536
|
+
Args:
|
|
537
|
+
slug: Custom slug for the public landing page URL.
|
|
538
|
+
seo_title: SEO title for the landing page (max 255 characters).
|
|
539
|
+
seo_description: SEO description for the landing page (max 255 characters).
|
|
540
|
+
use_case: Short description of what the task is used for.
|
|
541
|
+
timeout: Timeout for the API HTTP request.
|
|
542
|
+
|
|
543
|
+
Returns:
|
|
544
|
+
The published task.
|
|
545
|
+
"""
|
|
546
|
+
body = {
|
|
547
|
+
key: value
|
|
548
|
+
for key, value in {
|
|
549
|
+
'slug': slug,
|
|
550
|
+
'seoTitle': seo_title,
|
|
551
|
+
'seoDescription': seo_description,
|
|
552
|
+
'useCase': use_case,
|
|
553
|
+
}.items()
|
|
554
|
+
if value is not None
|
|
555
|
+
}
|
|
556
|
+
response = await self._http_client.call(
|
|
557
|
+
url=self._build_url('publish'),
|
|
558
|
+
method='POST',
|
|
559
|
+
params=self._build_params(),
|
|
560
|
+
json=body,
|
|
561
|
+
timeout=timeout,
|
|
562
|
+
)
|
|
563
|
+
return TaskResponse.model_validate(response.json()).data
|
|
564
|
+
|
|
565
|
+
async def unpublish(self, *, timeout: Timeout = 'short') -> Task:
|
|
566
|
+
"""Unpublish the task from its public landing page (Scrapely-specific endpoint).
|
|
567
|
+
|
|
568
|
+
Scrapely exposes a dedicated publication endpoint (`POST /v2/actor-tasks/{task_id}/unpublish`) rather than
|
|
569
|
+
Upstream-compatible task update with `isPublic`. The public display configuration is preserved, so the task can be
|
|
570
|
+
published again without re-entering it.
|
|
571
|
+
|
|
572
|
+
Args:
|
|
573
|
+
timeout: Timeout for the API HTTP request.
|
|
574
|
+
|
|
575
|
+
Returns:
|
|
576
|
+
The unpublished task.
|
|
577
|
+
"""
|
|
578
|
+
response = await self._http_client.call(
|
|
579
|
+
url=self._build_url('unpublish'),
|
|
580
|
+
method='POST',
|
|
581
|
+
params=self._build_params(),
|
|
582
|
+
timeout=timeout,
|
|
583
|
+
)
|
|
584
|
+
return TaskResponse.model_validate(response.json()).data
|
|
585
|
+
|
|
586
|
+
async def delete(self, *, timeout: Timeout = 'short') -> None:
|
|
587
|
+
"""Delete the task.
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
Args:
|
|
591
|
+
timeout: Timeout for the API HTTP request.
|
|
592
|
+
"""
|
|
593
|
+
await self._delete(timeout=timeout)
|
|
594
|
+
|
|
595
|
+
async def start(
|
|
596
|
+
self,
|
|
597
|
+
*,
|
|
598
|
+
task_input: TaskInputDict | TaskInput | None = None,
|
|
599
|
+
build: str | None = None,
|
|
600
|
+
max_items: int | None = None,
|
|
601
|
+
memory_mbytes: int | None = None,
|
|
602
|
+
run_timeout: timedelta | None = None,
|
|
603
|
+
restart_on_error: bool | None = None,
|
|
604
|
+
wait_for_finish: int | None = None,
|
|
605
|
+
webhooks: WebhooksList | None = None,
|
|
606
|
+
timeout: Timeout = 'medium',
|
|
607
|
+
) -> Run:
|
|
608
|
+
"""Start the task and immediately return the Run object.
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
Args:
|
|
612
|
+
task_input: Task input dictionary.
|
|
613
|
+
build: Specifies the Actor build to run. It can be either a build tag or build number. By default,
|
|
614
|
+
the run uses the build specified in the task settings (typically latest).
|
|
615
|
+
max_items: Maximum number of results that will be returned by this run. If the Actor is charged
|
|
616
|
+
per result, you will not be charged for more results than the given limit.
|
|
617
|
+
memory_mbytes: Memory limit for the run, in megabytes. By default, the run uses a memory limit specified
|
|
618
|
+
in the task settings.
|
|
619
|
+
run_timeout: Optional timeout for the run. By default, the run uses timeout specified
|
|
620
|
+
in the task settings.
|
|
621
|
+
restart_on_error: If true, the Task run process will be restarted whenever it exits with
|
|
622
|
+
a non-zero status code.
|
|
623
|
+
wait_for_finish: The maximum number of seconds the server waits for the run to finish. By default,
|
|
624
|
+
it is 0, the maximum value is 60.
|
|
625
|
+
webhooks: Optional ad-hoc webhooks associated with
|
|
626
|
+
the Actor run which can be used to receive a notification, e.g. when the Actor finished or failed.
|
|
627
|
+
If you already have a webhook set up for the Actor or task, you do not have to add it again here.
|
|
628
|
+
Each webhook is represented by a dictionary containing these items:
|
|
629
|
+
* `event_types`: List of `WebhookEventType` values which trigger the webhook.
|
|
630
|
+
* `request_url`: URL to which to send the webhook HTTP request.
|
|
631
|
+
* `payload_template`: Optional template for the request payload.
|
|
632
|
+
timeout: Timeout for the API HTTP request.
|
|
633
|
+
|
|
634
|
+
Returns:
|
|
635
|
+
The run object.
|
|
636
|
+
"""
|
|
637
|
+
if task_input is not None and not isinstance(task_input, TaskInput):
|
|
638
|
+
task_input = TaskInput.model_validate(task_input)
|
|
639
|
+
|
|
640
|
+
request_params = self._build_params(
|
|
641
|
+
build=build,
|
|
642
|
+
maxItems=max_items,
|
|
643
|
+
memory=memory_mbytes,
|
|
644
|
+
timeout=to_seconds(run_timeout, as_int=True),
|
|
645
|
+
restartOnError=restart_on_error,
|
|
646
|
+
waitForFinish=wait_for_finish,
|
|
647
|
+
webhooks=encode_webhooks_to_base64(webhooks),
|
|
648
|
+
)
|
|
649
|
+
|
|
650
|
+
response = await self._http_client.call(
|
|
651
|
+
url=self._build_url('runs'),
|
|
652
|
+
method='POST',
|
|
653
|
+
headers={'content-type': 'application/json; charset=utf-8'},
|
|
654
|
+
json=task_input.model_dump() if task_input is not None else None,
|
|
655
|
+
params=request_params,
|
|
656
|
+
timeout=timeout,
|
|
657
|
+
)
|
|
658
|
+
|
|
659
|
+
result = response_to_dict(response)
|
|
660
|
+
return RunResponse.model_validate(result).data
|
|
661
|
+
|
|
662
|
+
async def call(
|
|
663
|
+
self,
|
|
664
|
+
*,
|
|
665
|
+
task_input: TaskInputDict | TaskInput | None = None,
|
|
666
|
+
build: str | None = None,
|
|
667
|
+
max_items: int | None = None,
|
|
668
|
+
memory_mbytes: int | None = None,
|
|
669
|
+
run_timeout: timedelta | None = None,
|
|
670
|
+
restart_on_error: bool | None = None,
|
|
671
|
+
webhooks: WebhooksList | None = None,
|
|
672
|
+
wait_duration: timedelta | None = None,
|
|
673
|
+
timeout: Timeout = 'no_timeout',
|
|
674
|
+
) -> Run | None:
|
|
675
|
+
"""Start a task and wait for it to finish before returning the Run object.
|
|
676
|
+
|
|
677
|
+
It waits indefinitely, unless the wait_duration argument is provided.
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
Args:
|
|
681
|
+
task_input: Task input dictionary.
|
|
682
|
+
build: Specifies the Actor build to run. It can be either a build tag or build number. By default,
|
|
683
|
+
the run uses the build specified in the task settings (typically latest).
|
|
684
|
+
max_items: Maximum number of results that will be returned by this run. If the Actor is charged per result,
|
|
685
|
+
you will not be charged for more results than the given limit.
|
|
686
|
+
memory_mbytes: Memory limit for the run, in megabytes. By default, the run uses a memory limit specified
|
|
687
|
+
in the task settings.
|
|
688
|
+
run_timeout: Optional timeout for the run. By default, the run uses timeout specified
|
|
689
|
+
in the task settings.
|
|
690
|
+
restart_on_error: If true, the Task run process will be restarted whenever it exits with
|
|
691
|
+
a non-zero status code.
|
|
692
|
+
webhooks: Specifies optional webhooks associated with the Actor run, which can be used to receive
|
|
693
|
+
a notification e.g. when the Actor finished or failed. Note: if you already have a webhook set up for
|
|
694
|
+
the Actor or task, you do not have to add it again here.
|
|
695
|
+
wait_duration: The maximum time the server waits for the task run to finish. If not provided,
|
|
696
|
+
waits indefinitely.
|
|
697
|
+
timeout: Timeout for the API HTTP request.
|
|
698
|
+
|
|
699
|
+
Returns:
|
|
700
|
+
The run object.
|
|
701
|
+
"""
|
|
702
|
+
started_run = await self.start(
|
|
703
|
+
task_input=task_input,
|
|
704
|
+
build=build,
|
|
705
|
+
max_items=max_items,
|
|
706
|
+
memory_mbytes=memory_mbytes,
|
|
707
|
+
run_timeout=run_timeout,
|
|
708
|
+
restart_on_error=restart_on_error,
|
|
709
|
+
webhooks=webhooks,
|
|
710
|
+
timeout=timeout,
|
|
711
|
+
)
|
|
712
|
+
run_client = self._client_registry.run_client(
|
|
713
|
+
resource_id=started_run.id,
|
|
714
|
+
base_url=self._base_url,
|
|
715
|
+
public_base_url=self._public_base_url,
|
|
716
|
+
http_client=self._http_client,
|
|
717
|
+
client_registry=self._client_registry,
|
|
718
|
+
)
|
|
719
|
+
return await run_client.wait_for_finish(wait_duration=wait_duration)
|
|
720
|
+
|
|
721
|
+
async def get_input(self, *, timeout: Timeout = 'short') -> dict:
|
|
722
|
+
"""Retrieve the default input for this task.
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
Args:
|
|
726
|
+
timeout: Timeout for the API HTTP request.
|
|
727
|
+
|
|
728
|
+
Returns:
|
|
729
|
+
Retrieved task input.
|
|
730
|
+
|
|
731
|
+
Raises:
|
|
732
|
+
NotFoundError: If the task does not exist.
|
|
733
|
+
"""
|
|
734
|
+
response = await self._http_client.call(
|
|
735
|
+
url=self._build_url('input'),
|
|
736
|
+
method='GET',
|
|
737
|
+
params=self._build_params(),
|
|
738
|
+
timeout=timeout,
|
|
739
|
+
)
|
|
740
|
+
return response_to_dict(response)
|
|
741
|
+
|
|
742
|
+
async def update_input(self, *, task_input: TaskInputDict | TaskInput, timeout: Timeout = 'short') -> dict:
|
|
743
|
+
"""Update the default input for this task.
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
Args:
|
|
747
|
+
task_input: The new default input for this task.
|
|
748
|
+
timeout: Timeout for the API HTTP request.
|
|
749
|
+
|
|
750
|
+
Returns:
|
|
751
|
+
The updated task input.
|
|
752
|
+
"""
|
|
753
|
+
if task_input is not None and not isinstance(task_input, TaskInput):
|
|
754
|
+
task_input = TaskInput.model_validate(task_input)
|
|
755
|
+
|
|
756
|
+
response = await self._http_client.call(
|
|
757
|
+
url=self._build_url('input'),
|
|
758
|
+
method='PUT',
|
|
759
|
+
params=self._build_params(),
|
|
760
|
+
json=task_input.model_dump(),
|
|
761
|
+
timeout=timeout,
|
|
762
|
+
)
|
|
763
|
+
return response_to_dict(response)
|
|
764
|
+
|
|
765
|
+
def runs(self) -> RunCollectionClientAsync:
|
|
766
|
+
"""Retrieve a client for the runs of this task."""
|
|
767
|
+
return self._client_registry.run_collection_client(
|
|
768
|
+
resource_path='runs',
|
|
769
|
+
**self._base_client_kwargs,
|
|
770
|
+
)
|
|
771
|
+
|
|
772
|
+
def last_run(self, *, status: ActorJobStatus | None = None, origin: RunOrigin | None = None) -> RunClientAsync:
|
|
773
|
+
"""Retrieve the client for the last run of this task.
|
|
774
|
+
|
|
775
|
+
Last run is retrieved based on the start time of the runs.
|
|
776
|
+
|
|
777
|
+
Args:
|
|
778
|
+
status: Consider only runs with this status.
|
|
779
|
+
origin: Consider only runs started with this origin.
|
|
780
|
+
|
|
781
|
+
Returns:
|
|
782
|
+
The resource client for the last run of this task.
|
|
783
|
+
"""
|
|
784
|
+
return self._client_registry.run_client(
|
|
785
|
+
resource_id='last',
|
|
786
|
+
resource_path='runs',
|
|
787
|
+
params=self._build_params(status=status, origin=origin),
|
|
788
|
+
**self._base_client_kwargs,
|
|
789
|
+
)
|
|
790
|
+
|
|
791
|
+
def webhooks(self) -> WebhookCollectionClientAsync:
|
|
792
|
+
"""Retrieve a client for webhooks associated with this task."""
|
|
793
|
+
return self._client_registry.webhook_collection_client(**self._base_client_kwargs)
|