workly 0.1.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {workly-0.1.0 → workly-0.2.1}/PKG-INFO +31 -1
- {workly-0.1.0 → workly-0.2.1}/README.md +30 -0
- {workly-0.1.0 → workly-0.2.1}/pyproject.toml +1 -1
- {workly-0.1.0 → workly-0.2.1}/pyproject.toml.orig +1 -1
- {workly-0.1.0 → workly-0.2.1}/src/workly/__init__.py +21 -2
- {workly-0.1.0 → workly-0.2.1}/src/workly/_async_client.py +98 -2
- {workly-0.1.0 → workly-0.2.1}/src/workly/_client.py +97 -2
- {workly-0.1.0 → workly-0.2.1}/src/workly/_models.py +96 -1
- {workly-0.1.0 → workly-0.2.1}/src/workly/_requests.py +85 -1
- {workly-0.1.0 → workly-0.2.1}/LICENSE +0 -0
- {workly-0.1.0 → workly-0.2.1}/src/workly/_errors.py +0 -0
- {workly-0.1.0 → workly-0.2.1}/src/workly/_signing.py +0 -0
- {workly-0.1.0 → workly-0.2.1}/src/workly/_transport.py +0 -0
- {workly-0.1.0 → workly-0.2.1}/src/workly/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: workly
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Python SDK for Workly, the open-source HTTP task queue
|
|
5
5
|
Keywords: task queue,background jobs,webhooks,http
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -182,6 +182,36 @@ client.schedules.delete("nightly-report") # with all its runs
|
|
|
182
182
|
|
|
183
183
|
Runs are delivered with `Workly-Run-Id`, `Workly-Schedule` and `Workly-Scheduled-For` headers and the same signature, so `verify_signature` works unchanged.
|
|
184
184
|
|
|
185
|
+
## Task groups
|
|
186
|
+
|
|
187
|
+
A group tracks a set of tasks and enqueues a callback task once it's closed and they've all finished: a `POST` of the group's status and counts. See [Task groups](https://docs.workly.run/groups/). The callback goes to Workly's `_callbacks` queue unless it names a `queue`.
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
from workly import Callback
|
|
191
|
+
|
|
192
|
+
group = client.groups.create(
|
|
193
|
+
name="daily-stock-prices",
|
|
194
|
+
idempotency_key="stock-prices-2026-10-19", # creating it again returns this group
|
|
195
|
+
# or a dict: {"url": "https://api.example.com/jobs/refresh-views"}
|
|
196
|
+
on_complete=Callback(url="https://api.example.com/jobs/refresh-views"),
|
|
197
|
+
)
|
|
198
|
+
for company in companies:
|
|
199
|
+
client.tasks.enqueue(
|
|
200
|
+
"stock-prices",
|
|
201
|
+
url="https://api.example.com/jobs/update-stock-price",
|
|
202
|
+
payload={"symbol": company.symbol},
|
|
203
|
+
group_id=group.id,
|
|
204
|
+
)
|
|
205
|
+
client.groups.close(group.id) # no more tasks; it completes once they've finished
|
|
206
|
+
|
|
207
|
+
group = client.groups.get(group.id)
|
|
208
|
+
print(group.status, group.total, group.succeeded, group.failed, group.remaining)
|
|
209
|
+
for task in client.tasks.list(group_id=group.id, state="failed"):
|
|
210
|
+
print(task.id, task.last_error)
|
|
211
|
+
for group in client.groups.list(name="daily-stock-prices", status="failed"):
|
|
212
|
+
print(group.id, group.finished_at)
|
|
213
|
+
```
|
|
214
|
+
|
|
185
215
|
## Async
|
|
186
216
|
|
|
187
217
|
`AsyncWorkly` has the same methods, as coroutines:
|
|
@@ -162,6 +162,36 @@ client.schedules.delete("nightly-report") # with all its runs
|
|
|
162
162
|
|
|
163
163
|
Runs are delivered with `Workly-Run-Id`, `Workly-Schedule` and `Workly-Scheduled-For` headers and the same signature, so `verify_signature` works unchanged.
|
|
164
164
|
|
|
165
|
+
## Task groups
|
|
166
|
+
|
|
167
|
+
A group tracks a set of tasks and enqueues a callback task once it's closed and they've all finished: a `POST` of the group's status and counts. See [Task groups](https://docs.workly.run/groups/). The callback goes to Workly's `_callbacks` queue unless it names a `queue`.
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
from workly import Callback
|
|
171
|
+
|
|
172
|
+
group = client.groups.create(
|
|
173
|
+
name="daily-stock-prices",
|
|
174
|
+
idempotency_key="stock-prices-2026-10-19", # creating it again returns this group
|
|
175
|
+
# or a dict: {"url": "https://api.example.com/jobs/refresh-views"}
|
|
176
|
+
on_complete=Callback(url="https://api.example.com/jobs/refresh-views"),
|
|
177
|
+
)
|
|
178
|
+
for company in companies:
|
|
179
|
+
client.tasks.enqueue(
|
|
180
|
+
"stock-prices",
|
|
181
|
+
url="https://api.example.com/jobs/update-stock-price",
|
|
182
|
+
payload={"symbol": company.symbol},
|
|
183
|
+
group_id=group.id,
|
|
184
|
+
)
|
|
185
|
+
client.groups.close(group.id) # no more tasks; it completes once they've finished
|
|
186
|
+
|
|
187
|
+
group = client.groups.get(group.id)
|
|
188
|
+
print(group.status, group.total, group.succeeded, group.failed, group.remaining)
|
|
189
|
+
for task in client.tasks.list(group_id=group.id, state="failed"):
|
|
190
|
+
print(task.id, task.last_error)
|
|
191
|
+
for group in client.groups.list(name="daily-stock-prices", status="failed"):
|
|
192
|
+
print(group.id, group.finished_at)
|
|
193
|
+
```
|
|
194
|
+
|
|
165
195
|
## Async
|
|
166
196
|
|
|
167
197
|
`AsyncWorkly` has the same methods, as coroutines:
|
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from ._async_client import (
|
|
4
4
|
AsyncAlerts,
|
|
5
|
+
AsyncGroups,
|
|
5
6
|
AsyncQueues,
|
|
6
7
|
AsyncRuns,
|
|
7
8
|
AsyncSchedules,
|
|
8
9
|
AsyncTasks,
|
|
9
10
|
AsyncWorkly,
|
|
10
11
|
)
|
|
11
|
-
from ._client import Alerts, Queues, Runs, Schedules, Tasks, Workly
|
|
12
|
+
from ._client import Alerts, Groups, Queues, Runs, Schedules, Tasks, Workly
|
|
12
13
|
from ._errors import (
|
|
13
14
|
APIConnectionError,
|
|
14
15
|
APIError,
|
|
@@ -31,10 +32,15 @@ from ._models import (
|
|
|
31
32
|
AlertChannelType,
|
|
32
33
|
Attempt,
|
|
33
34
|
AttemptOutcome,
|
|
35
|
+
Callback,
|
|
34
36
|
CronPreview,
|
|
35
37
|
Delivery,
|
|
36
38
|
EnqueueResult,
|
|
37
39
|
FailureGroup,
|
|
40
|
+
Group,
|
|
41
|
+
GroupCallback,
|
|
42
|
+
GroupPage,
|
|
43
|
+
GroupStatus,
|
|
38
44
|
HTTPMethod,
|
|
39
45
|
LastRun,
|
|
40
46
|
MissedPolicy,
|
|
@@ -48,11 +54,13 @@ from ._models import (
|
|
|
48
54
|
RunTrigger,
|
|
49
55
|
Schedule,
|
|
50
56
|
SkipReason,
|
|
57
|
+
SortOrder,
|
|
51
58
|
Task,
|
|
52
59
|
TaskPage,
|
|
60
|
+
TaskSort,
|
|
53
61
|
TaskState,
|
|
54
62
|
)
|
|
55
|
-
from ._requests import TaskSpec, __version__
|
|
63
|
+
from ._requests import CallbackDict, GroupCallbackSpec, TaskSpec, __version__
|
|
56
64
|
from ._signing import sign, verify_signature
|
|
57
65
|
|
|
58
66
|
__all__ = [
|
|
@@ -66,6 +74,7 @@ __all__ = [
|
|
|
66
74
|
"AlertChannelType",
|
|
67
75
|
"Alerts",
|
|
68
76
|
"AsyncAlerts",
|
|
77
|
+
"AsyncGroups",
|
|
69
78
|
"AsyncQueues",
|
|
70
79
|
"AsyncRuns",
|
|
71
80
|
"AsyncSchedules",
|
|
@@ -75,11 +84,19 @@ __all__ = [
|
|
|
75
84
|
"AttemptOutcome",
|
|
76
85
|
"AuthenticationError",
|
|
77
86
|
"BadRequestError",
|
|
87
|
+
"Callback",
|
|
88
|
+
"CallbackDict",
|
|
78
89
|
"ConflictError",
|
|
79
90
|
"CronPreview",
|
|
80
91
|
"Delivery",
|
|
81
92
|
"EnqueueResult",
|
|
82
93
|
"FailureGroup",
|
|
94
|
+
"Group",
|
|
95
|
+
"GroupCallback",
|
|
96
|
+
"GroupCallbackSpec",
|
|
97
|
+
"GroupPage",
|
|
98
|
+
"GroupStatus",
|
|
99
|
+
"Groups",
|
|
83
100
|
"HTTPMethod",
|
|
84
101
|
"LastRun",
|
|
85
102
|
"MissedPolicy",
|
|
@@ -101,8 +118,10 @@ __all__ = [
|
|
|
101
118
|
"Schedules",
|
|
102
119
|
"SignatureVerificationError",
|
|
103
120
|
"SkipReason",
|
|
121
|
+
"SortOrder",
|
|
104
122
|
"Task",
|
|
105
123
|
"TaskPage",
|
|
124
|
+
"TaskSort",
|
|
106
125
|
"TaskSpec",
|
|
107
126
|
"TaskState",
|
|
108
127
|
"Tasks",
|
|
@@ -17,10 +17,14 @@ from ._models import (
|
|
|
17
17
|
AlertChannelList,
|
|
18
18
|
AlertChannelType,
|
|
19
19
|
Attempt,
|
|
20
|
+
Callback,
|
|
20
21
|
CronPreview,
|
|
21
22
|
Delivery,
|
|
22
23
|
EnqueueResult,
|
|
23
24
|
FailureGroup,
|
|
25
|
+
Group,
|
|
26
|
+
GroupPage,
|
|
27
|
+
GroupStatus,
|
|
24
28
|
HTTPMethod,
|
|
25
29
|
MissedPolicy,
|
|
26
30
|
OverlapPolicy,
|
|
@@ -30,8 +34,10 @@ from ._models import (
|
|
|
30
34
|
RunPage,
|
|
31
35
|
RunState,
|
|
32
36
|
Schedule,
|
|
37
|
+
SortOrder,
|
|
33
38
|
Task,
|
|
34
39
|
TaskPage,
|
|
40
|
+
TaskSort,
|
|
35
41
|
TaskState,
|
|
36
42
|
)
|
|
37
43
|
|
|
@@ -60,6 +66,7 @@ class AsyncWorkly:
|
|
|
60
66
|
self._max_retries = max_retries
|
|
61
67
|
self.queues = AsyncQueues(self)
|
|
62
68
|
self.tasks = AsyncTasks(self)
|
|
69
|
+
self.groups = AsyncGroups(self)
|
|
63
70
|
self.schedules = AsyncSchedules(self)
|
|
64
71
|
self.runs = AsyncRuns(self)
|
|
65
72
|
self.alerts = AsyncAlerts(self)
|
|
@@ -208,6 +215,7 @@ class AsyncTasks:
|
|
|
208
215
|
idempotency_key: str | None = None,
|
|
209
216
|
max_attempts: int | None = None,
|
|
210
217
|
timeout_seconds: float | None = None,
|
|
218
|
+
group_id: str | None = None,
|
|
211
219
|
traceparent: str | None = None,
|
|
212
220
|
) -> Task:
|
|
213
221
|
"""Enqueue a task to deliver to `url`.
|
|
@@ -215,7 +223,8 @@ class AsyncTasks:
|
|
|
215
223
|
`payload` is sent as JSON; `body` as raw text (use one). The queue is
|
|
216
224
|
created with default settings if it doesn't exist. If
|
|
217
225
|
`idempotency_key` matches a task already in the queue, that task is
|
|
218
|
-
returned and nothing is enqueued.
|
|
226
|
+
returned and nothing is enqueued. With `group_id`, the task joins
|
|
227
|
+
that group, which must be open.
|
|
219
228
|
"""
|
|
220
229
|
spec = {
|
|
221
230
|
"url": url,
|
|
@@ -229,6 +238,7 @@ class AsyncTasks:
|
|
|
229
238
|
"idempotency_key": idempotency_key,
|
|
230
239
|
"max_attempts": max_attempts,
|
|
231
240
|
"timeout_seconds": timeout_seconds,
|
|
241
|
+
"group_id": group_id,
|
|
232
242
|
}
|
|
233
243
|
return Task._from_json(await self._client._send(r.enqueue(queue, spec, traceparent)))
|
|
234
244
|
|
|
@@ -270,22 +280,33 @@ class AsyncTasks:
|
|
|
270
280
|
*,
|
|
271
281
|
queue: str | None = None,
|
|
272
282
|
state: TaskState | None = None,
|
|
283
|
+
group_id: str | None = None,
|
|
273
284
|
idempotency_key: str | None = None,
|
|
274
285
|
last_error: str | None = None,
|
|
275
286
|
created_after: datetime | None = None,
|
|
276
287
|
created_before: datetime | None = None,
|
|
288
|
+
sort: TaskSort | None = None,
|
|
289
|
+
order: SortOrder | None = None,
|
|
277
290
|
page_size: int | None = None,
|
|
278
291
|
) -> AsyncIterator[Task]:
|
|
279
|
-
"""Tasks matching every filter given,
|
|
292
|
+
"""Tasks matching every filter given, fetching pages as needed.
|
|
293
|
+
|
|
294
|
+
Newest first, unless `sort` and `order` say otherwise: `order` defaults
|
|
295
|
+
to "asc" (soonest first) for `sort="run_at"` and to "desc" for the rest.
|
|
296
|
+
List pending tasks by `run_at` to see them in the order they run.
|
|
297
|
+
"""
|
|
280
298
|
cursor: str | None = None
|
|
281
299
|
while True:
|
|
282
300
|
page = await self.list_page(
|
|
283
301
|
queue=queue,
|
|
284
302
|
state=state,
|
|
303
|
+
group_id=group_id,
|
|
285
304
|
idempotency_key=idempotency_key,
|
|
286
305
|
last_error=last_error,
|
|
287
306
|
created_after=created_after,
|
|
288
307
|
created_before=created_before,
|
|
308
|
+
sort=sort,
|
|
309
|
+
order=order,
|
|
289
310
|
cursor=cursor,
|
|
290
311
|
limit=page_size,
|
|
291
312
|
)
|
|
@@ -300,10 +321,13 @@ class AsyncTasks:
|
|
|
300
321
|
*,
|
|
301
322
|
queue: str | None = None,
|
|
302
323
|
state: TaskState | None = None,
|
|
324
|
+
group_id: str | None = None,
|
|
303
325
|
idempotency_key: str | None = None,
|
|
304
326
|
last_error: str | None = None,
|
|
305
327
|
created_after: datetime | None = None,
|
|
306
328
|
created_before: datetime | None = None,
|
|
329
|
+
sort: TaskSort | None = None,
|
|
330
|
+
order: SortOrder | None = None,
|
|
307
331
|
cursor: str | None = None,
|
|
308
332
|
limit: int | None = None,
|
|
309
333
|
) -> TaskPage:
|
|
@@ -311,10 +335,13 @@ class AsyncTasks:
|
|
|
311
335
|
req = r.search_tasks(
|
|
312
336
|
queue=queue,
|
|
313
337
|
state=state,
|
|
338
|
+
group_id=group_id,
|
|
314
339
|
idempotency_key=idempotency_key,
|
|
315
340
|
last_error=last_error,
|
|
316
341
|
created_after=created_after,
|
|
317
342
|
created_before=created_before,
|
|
343
|
+
sort=sort,
|
|
344
|
+
order=order,
|
|
318
345
|
cursor=cursor,
|
|
319
346
|
limit=limit,
|
|
320
347
|
)
|
|
@@ -328,6 +355,75 @@ class AsyncTasks:
|
|
|
328
355
|
return Activity._from_json(await self._client._send(r.get_activity(range)))
|
|
329
356
|
|
|
330
357
|
|
|
358
|
+
class AsyncGroups:
|
|
359
|
+
"""Task groups: a set of tasks, and a callback task once all have finished.
|
|
360
|
+
|
|
361
|
+
Create a group, enqueue tasks with its `group_id`, then close it. Once it
|
|
362
|
+
is closed and every task has succeeded, failed or been cancelled, it
|
|
363
|
+
completes and enqueues its `on_complete` callback, once.
|
|
364
|
+
"""
|
|
365
|
+
|
|
366
|
+
def __init__(self, client: AsyncWorkly) -> None:
|
|
367
|
+
self._client = client
|
|
368
|
+
|
|
369
|
+
async def create(
|
|
370
|
+
self,
|
|
371
|
+
*,
|
|
372
|
+
name: str | None = None,
|
|
373
|
+
idempotency_key: str | None = None,
|
|
374
|
+
on_complete: Callback | r.CallbackDict | None = None,
|
|
375
|
+
) -> Group:
|
|
376
|
+
"""Create an open group.
|
|
377
|
+
|
|
378
|
+
`on_complete` is the task to enqueue when the group completes, e.g.
|
|
379
|
+
`Callback(url="https://api.example.com/done")` or the same as a dict;
|
|
380
|
+
it is a POST of the group's summary as JSON. If `idempotency_key` matches an
|
|
381
|
+
existing group, that group is returned and nothing is created.
|
|
382
|
+
"""
|
|
383
|
+
req = r.create_group(name, idempotency_key, on_complete)
|
|
384
|
+
return Group._from_json(await self._client._send(req))
|
|
385
|
+
|
|
386
|
+
async def get(self, group_id: str) -> Group:
|
|
387
|
+
return Group._from_json(await self._client._send(r.get_group(group_id)))
|
|
388
|
+
|
|
389
|
+
async def close(self, group_id: str) -> Group:
|
|
390
|
+
"""Stop the group accepting tasks, so it completes once they have finished.
|
|
391
|
+
|
|
392
|
+
A group whose tasks have all finished completes at once. Closing a
|
|
393
|
+
closed group does nothing.
|
|
394
|
+
"""
|
|
395
|
+
return Group._from_json(await self._client._send(r.close_group(group_id)))
|
|
396
|
+
|
|
397
|
+
async def list(
|
|
398
|
+
self,
|
|
399
|
+
*,
|
|
400
|
+
status: GroupStatus | None = None,
|
|
401
|
+
name: str | None = None,
|
|
402
|
+
page_size: int | None = None,
|
|
403
|
+
) -> AsyncIterator[Group]:
|
|
404
|
+
"""Groups, newest first, fetching pages as needed."""
|
|
405
|
+
cursor: str | None = None
|
|
406
|
+
while True:
|
|
407
|
+
page = await self.list_page(status=status, name=name, cursor=cursor, limit=page_size)
|
|
408
|
+
for group in page.groups:
|
|
409
|
+
yield group
|
|
410
|
+
if page.next_cursor is None:
|
|
411
|
+
return
|
|
412
|
+
cursor = page.next_cursor
|
|
413
|
+
|
|
414
|
+
async def list_page(
|
|
415
|
+
self,
|
|
416
|
+
*,
|
|
417
|
+
status: GroupStatus | None = None,
|
|
418
|
+
name: str | None = None,
|
|
419
|
+
cursor: str | None = None,
|
|
420
|
+
limit: int | None = None,
|
|
421
|
+
) -> GroupPage:
|
|
422
|
+
"""One page of `list`. Pass the previous page's `next_cursor` as `cursor`."""
|
|
423
|
+
req = r.list_groups(status=status, name=name, cursor=cursor, limit=limit)
|
|
424
|
+
return GroupPage._from_json(await self._client._send(req))
|
|
425
|
+
|
|
426
|
+
|
|
331
427
|
class AsyncSchedules:
|
|
332
428
|
"""Cron schedules that call an endpoint. Separate from queues: each call is a run."""
|
|
333
429
|
|
|
@@ -17,10 +17,14 @@ from ._models import (
|
|
|
17
17
|
AlertChannelList,
|
|
18
18
|
AlertChannelType,
|
|
19
19
|
Attempt,
|
|
20
|
+
Callback,
|
|
20
21
|
CronPreview,
|
|
21
22
|
Delivery,
|
|
22
23
|
EnqueueResult,
|
|
23
24
|
FailureGroup,
|
|
25
|
+
Group,
|
|
26
|
+
GroupPage,
|
|
27
|
+
GroupStatus,
|
|
24
28
|
HTTPMethod,
|
|
25
29
|
MissedPolicy,
|
|
26
30
|
OverlapPolicy,
|
|
@@ -30,8 +34,10 @@ from ._models import (
|
|
|
30
34
|
RunPage,
|
|
31
35
|
RunState,
|
|
32
36
|
Schedule,
|
|
37
|
+
SortOrder,
|
|
33
38
|
Task,
|
|
34
39
|
TaskPage,
|
|
40
|
+
TaskSort,
|
|
35
41
|
TaskState,
|
|
36
42
|
)
|
|
37
43
|
|
|
@@ -60,6 +66,7 @@ class Workly:
|
|
|
60
66
|
self._max_retries = max_retries
|
|
61
67
|
self.queues = Queues(self)
|
|
62
68
|
self.tasks = Tasks(self)
|
|
69
|
+
self.groups = Groups(self)
|
|
63
70
|
self.schedules = Schedules(self)
|
|
64
71
|
self.runs = Runs(self)
|
|
65
72
|
self.alerts = Alerts(self)
|
|
@@ -208,6 +215,7 @@ class Tasks:
|
|
|
208
215
|
idempotency_key: str | None = None,
|
|
209
216
|
max_attempts: int | None = None,
|
|
210
217
|
timeout_seconds: float | None = None,
|
|
218
|
+
group_id: str | None = None,
|
|
211
219
|
traceparent: str | None = None,
|
|
212
220
|
) -> Task:
|
|
213
221
|
"""Enqueue a task to deliver to `url`.
|
|
@@ -215,7 +223,8 @@ class Tasks:
|
|
|
215
223
|
`payload` is sent as JSON; `body` as raw text (use one). The queue is
|
|
216
224
|
created with default settings if it doesn't exist. If
|
|
217
225
|
`idempotency_key` matches a task already in the queue, that task is
|
|
218
|
-
returned and nothing is enqueued.
|
|
226
|
+
returned and nothing is enqueued. With `group_id`, the task joins
|
|
227
|
+
that group, which must be open.
|
|
219
228
|
"""
|
|
220
229
|
spec = {
|
|
221
230
|
"url": url,
|
|
@@ -229,6 +238,7 @@ class Tasks:
|
|
|
229
238
|
"idempotency_key": idempotency_key,
|
|
230
239
|
"max_attempts": max_attempts,
|
|
231
240
|
"timeout_seconds": timeout_seconds,
|
|
241
|
+
"group_id": group_id,
|
|
232
242
|
}
|
|
233
243
|
return Task._from_json(self._client._send(r.enqueue(queue, spec, traceparent)))
|
|
234
244
|
|
|
@@ -270,22 +280,33 @@ class Tasks:
|
|
|
270
280
|
*,
|
|
271
281
|
queue: str | None = None,
|
|
272
282
|
state: TaskState | None = None,
|
|
283
|
+
group_id: str | None = None,
|
|
273
284
|
idempotency_key: str | None = None,
|
|
274
285
|
last_error: str | None = None,
|
|
275
286
|
created_after: datetime | None = None,
|
|
276
287
|
created_before: datetime | None = None,
|
|
288
|
+
sort: TaskSort | None = None,
|
|
289
|
+
order: SortOrder | None = None,
|
|
277
290
|
page_size: int | None = None,
|
|
278
291
|
) -> Iterator[Task]:
|
|
279
|
-
"""Tasks matching every filter given,
|
|
292
|
+
"""Tasks matching every filter given, fetching pages as needed.
|
|
293
|
+
|
|
294
|
+
Newest first, unless `sort` and `order` say otherwise: `order` defaults
|
|
295
|
+
to "asc" (soonest first) for `sort="run_at"` and to "desc" for the rest.
|
|
296
|
+
List pending tasks by `run_at` to see them in the order they run.
|
|
297
|
+
"""
|
|
280
298
|
cursor: str | None = None
|
|
281
299
|
while True:
|
|
282
300
|
page = self.list_page(
|
|
283
301
|
queue=queue,
|
|
284
302
|
state=state,
|
|
303
|
+
group_id=group_id,
|
|
285
304
|
idempotency_key=idempotency_key,
|
|
286
305
|
last_error=last_error,
|
|
287
306
|
created_after=created_after,
|
|
288
307
|
created_before=created_before,
|
|
308
|
+
sort=sort,
|
|
309
|
+
order=order,
|
|
289
310
|
cursor=cursor,
|
|
290
311
|
limit=page_size,
|
|
291
312
|
)
|
|
@@ -299,10 +320,13 @@ class Tasks:
|
|
|
299
320
|
*,
|
|
300
321
|
queue: str | None = None,
|
|
301
322
|
state: TaskState | None = None,
|
|
323
|
+
group_id: str | None = None,
|
|
302
324
|
idempotency_key: str | None = None,
|
|
303
325
|
last_error: str | None = None,
|
|
304
326
|
created_after: datetime | None = None,
|
|
305
327
|
created_before: datetime | None = None,
|
|
328
|
+
sort: TaskSort | None = None,
|
|
329
|
+
order: SortOrder | None = None,
|
|
306
330
|
cursor: str | None = None,
|
|
307
331
|
limit: int | None = None,
|
|
308
332
|
) -> TaskPage:
|
|
@@ -310,10 +334,13 @@ class Tasks:
|
|
|
310
334
|
req = r.search_tasks(
|
|
311
335
|
queue=queue,
|
|
312
336
|
state=state,
|
|
337
|
+
group_id=group_id,
|
|
313
338
|
idempotency_key=idempotency_key,
|
|
314
339
|
last_error=last_error,
|
|
315
340
|
created_after=created_after,
|
|
316
341
|
created_before=created_before,
|
|
342
|
+
sort=sort,
|
|
343
|
+
order=order,
|
|
317
344
|
cursor=cursor,
|
|
318
345
|
limit=limit,
|
|
319
346
|
)
|
|
@@ -327,6 +354,74 @@ class Tasks:
|
|
|
327
354
|
return Activity._from_json(self._client._send(r.get_activity(range)))
|
|
328
355
|
|
|
329
356
|
|
|
357
|
+
class Groups:
|
|
358
|
+
"""Task groups: a set of tasks, and a callback task once all have finished.
|
|
359
|
+
|
|
360
|
+
Create a group, enqueue tasks with its `group_id`, then close it. Once it
|
|
361
|
+
is closed and every task has succeeded, failed or been cancelled, it
|
|
362
|
+
completes and enqueues its `on_complete` callback, once.
|
|
363
|
+
"""
|
|
364
|
+
|
|
365
|
+
def __init__(self, client: Workly) -> None:
|
|
366
|
+
self._client = client
|
|
367
|
+
|
|
368
|
+
def create(
|
|
369
|
+
self,
|
|
370
|
+
*,
|
|
371
|
+
name: str | None = None,
|
|
372
|
+
idempotency_key: str | None = None,
|
|
373
|
+
on_complete: Callback | r.CallbackDict | None = None,
|
|
374
|
+
) -> Group:
|
|
375
|
+
"""Create an open group.
|
|
376
|
+
|
|
377
|
+
`on_complete` is the task to enqueue when the group completes, e.g.
|
|
378
|
+
`Callback(url="https://api.example.com/done")` or the same as a dict;
|
|
379
|
+
it is a POST of the group's summary as JSON. If `idempotency_key` matches an
|
|
380
|
+
existing group, that group is returned and nothing is created.
|
|
381
|
+
"""
|
|
382
|
+
req = r.create_group(name, idempotency_key, on_complete)
|
|
383
|
+
return Group._from_json(self._client._send(req))
|
|
384
|
+
|
|
385
|
+
def get(self, group_id: str) -> Group:
|
|
386
|
+
return Group._from_json(self._client._send(r.get_group(group_id)))
|
|
387
|
+
|
|
388
|
+
def close(self, group_id: str) -> Group:
|
|
389
|
+
"""Stop the group accepting tasks, so it completes once they have finished.
|
|
390
|
+
|
|
391
|
+
A group whose tasks have all finished completes at once. Closing a
|
|
392
|
+
closed group does nothing.
|
|
393
|
+
"""
|
|
394
|
+
return Group._from_json(self._client._send(r.close_group(group_id)))
|
|
395
|
+
|
|
396
|
+
def list(
|
|
397
|
+
self,
|
|
398
|
+
*,
|
|
399
|
+
status: GroupStatus | None = None,
|
|
400
|
+
name: str | None = None,
|
|
401
|
+
page_size: int | None = None,
|
|
402
|
+
) -> Iterator[Group]:
|
|
403
|
+
"""Groups, newest first, fetching pages as needed."""
|
|
404
|
+
cursor: str | None = None
|
|
405
|
+
while True:
|
|
406
|
+
page = self.list_page(status=status, name=name, cursor=cursor, limit=page_size)
|
|
407
|
+
yield from page.groups
|
|
408
|
+
if page.next_cursor is None:
|
|
409
|
+
return
|
|
410
|
+
cursor = page.next_cursor
|
|
411
|
+
|
|
412
|
+
def list_page(
|
|
413
|
+
self,
|
|
414
|
+
*,
|
|
415
|
+
status: GroupStatus | None = None,
|
|
416
|
+
name: str | None = None,
|
|
417
|
+
cursor: str | None = None,
|
|
418
|
+
limit: int | None = None,
|
|
419
|
+
) -> GroupPage:
|
|
420
|
+
"""One page of `list`. Pass the previous page's `next_cursor` as `cursor`."""
|
|
421
|
+
req = r.list_groups(status=status, name=name, cursor=cursor, limit=limit)
|
|
422
|
+
return GroupPage._from_json(self._client._send(req))
|
|
423
|
+
|
|
424
|
+
|
|
330
425
|
class Schedules:
|
|
331
426
|
"""Cron schedules that call an endpoint. Separate from queues: each call is a run."""
|
|
332
427
|
|
|
@@ -5,11 +5,15 @@ Unknown fields are ignored, so newer servers don't break older SDKs.
|
|
|
5
5
|
|
|
6
6
|
from __future__ import annotations
|
|
7
7
|
|
|
8
|
-
from
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from dataclasses import dataclass, field
|
|
9
10
|
from datetime import datetime
|
|
10
11
|
from typing import Any, Literal
|
|
11
12
|
|
|
12
13
|
TaskState = Literal["pending", "running", "succeeded", "failed", "cancelled"]
|
|
14
|
+
TaskSort = Literal["created_at", "run_at", "last_attempt_at", "attempt_count"]
|
|
15
|
+
"""What tasks are listed by. `run_at` is when a task runs next: pending tasks run in that order."""
|
|
16
|
+
SortOrder = Literal["asc", "desc"]
|
|
13
17
|
HTTPMethod = Literal["GET", "POST", "PUT", "PATCH", "DELETE"]
|
|
14
18
|
AttemptOutcome = Literal["success", "http_error", "timeout", "network_error", "lost"]
|
|
15
19
|
RunState = Literal["pending", "running", "succeeded", "failed", "cancelled", "skipped"]
|
|
@@ -17,6 +21,8 @@ RunTrigger = Literal["schedule", "manual"]
|
|
|
17
21
|
SkipReason = Literal["overlap", "missed"]
|
|
18
22
|
OverlapPolicy = Literal["skip", "allow"]
|
|
19
23
|
MissedPolicy = Literal["skip", "catch_up"]
|
|
24
|
+
GroupStatus = Literal["open", "running", "succeeded", "failed"]
|
|
25
|
+
"""`open` accepts tasks; `running` is closed with tasks unfinished; the other two are final."""
|
|
20
26
|
|
|
21
27
|
|
|
22
28
|
def _time(value: str) -> datetime:
|
|
@@ -218,6 +224,8 @@ class Task:
|
|
|
218
224
|
"""When the latest attempt finished."""
|
|
219
225
|
retried_at: datetime | None
|
|
220
226
|
"""When the task was last retried by hand."""
|
|
227
|
+
group_id: str | None = None
|
|
228
|
+
"""The group the task belongs to, if any."""
|
|
221
229
|
|
|
222
230
|
@classmethod
|
|
223
231
|
def _from_json(cls, d: dict[str, Any]) -> Task:
|
|
@@ -243,6 +251,7 @@ class Task:
|
|
|
243
251
|
finished_at=_opt_time(d.get("finished_at")),
|
|
244
252
|
last_attempt_at=_opt_time(d.get("last_attempt_at")),
|
|
245
253
|
retried_at=_opt_time(d.get("retried_at")),
|
|
254
|
+
group_id=d.get("group_id"),
|
|
246
255
|
)
|
|
247
256
|
|
|
248
257
|
|
|
@@ -500,3 +509,89 @@ class AlertChannelList:
|
|
|
500
509
|
channels=[AlertChannel._from_json(c) for c in d["channels"]],
|
|
501
510
|
email_enabled=d["email_enabled"],
|
|
502
511
|
)
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
CALLBACK_QUEUE = "_callbacks"
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
518
|
+
class Callback:
|
|
519
|
+
"""A task Workly enqueues when something completes: a POST to `url` saying what completed.
|
|
520
|
+
|
|
521
|
+
A group's callback has the group's summary as its JSON body. It is
|
|
522
|
+
delivered like any task in its queue, with that queue's retries.
|
|
523
|
+
"""
|
|
524
|
+
|
|
525
|
+
url: str
|
|
526
|
+
queue: str = CALLBACK_QUEUE
|
|
527
|
+
"""The queue to enqueue the callback into. `_callbacks`, the default, is
|
|
528
|
+
created by Workly; configure it like any queue to change its retries."""
|
|
529
|
+
headers: Mapping[str, str] = field(default_factory=dict[str, str])
|
|
530
|
+
"""Headers sent with the callback. `Workly-*` headers are reserved."""
|
|
531
|
+
|
|
532
|
+
@classmethod
|
|
533
|
+
def _from_json(cls, d: dict[str, Any]) -> Callback:
|
|
534
|
+
return cls(
|
|
535
|
+
url=d["url"], queue=d.get("queue") or CALLBACK_QUEUE, headers=d.get("headers") or {}
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
GroupCallback = Callback
|
|
540
|
+
"""Deprecated: use `Callback`."""
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
@dataclass(frozen=True, slots=True)
|
|
544
|
+
class Group:
|
|
545
|
+
"""A set of tasks tracked together, with a callback once all have finished."""
|
|
546
|
+
|
|
547
|
+
id: str
|
|
548
|
+
name: str | None
|
|
549
|
+
status: GroupStatus
|
|
550
|
+
idempotency_key: str | None
|
|
551
|
+
total: int
|
|
552
|
+
succeeded: int
|
|
553
|
+
failed: int
|
|
554
|
+
cancelled: int
|
|
555
|
+
remaining: int
|
|
556
|
+
"""Tasks still pending or running."""
|
|
557
|
+
on_complete: Callback | None
|
|
558
|
+
callback_task_id: str | None
|
|
559
|
+
"""The callback task, once the group has completed."""
|
|
560
|
+
created_at: datetime
|
|
561
|
+
closed_at: datetime | None
|
|
562
|
+
finished_at: datetime | None
|
|
563
|
+
"""When the group completed."""
|
|
564
|
+
|
|
565
|
+
@classmethod
|
|
566
|
+
def _from_json(cls, d: dict[str, Any]) -> Group:
|
|
567
|
+
cb = d.get("on_complete")
|
|
568
|
+
return cls(
|
|
569
|
+
id=d["id"],
|
|
570
|
+
name=d.get("name"),
|
|
571
|
+
status=d["status"],
|
|
572
|
+
idempotency_key=d.get("idempotency_key"),
|
|
573
|
+
total=d["total"],
|
|
574
|
+
succeeded=d["succeeded"],
|
|
575
|
+
failed=d["failed"],
|
|
576
|
+
cancelled=d["cancelled"],
|
|
577
|
+
remaining=d["remaining"],
|
|
578
|
+
on_complete=None if cb is None else Callback._from_json(cb),
|
|
579
|
+
callback_task_id=d.get("callback_task_id"),
|
|
580
|
+
created_at=_time(d["created_at"]),
|
|
581
|
+
closed_at=_opt_time(d.get("closed_at")),
|
|
582
|
+
finished_at=_opt_time(d.get("finished_at")),
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
@dataclass(frozen=True, slots=True)
|
|
587
|
+
class GroupPage:
|
|
588
|
+
groups: list[Group]
|
|
589
|
+
next_cursor: str | None
|
|
590
|
+
"""Pass as `cursor` to get the next page. None on the last page."""
|
|
591
|
+
|
|
592
|
+
@classmethod
|
|
593
|
+
def _from_json(cls, d: dict[str, Any]) -> GroupPage:
|
|
594
|
+
return cls(
|
|
595
|
+
groups=[Group._from_json(g) for g in d["groups"]],
|
|
596
|
+
next_cursor=d.get("next_cursor"),
|
|
597
|
+
)
|
|
@@ -14,7 +14,17 @@ from importlib.metadata import PackageNotFoundError, version
|
|
|
14
14
|
from typing import Any, Required, TypedDict
|
|
15
15
|
from urllib.parse import quote
|
|
16
16
|
|
|
17
|
-
from ._models import
|
|
17
|
+
from ._models import (
|
|
18
|
+
CALLBACK_QUEUE,
|
|
19
|
+
AlertChannelType,
|
|
20
|
+
Callback,
|
|
21
|
+
GroupStatus,
|
|
22
|
+
HTTPMethod,
|
|
23
|
+
RunState,
|
|
24
|
+
SortOrder,
|
|
25
|
+
TaskSort,
|
|
26
|
+
TaskState,
|
|
27
|
+
)
|
|
18
28
|
|
|
19
29
|
DEFAULT_URL = "http://localhost:7337"
|
|
20
30
|
|
|
@@ -64,6 +74,32 @@ class TaskSpec(TypedDict, total=False):
|
|
|
64
74
|
idempotency_key: str
|
|
65
75
|
max_attempts: int
|
|
66
76
|
timeout_seconds: float
|
|
77
|
+
group_id: str
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class CallbackDict(TypedDict, total=False):
|
|
81
|
+
"""A `Callback` as a dict, with the same fields."""
|
|
82
|
+
|
|
83
|
+
url: Required[str]
|
|
84
|
+
queue: str
|
|
85
|
+
headers: Mapping[str, str]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
GroupCallbackSpec = CallbackDict
|
|
89
|
+
"""Deprecated: use `Callback` or `CallbackDict`."""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def callback_body(cb: Callback | CallbackDict) -> dict[str, Any]:
|
|
93
|
+
if isinstance(cb, Callback):
|
|
94
|
+
url, queue, headers = cb.url, cb.queue, cb.headers
|
|
95
|
+
else:
|
|
96
|
+
url, queue, headers = cb["url"], cb.get("queue", CALLBACK_QUEUE), cb.get("headers", {})
|
|
97
|
+
body: dict[str, Any] = {"url": url}
|
|
98
|
+
if queue != CALLBACK_QUEUE:
|
|
99
|
+
body["queue"] = queue
|
|
100
|
+
if headers:
|
|
101
|
+
body["headers"] = dict(headers)
|
|
102
|
+
return body
|
|
67
103
|
|
|
68
104
|
|
|
69
105
|
def task_body(spec: Mapping[str, Any]) -> dict[str, Any]:
|
|
@@ -178,10 +214,13 @@ def search_tasks(
|
|
|
178
214
|
*,
|
|
179
215
|
queue: str | None,
|
|
180
216
|
state: TaskState | None,
|
|
217
|
+
group_id: str | None,
|
|
181
218
|
idempotency_key: str | None,
|
|
182
219
|
last_error: str | None,
|
|
183
220
|
created_after: datetime | None,
|
|
184
221
|
created_before: datetime | None,
|
|
222
|
+
sort: TaskSort | None,
|
|
223
|
+
order: SortOrder | None,
|
|
185
224
|
cursor: str | None,
|
|
186
225
|
limit: int | None,
|
|
187
226
|
) -> Request:
|
|
@@ -190,6 +229,8 @@ def search_tasks(
|
|
|
190
229
|
params["queue"] = queue
|
|
191
230
|
if state is not None:
|
|
192
231
|
params["state"] = state
|
|
232
|
+
if group_id is not None:
|
|
233
|
+
params["group_id"] = group_id
|
|
193
234
|
if idempotency_key is not None:
|
|
194
235
|
params["idempotency_key"] = idempotency_key
|
|
195
236
|
if last_error is not None:
|
|
@@ -198,6 +239,10 @@ def search_tasks(
|
|
|
198
239
|
params["created_after"] = _timestamp(created_after, "created_after")
|
|
199
240
|
if created_before is not None:
|
|
200
241
|
params["created_before"] = _timestamp(created_before, "created_before")
|
|
242
|
+
if sort is not None:
|
|
243
|
+
params["sort"] = sort
|
|
244
|
+
if order is not None:
|
|
245
|
+
params["order"] = order
|
|
201
246
|
if cursor is not None:
|
|
202
247
|
params["cursor"] = cursor
|
|
203
248
|
if limit is not None:
|
|
@@ -226,6 +271,45 @@ def retry_task(task_id: str) -> Request:
|
|
|
226
271
|
return Request("POST", f"/v1/tasks/{_q(task_id)}/retry")
|
|
227
272
|
|
|
228
273
|
|
|
274
|
+
# Groups
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def create_group(
|
|
278
|
+
name: str | None, idempotency_key: str | None, on_complete: Callback | CallbackDict | None
|
|
279
|
+
) -> Request:
|
|
280
|
+
body: dict[str, Any] = {}
|
|
281
|
+
if name is not None:
|
|
282
|
+
body["name"] = name
|
|
283
|
+
if idempotency_key is not None:
|
|
284
|
+
body["idempotency_key"] = idempotency_key
|
|
285
|
+
if on_complete is not None:
|
|
286
|
+
body["on_complete"] = callback_body(on_complete)
|
|
287
|
+
return Request("POST", "/v1/groups", json=body, idempotent=idempotency_key is not None)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def get_group(group_id: str) -> Request:
|
|
291
|
+
return Request("GET", f"/v1/groups/{_q(group_id)}", idempotent=True)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def list_groups(
|
|
295
|
+
*, status: GroupStatus | None, name: str | None, cursor: str | None, limit: int | None
|
|
296
|
+
) -> Request:
|
|
297
|
+
params: dict[str, str] = {}
|
|
298
|
+
if status is not None:
|
|
299
|
+
params["status"] = status
|
|
300
|
+
if name is not None:
|
|
301
|
+
params["name"] = name
|
|
302
|
+
if cursor is not None:
|
|
303
|
+
params["cursor"] = cursor
|
|
304
|
+
if limit is not None:
|
|
305
|
+
params["limit"] = str(limit)
|
|
306
|
+
return Request("GET", "/v1/groups", params=params, idempotent=True)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def close_group(group_id: str) -> Request:
|
|
310
|
+
return Request("POST", f"/v1/groups/{_q(group_id)}/close", idempotent=True)
|
|
311
|
+
|
|
312
|
+
|
|
229
313
|
# Schedules and runs
|
|
230
314
|
|
|
231
315
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|