tasktronaut 0.3.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/PKG-INFO +12 -1
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/README.md +7 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/pyproject.toml +14 -3
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/backends/rq.py +143 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut.egg-info/PKG-INFO +12 -1
- tasktronaut-0.5.0/src/tasktronaut.egg-info/requires.txt +5 -0
- tasktronaut-0.3.0/src/tasktronaut.egg-info/requires.txt +0 -1
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/LICENSE +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/setup.cfg +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/__init__.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/backend.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/backends/__init__.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/builder.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/context.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/decorators.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/errors.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/process.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/py.typed +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/steps.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/types.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/utils.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut.egg-info/SOURCES.txt +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut.egg-info/dependency_links.txt +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut.egg-info/top_level.txt +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_backend.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_builder.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_context.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_decorators.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_errors.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_examples.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_process.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_steps.py +0 -0
- {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_utils.py +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tasktronaut
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: A simple task orchestration library for running complex processes or workflows in Python.
|
|
5
5
|
Author-email: Chris Stefano <virtualstaticvoid@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: homepage, https://github.com/virtualstaticvoid/tasktronaut
|
|
8
|
+
Project-URL: documentation, https://www.virtualstaticvoid.com/tasktronaut/
|
|
8
9
|
Project-URL: repository, https://github.com/virtualstaticvoid/tasktronaut.git
|
|
9
10
|
Project-URL: changelog, https://github.com/virtualstaticvoid/tasktronaut/blob/main/CHANGELOG.md
|
|
10
11
|
Classifier: Development Status :: 5 - Production/Stable
|
|
@@ -15,6 +16,9 @@ Requires-Python: >=3.10
|
|
|
15
16
|
Description-Content-Type: text/markdown
|
|
16
17
|
License-File: LICENSE
|
|
17
18
|
Requires-Dist: pydantic>=2.12.4
|
|
19
|
+
Provides-Extra: rq
|
|
20
|
+
Requires-Dist: redis>=7.0.1; extra == "rq"
|
|
21
|
+
Requires-Dist: rq>=3.0.0; extra == "rq"
|
|
18
22
|
Dynamic: license-file
|
|
19
23
|
|
|
20
24
|
# tasktronaut
|
|
@@ -28,6 +32,13 @@ A lightweight task orchestration library for building and running complex workfl
|
|
|
28
32
|
Define workflows using a simple expressive DSL, with minimal boilerplate. Once defined, processes
|
|
29
33
|
can be queued and executed with support for both sequential and concurrent DAG based execution.
|
|
30
34
|
|
|
35
|
+
Checkout the [documentation][documentation] for more information.
|
|
36
|
+
|
|
31
37
|
## License
|
|
32
38
|
|
|
33
39
|
MIT Copyright (c) 2025 Chris Stefano
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
<!-- links -->
|
|
43
|
+
|
|
44
|
+
[documentation]: https://www.virtualstaticvoid.com/tasktronaut/
|
|
@@ -9,6 +9,13 @@ A lightweight task orchestration library for building and running complex workfl
|
|
|
9
9
|
Define workflows using a simple expressive DSL, with minimal boilerplate. Once defined, processes
|
|
10
10
|
can be queued and executed with support for both sequential and concurrent DAG based execution.
|
|
11
11
|
|
|
12
|
+
Checkout the [documentation][documentation] for more information.
|
|
13
|
+
|
|
12
14
|
## License
|
|
13
15
|
|
|
14
16
|
MIT Copyright (c) 2025 Chris Stefano
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
<!-- links -->
|
|
20
|
+
|
|
21
|
+
[documentation]: https://www.virtualstaticvoid.com/tasktronaut/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tasktronaut"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.5.0"
|
|
4
4
|
description = "A simple task orchestration library for running complex processes or workflows in Python."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
@@ -19,6 +19,15 @@ dependencies = [
|
|
|
19
19
|
"pydantic>=2.12.4",
|
|
20
20
|
]
|
|
21
21
|
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
rq = [
|
|
24
|
+
"redis>=7.0.1",
|
|
25
|
+
# rq < 3.0 can leave finished jobs in the started job registry when the
|
|
26
|
+
# success-handling transaction is retried (WatchError), so they are later
|
|
27
|
+
# marked failed with AbandonedJobError; rq 3.0 fixes this.
|
|
28
|
+
"rq>=3.0.0",
|
|
29
|
+
]
|
|
30
|
+
|
|
22
31
|
[dependency-groups]
|
|
23
32
|
dev = [
|
|
24
33
|
{ include-group = "build" },
|
|
@@ -33,15 +42,16 @@ dev = [
|
|
|
33
42
|
]
|
|
34
43
|
rq = [
|
|
35
44
|
"redis>=7.0.1",
|
|
36
|
-
"rq>=
|
|
45
|
+
"rq>=3.0.0",
|
|
37
46
|
]
|
|
38
47
|
django-rq = [
|
|
39
|
-
"django-rq>=
|
|
48
|
+
"django-rq>=4.2.0",
|
|
40
49
|
]
|
|
41
50
|
dagster = [
|
|
42
51
|
"dagster>=1.12.0",
|
|
43
52
|
]
|
|
44
53
|
test = [
|
|
54
|
+
"fakeredis>=2.40.0",
|
|
45
55
|
"pytest>=8.4.2",
|
|
46
56
|
"pytest-cov>=7.0.0",
|
|
47
57
|
]
|
|
@@ -57,6 +67,7 @@ docs = [
|
|
|
57
67
|
|
|
58
68
|
[project.urls]
|
|
59
69
|
homepage = "https://github.com/virtualstaticvoid/tasktronaut"
|
|
70
|
+
documentation = "https://www.virtualstaticvoid.com/tasktronaut/"
|
|
60
71
|
repository = "https://github.com/virtualstaticvoid/tasktronaut.git"
|
|
61
72
|
changelog = "https://github.com/virtualstaticvoid/tasktronaut/blob/main/CHANGELOG.md"
|
|
62
73
|
|
|
@@ -5,15 +5,158 @@ This module provides a concrete implementation of the Backend abstract base clas
|
|
|
5
5
|
using the RQ (Redis Queue) job queue library. It enables asynchronous execution
|
|
6
6
|
of Tasktronaut processes with support for job dependencies and Redis-backed
|
|
7
7
|
job persistence.
|
|
8
|
+
|
|
9
|
+
Workers executing Tasktronaut processes should use :class:`RqQueue` (or a queue class
|
|
10
|
+
including :class:`JobScopedReadyJobRegistryMixin`) as their queue class, so that the
|
|
11
|
+
jobs of a process are enqueued as soon as the jobs they depend on have finished. For
|
|
12
|
+
example, ``rq worker --queue-class tasktronaut.backends.rq.RqQueue``.
|
|
8
13
|
"""
|
|
9
14
|
|
|
15
|
+
import logging
|
|
10
16
|
from typing import Any, Dict, List, Optional, Union, cast
|
|
11
17
|
|
|
12
18
|
import rq
|
|
19
|
+
from redis.exceptions import WatchError
|
|
20
|
+
from rq.job import JobStatus
|
|
21
|
+
from rq.registry import ReadyJobRegistry
|
|
22
|
+
from rq.utils import as_text
|
|
13
23
|
|
|
14
24
|
from ..backend import Backend, Job
|
|
15
25
|
from ..types import Description
|
|
16
26
|
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class JobScopedReadyJobRegistry(ReadyJobRegistry):
|
|
31
|
+
"""
|
|
32
|
+
Ready job registry whose claim of a job only conflicts with changes to that job.
|
|
33
|
+
|
|
34
|
+
RQ 3.0 claims each ready dependent job in an optimistic transaction which watches
|
|
35
|
+
the queue's shared ready job registry as well as the job. Every worker writes to
|
|
36
|
+
that registry when readying or enqueueing any dependent job, so concurrent
|
|
37
|
+
hand-offs of unrelated jobs abort each other, and the aborted job waits for worker
|
|
38
|
+
maintenance, which can be up to ten minutes later. Meanwhile, the subsequent steps
|
|
39
|
+
of the process sit in the deferred job registry.
|
|
40
|
+
|
|
41
|
+
Watching only the job keeps the guarantee against enqueueing a job twice: every
|
|
42
|
+
successful claim, and every deletion or cancellation of a job, writes the job's
|
|
43
|
+
hash in its transaction, so a competing change to the same job still aborts the
|
|
44
|
+
claim.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def enqueue_jobs(self, job_ids: List[str]) -> List[rq.job.Job]:
|
|
48
|
+
"""
|
|
49
|
+
Move ready jobs from the registry onto the queue.
|
|
50
|
+
|
|
51
|
+
Mirrors RQ 3.0's :meth:`rq.registry.ReadyJobRegistry.enqueue_jobs` except for
|
|
52
|
+
the keys it watches; re-check it against RQ's implementation when upgrading
|
|
53
|
+
RQ. A job whose claim conflicts with a concurrent change to that job is
|
|
54
|
+
skipped and left for :meth:`cleanup`.
|
|
55
|
+
|
|
56
|
+
:param job_ids: Ids of the jobs to enqueue, in enqueue order
|
|
57
|
+
:type job_ids: List[str]
|
|
58
|
+
:return: The jobs that were enqueued
|
|
59
|
+
:rtype: List[rq.job.Job]
|
|
60
|
+
"""
|
|
61
|
+
if not job_ids:
|
|
62
|
+
return []
|
|
63
|
+
|
|
64
|
+
queue = rq.Queue(
|
|
65
|
+
name=self.name,
|
|
66
|
+
connection=self.connection,
|
|
67
|
+
job_class=self.job_class,
|
|
68
|
+
serializer=self.serializer,
|
|
69
|
+
)
|
|
70
|
+
fetched = self.job_class.fetch_many(
|
|
71
|
+
job_ids, connection=self.connection, serializer=self.serializer
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
enqueued: List[rq.job.Job] = []
|
|
75
|
+
for job_id, job in zip(job_ids, fetched):
|
|
76
|
+
if job is None:
|
|
77
|
+
self.connection.zrem(self.key, job_id)
|
|
78
|
+
continue
|
|
79
|
+
if job.get_status(refresh=False) != JobStatus.READY_TO_ENQUEUE:
|
|
80
|
+
self.connection.zrem(self.key, job_id)
|
|
81
|
+
continue
|
|
82
|
+
try:
|
|
83
|
+
with self.connection.pipeline() as pipe:
|
|
84
|
+
# NOTE: RQ also watches `self.key`, the shared registry
|
|
85
|
+
pipe.watch(job.key)
|
|
86
|
+
|
|
87
|
+
if pipe.zscore(self.key, job.id) is None:
|
|
88
|
+
pipe.unwatch()
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
status = cast(
|
|
92
|
+
Optional[Union[bytes, str]], pipe.hget(job.key, "status")
|
|
93
|
+
)
|
|
94
|
+
if (
|
|
95
|
+
not status
|
|
96
|
+
or JobStatus(as_text(status)) != JobStatus.READY_TO_ENQUEUE
|
|
97
|
+
):
|
|
98
|
+
pipe.multi()
|
|
99
|
+
self.remove(job, pipeline=pipe)
|
|
100
|
+
pipe.execute()
|
|
101
|
+
continue
|
|
102
|
+
|
|
103
|
+
pipe.multi()
|
|
104
|
+
self.remove(job, pipeline=pipe)
|
|
105
|
+
# pylint: disable=protected-access
|
|
106
|
+
if job.has_rate_limit:
|
|
107
|
+
queue._enqueue_rate_limited_job(job, pipeline=pipe)
|
|
108
|
+
else:
|
|
109
|
+
queue._enqueue_job(
|
|
110
|
+
job,
|
|
111
|
+
pipeline=pipe,
|
|
112
|
+
at_front=job.should_enqueue_at_front(),
|
|
113
|
+
)
|
|
114
|
+
pipe.execute()
|
|
115
|
+
except WatchError:
|
|
116
|
+
logger.info(
|
|
117
|
+
"Ready job %s changed while enqueueing; skipping for now", job_id
|
|
118
|
+
)
|
|
119
|
+
continue
|
|
120
|
+
except Exception: # pylint: disable=broad-exception-caught
|
|
121
|
+
logger.exception(
|
|
122
|
+
"Failed to enqueue ready job %s; leaving in ReadyJobRegistry",
|
|
123
|
+
job_id,
|
|
124
|
+
)
|
|
125
|
+
continue
|
|
126
|
+
enqueued.append(job)
|
|
127
|
+
return enqueued
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class JobScopedReadyJobRegistryMixin: # pylint: disable=too-few-public-methods
|
|
131
|
+
"""
|
|
132
|
+
Queue mixin which hands off ready dependent jobs via the job-scoped registry.
|
|
133
|
+
|
|
134
|
+
A worker enqueues the dependents of a finished job through the
|
|
135
|
+
``ready_job_registry`` of its queue class, so mix this into the queue class
|
|
136
|
+
workers are configured with. For example, with ``django-rq``::
|
|
137
|
+
|
|
138
|
+
class ApplicationQueue(JobScopedReadyJobRegistryMixin, DjangoRQ):
|
|
139
|
+
pass
|
|
140
|
+
|
|
141
|
+
RQ = {"QUEUE_CLASS": "myapp.queues.ApplicationQueue"}
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
@property
|
|
145
|
+
def ready_job_registry(self) -> JobScopedReadyJobRegistry:
|
|
146
|
+
queue = cast(rq.Queue, self)
|
|
147
|
+
return JobScopedReadyJobRegistry(
|
|
148
|
+
queue=queue, job_class=queue.job_class, serializer=queue.serializer
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class RqQueue(JobScopedReadyJobRegistryMixin, rq.Queue):
|
|
153
|
+
"""
|
|
154
|
+
RQ queue which hands off ready dependent jobs via :class:`JobScopedReadyJobRegistry`.
|
|
155
|
+
|
|
156
|
+
Use it as the queue class of workers executing Tasktronaut processes, e.g.
|
|
157
|
+
``rq worker --queue-class tasktronaut.backends.rq.RqQueue``.
|
|
158
|
+
"""
|
|
159
|
+
|
|
17
160
|
|
|
18
161
|
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
19
162
|
class RqBackend(Backend):
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tasktronaut
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: A simple task orchestration library for running complex processes or workflows in Python.
|
|
5
5
|
Author-email: Chris Stefano <virtualstaticvoid@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: homepage, https://github.com/virtualstaticvoid/tasktronaut
|
|
8
|
+
Project-URL: documentation, https://www.virtualstaticvoid.com/tasktronaut/
|
|
8
9
|
Project-URL: repository, https://github.com/virtualstaticvoid/tasktronaut.git
|
|
9
10
|
Project-URL: changelog, https://github.com/virtualstaticvoid/tasktronaut/blob/main/CHANGELOG.md
|
|
10
11
|
Classifier: Development Status :: 5 - Production/Stable
|
|
@@ -15,6 +16,9 @@ Requires-Python: >=3.10
|
|
|
15
16
|
Description-Content-Type: text/markdown
|
|
16
17
|
License-File: LICENSE
|
|
17
18
|
Requires-Dist: pydantic>=2.12.4
|
|
19
|
+
Provides-Extra: rq
|
|
20
|
+
Requires-Dist: redis>=7.0.1; extra == "rq"
|
|
21
|
+
Requires-Dist: rq>=3.0.0; extra == "rq"
|
|
18
22
|
Dynamic: license-file
|
|
19
23
|
|
|
20
24
|
# tasktronaut
|
|
@@ -28,6 +32,13 @@ A lightweight task orchestration library for building and running complex workfl
|
|
|
28
32
|
Define workflows using a simple expressive DSL, with minimal boilerplate. Once defined, processes
|
|
29
33
|
can be queued and executed with support for both sequential and concurrent DAG based execution.
|
|
30
34
|
|
|
35
|
+
Checkout the [documentation][documentation] for more information.
|
|
36
|
+
|
|
31
37
|
## License
|
|
32
38
|
|
|
33
39
|
MIT Copyright (c) 2025 Chris Stefano
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
<!-- links -->
|
|
43
|
+
|
|
44
|
+
[documentation]: https://www.virtualstaticvoid.com/tasktronaut/
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
pydantic>=2.12.4
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|