tasktronaut 0.3.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/PKG-INFO +12 -1
  2. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/README.md +7 -0
  3. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/pyproject.toml +14 -3
  4. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/backends/rq.py +143 -0
  5. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut.egg-info/PKG-INFO +12 -1
  6. tasktronaut-0.5.0/src/tasktronaut.egg-info/requires.txt +5 -0
  7. tasktronaut-0.3.0/src/tasktronaut.egg-info/requires.txt +0 -1
  8. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/LICENSE +0 -0
  9. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/setup.cfg +0 -0
  10. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/__init__.py +0 -0
  11. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/backend.py +0 -0
  12. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/backends/__init__.py +0 -0
  13. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/builder.py +0 -0
  14. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/context.py +0 -0
  15. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/decorators.py +0 -0
  16. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/errors.py +0 -0
  17. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/process.py +0 -0
  18. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/py.typed +0 -0
  19. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/steps.py +0 -0
  20. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/types.py +0 -0
  21. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut/utils.py +0 -0
  22. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut.egg-info/SOURCES.txt +0 -0
  23. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut.egg-info/dependency_links.txt +0 -0
  24. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/src/tasktronaut.egg-info/top_level.txt +0 -0
  25. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_backend.py +0 -0
  26. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_builder.py +0 -0
  27. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_context.py +0 -0
  28. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_decorators.py +0 -0
  29. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_errors.py +0 -0
  30. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_examples.py +0 -0
  31. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_process.py +0 -0
  32. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_steps.py +0 -0
  33. {tasktronaut-0.3.0 → tasktronaut-0.5.0}/tests/test_utils.py +0 -0
@@ -1,10 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tasktronaut
3
- Version: 0.3.0
3
+ Version: 0.5.0
4
4
  Summary: A simple task orchestration library for running complex processes or workflows in Python.
5
5
  Author-email: Chris Stefano <virtualstaticvoid@gmail.com>
6
6
  License-Expression: MIT
7
7
  Project-URL: homepage, https://github.com/virtualstaticvoid/tasktronaut
8
+ Project-URL: documentation, https://www.virtualstaticvoid.com/tasktronaut/
8
9
  Project-URL: repository, https://github.com/virtualstaticvoid/tasktronaut.git
9
10
  Project-URL: changelog, https://github.com/virtualstaticvoid/tasktronaut/blob/main/CHANGELOG.md
10
11
  Classifier: Development Status :: 5 - Production/Stable
@@ -15,6 +16,9 @@ Requires-Python: >=3.10
15
16
  Description-Content-Type: text/markdown
16
17
  License-File: LICENSE
17
18
  Requires-Dist: pydantic>=2.12.4
19
+ Provides-Extra: rq
20
+ Requires-Dist: redis>=7.0.1; extra == "rq"
21
+ Requires-Dist: rq>=3.0.0; extra == "rq"
18
22
  Dynamic: license-file
19
23
 
20
24
  # tasktronaut
@@ -28,6 +32,13 @@ A lightweight task orchestration library for building and running complex workfl
28
32
  Define workflows using a simple expressive DSL, with minimal boilerplate. Once defined, processes
29
33
  can be queued and executed with support for both sequential and concurrent DAG based execution.
30
34
 
35
+ Checkout the [documentation][documentation] for more information.
36
+
31
37
  ## License
32
38
 
33
39
  MIT Copyright (c) 2025 Chris Stefano
40
+
41
+
42
+ <!-- links -->
43
+
44
+ [documentation]: https://www.virtualstaticvoid.com/tasktronaut/
@@ -9,6 +9,13 @@ A lightweight task orchestration library for building and running complex workfl
9
9
  Define workflows using a simple expressive DSL, with minimal boilerplate. Once defined, processes
10
10
  can be queued and executed with support for both sequential and concurrent DAG based execution.
11
11
 
12
+ Checkout the [documentation][documentation] for more information.
13
+
12
14
  ## License
13
15
 
14
16
  MIT Copyright (c) 2025 Chris Stefano
17
+
18
+
19
+ <!-- links -->
20
+
21
+ [documentation]: https://www.virtualstaticvoid.com/tasktronaut/
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tasktronaut"
3
- version = "0.3.0"
3
+ version = "0.5.0"
4
4
  description = "A simple task orchestration library for running complex processes or workflows in Python."
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.10"
@@ -19,6 +19,15 @@ dependencies = [
19
19
  "pydantic>=2.12.4",
20
20
  ]
21
21
 
22
+ [project.optional-dependencies]
23
+ rq = [
24
+ "redis>=7.0.1",
25
+ # rq < 3.0 can leave finished jobs in the started job registry when the
26
+ # success-handling transaction is retried (WatchError), so they are later
27
+ # marked failed with AbandonedJobError; rq 3.0 fixes this.
28
+ "rq>=3.0.0",
29
+ ]
30
+
22
31
  [dependency-groups]
23
32
  dev = [
24
33
  { include-group = "build" },
@@ -33,15 +42,16 @@ dev = [
33
42
  ]
34
43
  rq = [
35
44
  "redis>=7.0.1",
36
- "rq>=2.6.0",
45
+ "rq>=3.0.0",
37
46
  ]
38
47
  django-rq = [
39
- "django-rq>=3.1",
48
+ "django-rq>=4.2.0",
40
49
  ]
41
50
  dagster = [
42
51
  "dagster>=1.12.0",
43
52
  ]
44
53
  test = [
54
+ "fakeredis>=2.40.0",
45
55
  "pytest>=8.4.2",
46
56
  "pytest-cov>=7.0.0",
47
57
  ]
@@ -57,6 +67,7 @@ docs = [
57
67
 
58
68
  [project.urls]
59
69
  homepage = "https://github.com/virtualstaticvoid/tasktronaut"
70
+ documentation = "https://www.virtualstaticvoid.com/tasktronaut/"
60
71
  repository = "https://github.com/virtualstaticvoid/tasktronaut.git"
61
72
  changelog = "https://github.com/virtualstaticvoid/tasktronaut/blob/main/CHANGELOG.md"
62
73
 
@@ -5,15 +5,158 @@ This module provides a concrete implementation of the Backend abstract base clas
5
5
  using the RQ (Redis Queue) job queue library. It enables asynchronous execution
6
6
  of Tasktronaut processes with support for job dependencies and Redis-backed
7
7
  job persistence.
8
+
9
+ Workers executing Tasktronaut processes should use :class:`RqQueue` (or a queue class
10
+ including :class:`JobScopedReadyJobRegistryMixin`) as their queue class, so that the
11
+ jobs of a process are enqueued as soon as the jobs they depend on have finished. For
12
+ example, ``rq worker --queue-class tasktronaut.backends.rq.RqQueue``.
8
13
  """
9
14
 
15
+ import logging
10
16
  from typing import Any, Dict, List, Optional, Union, cast
11
17
 
12
18
  import rq
19
+ from redis.exceptions import WatchError
20
+ from rq.job import JobStatus
21
+ from rq.registry import ReadyJobRegistry
22
+ from rq.utils import as_text
13
23
 
14
24
  from ..backend import Backend, Job
15
25
  from ..types import Description
16
26
 
27
+ logger = logging.getLogger(__name__)
28
+
29
+
30
+ class JobScopedReadyJobRegistry(ReadyJobRegistry):
31
+ """
32
+ Ready job registry whose claim of a job only conflicts with changes to that job.
33
+
34
+ RQ 3.0 claims each ready dependent job in an optimistic transaction which watches
35
+ the queue's shared ready job registry as well as the job. Every worker writes to
36
+ that registry when readying or enqueueing any dependent job, so concurrent
37
+ hand-offs of unrelated jobs abort each other, and the aborted job waits for worker
38
+ maintenance, which can be up to ten minutes later. Meanwhile, the subsequent steps
39
+ of the process sit in the deferred job registry.
40
+
41
+ Watching only the job keeps the guarantee against enqueueing a job twice: every
42
+ successful claim, and every deletion or cancellation of a job, writes the job's
43
+ hash in its transaction, so a competing change to the same job still aborts the
44
+ claim.
45
+ """
46
+
47
+ def enqueue_jobs(self, job_ids: List[str]) -> List[rq.job.Job]:
48
+ """
49
+ Move ready jobs from the registry onto the queue.
50
+
51
+ Mirrors RQ 3.0's :meth:`rq.registry.ReadyJobRegistry.enqueue_jobs` except for
52
+ the keys it watches; re-check it against RQ's implementation when upgrading
53
+ RQ. A job whose claim conflicts with a concurrent change to that job is
54
+ skipped and left for :meth:`cleanup`.
55
+
56
+ :param job_ids: Ids of the jobs to enqueue, in enqueue order
57
+ :type job_ids: List[str]
58
+ :return: The jobs that were enqueued
59
+ :rtype: List[rq.job.Job]
60
+ """
61
+ if not job_ids:
62
+ return []
63
+
64
+ queue = rq.Queue(
65
+ name=self.name,
66
+ connection=self.connection,
67
+ job_class=self.job_class,
68
+ serializer=self.serializer,
69
+ )
70
+ fetched = self.job_class.fetch_many(
71
+ job_ids, connection=self.connection, serializer=self.serializer
72
+ )
73
+
74
+ enqueued: List[rq.job.Job] = []
75
+ for job_id, job in zip(job_ids, fetched):
76
+ if job is None:
77
+ self.connection.zrem(self.key, job_id)
78
+ continue
79
+ if job.get_status(refresh=False) != JobStatus.READY_TO_ENQUEUE:
80
+ self.connection.zrem(self.key, job_id)
81
+ continue
82
+ try:
83
+ with self.connection.pipeline() as pipe:
84
+ # NOTE: RQ also watches `self.key`, the shared registry
85
+ pipe.watch(job.key)
86
+
87
+ if pipe.zscore(self.key, job.id) is None:
88
+ pipe.unwatch()
89
+ continue
90
+
91
+ status = cast(
92
+ Optional[Union[bytes, str]], pipe.hget(job.key, "status")
93
+ )
94
+ if (
95
+ not status
96
+ or JobStatus(as_text(status)) != JobStatus.READY_TO_ENQUEUE
97
+ ):
98
+ pipe.multi()
99
+ self.remove(job, pipeline=pipe)
100
+ pipe.execute()
101
+ continue
102
+
103
+ pipe.multi()
104
+ self.remove(job, pipeline=pipe)
105
+ # pylint: disable=protected-access
106
+ if job.has_rate_limit:
107
+ queue._enqueue_rate_limited_job(job, pipeline=pipe)
108
+ else:
109
+ queue._enqueue_job(
110
+ job,
111
+ pipeline=pipe,
112
+ at_front=job.should_enqueue_at_front(),
113
+ )
114
+ pipe.execute()
115
+ except WatchError:
116
+ logger.info(
117
+ "Ready job %s changed while enqueueing; skipping for now", job_id
118
+ )
119
+ continue
120
+ except Exception: # pylint: disable=broad-exception-caught
121
+ logger.exception(
122
+ "Failed to enqueue ready job %s; leaving in ReadyJobRegistry",
123
+ job_id,
124
+ )
125
+ continue
126
+ enqueued.append(job)
127
+ return enqueued
128
+
129
+
130
+ class JobScopedReadyJobRegistryMixin: # pylint: disable=too-few-public-methods
131
+ """
132
+ Queue mixin which hands off ready dependent jobs via the job-scoped registry.
133
+
134
+ A worker enqueues the dependents of a finished job through the
135
+ ``ready_job_registry`` of its queue class, so mix this into the queue class
136
+ workers are configured with. For example, with ``django-rq``::
137
+
138
+ class ApplicationQueue(JobScopedReadyJobRegistryMixin, DjangoRQ):
139
+ pass
140
+
141
+ RQ = {"QUEUE_CLASS": "myapp.queues.ApplicationQueue"}
142
+ """
143
+
144
+ @property
145
+ def ready_job_registry(self) -> JobScopedReadyJobRegistry:
146
+ queue = cast(rq.Queue, self)
147
+ return JobScopedReadyJobRegistry(
148
+ queue=queue, job_class=queue.job_class, serializer=queue.serializer
149
+ )
150
+
151
+
152
+ class RqQueue(JobScopedReadyJobRegistryMixin, rq.Queue):
153
+ """
154
+ RQ queue which hands off ready dependent jobs via :class:`JobScopedReadyJobRegistry`.
155
+
156
+ Use it as the queue class of workers executing Tasktronaut processes, e.g.
157
+ ``rq worker --queue-class tasktronaut.backends.rq.RqQueue``.
158
+ """
159
+
17
160
 
18
161
  # pylint: disable=too-many-arguments,too-many-positional-arguments
19
162
  class RqBackend(Backend):
@@ -1,10 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tasktronaut
3
- Version: 0.3.0
3
+ Version: 0.5.0
4
4
  Summary: A simple task orchestration library for running complex processes or workflows in Python.
5
5
  Author-email: Chris Stefano <virtualstaticvoid@gmail.com>
6
6
  License-Expression: MIT
7
7
  Project-URL: homepage, https://github.com/virtualstaticvoid/tasktronaut
8
+ Project-URL: documentation, https://www.virtualstaticvoid.com/tasktronaut/
8
9
  Project-URL: repository, https://github.com/virtualstaticvoid/tasktronaut.git
9
10
  Project-URL: changelog, https://github.com/virtualstaticvoid/tasktronaut/blob/main/CHANGELOG.md
10
11
  Classifier: Development Status :: 5 - Production/Stable
@@ -15,6 +16,9 @@ Requires-Python: >=3.10
15
16
  Description-Content-Type: text/markdown
16
17
  License-File: LICENSE
17
18
  Requires-Dist: pydantic>=2.12.4
19
+ Provides-Extra: rq
20
+ Requires-Dist: redis>=7.0.1; extra == "rq"
21
+ Requires-Dist: rq>=3.0.0; extra == "rq"
18
22
  Dynamic: license-file
19
23
 
20
24
  # tasktronaut
@@ -28,6 +32,13 @@ A lightweight task orchestration library for building and running complex workfl
28
32
  Define workflows using a simple expressive DSL, with minimal boilerplate. Once defined, processes
29
33
  can be queued and executed with support for both sequential and concurrent DAG based execution.
30
34
 
35
+ Checkout the [documentation][documentation] for more information.
36
+
31
37
  ## License
32
38
 
33
39
  MIT Copyright (c) 2025 Chris Stefano
40
+
41
+
42
+ <!-- links -->
43
+
44
+ [documentation]: https://www.virtualstaticvoid.com/tasktronaut/
@@ -0,0 +1,5 @@
1
+ pydantic>=2.12.4
2
+
3
+ [rq]
4
+ redis>=7.0.1
5
+ rq>=3.0.0
@@ -1 +0,0 @@
1
- pydantic>=2.12.4
File without changes
File without changes