tasktronaut 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/PKG-INFO +12 -1
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/README.md +7 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/pyproject.toml +14 -3
- tasktronaut-0.4.0/src/tasktronaut/__init__.py +33 -0
- tasktronaut-0.4.0/src/tasktronaut/backend.py +299 -0
- tasktronaut-0.4.0/src/tasktronaut/backends/rq.py +199 -0
- tasktronaut-0.4.0/src/tasktronaut/builder.py +385 -0
- tasktronaut-0.4.0/src/tasktronaut/context.py +17 -0
- tasktronaut-0.4.0/src/tasktronaut/decorators.py +70 -0
- tasktronaut-0.4.0/src/tasktronaut/errors.py +31 -0
- tasktronaut-0.4.0/src/tasktronaut/process.py +457 -0
- tasktronaut-0.4.0/src/tasktronaut/steps.py +85 -0
- tasktronaut-0.4.0/src/tasktronaut/types.py +171 -0
- tasktronaut-0.4.0/src/tasktronaut/utils.py +81 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/src/tasktronaut.egg-info/PKG-INFO +12 -1
- tasktronaut-0.4.0/src/tasktronaut.egg-info/requires.txt +5 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/tests/test_backend.py +47 -8
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/tests/test_process.py +3 -8
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/tests/test_utils.py +5 -1
- tasktronaut-0.2.0/src/tasktronaut/__init__.py +0 -14
- tasktronaut-0.2.0/src/tasktronaut/backend.py +0 -129
- tasktronaut-0.2.0/src/tasktronaut/backends/rq.py +0 -86
- tasktronaut-0.2.0/src/tasktronaut/builder.py +0 -151
- tasktronaut-0.2.0/src/tasktronaut/context.py +0 -2
- tasktronaut-0.2.0/src/tasktronaut/decorators.py +0 -28
- tasktronaut-0.2.0/src/tasktronaut/errors.py +0 -6
- tasktronaut-0.2.0/src/tasktronaut/process.py +0 -203
- tasktronaut-0.2.0/src/tasktronaut/steps.py +0 -32
- tasktronaut-0.2.0/src/tasktronaut/types.py +0 -53
- tasktronaut-0.2.0/src/tasktronaut/utils.py +0 -26
- tasktronaut-0.2.0/src/tasktronaut.egg-info/requires.txt +0 -1
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/LICENSE +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/setup.cfg +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/src/tasktronaut/backends/__init__.py +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/src/tasktronaut/py.typed +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/src/tasktronaut.egg-info/SOURCES.txt +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/src/tasktronaut.egg-info/dependency_links.txt +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/src/tasktronaut.egg-info/top_level.txt +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/tests/test_builder.py +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/tests/test_context.py +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/tests/test_decorators.py +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/tests/test_errors.py +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/tests/test_examples.py +0 -0
- {tasktronaut-0.2.0 → tasktronaut-0.4.0}/tests/test_steps.py +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tasktronaut
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: A simple task orchestration library for running complex processes or workflows in Python.
|
|
5
5
|
Author-email: Chris Stefano <virtualstaticvoid@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: homepage, https://github.com/virtualstaticvoid/tasktronaut
|
|
8
|
+
Project-URL: documentation, https://www.virtualstaticvoid.com/tasktronaut/
|
|
8
9
|
Project-URL: repository, https://github.com/virtualstaticvoid/tasktronaut.git
|
|
9
10
|
Project-URL: changelog, https://github.com/virtualstaticvoid/tasktronaut/blob/main/CHANGELOG.md
|
|
10
11
|
Classifier: Development Status :: 5 - Production/Stable
|
|
@@ -15,6 +16,9 @@ Requires-Python: >=3.10
|
|
|
15
16
|
Description-Content-Type: text/markdown
|
|
16
17
|
License-File: LICENSE
|
|
17
18
|
Requires-Dist: pydantic>=2.12.4
|
|
19
|
+
Provides-Extra: rq
|
|
20
|
+
Requires-Dist: redis>=7.0.1; extra == "rq"
|
|
21
|
+
Requires-Dist: rq>=3.0.0; extra == "rq"
|
|
18
22
|
Dynamic: license-file
|
|
19
23
|
|
|
20
24
|
# tasktronaut
|
|
@@ -28,6 +32,13 @@ A lightweight task orchestration library for building and running complex workfl
|
|
|
28
32
|
Define workflows using a simple expressive DSL, with minimal boilerplate. Once defined, processes
|
|
29
33
|
can be queued and executed with support for both sequential and concurrent DAG based execution.
|
|
30
34
|
|
|
35
|
+
Checkout the [documentation][documentation] for more information.
|
|
36
|
+
|
|
31
37
|
## License
|
|
32
38
|
|
|
33
39
|
MIT Copyright (c) 2025 Chris Stefano
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
<!-- links -->
|
|
43
|
+
|
|
44
|
+
[documentation]: https://www.virtualstaticvoid.com/tasktronaut/
|
|
@@ -9,6 +9,13 @@ A lightweight task orchestration library for building and running complex workfl
|
|
|
9
9
|
Define workflows using a simple expressive DSL, with minimal boilerplate. Once defined, processes
|
|
10
10
|
can be queued and executed with support for both sequential and concurrent DAG based execution.
|
|
11
11
|
|
|
12
|
+
Checkout the [documentation][documentation] for more information.
|
|
13
|
+
|
|
12
14
|
## License
|
|
13
15
|
|
|
14
16
|
MIT Copyright (c) 2025 Chris Stefano
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
<!-- links -->
|
|
20
|
+
|
|
21
|
+
[documentation]: https://www.virtualstaticvoid.com/tasktronaut/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tasktronaut"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
description = "A simple task orchestration library for running complex processes or workflows in Python."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
@@ -19,6 +19,15 @@ dependencies = [
|
|
|
19
19
|
"pydantic>=2.12.4",
|
|
20
20
|
]
|
|
21
21
|
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
rq = [
|
|
24
|
+
"redis>=7.0.1",
|
|
25
|
+
# rq < 3.0 can leave finished jobs in the started job registry when the
|
|
26
|
+
# success-handling transaction is retried (WatchError), so they are later
|
|
27
|
+
# marked failed with AbandonedJobError; rq 3.0 fixes this.
|
|
28
|
+
"rq>=3.0.0",
|
|
29
|
+
]
|
|
30
|
+
|
|
22
31
|
[dependency-groups]
|
|
23
32
|
dev = [
|
|
24
33
|
{ include-group = "build" },
|
|
@@ -26,16 +35,17 @@ dev = [
|
|
|
26
35
|
{ include-group = "django-rq" },
|
|
27
36
|
{ include-group = "rq" },
|
|
28
37
|
{ include-group = "test" },
|
|
38
|
+
{ include-group = "docs" },
|
|
29
39
|
"ruff>=0.14.4",
|
|
30
40
|
"pylint>=4.0.2",
|
|
31
41
|
"mypy>=1.18.2",
|
|
32
42
|
]
|
|
33
43
|
rq = [
|
|
34
44
|
"redis>=7.0.1",
|
|
35
|
-
"rq>=
|
|
45
|
+
"rq>=3.0.0",
|
|
36
46
|
]
|
|
37
47
|
django-rq = [
|
|
38
|
-
"django-rq>=
|
|
48
|
+
"django-rq>=4.2.0",
|
|
39
49
|
]
|
|
40
50
|
dagster = [
|
|
41
51
|
"dagster>=1.12.0",
|
|
@@ -56,6 +66,7 @@ docs = [
|
|
|
56
66
|
|
|
57
67
|
[project.urls]
|
|
58
68
|
homepage = "https://github.com/virtualstaticvoid/tasktronaut"
|
|
69
|
+
documentation = "https://www.virtualstaticvoid.com/tasktronaut/"
|
|
59
70
|
repository = "https://github.com/virtualstaticvoid/tasktronaut.git"
|
|
60
71
|
changelog = "https://github.com/virtualstaticvoid/tasktronaut/blob/main/CHANGELOG.md"
|
|
61
72
|
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Tasktronaut: A Python library for defining and executing process workflows.
|
|
3
|
+
|
|
4
|
+
The tasktronaut package provides a framework for building complex process definitions
|
|
5
|
+
with support for sequential and concurrent execution, sub-processes, conditional logic,
|
|
6
|
+
and iterative patterns.
|
|
7
|
+
|
|
8
|
+
This module serves as the main entry point, exposing the primary public API for
|
|
9
|
+
the library.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
13
|
+
|
|
14
|
+
from .backend import Backend
|
|
15
|
+
from .builder import Builder
|
|
16
|
+
from .context import Context
|
|
17
|
+
from .decorators import task
|
|
18
|
+
from .process import ExecutionMode, ProcessDefinition
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
__version__ = version(__name__)
|
|
22
|
+
except PackageNotFoundError:
|
|
23
|
+
__version__ = "0.0.0.dev"
|
|
24
|
+
|
|
25
|
+
__all__ = (
|
|
26
|
+
"Backend",
|
|
27
|
+
"Builder",
|
|
28
|
+
"Context",
|
|
29
|
+
"ExecutionMode",
|
|
30
|
+
"ProcessDefinition",
|
|
31
|
+
"task",
|
|
32
|
+
"__version__",
|
|
33
|
+
)
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Backend module.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import inspect
|
|
6
|
+
import logging
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
from typing import Any, Dict, List, Optional, Protocol, Union
|
|
9
|
+
|
|
10
|
+
from .context import Context
|
|
11
|
+
from .errors import CancelProcessError, NonRetryableProcessError
|
|
12
|
+
from .types import Description
|
|
13
|
+
from .utils import load_definition
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Job(Protocol):
|
|
19
|
+
"""
|
|
20
|
+
Protocol defining the interface for a job object.
|
|
21
|
+
|
|
22
|
+
A job represents an enqueued unit of work in the backend system and provides
|
|
23
|
+
methods to manage its lifecycle.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def cancel(self): # pragma: no cover
|
|
27
|
+
"""
|
|
28
|
+
Cancel the execution of this job.
|
|
29
|
+
|
|
30
|
+
Stops the job from being executed if it hasn't already started.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def delete_dependents(self): # pragma: no cover
|
|
34
|
+
"""
|
|
35
|
+
Delete all jobs that depend on this job.
|
|
36
|
+
|
|
37
|
+
Removes any downstream jobs that were scheduled to run after completion
|
|
38
|
+
of this job.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
43
|
+
class Backend(ABC):
|
|
44
|
+
"""
|
|
45
|
+
Abstract base class defining the interface for process execution backends.
|
|
46
|
+
|
|
47
|
+
Backends are responsible for enqueueing tasks, managing job dependencies, and
|
|
48
|
+
executing process lifecycle callbacks. Subclasses must implement the abstract
|
|
49
|
+
enqueue methods to integrate with specific job queue systems.
|
|
50
|
+
|
|
51
|
+
The backend handles three main phases of process execution:
|
|
52
|
+
- Process start callbacks
|
|
53
|
+
- Individual task execution with error handling
|
|
54
|
+
- Process completion callbacks
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
@abstractmethod
|
|
58
|
+
def enqueue_perform_start(
|
|
59
|
+
self,
|
|
60
|
+
identifier: str,
|
|
61
|
+
module_name: str,
|
|
62
|
+
definition_class: str,
|
|
63
|
+
depends_on: Optional[Job] = None,
|
|
64
|
+
) -> Job: # pragma: no cover
|
|
65
|
+
"""
|
|
66
|
+
Enqueue a process start callback.
|
|
67
|
+
|
|
68
|
+
Schedules the execution of the process start callback, which allows the
|
|
69
|
+
process definition to perform initialization logic when execution begins.
|
|
70
|
+
|
|
71
|
+
:param identifier: Unique identifier for the process execution instance
|
|
72
|
+
:type identifier: str
|
|
73
|
+
:param module_name: Fully qualified module name containing the process definition
|
|
74
|
+
:type module_name: str
|
|
75
|
+
:param definition_class: Name of the process definition class to instantiate
|
|
76
|
+
:type definition_class: str
|
|
77
|
+
:param depends_on:
|
|
78
|
+
Optional job that this job depends on. If provided,
|
|
79
|
+
this job will not execute until the dependency completes.
|
|
80
|
+
:type depends_on: Optional[Job]
|
|
81
|
+
:return: Job object representing the enqueued task
|
|
82
|
+
:rtype: Job
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
@staticmethod
|
|
86
|
+
def perform_start(
|
|
87
|
+
identifier: str,
|
|
88
|
+
module_name: str,
|
|
89
|
+
definition_class: str,
|
|
90
|
+
):
|
|
91
|
+
"""
|
|
92
|
+
Execute the process start callback.
|
|
93
|
+
|
|
94
|
+
Loads the process definition and invokes its start callback. This method
|
|
95
|
+
is typically called by the backend after the job has been dequeued.
|
|
96
|
+
|
|
97
|
+
It is implemented as a staticmethod, to allow for the most flexibility in
|
|
98
|
+
backend implementations.
|
|
99
|
+
|
|
100
|
+
:param identifier: Unique identifier for the process execution instance
|
|
101
|
+
:type identifier: str
|
|
102
|
+
:param module_name: Fully qualified module name containing the process definition
|
|
103
|
+
:type module_name: str
|
|
104
|
+
:param definition_class: Name of the process definition class to instantiate
|
|
105
|
+
:type definition_class: str
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
definition = load_definition(module_name, definition_class)
|
|
109
|
+
definition.on_started(identifier)
|
|
110
|
+
|
|
111
|
+
@abstractmethod
|
|
112
|
+
def enqueue_perform_task(
|
|
113
|
+
self,
|
|
114
|
+
identifier: str,
|
|
115
|
+
module_name: str,
|
|
116
|
+
definition_class: str,
|
|
117
|
+
function_name: str,
|
|
118
|
+
description: Description,
|
|
119
|
+
kwargs: Dict[str, Any],
|
|
120
|
+
depends_on: Optional[Job] = None,
|
|
121
|
+
) -> Job: # pragma: no cover
|
|
122
|
+
"""
|
|
123
|
+
Enqueue a process task for execution.
|
|
124
|
+
|
|
125
|
+
Schedules the execution of a specific task function within a process definition.
|
|
126
|
+
The task will be executed with the provided keyword arguments and can optionally
|
|
127
|
+
depend on another job.
|
|
128
|
+
|
|
129
|
+
:param identifier: Unique identifier for the process execution instance
|
|
130
|
+
:type identifier: str
|
|
131
|
+
:param module_name: Fully qualified module name containing the process definition
|
|
132
|
+
:type module_name: str
|
|
133
|
+
:param definition_class: Name of the process definition class to instantiate
|
|
134
|
+
:type definition_class: str
|
|
135
|
+
:param function_name: Name of the task method to execute
|
|
136
|
+
:type function_name: str
|
|
137
|
+
:param description: Human-readable description of the task
|
|
138
|
+
:type description: Description
|
|
139
|
+
:param kwargs: Keyword arguments to pass to the task function
|
|
140
|
+
:type kwargs: Dict[str, Any]
|
|
141
|
+
:param depends_on:
|
|
142
|
+
Optional job that this job depends on. If provided,
|
|
143
|
+
this job will not execute until the dependency completes.
|
|
144
|
+
:type depends_on: Optional[Job]
|
|
145
|
+
:return: Job object representing the enqueued task
|
|
146
|
+
:rtype: Job
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
@classmethod
|
|
150
|
+
def perform_task(
|
|
151
|
+
cls,
|
|
152
|
+
job: Job,
|
|
153
|
+
identifier: str,
|
|
154
|
+
module_name: str,
|
|
155
|
+
definition_class: str,
|
|
156
|
+
function_name: str,
|
|
157
|
+
description: Description,
|
|
158
|
+
kwargs: Dict[str, Any],
|
|
159
|
+
):
|
|
160
|
+
"""
|
|
161
|
+
Execute a process task with comprehensive error handling.
|
|
162
|
+
|
|
163
|
+
Loads the process definition, creates an execution context, and invokes the
|
|
164
|
+
specified task function. Handles three categories of errors:
|
|
165
|
+
|
|
166
|
+
- NonRetryableProcessError: Logs error, calls on_failed callback, and re-raises
|
|
167
|
+
- CancelProcessError: Cancels the job and its dependents, calls on_cancelled callback
|
|
168
|
+
- Other exceptions: Logs error, calls on_failed callback, and re-raises for retry
|
|
169
|
+
|
|
170
|
+
The task execution is wrapped with the definition's around_task context manager,
|
|
171
|
+
allowing the definition to perform setup and teardown logic.
|
|
172
|
+
|
|
173
|
+
:param job: Job object that can be cancelled or have dependents deleted
|
|
174
|
+
:type job: Job
|
|
175
|
+
:param identifier: Unique identifier for the process execution instance
|
|
176
|
+
:type identifier: str
|
|
177
|
+
:param module_name: Fully qualified module name containing the process definition
|
|
178
|
+
:type module_name: str
|
|
179
|
+
:param definition_class: Name of the process definition class to instantiate
|
|
180
|
+
:type definition_class: str
|
|
181
|
+
:param function_name: Name of the task method to execute
|
|
182
|
+
:type function_name: str
|
|
183
|
+
:param description: Human-readable description of the task
|
|
184
|
+
:type description: Description
|
|
185
|
+
:param kwargs: Keyword arguments to pass to the task function
|
|
186
|
+
:type kwargs: Dict[str, Any]
|
|
187
|
+
|
|
188
|
+
:raises NonRetryableProcessError: When task execution raises a non-retryable error
|
|
189
|
+
:raises Exception: When task execution raises an unhandled error (retryable)
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
definition = load_definition(module_name, definition_class)
|
|
193
|
+
task_func = getattr(definition, function_name)
|
|
194
|
+
|
|
195
|
+
signature = inspect.signature(task_func)
|
|
196
|
+
has_context = "context" in signature.parameters
|
|
197
|
+
|
|
198
|
+
context = Context()
|
|
199
|
+
try:
|
|
200
|
+
with definition.around_task(
|
|
201
|
+
identifier=identifier,
|
|
202
|
+
step=function_name,
|
|
203
|
+
description=description,
|
|
204
|
+
context=context,
|
|
205
|
+
):
|
|
206
|
+
if has_context:
|
|
207
|
+
task_func(context=context, **kwargs)
|
|
208
|
+
else:
|
|
209
|
+
task_func(**kwargs)
|
|
210
|
+
|
|
211
|
+
except NonRetryableProcessError as e:
|
|
212
|
+
logger.error(
|
|
213
|
+
"Failed to execute task '%s', a non-retryable error was raised.",
|
|
214
|
+
function_name,
|
|
215
|
+
exc_info=True,
|
|
216
|
+
stack_info=True,
|
|
217
|
+
)
|
|
218
|
+
definition.on_failed(identifier, function_name, e)
|
|
219
|
+
# re-raise error to fail job
|
|
220
|
+
raise
|
|
221
|
+
|
|
222
|
+
except CancelProcessError:
|
|
223
|
+
logger.error(
|
|
224
|
+
"Failed to execute task '%s', the process was cancelled.",
|
|
225
|
+
function_name,
|
|
226
|
+
exc_info=True,
|
|
227
|
+
stack_info=True,
|
|
228
|
+
)
|
|
229
|
+
job.cancel()
|
|
230
|
+
job.delete_dependents() # clean up
|
|
231
|
+
definition.on_cancelled(identifier)
|
|
232
|
+
# ignore error
|
|
233
|
+
|
|
234
|
+
except Exception as e:
|
|
235
|
+
logger.error(
|
|
236
|
+
"Failed to execute task '%s' due to an unhandled error. "
|
|
237
|
+
"The task will be retried.",
|
|
238
|
+
function_name,
|
|
239
|
+
exc_info=True,
|
|
240
|
+
stack_info=True,
|
|
241
|
+
)
|
|
242
|
+
definition.on_failed(identifier, function_name, e)
|
|
243
|
+
# re-raise error to fail job
|
|
244
|
+
raise
|
|
245
|
+
|
|
246
|
+
@abstractmethod
|
|
247
|
+
def enqueue_perform_complete(
|
|
248
|
+
self,
|
|
249
|
+
identifier: str,
|
|
250
|
+
module_name: str,
|
|
251
|
+
definition_class: str,
|
|
252
|
+
depends_on: Optional[Union[Job, List[Job]]] = None,
|
|
253
|
+
) -> Job: # pragma: no cover
|
|
254
|
+
"""
|
|
255
|
+
Enqueue a process completion callback.
|
|
256
|
+
|
|
257
|
+
Schedules the execution of the process completion callback, which allows the
|
|
258
|
+
process definition to perform finalization logic when all tasks have completed.
|
|
259
|
+
Can depend on one or more jobs.
|
|
260
|
+
|
|
261
|
+
:param identifier: Unique identifier for the process execution instance
|
|
262
|
+
:type identifier: str
|
|
263
|
+
:param module_name: Fully qualified module name containing the process definition
|
|
264
|
+
:type module_name: str
|
|
265
|
+
:param definition_class: Name of the process definition class to instantiate
|
|
266
|
+
:type definition_class: str
|
|
267
|
+
:param depends_on: Optional job or list of jobs that this job depends on.
|
|
268
|
+
If provided, this job will not execute until all dependencies complete.
|
|
269
|
+
:type depends_on: Optional[Union[Job, List[Job]]]
|
|
270
|
+
:return: Job object representing the enqueued task
|
|
271
|
+
:rtype: Job
|
|
272
|
+
"""
|
|
273
|
+
|
|
274
|
+
@staticmethod
|
|
275
|
+
def perform_complete(
|
|
276
|
+
identifier: str,
|
|
277
|
+
module_name: str,
|
|
278
|
+
definition_class: str,
|
|
279
|
+
):
|
|
280
|
+
"""
|
|
281
|
+
Execute the process completion callback.
|
|
282
|
+
|
|
283
|
+
Loads the process definition and invokes its completion callback. This method
|
|
284
|
+
is typically called by the backend after all process tasks have completed
|
|
285
|
+
successfully.
|
|
286
|
+
|
|
287
|
+
It is implemented as a staticmethod, to allow for the most flexibility in
|
|
288
|
+
backend implementations.
|
|
289
|
+
|
|
290
|
+
:param identifier: Unique identifier for the process execution instance
|
|
291
|
+
:type identifier: str
|
|
292
|
+
:param module_name: Fully qualified module name containing the process definition
|
|
293
|
+
:type module_name: str
|
|
294
|
+
:param definition_class: Name of the process definition class to instantiate
|
|
295
|
+
:type definition_class: str
|
|
296
|
+
"""
|
|
297
|
+
|
|
298
|
+
definition = load_definition(module_name, definition_class)
|
|
299
|
+
definition.on_completed(identifier)
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""
|
|
2
|
+
RQ Backend implementation for Tasktronaut process execution.
|
|
3
|
+
|
|
4
|
+
This module provides a concrete implementation of the Backend abstract base class
|
|
5
|
+
using the RQ (Redis Queue) job queue library. It enables asynchronous execution
|
|
6
|
+
of Tasktronaut processes with support for job dependencies and Redis-backed
|
|
7
|
+
job persistence.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from typing import Any, Dict, List, Optional, Union, cast
|
|
11
|
+
|
|
12
|
+
import rq
|
|
13
|
+
|
|
14
|
+
from ..backend import Backend, Job
|
|
15
|
+
from ..types import Description
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
19
|
+
class RqBackend(Backend):
|
|
20
|
+
"""
|
|
21
|
+
Backend implementation using RQ (Redis Queue) for process execution.
|
|
22
|
+
|
|
23
|
+
This backend enqueues process start, task, and completion callbacks to an RQ queue
|
|
24
|
+
for asynchronous execution. It leverages RQ's job dependency features to ensure
|
|
25
|
+
proper task ordering and provides Redis-backed persistence of job state.
|
|
26
|
+
|
|
27
|
+
:param queue: The RQ Queue instance to which jobs will be enqueued
|
|
28
|
+
:type queue: rq.Queue
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, queue: rq.Queue):
|
|
32
|
+
"""
|
|
33
|
+
Initialize the RQ backend with a queue instance.
|
|
34
|
+
|
|
35
|
+
:param queue: The RQ Queue instance to which jobs will be enqueued
|
|
36
|
+
:type queue: rq.Queue
|
|
37
|
+
"""
|
|
38
|
+
self.queue = queue
|
|
39
|
+
|
|
40
|
+
def enqueue_perform_start(
|
|
41
|
+
self,
|
|
42
|
+
identifier: str,
|
|
43
|
+
module_name: str,
|
|
44
|
+
definition_class: str,
|
|
45
|
+
depends_on: Optional[Job] = None,
|
|
46
|
+
) -> Job:
|
|
47
|
+
"""
|
|
48
|
+
Enqueue a process start callback.
|
|
49
|
+
|
|
50
|
+
Schedules the execution of the process start callback, which allows the
|
|
51
|
+
process definition to perform initialization logic when execution begins.
|
|
52
|
+
|
|
53
|
+
:param identifier: Unique identifier for the process execution instance
|
|
54
|
+
:type identifier: str
|
|
55
|
+
:param module_name: Fully qualified module name containing the process definition
|
|
56
|
+
:type module_name: str
|
|
57
|
+
:param definition_class: Name of the process definition class to instantiate
|
|
58
|
+
:type definition_class: str
|
|
59
|
+
:param depends_on:
|
|
60
|
+
Optional job that this job depends on. If provided,
|
|
61
|
+
this job will not execute until the dependency completes.
|
|
62
|
+
:type depends_on: Optional[Job]
|
|
63
|
+
:return: Job object representing the enqueued task
|
|
64
|
+
:rtype: Job
|
|
65
|
+
"""
|
|
66
|
+
return self.queue.enqueue(
|
|
67
|
+
self.perform_start,
|
|
68
|
+
identifier=identifier,
|
|
69
|
+
module_name=module_name,
|
|
70
|
+
definition_class=definition_class,
|
|
71
|
+
depends_on=depends_on,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
def enqueue_perform_task(
|
|
75
|
+
self,
|
|
76
|
+
identifier: str,
|
|
77
|
+
module_name: str,
|
|
78
|
+
definition_class: str,
|
|
79
|
+
function_name: str,
|
|
80
|
+
description: Optional[str],
|
|
81
|
+
kwargs: Dict[str, Any],
|
|
82
|
+
depends_on: Optional[Job] = None,
|
|
83
|
+
) -> Job:
|
|
84
|
+
"""
|
|
85
|
+
Enqueue a process task for execution to the RQ queue.
|
|
86
|
+
|
|
87
|
+
Schedules the execution of a specific task function within a process definition
|
|
88
|
+
by enqueueing the rq_perform_task method to the configured RQ queue. Uses a
|
|
89
|
+
wrapper method to handle RQ-specific parameter handling.
|
|
90
|
+
|
|
91
|
+
:param identifier: Unique identifier for the process execution instance
|
|
92
|
+
:type identifier: str
|
|
93
|
+
:param module_name: Fully qualified module name containing the process definition
|
|
94
|
+
:type module_name: str
|
|
95
|
+
:param definition_class: Name of the process definition class to instantiate
|
|
96
|
+
:type definition_class: str
|
|
97
|
+
:param function_name: Name of the task method to execute
|
|
98
|
+
:type function_name: str
|
|
99
|
+
:param description: Human-readable description of the task
|
|
100
|
+
:type description: Optional[str]
|
|
101
|
+
:param kwargs: Keyword arguments to pass to the task function
|
|
102
|
+
:type kwargs: Dict[str, Any]
|
|
103
|
+
:param depends_on:
|
|
104
|
+
Optional job that this job depends on. If provided,
|
|
105
|
+
this job will not execute until the dependency completes.
|
|
106
|
+
:type depends_on: Optional[Job]
|
|
107
|
+
:return: RQ Job object representing the enqueued task
|
|
108
|
+
:rtype: Job
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
# NOTE: `description` and `kwargs` are used by rq
|
|
112
|
+
return self.queue.enqueue(
|
|
113
|
+
self.rq_perform_task,
|
|
114
|
+
identifier=identifier,
|
|
115
|
+
module_name=module_name,
|
|
116
|
+
definition_class=definition_class,
|
|
117
|
+
function_name=function_name,
|
|
118
|
+
task_description=description,
|
|
119
|
+
task_kwargs=kwargs,
|
|
120
|
+
depends_on=depends_on,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
@classmethod
|
|
124
|
+
def rq_perform_task(
|
|
125
|
+
cls,
|
|
126
|
+
identifier: str,
|
|
127
|
+
module_name: str,
|
|
128
|
+
definition_class: str,
|
|
129
|
+
function_name: str,
|
|
130
|
+
task_description: Description,
|
|
131
|
+
task_kwargs: Dict[str, Any],
|
|
132
|
+
):
|
|
133
|
+
"""
|
|
134
|
+
Execute a process task from within an RQ job context.
|
|
135
|
+
|
|
136
|
+
This classmethod serves as the RQ job entry point for task execution. It retrieves
|
|
137
|
+
the current RQ job, and delegates to the perform_task method.
|
|
138
|
+
|
|
139
|
+
:param identifier: Unique identifier for the process execution instance
|
|
140
|
+
:type identifier: str
|
|
141
|
+
:param module_name: Fully qualified module name containing the process definition
|
|
142
|
+
:type module_name: str
|
|
143
|
+
:param definition_class: Name of the process definition class to instantiate
|
|
144
|
+
:type definition_class: str
|
|
145
|
+
:param function_name: Name of the task method to execute
|
|
146
|
+
:type function_name: str
|
|
147
|
+
:param task_description: Human-readable description of the task
|
|
148
|
+
:type task_description: Description
|
|
149
|
+
:param task_kwargs: Keyword arguments to pass to the task function
|
|
150
|
+
:type task_kwargs: Dict[str, Any]
|
|
151
|
+
"""
|
|
152
|
+
|
|
153
|
+
job = cast(Job, rq.get_current_job())
|
|
154
|
+
|
|
155
|
+
super().perform_task(
|
|
156
|
+
job=job,
|
|
157
|
+
identifier=identifier,
|
|
158
|
+
module_name=module_name,
|
|
159
|
+
definition_class=definition_class,
|
|
160
|
+
function_name=function_name,
|
|
161
|
+
description=task_description,
|
|
162
|
+
kwargs=task_kwargs,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
def enqueue_perform_complete(
|
|
166
|
+
self,
|
|
167
|
+
identifier: str,
|
|
168
|
+
module_name: str,
|
|
169
|
+
definition_class: str,
|
|
170
|
+
depends_on: Optional[Union[Job, List[Job]]] = None,
|
|
171
|
+
) -> Job:
|
|
172
|
+
"""
|
|
173
|
+
Enqueue a process completion callback to the RQ queue.
|
|
174
|
+
|
|
175
|
+
Schedules the execution of the process completion callback by enqueueing the
|
|
176
|
+
perform_complete method to the configured RQ queue. The job will optionally
|
|
177
|
+
depend on one or more other jobs before execution.
|
|
178
|
+
|
|
179
|
+
:param identifier: Unique identifier for the process execution instance
|
|
180
|
+
:type identifier: str
|
|
181
|
+
:param module_name: Fully qualified module name containing the process definition
|
|
182
|
+
:type module_name: str
|
|
183
|
+
:param definition_class: Name of the process definition class to instantiate
|
|
184
|
+
:type definition_class: str
|
|
185
|
+
:param depends_on:
|
|
186
|
+
Optional job or list of jobs that this job depends on.
|
|
187
|
+
If provided, this job will not execute until all dependencies complete.
|
|
188
|
+
:type depends_on: Optional[Union[Job, List[Job]]]
|
|
189
|
+
:return: RQ Job object representing the enqueued task
|
|
190
|
+
:rtype: Job
|
|
191
|
+
"""
|
|
192
|
+
|
|
193
|
+
return self.queue.enqueue(
|
|
194
|
+
self.perform_complete,
|
|
195
|
+
identifier=identifier,
|
|
196
|
+
module_name=module_name,
|
|
197
|
+
definition_class=definition_class,
|
|
198
|
+
depends_on=depends_on,
|
|
199
|
+
)
|