tasktronaut 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasktronaut/__init__.py +14 -0
- tasktronaut/backend.py +129 -0
- tasktronaut/backends/__init__.py +0 -0
- tasktronaut/backends/rq.py +86 -0
- tasktronaut/builder.py +151 -0
- tasktronaut/context.py +2 -0
- tasktronaut/decorators.py +28 -0
- tasktronaut/errors.py +6 -0
- tasktronaut/process.py +203 -0
- tasktronaut/py.typed +0 -0
- tasktronaut/steps.py +32 -0
- tasktronaut/types.py +53 -0
- tasktronaut/utils.py +26 -0
- tasktronaut-0.1.0.dist-info/METADATA +30 -0
- tasktronaut-0.1.0.dist-info/RECORD +18 -0
- tasktronaut-0.1.0.dist-info/WHEEL +5 -0
- tasktronaut-0.1.0.dist-info/licenses/LICENSE +21 -0
- tasktronaut-0.1.0.dist-info/top_level.txt +1 -0
tasktronaut/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from .backend import Backend
|
|
2
|
+
from .builder import Builder
|
|
3
|
+
from .context import Context
|
|
4
|
+
from .decorators import task
|
|
5
|
+
from .process import ExecutionMode, ProcessDefinition
|
|
6
|
+
|
|
7
|
+
__all__ = (
|
|
8
|
+
"Backend",
|
|
9
|
+
"Builder",
|
|
10
|
+
"Context",
|
|
11
|
+
"ExecutionMode",
|
|
12
|
+
"ProcessDefinition",
|
|
13
|
+
"task",
|
|
14
|
+
)
|
tasktronaut/backend.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
from typing import Any, Dict, List, Optional, Protocol, Union
|
|
4
|
+
|
|
5
|
+
from .context import Context
|
|
6
|
+
from .errors import CancelProcessError, NonRetryableProcessError
|
|
7
|
+
from .types import Description
|
|
8
|
+
from .utils import load_definition
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Job(Protocol):
|
|
14
|
+
def cancel(self): # pragma: no cover
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
def delete_dependents(self): # pragma: no cover
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
22
|
+
class Backend(ABC):
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def enqueue_perform_start(
|
|
25
|
+
self,
|
|
26
|
+
identifier: str,
|
|
27
|
+
module_name: str,
|
|
28
|
+
definition_class: str,
|
|
29
|
+
depends_on: Optional[Job] = None,
|
|
30
|
+
) -> Job: # pragma: no cover
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
@staticmethod
|
|
34
|
+
def perform_start(
|
|
35
|
+
identifier: str,
|
|
36
|
+
module_name: str,
|
|
37
|
+
definition_class: str,
|
|
38
|
+
):
|
|
39
|
+
definition = load_definition(module_name, definition_class)
|
|
40
|
+
definition.on_started(identifier)
|
|
41
|
+
|
|
42
|
+
@abstractmethod
|
|
43
|
+
def enqueue_perform_task(
|
|
44
|
+
self,
|
|
45
|
+
identifier: str,
|
|
46
|
+
module_name: str,
|
|
47
|
+
definition_class: str,
|
|
48
|
+
function_name: str,
|
|
49
|
+
description: Description,
|
|
50
|
+
kwargs: Dict[str, Any],
|
|
51
|
+
depends_on: Optional[Job] = None,
|
|
52
|
+
) -> Job: # pragma: no cover
|
|
53
|
+
pass
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def perform_task(
|
|
57
|
+
cls,
|
|
58
|
+
job: Job,
|
|
59
|
+
identifier: str,
|
|
60
|
+
module_name: str,
|
|
61
|
+
definition_class: str,
|
|
62
|
+
function_name: str,
|
|
63
|
+
description: Description,
|
|
64
|
+
kwargs: Dict[str, Any],
|
|
65
|
+
):
|
|
66
|
+
definition = load_definition(module_name, definition_class)
|
|
67
|
+
task_func = getattr(definition, function_name)
|
|
68
|
+
context = Context()
|
|
69
|
+
try:
|
|
70
|
+
with definition.around_task(
|
|
71
|
+
identifier=identifier,
|
|
72
|
+
step=function_name,
|
|
73
|
+
description=description,
|
|
74
|
+
):
|
|
75
|
+
task_func(context=context, **kwargs)
|
|
76
|
+
|
|
77
|
+
except NonRetryableProcessError as e:
|
|
78
|
+
logger.error(
|
|
79
|
+
"Failed to execute task '%s', a non-retryable error was raised.",
|
|
80
|
+
function_name,
|
|
81
|
+
exc_info=True,
|
|
82
|
+
stack_info=True,
|
|
83
|
+
)
|
|
84
|
+
definition.on_failed(identifier, function_name, e)
|
|
85
|
+
# re-raise error to fail job
|
|
86
|
+
raise
|
|
87
|
+
|
|
88
|
+
except CancelProcessError:
|
|
89
|
+
logger.error(
|
|
90
|
+
"Failed to execute task '%s', the process was cancelled.",
|
|
91
|
+
function_name,
|
|
92
|
+
exc_info=True,
|
|
93
|
+
stack_info=True,
|
|
94
|
+
)
|
|
95
|
+
job.cancel()
|
|
96
|
+
job.delete_dependents() # clean up
|
|
97
|
+
definition.on_cancelled(identifier)
|
|
98
|
+
# ignore error
|
|
99
|
+
|
|
100
|
+
except Exception as e:
|
|
101
|
+
logger.error(
|
|
102
|
+
"Failed to execute task '%s' due to an unhandled error. "
|
|
103
|
+
"The task will be retried.",
|
|
104
|
+
function_name,
|
|
105
|
+
exc_info=True,
|
|
106
|
+
stack_info=True,
|
|
107
|
+
)
|
|
108
|
+
definition.on_failed(identifier, function_name, e)
|
|
109
|
+
# re-raise error to fail job
|
|
110
|
+
raise
|
|
111
|
+
|
|
112
|
+
@abstractmethod
|
|
113
|
+
def enqueue_perform_complete(
|
|
114
|
+
self,
|
|
115
|
+
identifier: str,
|
|
116
|
+
module_name: str,
|
|
117
|
+
definition_class: str,
|
|
118
|
+
depends_on: Optional[Union[Job, List[Job]]] = None,
|
|
119
|
+
) -> Job: # pragma: no cover
|
|
120
|
+
pass
|
|
121
|
+
|
|
122
|
+
@staticmethod
|
|
123
|
+
def perform_complete(
|
|
124
|
+
identifier: str,
|
|
125
|
+
module_name: str,
|
|
126
|
+
definition_class: str,
|
|
127
|
+
):
|
|
128
|
+
definition = load_definition(module_name, definition_class)
|
|
129
|
+
definition.on_completed(identifier)
|
|
File without changes
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
from typing import Any, Dict, List, Optional, Union, cast
|
|
2
|
+
|
|
3
|
+
import rq
|
|
4
|
+
|
|
5
|
+
from ..backend import Backend, Job
|
|
6
|
+
from ..types import Description
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
10
|
+
class RqBackend(Backend):
|
|
11
|
+
def __init__(self, queue: rq.Queue):
|
|
12
|
+
self.queue = queue
|
|
13
|
+
|
|
14
|
+
def enqueue_perform_start(
|
|
15
|
+
self,
|
|
16
|
+
identifier: str,
|
|
17
|
+
module_name: str,
|
|
18
|
+
definition_class: str,
|
|
19
|
+
depends_on: Optional[Job] = None,
|
|
20
|
+
) -> Job:
|
|
21
|
+
return self.queue.enqueue(
|
|
22
|
+
self.perform_start,
|
|
23
|
+
identifier=identifier,
|
|
24
|
+
module_name=module_name,
|
|
25
|
+
definition_class=definition_class,
|
|
26
|
+
depends_on=depends_on,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
def enqueue_perform_task(
|
|
30
|
+
self,
|
|
31
|
+
identifier: str,
|
|
32
|
+
module_name: str,
|
|
33
|
+
definition_class: str,
|
|
34
|
+
function_name: str,
|
|
35
|
+
description: Optional[str],
|
|
36
|
+
kwargs: Dict[str, Any],
|
|
37
|
+
depends_on: Optional[Job] = None,
|
|
38
|
+
) -> Job:
|
|
39
|
+
# NOTE: `description` and `kwargs` are used by rq
|
|
40
|
+
return self.queue.enqueue(
|
|
41
|
+
self.rq_perform_task,
|
|
42
|
+
identifier=identifier,
|
|
43
|
+
module_name=module_name,
|
|
44
|
+
definition_class=definition_class,
|
|
45
|
+
function_name=function_name,
|
|
46
|
+
task_description=description,
|
|
47
|
+
task_kwargs=kwargs,
|
|
48
|
+
depends_on=depends_on,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def rq_perform_task(
|
|
53
|
+
cls,
|
|
54
|
+
identifier: str,
|
|
55
|
+
module_name: str,
|
|
56
|
+
definition_class: str,
|
|
57
|
+
function_name: str,
|
|
58
|
+
task_description: Description,
|
|
59
|
+
task_kwargs: Dict[str, Any],
|
|
60
|
+
):
|
|
61
|
+
job = cast(Job, rq.get_current_job())
|
|
62
|
+
|
|
63
|
+
super().perform_task(
|
|
64
|
+
job=job,
|
|
65
|
+
identifier=identifier,
|
|
66
|
+
module_name=module_name,
|
|
67
|
+
definition_class=definition_class,
|
|
68
|
+
function_name=function_name,
|
|
69
|
+
description=task_description,
|
|
70
|
+
kwargs=task_kwargs,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
def enqueue_perform_complete(
|
|
74
|
+
self,
|
|
75
|
+
identifier: str,
|
|
76
|
+
module_name: str,
|
|
77
|
+
definition_class: str,
|
|
78
|
+
depends_on: Optional[Union[Job, List[Job]]] = None,
|
|
79
|
+
) -> Job:
|
|
80
|
+
return self.queue.enqueue(
|
|
81
|
+
self.perform_complete,
|
|
82
|
+
identifier=identifier,
|
|
83
|
+
module_name=module_name,
|
|
84
|
+
definition_class=definition_class,
|
|
85
|
+
depends_on=depends_on,
|
|
86
|
+
)
|
tasktronaut/builder.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
from contextlib import contextmanager
|
|
2
|
+
from types import SimpleNamespace
|
|
3
|
+
from typing import Any, Dict, Optional, TYPE_CHECKING, Type, cast
|
|
4
|
+
|
|
5
|
+
from pydantic import TypeAdapter, ValidationError
|
|
6
|
+
|
|
7
|
+
from .steps import Step
|
|
8
|
+
from .types import (
|
|
9
|
+
BuilderIterator,
|
|
10
|
+
Description,
|
|
11
|
+
ForEachMethod,
|
|
12
|
+
Options,
|
|
13
|
+
ProcessDefinitionType,
|
|
14
|
+
TaskMethod,
|
|
15
|
+
TransformMethod,
|
|
16
|
+
Value,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from .process import Process
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Builder:
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
process: "Process",
|
|
27
|
+
kwargs: Dict[str, Any],
|
|
28
|
+
options: Optional[Options] = None,
|
|
29
|
+
description: Description = None,
|
|
30
|
+
):
|
|
31
|
+
self.process = process
|
|
32
|
+
self.description = description
|
|
33
|
+
self.options = self._convert_options(options)
|
|
34
|
+
self.kwargs = kwargs
|
|
35
|
+
|
|
36
|
+
@staticmethod
|
|
37
|
+
def _convert_options(options: Optional[Options] = None) -> SimpleNamespace:
|
|
38
|
+
if options:
|
|
39
|
+
if isinstance(options, SimpleNamespace):
|
|
40
|
+
return options
|
|
41
|
+
if isinstance(options, dict):
|
|
42
|
+
return SimpleNamespace(**options)
|
|
43
|
+
return SimpleNamespace()
|
|
44
|
+
|
|
45
|
+
def expected_arguments(self, **kwargs: Optional[Type]):
|
|
46
|
+
# NOTE: this doesn't check for additional arguments
|
|
47
|
+
for name, expected_type in kwargs.items():
|
|
48
|
+
if name not in self.kwargs:
|
|
49
|
+
raise TypeError(f"Argument '{name}' not found.")
|
|
50
|
+
|
|
51
|
+
if expected_type is not None:
|
|
52
|
+
arg_type = self.kwargs[name]
|
|
53
|
+
type_adapter = TypeAdapter(expected_type)
|
|
54
|
+
try:
|
|
55
|
+
type_adapter.validate_python(arg_type)
|
|
56
|
+
except ValidationError as e:
|
|
57
|
+
raise ValueError(
|
|
58
|
+
f"Argument '{name}' is expected to be of type '{arg_type}'."
|
|
59
|
+
) from e
|
|
60
|
+
|
|
61
|
+
@contextmanager
|
|
62
|
+
def concurrent(self, description: Description = None) -> BuilderIterator:
|
|
63
|
+
from .process import ConcurrentProcess # pylint: disable=import-outside-toplevel
|
|
64
|
+
|
|
65
|
+
yield from self._build_process(ConcurrentProcess, description)
|
|
66
|
+
|
|
67
|
+
@contextmanager
|
|
68
|
+
def sequential(self, description: Description = None) -> BuilderIterator:
|
|
69
|
+
from .process import SequentialProcess # pylint: disable=import-outside-toplevel
|
|
70
|
+
|
|
71
|
+
yield from self._build_process(SequentialProcess, description)
|
|
72
|
+
|
|
73
|
+
def _build_process(
|
|
74
|
+
self, process_type: Type["Process"], description: Description = None
|
|
75
|
+
) -> BuilderIterator:
|
|
76
|
+
builder = Builder(
|
|
77
|
+
process=process_type(
|
|
78
|
+
identifier=self.process.identifier,
|
|
79
|
+
definition=self.process.definition,
|
|
80
|
+
),
|
|
81
|
+
options=self.options,
|
|
82
|
+
description=description or self.description,
|
|
83
|
+
kwargs=self.kwargs,
|
|
84
|
+
)
|
|
85
|
+
yield builder
|
|
86
|
+
self.process.steps.append(builder.process)
|
|
87
|
+
|
|
88
|
+
def task(
|
|
89
|
+
self,
|
|
90
|
+
func: TaskMethod,
|
|
91
|
+
description: Description = None,
|
|
92
|
+
):
|
|
93
|
+
self.process.steps.append(
|
|
94
|
+
Step(
|
|
95
|
+
func=func,
|
|
96
|
+
description=(
|
|
97
|
+
description or getattr(func, "description", self.description)
|
|
98
|
+
),
|
|
99
|
+
kwargs=self.kwargs,
|
|
100
|
+
)
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
def sub_process(
|
|
104
|
+
self,
|
|
105
|
+
definition: ProcessDefinitionType,
|
|
106
|
+
description: Description = None,
|
|
107
|
+
):
|
|
108
|
+
builder = Builder(
|
|
109
|
+
process=definition.create_process(
|
|
110
|
+
identifier=self.process.identifier,
|
|
111
|
+
definition=definition,
|
|
112
|
+
),
|
|
113
|
+
options=self.options,
|
|
114
|
+
description=description or self.description,
|
|
115
|
+
kwargs=self.kwargs,
|
|
116
|
+
)
|
|
117
|
+
definition().define_process(builder)
|
|
118
|
+
self.process.steps.append(builder.process)
|
|
119
|
+
|
|
120
|
+
def each(
|
|
121
|
+
self,
|
|
122
|
+
func: ForEachMethod,
|
|
123
|
+
description: Description = None,
|
|
124
|
+
) -> BuilderIterator:
|
|
125
|
+
for kwargs in func(**self.kwargs.copy()):
|
|
126
|
+
kwargs, desc = kwargs if isinstance(kwargs, tuple) else (kwargs, None)
|
|
127
|
+
yield Builder(
|
|
128
|
+
process=self.process,
|
|
129
|
+
options=self.options,
|
|
130
|
+
description=desc or description or self.description,
|
|
131
|
+
kwargs=kwargs,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
@contextmanager
|
|
135
|
+
def transform(
|
|
136
|
+
self,
|
|
137
|
+
func: TransformMethod,
|
|
138
|
+
description: Description = None,
|
|
139
|
+
) -> BuilderIterator:
|
|
140
|
+
kwargs = func(**self.kwargs.copy())
|
|
141
|
+
kwargs, desc = kwargs if isinstance(kwargs, tuple) else (kwargs, None)
|
|
142
|
+
|
|
143
|
+
yield Builder(
|
|
144
|
+
process=self.process,
|
|
145
|
+
options=self.options,
|
|
146
|
+
description=desc or description or self.description,
|
|
147
|
+
kwargs=kwargs,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
def option(self, name: str, default: Optional[Value] = None) -> Value:
|
|
151
|
+
return cast(Value, getattr(self.options, name, default))
|
tasktronaut/context.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from typing import Callable, Optional, Union, cast, overload
|
|
2
|
+
|
|
3
|
+
from .types import DecoratedTaskMethod, Description, UndecoratedTaskMethod
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@overload
|
|
7
|
+
def task(func: UndecoratedTaskMethod, /) -> DecoratedTaskMethod: ...
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@overload
|
|
11
|
+
def task(
|
|
12
|
+
*, description: Description = None
|
|
13
|
+
) -> Callable[[UndecoratedTaskMethod], DecoratedTaskMethod]: ...
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def task(
|
|
17
|
+
func: Optional[UndecoratedTaskMethod] = None,
|
|
18
|
+
/,
|
|
19
|
+
*,
|
|
20
|
+
description: Description = None,
|
|
21
|
+
) -> Union[Callable[[UndecoratedTaskMethod], DecoratedTaskMethod], DecoratedTaskMethod]:
|
|
22
|
+
def _wrapper(f: UndecoratedTaskMethod) -> DecoratedTaskMethod:
|
|
23
|
+
setattr(f, "description", description)
|
|
24
|
+
return cast(DecoratedTaskMethod, f)
|
|
25
|
+
|
|
26
|
+
if func is not None:
|
|
27
|
+
return _wrapper(func)
|
|
28
|
+
return _wrapper
|
tasktronaut/errors.py
ADDED
tasktronaut/process.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import enum
|
|
2
|
+
import logging
|
|
3
|
+
import uuid
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
from typing import Generator, Optional, Type, cast
|
|
7
|
+
|
|
8
|
+
from .backend import Backend, Job
|
|
9
|
+
from .builder import Builder
|
|
10
|
+
from .steps import Steps
|
|
11
|
+
from .types import Description, Options
|
|
12
|
+
from .utils import to_dict
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Process(ABC):
|
|
18
|
+
def __init__(self, identifier: str, definition: Type["ProcessDefinition"]):
|
|
19
|
+
self.identifier = identifier
|
|
20
|
+
self.definition: Type[ProcessDefinition] = definition
|
|
21
|
+
self.steps = Steps()
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def is_process(self) -> bool:
|
|
25
|
+
return True
|
|
26
|
+
|
|
27
|
+
@abstractmethod
|
|
28
|
+
def enqueue(
|
|
29
|
+
self, backend: Backend, start_job: Optional[Job] = None
|
|
30
|
+
) -> Job: # pragma: no cover
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
def __str__(self):
|
|
34
|
+
return (
|
|
35
|
+
f"{self.__class__.__name__}("
|
|
36
|
+
f"id={self.identifier},"
|
|
37
|
+
f"definition={self.definition.__name__},"
|
|
38
|
+
f"steps={len(self.steps)},"
|
|
39
|
+
")"
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
def __repr__(self):
|
|
43
|
+
return (
|
|
44
|
+
f"{self.__class__.__name__}("
|
|
45
|
+
f"id={self.identifier},"
|
|
46
|
+
f"definition={self.definition.__name__},"
|
|
47
|
+
f"steps={len(self.steps)},"
|
|
48
|
+
")"
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class SequentialProcess(Process):
|
|
53
|
+
def enqueue(self, backend: Backend, start_job: Optional[Job] = None) -> Job:
|
|
54
|
+
base_kwargs = to_dict(
|
|
55
|
+
identifier=self.identifier,
|
|
56
|
+
module_name=self.definition.__module__,
|
|
57
|
+
definition_class=self.definition.__qualname__,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
job = start_job or backend.enqueue_perform_start(**base_kwargs)
|
|
61
|
+
|
|
62
|
+
for step in self.steps:
|
|
63
|
+
if isinstance(step, Process):
|
|
64
|
+
job = step.enqueue(backend=backend, start_job=job)
|
|
65
|
+
continue
|
|
66
|
+
|
|
67
|
+
kwargs = base_kwargs.copy()
|
|
68
|
+
kwargs.update(
|
|
69
|
+
**to_dict(
|
|
70
|
+
function_name=step.func.__name__,
|
|
71
|
+
depends_on=job,
|
|
72
|
+
),
|
|
73
|
+
description=step.description,
|
|
74
|
+
kwargs=step.kwargs,
|
|
75
|
+
)
|
|
76
|
+
job = backend.enqueue_perform_task(**kwargs)
|
|
77
|
+
|
|
78
|
+
return backend.enqueue_perform_complete(
|
|
79
|
+
depends_on=job,
|
|
80
|
+
**base_kwargs,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class ConcurrentProcess(Process):
|
|
85
|
+
def enqueue(self, backend: Backend, start_job: Optional[Job] = None) -> Job:
|
|
86
|
+
base_kwargs = to_dict(
|
|
87
|
+
identifier=self.identifier,
|
|
88
|
+
module_name=self.definition.__module__,
|
|
89
|
+
definition_class=self.definition.__qualname__,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
job = backend.enqueue_perform_start(**base_kwargs)
|
|
93
|
+
|
|
94
|
+
jobs = []
|
|
95
|
+
for step in self.steps:
|
|
96
|
+
if isinstance(step, Process):
|
|
97
|
+
jobs.append(step.enqueue(backend=backend, start_job=job))
|
|
98
|
+
continue
|
|
99
|
+
|
|
100
|
+
kwargs = base_kwargs.copy()
|
|
101
|
+
kwargs.update(
|
|
102
|
+
**to_dict(
|
|
103
|
+
function_name=step.func.__name__,
|
|
104
|
+
depends_on=job,
|
|
105
|
+
),
|
|
106
|
+
description=step.description,
|
|
107
|
+
kwargs=step.kwargs,
|
|
108
|
+
)
|
|
109
|
+
jobs.append(backend.enqueue_perform_task(**kwargs))
|
|
110
|
+
|
|
111
|
+
return backend.enqueue_perform_complete(
|
|
112
|
+
depends_on=jobs,
|
|
113
|
+
**base_kwargs,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class ExecutionMode(enum.Enum):
|
|
118
|
+
SEQUENTIAL = SequentialProcess
|
|
119
|
+
CONCURRENT = ConcurrentProcess
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class ProcessDefinition(ABC):
|
|
123
|
+
description: Description = None
|
|
124
|
+
execution_mode: ExecutionMode = ExecutionMode.SEQUENTIAL
|
|
125
|
+
|
|
126
|
+
@classmethod
|
|
127
|
+
def create_process(
|
|
128
|
+
cls,
|
|
129
|
+
identifier: str,
|
|
130
|
+
definition: Type["ProcessDefinition"],
|
|
131
|
+
) -> Process:
|
|
132
|
+
return cast(
|
|
133
|
+
Process,
|
|
134
|
+
cls.execution_mode.value(
|
|
135
|
+
identifier=identifier,
|
|
136
|
+
definition=definition,
|
|
137
|
+
),
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
@classmethod
|
|
141
|
+
def build(
|
|
142
|
+
cls,
|
|
143
|
+
identifier: Optional[str] = None,
|
|
144
|
+
options: Optional[Options] = None,
|
|
145
|
+
**kwargs,
|
|
146
|
+
) -> Process:
|
|
147
|
+
builder = Builder(
|
|
148
|
+
process=cls.create_process(
|
|
149
|
+
identifier=identifier or uuid.uuid4().hex,
|
|
150
|
+
definition=cls,
|
|
151
|
+
),
|
|
152
|
+
options=options,
|
|
153
|
+
kwargs=kwargs,
|
|
154
|
+
)
|
|
155
|
+
cls().define_process(builder)
|
|
156
|
+
return builder.process
|
|
157
|
+
|
|
158
|
+
@abstractmethod
|
|
159
|
+
def define_process(self, builder: Builder): # pragma: no cover
|
|
160
|
+
pass
|
|
161
|
+
|
|
162
|
+
def on_started(self, identifier: str):
|
|
163
|
+
logger.info("%s started [%s].", self.__class__.__name__, identifier)
|
|
164
|
+
|
|
165
|
+
@contextmanager
|
|
166
|
+
def around_task(
|
|
167
|
+
self,
|
|
168
|
+
identifier: str,
|
|
169
|
+
step: str,
|
|
170
|
+
description: Description,
|
|
171
|
+
) -> Generator[None, None, None]:
|
|
172
|
+
logger.info(
|
|
173
|
+
"Executing '%s' (%s) of %s process [%s].",
|
|
174
|
+
description,
|
|
175
|
+
step,
|
|
176
|
+
self.__class__.__name__,
|
|
177
|
+
identifier,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
yield
|
|
181
|
+
|
|
182
|
+
logger.info(
|
|
183
|
+
"Completed '%s' (%s) of %s process [%s].",
|
|
184
|
+
description,
|
|
185
|
+
step,
|
|
186
|
+
self.__class__.__name__,
|
|
187
|
+
identifier,
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
def on_completed(self, identifier: str):
|
|
191
|
+
logger.info("%s completed [%s].", self.__class__.__name__, identifier)
|
|
192
|
+
|
|
193
|
+
def on_failed(self, identifier: str, step: str, e: Exception):
|
|
194
|
+
logger.error(
|
|
195
|
+
"%s failed executing %s step [%s].\n%s",
|
|
196
|
+
self.__class__.__name__,
|
|
197
|
+
step,
|
|
198
|
+
identifier,
|
|
199
|
+
e,
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
def on_cancelled(self, identifier: str):
|
|
203
|
+
logger.warning("%s cancelled [%s].", self.__class__.__name__, identifier)
|
tasktronaut/py.typed
ADDED
|
File without changes
|
tasktronaut/steps.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from typing import Any, Dict, TYPE_CHECKING, Union
|
|
2
|
+
|
|
3
|
+
from .types import Description, TaskMethod
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
from .process import Process # noqa
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Step:
|
|
10
|
+
def __init__(
|
|
11
|
+
self,
|
|
12
|
+
func: TaskMethod,
|
|
13
|
+
description: Description,
|
|
14
|
+
kwargs: Dict[str, Any],
|
|
15
|
+
):
|
|
16
|
+
self.func = func
|
|
17
|
+
self.description = description
|
|
18
|
+
self.kwargs = kwargs
|
|
19
|
+
|
|
20
|
+
@property
|
|
21
|
+
def is_process(self) -> bool:
|
|
22
|
+
return False
|
|
23
|
+
|
|
24
|
+
def __str__(self) -> str:
|
|
25
|
+
return f"{self.__class__.__name__}(func={self.func.__name__},kwargs={self.kwargs!r},)"
|
|
26
|
+
|
|
27
|
+
def __repr__(self) -> str:
|
|
28
|
+
return f"{self.__class__.__name__}(func={self.func.__name__},kwargs={self.kwargs!r},)"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Steps(list[Union[Step, "Process"]]): # pragma: no cover
|
|
32
|
+
pass
|
tasktronaut/types.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from types import SimpleNamespace
|
|
2
|
+
from typing import (
|
|
3
|
+
Any,
|
|
4
|
+
Callable,
|
|
5
|
+
Dict,
|
|
6
|
+
Iterator,
|
|
7
|
+
Optional,
|
|
8
|
+
Protocol,
|
|
9
|
+
TYPE_CHECKING,
|
|
10
|
+
Tuple,
|
|
11
|
+
Type,
|
|
12
|
+
TypeAlias,
|
|
13
|
+
TypeVar,
|
|
14
|
+
Union,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from .builder import Builder
|
|
19
|
+
from .process import ProcessDefinition
|
|
20
|
+
from .context import Context
|
|
21
|
+
|
|
22
|
+
UndecoratedTaskMethod: TypeAlias = Callable[..., None]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DecoratedTaskMethod(Protocol): # pylint: disable=too-few-public-methods
|
|
26
|
+
__name__: str
|
|
27
|
+
description: Optional[str]
|
|
28
|
+
|
|
29
|
+
def __call__(
|
|
30
|
+
self, *args, context: Optional["Context"] = None, **kwargs
|
|
31
|
+
): # pragma: no cover
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
TaskMethod: TypeAlias = Union[UndecoratedTaskMethod, DecoratedTaskMethod]
|
|
36
|
+
|
|
37
|
+
Description: TypeAlias = Optional[str]
|
|
38
|
+
|
|
39
|
+
TaskArgs: TypeAlias = Union[Dict[str, Any], Tuple[Dict[str, Any], Description]]
|
|
40
|
+
|
|
41
|
+
ForEachReturn: TypeAlias = Iterator[TaskArgs]
|
|
42
|
+
|
|
43
|
+
ForEachMethod: TypeAlias = Callable[..., ForEachReturn]
|
|
44
|
+
|
|
45
|
+
TransformMethod: TypeAlias = Callable[..., TaskArgs]
|
|
46
|
+
|
|
47
|
+
BuilderIterator: TypeAlias = Iterator["Builder"]
|
|
48
|
+
|
|
49
|
+
ProcessDefinitionType: TypeAlias = Type["ProcessDefinition"]
|
|
50
|
+
|
|
51
|
+
Options: TypeAlias = Union[SimpleNamespace, Dict[str, Any]]
|
|
52
|
+
|
|
53
|
+
Value = TypeVar("Value")
|
tasktronaut/utils.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
|
|
3
|
+
from typing import Any, TYPE_CHECKING, cast
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
from .process import ProcessDefinition # noqa
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def to_dict(**kwargs: Any) -> dict[str, Any]:
|
|
10
|
+
return kwargs
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def to_kwargs(base: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
|
|
14
|
+
d = base.copy()
|
|
15
|
+
d.update(kwargs)
|
|
16
|
+
return d
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def load_definition(
|
|
20
|
+
module_name: str,
|
|
21
|
+
definition_class: str,
|
|
22
|
+
) -> "ProcessDefinition":
|
|
23
|
+
module = importlib.import_module(module_name)
|
|
24
|
+
definition_type = getattr(module, definition_class)
|
|
25
|
+
# return new instance of definition type
|
|
26
|
+
return cast("ProcessDefinition", definition_type())
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tasktronaut
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A simple task orchestration library for running complex processes or workflows in Python.
|
|
5
|
+
Author-email: Chris Stefano <virtualstaticvoid@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: homepage, https://github.com/virtualstaticvoid/tasktronaut
|
|
8
|
+
Project-URL: documentation, https://pytorch-geometric.readthedocs.io
|
|
9
|
+
Project-URL: repository, https://github.com/virtualstaticvoid/tasktronaut.git
|
|
10
|
+
Project-URL: changelog, https://github.com/virtualstaticvoid/tasktronaut/blob/main/CHANGELOG.md
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Programming Language :: Python
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: pydantic>=2.12.4
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# tasktronaut
|
|
22
|
+
|
|
23
|
+
A lightweight task orchestration library for building and running complex workflows in Python.
|
|
24
|
+
|
|
25
|
+
Define workflows using a simple expressive DSL, with minimal boilerplate. Once defined, processes
|
|
26
|
+
can be queued and executed with support for both sequential and concurrent DAG based execution.
|
|
27
|
+
|
|
28
|
+
## License
|
|
29
|
+
|
|
30
|
+
MIT Copyright (c) 2025 Chris Stefano
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
tasktronaut/__init__.py,sha256=AgvojZGBMOkUF97NNzD5ceBv-JMqnCZTx7zbNZuDLaI,288
|
|
2
|
+
tasktronaut/backend.py,sha256=KOhJ1j23BlXh_CqqIAohz_S2mmeR_MyN1rrGxN0xojU,3688
|
|
3
|
+
tasktronaut/builder.py,sha256=821ah4fHVdH2ksF-F0V3C_uagwr0vqfh9OpS22aWUFI,4846
|
|
4
|
+
tasktronaut/context.py,sha256=i24tAgnqQQgD0BerfVPNy2n9faigI9q1rtljhJFvT4o,66
|
|
5
|
+
tasktronaut/decorators.py,sha256=HN0PkCcBLUXISLnhuUd5eR8MBJW8uiQH2kOMh2XyoHM,780
|
|
6
|
+
tasktronaut/errors.py,sha256=qzw0KjOGUduV68yBgJkvrJCK4k_IGpGL8Q-GH8QSRpA,140
|
|
7
|
+
tasktronaut/process.py,sha256=4FglJZ0pHRPdZCjdz58pIPnKAscYbctOk05oBLt1Dd4,5676
|
|
8
|
+
tasktronaut/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
tasktronaut/steps.py,sha256=CbZKzgvLy9mT8t05WxHCfR2ECln8XHZqP0h8giexmIk,790
|
|
10
|
+
tasktronaut/types.py,sha256=ZI2yXjdzbWT7xpHBum4vUQ0fLKEcYYRZTaSGP8NrZrQ,1205
|
|
11
|
+
tasktronaut/utils.py,sha256=UvEZsSaojtCXQQauJpdMuJV0lTabxQw7DOheowP4lus,629
|
|
12
|
+
tasktronaut/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
tasktronaut/backends/rq.py,sha256=kpuZI6M6-8otkH5JwzQyC0da33xVHinPl5KYun5t5as,2430
|
|
14
|
+
tasktronaut-0.1.0.dist-info/licenses/LICENSE,sha256=3uRRh4AHs8khJwXHIHoHEx19qbvwNK89SRl37o4n9f8,1070
|
|
15
|
+
tasktronaut-0.1.0.dist-info/METADATA,sha256=XIJIFrMxGeg_D8ZdI8neOtUZ9wY2dIPI_PXwjiOB0tw,1253
|
|
16
|
+
tasktronaut-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
17
|
+
tasktronaut-0.1.0.dist-info/top_level.txt,sha256=x9wVhfHbQp0kJ0v8-T0A433J-5Lyfh9OXwX-0VikR-A,12
|
|
18
|
+
tasktronaut-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Chris Stefano
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
tasktronaut
|