dotflow 0.12.0.dev3__tar.gz → 0.13.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/PKG-INFO +2 -1
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/__init__.py +1 -1
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/abc/flow.py +1 -1
- dotflow-0.13.0.dev1/dotflow/abc/log.py +17 -0
- dotflow-0.13.0.dev1/dotflow/abc/notify.py +13 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/action.py +53 -21
- dotflow-0.13.0.dev1/dotflow/core/config.py +54 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/dotflow.py +4 -2
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/exception.py +5 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/execution.py +17 -12
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/task.py +9 -25
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/types/__init__.py +2 -2
- dotflow-0.13.0.dev1/dotflow/core/types/status.py +31 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/workflow.py +11 -11
- dotflow-0.13.0.dev1/dotflow/notify.py +6 -0
- dotflow-0.13.0.dev1/dotflow/providers/log_default.py +30 -0
- dotflow-0.13.0.dev1/dotflow/providers/notify_default.py +11 -0
- dotflow-0.13.0.dev1/dotflow/providers/notify_telegram.py +56 -0
- dotflow-0.13.0.dev1/dotflow/providers.py +15 -0
- dotflow-0.13.0.dev1/dotflow/types.py +9 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/pyproject.toml +6 -3
- dotflow-0.12.0.dev3/dotflow/core/config.py +0 -28
- dotflow-0.12.0.dev3/dotflow/core/types/status.py +0 -19
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/LICENSE +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/README.md +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/abc/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/abc/file.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/abc/http.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/abc/storage.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/abc/tcp.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/cli/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/cli/command.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/cli/commands/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/cli/commands/init.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/cli/commands/log.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/cli/commands/start.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/cli/setup.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/cli/validators/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/cli/validators/start.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/context.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/decorators/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/decorators/time.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/module.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/serializers/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/serializers/task.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/serializers/transport.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/serializers/workflow.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/types/execution.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/types/storage.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/core/types/worflow.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/logging.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/main.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/providers/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/providers/storage_default.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/providers/storage_file.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/settings.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/storage.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/utils/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/utils/basic_functions.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/utils/error_handler.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev1}/dotflow/utils/tools.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: dotflow
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0.dev1
|
|
4
4
|
Summary: 🎲 Dotflow turns an idea into flow!
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -37,6 +37,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
37
37
|
Provides-Extra: mongodb
|
|
38
38
|
Requires-Dist: dotflow-mongodb ; extra == "mongodb"
|
|
39
39
|
Requires-Dist: pydantic
|
|
40
|
+
Requires-Dist: requests
|
|
40
41
|
Requires-Dist: rich
|
|
41
42
|
Requires-Dist: typing-extensions
|
|
42
43
|
Project-URL: Documentation, https://github.com/dotflow-io/dotflow/blob/master/README.md
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Notify ABC"""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Log(ABC):
|
|
9
|
+
"""Log"""
|
|
10
|
+
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def info(self, task: Any) -> None:
|
|
13
|
+
"""Info"""
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def error(self, task: Any) -> None:
|
|
17
|
+
"""Error"""
|
|
@@ -1,11 +1,25 @@
|
|
|
1
1
|
"""Action module"""
|
|
2
2
|
|
|
3
|
+
from time import sleep
|
|
4
|
+
|
|
5
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
3
6
|
from typing import Callable, Dict
|
|
4
7
|
from types import FunctionType
|
|
5
8
|
|
|
9
|
+
from dotflow.core.exception import ExecutionWithClassError
|
|
6
10
|
from dotflow.core.context import Context
|
|
7
11
|
|
|
8
12
|
|
|
13
|
+
def is_execution_with_class_internal_error(error: Exception) -> bool:
|
|
14
|
+
message = str(error)
|
|
15
|
+
patterns = [
|
|
16
|
+
"initial_context",
|
|
17
|
+
"previous_context",
|
|
18
|
+
"missing 1 required positional argument: 'self'",
|
|
19
|
+
]
|
|
20
|
+
return any(pattern in message for pattern in patterns)
|
|
21
|
+
|
|
22
|
+
|
|
9
23
|
class Action(object):
|
|
10
24
|
"""
|
|
11
25
|
Import:
|
|
@@ -39,14 +53,20 @@ class Action(object):
|
|
|
39
53
|
"""
|
|
40
54
|
|
|
41
55
|
def __init__(
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
56
|
+
self,
|
|
57
|
+
func: Callable = None,
|
|
58
|
+
task: Callable = None,
|
|
59
|
+
retry: int = 1,
|
|
60
|
+
timeout: int = 0,
|
|
61
|
+
retry_delay: int = 1,
|
|
62
|
+
backoff: bool = False,
|
|
46
63
|
) -> None:
|
|
47
64
|
self.func = func
|
|
48
65
|
self.task = task
|
|
49
66
|
self.retry = retry
|
|
67
|
+
self.timeout = timeout
|
|
68
|
+
self.retry_delay = retry_delay
|
|
69
|
+
self.backoff = backoff
|
|
50
70
|
self.params = []
|
|
51
71
|
|
|
52
72
|
def __call__(self, *args, **kwargs):
|
|
@@ -59,15 +79,15 @@ class Action(object):
|
|
|
59
79
|
|
|
60
80
|
if contexts:
|
|
61
81
|
return Context(
|
|
62
|
-
storage=self.
|
|
82
|
+
storage=self._run_action(*args, **contexts),
|
|
63
83
|
task_id=task.task_id,
|
|
64
|
-
workflow_id=task.workflow_id
|
|
84
|
+
workflow_id=task.workflow_id,
|
|
65
85
|
)
|
|
66
86
|
|
|
67
87
|
return Context(
|
|
68
|
-
storage=self.
|
|
88
|
+
storage=self._run_action(*args),
|
|
69
89
|
task_id=task.task_id,
|
|
70
|
-
workflow_id=task.workflow_id
|
|
90
|
+
workflow_id=task.workflow_id,
|
|
71
91
|
)
|
|
72
92
|
|
|
73
93
|
# No parameters
|
|
@@ -80,31 +100,41 @@ class Action(object):
|
|
|
80
100
|
|
|
81
101
|
if contexts:
|
|
82
102
|
return Context(
|
|
83
|
-
storage=self.
|
|
103
|
+
storage=self._run_action(*_args, **contexts),
|
|
84
104
|
task_id=task.task_id,
|
|
85
|
-
workflow_id=task.workflow_id
|
|
105
|
+
workflow_id=task.workflow_id,
|
|
86
106
|
)
|
|
87
107
|
|
|
88
108
|
return Context(
|
|
89
|
-
storage=self.
|
|
109
|
+
storage=self._run_action(*_args),
|
|
90
110
|
task_id=task.task_id,
|
|
91
|
-
workflow_id=task.workflow_id
|
|
111
|
+
workflow_id=task.workflow_id,
|
|
92
112
|
)
|
|
93
113
|
|
|
94
114
|
return action
|
|
95
115
|
|
|
96
|
-
def
|
|
97
|
-
attempt
|
|
98
|
-
exception = Exception()
|
|
99
|
-
|
|
100
|
-
while self.retry > attempt:
|
|
116
|
+
def _run_action(self, *args, **kwargs):
|
|
117
|
+
for attempt in range(1, self.retry + 1):
|
|
101
118
|
try:
|
|
119
|
+
if self.timeout:
|
|
120
|
+
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
121
|
+
future = executor.submit(self.func, *args, **kwargs)
|
|
122
|
+
return future.result(timeout=self.timeout)
|
|
123
|
+
|
|
102
124
|
return self.func(*args, **kwargs)
|
|
125
|
+
|
|
103
126
|
except Exception as error:
|
|
104
|
-
|
|
105
|
-
|
|
127
|
+
last_exception = error
|
|
128
|
+
|
|
129
|
+
if is_execution_with_class_internal_error(error=last_exception):
|
|
130
|
+
raise ExecutionWithClassError()
|
|
131
|
+
|
|
132
|
+
if attempt == self.retry:
|
|
133
|
+
raise last_exception
|
|
106
134
|
|
|
107
|
-
|
|
135
|
+
sleep(self.retry_delay)
|
|
136
|
+
if self.backoff:
|
|
137
|
+
self.retry_delay *= 2
|
|
108
138
|
|
|
109
139
|
def _set_params(self):
|
|
110
140
|
if isinstance(self.func, FunctionType):
|
|
@@ -113,7 +143,9 @@ class Action(object):
|
|
|
113
143
|
if type(self.func) is type:
|
|
114
144
|
if hasattr(self.func, "__init__"):
|
|
115
145
|
if hasattr(self.func.__init__, "__code__"):
|
|
116
|
-
self.params = [
|
|
146
|
+
self.params = [
|
|
147
|
+
param for param in self.func.__init__.__code__.co_varnames
|
|
148
|
+
]
|
|
117
149
|
|
|
118
150
|
def _get_context(self, kwargs: Dict):
|
|
119
151
|
context = {}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Config module"""
|
|
2
|
+
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
from dotflow.abc.log import Log
|
|
6
|
+
from dotflow.abc.storage import Storage
|
|
7
|
+
from dotflow.abc.notify import Notify
|
|
8
|
+
|
|
9
|
+
from dotflow.providers.log_default import LogDefault
|
|
10
|
+
from dotflow.providers.storage_default import StorageDefault
|
|
11
|
+
from dotflow.providers.notify_default import NotifyDefault
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Config:
|
|
15
|
+
"""
|
|
16
|
+
Import:
|
|
17
|
+
You can import the **Config** class with:
|
|
18
|
+
|
|
19
|
+
from dotflow import Config
|
|
20
|
+
from dotflow.providers import (
|
|
21
|
+
StorageDefault,
|
|
22
|
+
NotifyDefault,
|
|
23
|
+
LogDefault
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
Example:
|
|
27
|
+
`class` dotflow.core.config.Config
|
|
28
|
+
|
|
29
|
+
config = Config(
|
|
30
|
+
storage=StorageFile(path=".output"),
|
|
31
|
+
notify=NotifyDefault(),
|
|
32
|
+
log=LogDefault()
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
storage (Optional[Storage]): Type of the storage.
|
|
37
|
+
notify (Optional[Notify]): Type of the notify.
|
|
38
|
+
log (Optional[Log]): Type of the notify.
|
|
39
|
+
|
|
40
|
+
Attributes:
|
|
41
|
+
storage (Optional[Storage]):
|
|
42
|
+
notify (Optional[Notify]):
|
|
43
|
+
log (Optional[Log]):
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
storage: Optional[Storage] = None,
|
|
49
|
+
notify: Optional[Notify] = None,
|
|
50
|
+
log: Optional[Log] = None,
|
|
51
|
+
) -> None:
|
|
52
|
+
self.storage = storage if storage else StorageDefault()
|
|
53
|
+
self.notify = notify if notify else NotifyDefault()
|
|
54
|
+
self.log = log if log else LogDefault()
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from uuid import uuid4
|
|
4
4
|
from functools import partial
|
|
5
|
+
from typing import Optional
|
|
5
6
|
|
|
6
7
|
from dotflow.core.config import Config
|
|
7
8
|
from dotflow.core.workflow import Manager
|
|
@@ -26,7 +27,7 @@ class DotFlow:
|
|
|
26
27
|
workflow = DotFlow(config=config)
|
|
27
28
|
|
|
28
29
|
Args:
|
|
29
|
-
config (Config): Configuration class.
|
|
30
|
+
config (Optional[Config]): Configuration class.
|
|
30
31
|
|
|
31
32
|
Attributes:
|
|
32
33
|
workflow_id (UUID):
|
|
@@ -38,9 +39,10 @@ class DotFlow:
|
|
|
38
39
|
|
|
39
40
|
def __init__(
|
|
40
41
|
self,
|
|
41
|
-
config: Config =
|
|
42
|
+
config: Optional[Config] = None
|
|
42
43
|
) -> None:
|
|
43
44
|
self.workflow_id = uuid4()
|
|
45
|
+
config = config if config else Config()
|
|
44
46
|
|
|
45
47
|
self.task = TaskBuilder(
|
|
46
48
|
config=config,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Execution module"""
|
|
2
2
|
|
|
3
3
|
from uuid import UUID
|
|
4
|
+
from datetime import datetime
|
|
4
5
|
from typing import Callable, List, Tuple
|
|
5
6
|
from inspect import getsourcelines
|
|
6
7
|
from types import FunctionType
|
|
@@ -10,13 +11,13 @@ try:
|
|
|
10
11
|
except ImportError:
|
|
11
12
|
NoneType = type(None)
|
|
12
13
|
|
|
14
|
+
from dotflow.core.exception import ExecutionWithClassError
|
|
13
15
|
from dotflow.logging import logger
|
|
14
16
|
from dotflow.core.action import Action
|
|
15
17
|
from dotflow.core.context import Context
|
|
16
18
|
from dotflow.core.task import Task
|
|
17
|
-
from dotflow.core.types import
|
|
19
|
+
from dotflow.core.types import TypeStatus
|
|
18
20
|
|
|
19
|
-
from dotflow.core.decorators import time
|
|
20
21
|
from dotflow.utils import basic_callback
|
|
21
22
|
|
|
22
23
|
|
|
@@ -46,14 +47,14 @@ class Execution:
|
|
|
46
47
|
task: Task,
|
|
47
48
|
workflow_id: UUID,
|
|
48
49
|
previous_context: Context = None,
|
|
49
|
-
|
|
50
|
+
_flow_callback: Callable = basic_callback,
|
|
50
51
|
) -> None:
|
|
51
52
|
self.task = task
|
|
52
|
-
self.task.status =
|
|
53
|
+
self.task.status = TypeStatus.IN_PROGRESS
|
|
53
54
|
self.task.previous_context = previous_context
|
|
54
55
|
self.task.workflow_id = workflow_id
|
|
55
56
|
|
|
56
|
-
self._excution(
|
|
57
|
+
self._excution(_flow_callback)
|
|
57
58
|
|
|
58
59
|
def _is_action(self, class_instance: Callable, func: Callable):
|
|
59
60
|
try:
|
|
@@ -116,7 +117,10 @@ class Execution:
|
|
|
116
117
|
new_context.storage.append(subcontext)
|
|
117
118
|
previous_context = subcontext
|
|
118
119
|
|
|
119
|
-
except Exception:
|
|
120
|
+
except Exception as error:
|
|
121
|
+
if not isinstance(error, ExecutionWithClassError):
|
|
122
|
+
raise error
|
|
123
|
+
|
|
120
124
|
subcontext = new_object(
|
|
121
125
|
class_instance,
|
|
122
126
|
initial_context=self.task.initial_context,
|
|
@@ -131,9 +135,9 @@ class Execution:
|
|
|
131
135
|
|
|
132
136
|
return new_context
|
|
133
137
|
|
|
134
|
-
|
|
135
|
-
def _excution(self, _internal_callback):
|
|
138
|
+
def _excution(self, _flow_callback):
|
|
136
139
|
try:
|
|
140
|
+
start = datetime.now()
|
|
137
141
|
current_context = self.task.step(
|
|
138
142
|
initial_context=self.task.initial_context,
|
|
139
143
|
previous_context=self.task.previous_context,
|
|
@@ -145,19 +149,20 @@ class Execution:
|
|
|
145
149
|
class_instance=current_context.storage
|
|
146
150
|
)
|
|
147
151
|
|
|
148
|
-
self.task.status = TaskStatus.COMPLETED
|
|
149
152
|
self.task.current_context = current_context
|
|
153
|
+
self.task.duration = (datetime.now() - start).total_seconds()
|
|
154
|
+
self.task.status = TypeStatus.COMPLETED
|
|
150
155
|
|
|
151
156
|
except AssertionError as err:
|
|
152
157
|
raise err
|
|
153
158
|
|
|
154
159
|
except Exception as err:
|
|
155
|
-
self.task.status = TaskStatus.FAILED
|
|
156
|
-
self.task.current_context = None
|
|
157
160
|
self.task.error = err
|
|
161
|
+
self.task.current_context = None
|
|
162
|
+
self.task.status = TypeStatus.FAILED
|
|
158
163
|
|
|
159
164
|
finally:
|
|
160
165
|
self.task.callback(task=self.task)
|
|
161
|
-
|
|
166
|
+
_flow_callback(task=self.task)
|
|
162
167
|
|
|
163
168
|
return self.task
|
|
@@ -5,9 +5,6 @@ import json
|
|
|
5
5
|
from uuid import UUID
|
|
6
6
|
from typing import Any, Callable, List
|
|
7
7
|
|
|
8
|
-
from rich.console import Console # type: ignore
|
|
9
|
-
|
|
10
|
-
from dotflow.logging import logger
|
|
11
8
|
from dotflow.core.config import Config
|
|
12
9
|
from dotflow.core.action import Action
|
|
13
10
|
from dotflow.core.context import Context
|
|
@@ -15,7 +12,7 @@ from dotflow.core.module import Module
|
|
|
15
12
|
from dotflow.core.serializers.task import SerializerTask
|
|
16
13
|
from dotflow.core.serializers.workflow import SerializerWorkflow
|
|
17
14
|
from dotflow.core.exception import MissingActionDecorator, NotCallableObject
|
|
18
|
-
from dotflow.core.types.status import
|
|
15
|
+
from dotflow.core.types.status import TypeStatus
|
|
19
16
|
from dotflow.utils import (
|
|
20
17
|
basic_callback,
|
|
21
18
|
traceback_error,
|
|
@@ -103,14 +100,14 @@ class Task(TaskInstance):
|
|
|
103
100
|
config,
|
|
104
101
|
group_name
|
|
105
102
|
)
|
|
103
|
+
self.config = config
|
|
104
|
+
self.group_name = group_name
|
|
106
105
|
self.task_id = task_id
|
|
107
106
|
self.workflow_id = workflow_id
|
|
108
107
|
self.step = step
|
|
109
108
|
self.callback = callback
|
|
110
109
|
self.initial_context = initial_context
|
|
111
|
-
self.status =
|
|
112
|
-
self.config = config
|
|
113
|
-
self.group_name = group_name
|
|
110
|
+
self.status = TypeStatus.NOT_STARTED
|
|
114
111
|
|
|
115
112
|
@property
|
|
116
113
|
def step(self):
|
|
@@ -206,33 +203,20 @@ class Task(TaskInstance):
|
|
|
206
203
|
task_error = TaskError(value)
|
|
207
204
|
self._error = task_error
|
|
208
205
|
|
|
209
|
-
|
|
210
|
-
"ID %s - %s - %s \n %s",
|
|
211
|
-
self.workflow_id,
|
|
212
|
-
self.task_id,
|
|
213
|
-
self.status,
|
|
214
|
-
task_error.traceback,
|
|
215
|
-
)
|
|
216
|
-
|
|
217
|
-
console = Console()
|
|
218
|
-
console.print_exception(show_locals=True)
|
|
206
|
+
self.config.log.error(task=self)
|
|
219
207
|
|
|
220
208
|
@property
|
|
221
209
|
def status(self):
|
|
222
210
|
if not self._status:
|
|
223
|
-
return
|
|
211
|
+
return TypeStatus.NOT_STARTED
|
|
224
212
|
return self._status
|
|
225
213
|
|
|
226
214
|
@status.setter
|
|
227
|
-
def status(self, value:
|
|
215
|
+
def status(self, value: TypeStatus) -> None:
|
|
228
216
|
self._status = value
|
|
229
217
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
self.workflow_id,
|
|
233
|
-
self.task_id,
|
|
234
|
-
self.status,
|
|
235
|
-
)
|
|
218
|
+
self.config.notify.send(task=self)
|
|
219
|
+
self.config.log.info(task=self)
|
|
236
220
|
|
|
237
221
|
@property
|
|
238
222
|
def config(self):
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
"""Types __init__ module."""
|
|
2
2
|
|
|
3
3
|
from dotflow.core.types.execution import TypeExecution
|
|
4
|
-
from dotflow.core.types.status import
|
|
4
|
+
from dotflow.core.types.status import TypeStatus
|
|
5
5
|
from dotflow.core.types.storage import TypeStorage
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
__all__ = [
|
|
9
9
|
"TypeExecution",
|
|
10
|
-
"
|
|
10
|
+
"TypeStatus",
|
|
11
11
|
"TypeStorage"
|
|
12
12
|
]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Type TypeStatus mode module"""
|
|
2
|
+
|
|
3
|
+
from typing_extensions import Annotated, Doc
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TypeStatus:
|
|
7
|
+
"""
|
|
8
|
+
Import:
|
|
9
|
+
You can import the **TypeStatus** class with:
|
|
10
|
+
|
|
11
|
+
from dotflow.core.types import TypeStatus
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
NOT_STARTED: Annotated[str, Doc("Status not started.")] = "Not started"
|
|
15
|
+
IN_PROGRESS: Annotated[str, Doc("Status in progress.")] = "In progress"
|
|
16
|
+
COMPLETED: Annotated[str, Doc("Status completed.")] = "Completed"
|
|
17
|
+
PAUSED: Annotated[str, Doc("Status paused.")] = "Paused"
|
|
18
|
+
RETRY: Annotated[str, Doc("Status retry.")] = "Retry"
|
|
19
|
+
FAILED: Annotated[str, Doc("Status failed.")] = "Failed"
|
|
20
|
+
|
|
21
|
+
@classmethod
|
|
22
|
+
def get_symbol(cls, value: str) -> str:
|
|
23
|
+
status = {
|
|
24
|
+
TypeStatus.NOT_STARTED: "⚪",
|
|
25
|
+
TypeStatus.IN_PROGRESS: "🔵",
|
|
26
|
+
TypeStatus.COMPLETED: "✅",
|
|
27
|
+
TypeStatus.PAUSED: "◼️",
|
|
28
|
+
TypeStatus.RETRY: "❗",
|
|
29
|
+
TypeStatus.FAILED: "❌"
|
|
30
|
+
}
|
|
31
|
+
return status.get(value)
|
|
@@ -14,7 +14,7 @@ from dotflow.abc.flow import Flow
|
|
|
14
14
|
from dotflow.core.context import Context
|
|
15
15
|
from dotflow.core.execution import Execution
|
|
16
16
|
from dotflow.core.exception import ExecutionModeNotExist
|
|
17
|
-
from dotflow.core.types import TypeExecution,
|
|
17
|
+
from dotflow.core.types import TypeExecution, TypeStatus
|
|
18
18
|
from dotflow.core.task import Task
|
|
19
19
|
from dotflow.utils import basic_callback
|
|
20
20
|
|
|
@@ -117,7 +117,7 @@ class Manager:
|
|
|
117
117
|
def _callback_workflow(self, tasks: List[Task]):
|
|
118
118
|
final_status = [task.status for task in tasks]
|
|
119
119
|
|
|
120
|
-
if
|
|
120
|
+
if TypeStatus.FAILED in final_status:
|
|
121
121
|
self.failure(tasks=tasks)
|
|
122
122
|
else:
|
|
123
123
|
self.success(tasks=tasks)
|
|
@@ -161,7 +161,7 @@ class Sequential(Flow):
|
|
|
161
161
|
def get_tasks(self) -> List[Task]:
|
|
162
162
|
return self.queue
|
|
163
163
|
|
|
164
|
-
def
|
|
164
|
+
def _flow_callback(self, task: Task) -> None:
|
|
165
165
|
self.queue.append(task)
|
|
166
166
|
|
|
167
167
|
def run(self) -> None:
|
|
@@ -172,14 +172,14 @@ class Sequential(Flow):
|
|
|
172
172
|
task=task,
|
|
173
173
|
workflow_id=self.workflow_id,
|
|
174
174
|
previous_context=previous_context,
|
|
175
|
-
|
|
175
|
+
_flow_callback=self._flow_callback,
|
|
176
176
|
)
|
|
177
177
|
|
|
178
178
|
previous_context = task.config.storage.get(
|
|
179
179
|
key=task.config.storage.key(task=task)
|
|
180
180
|
)
|
|
181
181
|
|
|
182
|
-
if not self.ignore and task.status ==
|
|
182
|
+
if not self.ignore and task.status == TypeStatus.FAILED:
|
|
183
183
|
break
|
|
184
184
|
|
|
185
185
|
|
|
@@ -204,7 +204,7 @@ class SequentialGroup(Flow):
|
|
|
204
204
|
|
|
205
205
|
return self.tasks
|
|
206
206
|
|
|
207
|
-
def
|
|
207
|
+
def _flow_callback(self, task: Task) -> None:
|
|
208
208
|
current_task = {
|
|
209
209
|
task.task_id: {
|
|
210
210
|
"current_context": task.current_context,
|
|
@@ -249,14 +249,14 @@ class SequentialGroup(Flow):
|
|
|
249
249
|
task=task,
|
|
250
250
|
workflow_id=self.workflow_id,
|
|
251
251
|
previous_context=previous_context,
|
|
252
|
-
|
|
252
|
+
_flow_callback=self._flow_callback,
|
|
253
253
|
)
|
|
254
254
|
|
|
255
255
|
previous_context = task.config.storage.get(
|
|
256
256
|
key=task.config.storage.key(task=task)
|
|
257
257
|
)
|
|
258
258
|
|
|
259
|
-
if not self.ignore and task.status ==
|
|
259
|
+
if not self.ignore and task.status == TypeStatus.FAILED:
|
|
260
260
|
break
|
|
261
261
|
|
|
262
262
|
|
|
@@ -269,7 +269,7 @@ class Background(Flow):
|
|
|
269
269
|
def get_tasks(self) -> List[Task]:
|
|
270
270
|
return self.tasks
|
|
271
271
|
|
|
272
|
-
def
|
|
272
|
+
def _flow_callback(self, task: Task) -> None:
|
|
273
273
|
pass
|
|
274
274
|
|
|
275
275
|
def run(self) -> None:
|
|
@@ -306,7 +306,7 @@ class Parallel(Flow):
|
|
|
306
306
|
|
|
307
307
|
return self.tasks
|
|
308
308
|
|
|
309
|
-
def
|
|
309
|
+
def _flow_callback(self, task: Task) -> None:
|
|
310
310
|
current_task = {
|
|
311
311
|
task.task_id: {
|
|
312
312
|
"current_context": task.current_context,
|
|
@@ -324,7 +324,7 @@ class Parallel(Flow):
|
|
|
324
324
|
for task in self.tasks:
|
|
325
325
|
process = Process(
|
|
326
326
|
target=Execution,
|
|
327
|
-
args=(task, self.workflow_id, previous_context, self.
|
|
327
|
+
args=(task, self.workflow_id, previous_context, self._flow_callback),
|
|
328
328
|
)
|
|
329
329
|
process.start()
|
|
330
330
|
processes.append(process)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Notify Default"""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from rich.console import Console # type: ignore
|
|
6
|
+
|
|
7
|
+
from dotflow.abc.log import Log
|
|
8
|
+
from dotflow.logging import logger
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class LogDefault(Log):
|
|
12
|
+
|
|
13
|
+
def info(self, task: Any) -> None:
|
|
14
|
+
logger.info(
|
|
15
|
+
"ID %s - %s - %s",
|
|
16
|
+
task.workflow_id,
|
|
17
|
+
task.task_id,
|
|
18
|
+
task.status,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
def error(self, task: Any) -> None:
|
|
22
|
+
logger.error(
|
|
23
|
+
"ID %s - %s - %s \n %s",
|
|
24
|
+
task.workflow_id,
|
|
25
|
+
task.task_id,
|
|
26
|
+
task.status,
|
|
27
|
+
task.error.traceback,
|
|
28
|
+
)
|
|
29
|
+
console = Console()
|
|
30
|
+
console.print_exception(show_locals=True)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Notify Default"""
|
|
2
|
+
|
|
3
|
+
from json import dumps
|
|
4
|
+
from typing import Any, Optional
|
|
5
|
+
|
|
6
|
+
from requests import post
|
|
7
|
+
|
|
8
|
+
from dotflow.core.types.status import TypeStatus
|
|
9
|
+
from dotflow.abc.notify import Notify
|
|
10
|
+
from dotflow.logging import logger
|
|
11
|
+
|
|
12
|
+
MESSAGE = "{symbol} {status}\n```json\n{task}```\n{workflow_id}-{task_id}"
|
|
13
|
+
API_TELEGRAM = "https://api.telegram.org/bot{token}/sendMessage"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class NotifyTelegram(Notify):
|
|
17
|
+
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
token: str,
|
|
21
|
+
chat_id: int,
|
|
22
|
+
notification_type: Optional[TypeStatus] = None
|
|
23
|
+
):
|
|
24
|
+
self.token = token
|
|
25
|
+
self.chat_id = chat_id
|
|
26
|
+
self.notification_type = notification_type
|
|
27
|
+
|
|
28
|
+
def send(self, task: Any) -> None:
|
|
29
|
+
if not self.notification_type or self.notification_type == task.status:
|
|
30
|
+
data = {
|
|
31
|
+
"chat_id": self.chat_id,
|
|
32
|
+
"text": self._get_text(task=task),
|
|
33
|
+
"parse_mode": "markdown",
|
|
34
|
+
}
|
|
35
|
+
try:
|
|
36
|
+
response = post(
|
|
37
|
+
url=API_TELEGRAM.format(token=self.token),
|
|
38
|
+
headers={"Content-Type": "application/json"},
|
|
39
|
+
data=dumps(data),
|
|
40
|
+
timeout=5
|
|
41
|
+
)
|
|
42
|
+
response.raise_for_status()
|
|
43
|
+
except Exception as error:
|
|
44
|
+
logger.error(
|
|
45
|
+
"Internal problem sending notification on Telegram: %s",
|
|
46
|
+
str(error),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
def _get_text(self, task: Any) -> str:
|
|
50
|
+
return MESSAGE.format(
|
|
51
|
+
symbol=TypeStatus.get_symbol(task.status),
|
|
52
|
+
status=task.status,
|
|
53
|
+
workflow_id=task.workflow_id,
|
|
54
|
+
task_id=task.task_id,
|
|
55
|
+
task=task.result(),
|
|
56
|
+
)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Providers module"""
|
|
2
|
+
|
|
3
|
+
from dotflow.providers.log_default import LogDefault
|
|
4
|
+
from dotflow.providers.notify_default import NotifyDefault
|
|
5
|
+
from dotflow.providers.notify_telegram import NotifyTelegram
|
|
6
|
+
from dotflow.providers.storage_default import StorageDefault
|
|
7
|
+
from dotflow.providers.storage_file import StorageFile
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"LogDefault",
|
|
11
|
+
"NotifyDefault",
|
|
12
|
+
"NotifyTelegram",
|
|
13
|
+
"StorageDefault",
|
|
14
|
+
"StorageFile"
|
|
15
|
+
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dotflow"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.13.0.dev1"
|
|
4
4
|
authors = [
|
|
5
5
|
{ name="Fernando Celmer", email="email@fernandocelmer.com" },
|
|
6
6
|
]
|
|
@@ -11,7 +11,8 @@ requires-python = ">=3.9"
|
|
|
11
11
|
dependencies = [
|
|
12
12
|
"rich",
|
|
13
13
|
"pydantic",
|
|
14
|
-
"typing-extensions"
|
|
14
|
+
"typing-extensions",
|
|
15
|
+
"requests"
|
|
15
16
|
]
|
|
16
17
|
classifiers = [
|
|
17
18
|
'Development Status :: 4 - Beta',
|
|
@@ -35,7 +36,7 @@ mongodb = ["dotflow-mongodb"]
|
|
|
35
36
|
|
|
36
37
|
[tool.poetry]
|
|
37
38
|
name = "dotflow"
|
|
38
|
-
version = "0.
|
|
39
|
+
version = "0.13.0.dev1"
|
|
39
40
|
description = "🎲 Dotflow turns an idea into flow!"
|
|
40
41
|
authors = ["Fernando Celmer <email@fernandocelmer.com>"]
|
|
41
42
|
readme = "README.md"
|
|
@@ -63,6 +64,8 @@ python = ">=3.9.0"
|
|
|
63
64
|
rich = "^13.9.4"
|
|
64
65
|
pydantic = "^2.10.6"
|
|
65
66
|
typing-extensions = "^4.12.2"
|
|
67
|
+
python-dotenv = "^1.1.0"
|
|
68
|
+
requests = "^2.32.3"
|
|
66
69
|
|
|
67
70
|
[tool.poetry.group.dev.dependencies]
|
|
68
71
|
build = "^1.2.2.post1"
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
"""Config module"""
|
|
2
|
-
|
|
3
|
-
from dotflow.abc.storage import Storage
|
|
4
|
-
from dotflow.providers.storage_default import StorageDefault
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class Config:
|
|
8
|
-
"""
|
|
9
|
-
Import:
|
|
10
|
-
You can import the **Config** class with:
|
|
11
|
-
|
|
12
|
-
from dotflow import Config
|
|
13
|
-
from dotflow.storage import StorageDefault
|
|
14
|
-
|
|
15
|
-
Example:
|
|
16
|
-
`class` dotflow.core.config.Config
|
|
17
|
-
|
|
18
|
-
config = Config(storage=StorageDefault)
|
|
19
|
-
|
|
20
|
-
Args:
|
|
21
|
-
storage (Storage): Type of the storage.
|
|
22
|
-
|
|
23
|
-
Attributes:
|
|
24
|
-
storage (Storage):
|
|
25
|
-
"""
|
|
26
|
-
|
|
27
|
-
def __init__(self, storage: Storage = StorageDefault()) -> None:
|
|
28
|
-
self.storage = storage
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
"""Type TaskStatus mode module"""
|
|
2
|
-
|
|
3
|
-
from typing_extensions import Annotated, Doc
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class TaskStatus:
|
|
7
|
-
"""
|
|
8
|
-
Import:
|
|
9
|
-
You can import the **TaskStatus** class with:
|
|
10
|
-
|
|
11
|
-
from dotflow.core.types import TaskStatus
|
|
12
|
-
"""
|
|
13
|
-
|
|
14
|
-
NOT_STARTED: Annotated[str, Doc("Status not started.")] = "Not started"
|
|
15
|
-
IN_PROGRESS: Annotated[str, Doc("Status in progress.")] = "In progress"
|
|
16
|
-
COMPLETED: Annotated[str, Doc("Status completed.")] = "Completed"
|
|
17
|
-
PAUSED: Annotated[str, Doc("Status paused.")] = "Paused"
|
|
18
|
-
RETRY: Annotated[str, Doc("Status retry.")] = "Retry"
|
|
19
|
-
FAILED: Annotated[str, Doc("Status failed.")] = "Failed"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|