dotflow 0.12.0.dev2__tar.gz → 0.13.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/PKG-INFO +2 -1
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/__init__.py +1 -1
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/abc/flow.py +1 -1
- dotflow-0.13.0.dev1/dotflow/abc/log.py +17 -0
- dotflow-0.13.0.dev1/dotflow/abc/notify.py +13 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/action.py +53 -21
- dotflow-0.13.0.dev1/dotflow/core/config.py +54 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/dotflow.py +5 -3
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/exception.py +5 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/execution.py +25 -19
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/serializers/task.py +1 -1
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/task.py +17 -29
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/types/__init__.py +2 -2
- dotflow-0.13.0.dev1/dotflow/core/types/status.py +31 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/workflow.py +11 -11
- dotflow-0.13.0.dev1/dotflow/notify.py +6 -0
- dotflow-0.13.0.dev1/dotflow/providers/log_default.py +30 -0
- dotflow-0.13.0.dev1/dotflow/providers/notify_default.py +11 -0
- dotflow-0.13.0.dev1/dotflow/providers/notify_telegram.py +56 -0
- dotflow-0.13.0.dev1/dotflow/providers.py +15 -0
- dotflow-0.13.0.dev1/dotflow/types.py +9 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/pyproject.toml +6 -3
- dotflow-0.12.0.dev2/dotflow/core/config.py +0 -28
- dotflow-0.12.0.dev2/dotflow/core/types/status.py +0 -19
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/LICENSE +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/README.md +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/abc/__init__.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/abc/file.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/abc/http.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/abc/storage.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/abc/tcp.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/cli/__init__.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/cli/command.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/cli/commands/__init__.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/cli/commands/init.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/cli/commands/log.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/cli/commands/start.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/cli/setup.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/cli/validators/__init__.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/cli/validators/start.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/__init__.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/context.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/decorators/__init__.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/decorators/time.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/module.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/serializers/__init__.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/serializers/transport.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/serializers/workflow.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/types/execution.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/types/storage.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/core/types/worflow.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/logging.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/main.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/providers/__init__.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/providers/storage_default.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/providers/storage_file.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/settings.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/storage.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/utils/__init__.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/utils/basic_functions.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/utils/error_handler.py +0 -0
- {dotflow-0.12.0.dev2 → dotflow-0.13.0.dev1}/dotflow/utils/tools.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: dotflow
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0.dev1
|
|
4
4
|
Summary: 🎲 Dotflow turns an idea into flow!
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -37,6 +37,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
37
37
|
Provides-Extra: mongodb
|
|
38
38
|
Requires-Dist: dotflow-mongodb ; extra == "mongodb"
|
|
39
39
|
Requires-Dist: pydantic
|
|
40
|
+
Requires-Dist: requests
|
|
40
41
|
Requires-Dist: rich
|
|
41
42
|
Requires-Dist: typing-extensions
|
|
42
43
|
Project-URL: Documentation, https://github.com/dotflow-io/dotflow/blob/master/README.md
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Notify ABC"""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Log(ABC):
|
|
9
|
+
"""Log"""
|
|
10
|
+
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def info(self, task: Any) -> None:
|
|
13
|
+
"""Info"""
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def error(self, task: Any) -> None:
|
|
17
|
+
"""Error"""
|
|
@@ -1,11 +1,25 @@
|
|
|
1
1
|
"""Action module"""
|
|
2
2
|
|
|
3
|
+
from time import sleep
|
|
4
|
+
|
|
5
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
3
6
|
from typing import Callable, Dict
|
|
4
7
|
from types import FunctionType
|
|
5
8
|
|
|
9
|
+
from dotflow.core.exception import ExecutionWithClassError
|
|
6
10
|
from dotflow.core.context import Context
|
|
7
11
|
|
|
8
12
|
|
|
13
|
+
def is_execution_with_class_internal_error(error: Exception) -> bool:
|
|
14
|
+
message = str(error)
|
|
15
|
+
patterns = [
|
|
16
|
+
"initial_context",
|
|
17
|
+
"previous_context",
|
|
18
|
+
"missing 1 required positional argument: 'self'",
|
|
19
|
+
]
|
|
20
|
+
return any(pattern in message for pattern in patterns)
|
|
21
|
+
|
|
22
|
+
|
|
9
23
|
class Action(object):
|
|
10
24
|
"""
|
|
11
25
|
Import:
|
|
@@ -39,14 +53,20 @@ class Action(object):
|
|
|
39
53
|
"""
|
|
40
54
|
|
|
41
55
|
def __init__(
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
56
|
+
self,
|
|
57
|
+
func: Callable = None,
|
|
58
|
+
task: Callable = None,
|
|
59
|
+
retry: int = 1,
|
|
60
|
+
timeout: int = 0,
|
|
61
|
+
retry_delay: int = 1,
|
|
62
|
+
backoff: bool = False,
|
|
46
63
|
) -> None:
|
|
47
64
|
self.func = func
|
|
48
65
|
self.task = task
|
|
49
66
|
self.retry = retry
|
|
67
|
+
self.timeout = timeout
|
|
68
|
+
self.retry_delay = retry_delay
|
|
69
|
+
self.backoff = backoff
|
|
50
70
|
self.params = []
|
|
51
71
|
|
|
52
72
|
def __call__(self, *args, **kwargs):
|
|
@@ -59,15 +79,15 @@ class Action(object):
|
|
|
59
79
|
|
|
60
80
|
if contexts:
|
|
61
81
|
return Context(
|
|
62
|
-
storage=self.
|
|
82
|
+
storage=self._run_action(*args, **contexts),
|
|
63
83
|
task_id=task.task_id,
|
|
64
|
-
workflow_id=task.workflow_id
|
|
84
|
+
workflow_id=task.workflow_id,
|
|
65
85
|
)
|
|
66
86
|
|
|
67
87
|
return Context(
|
|
68
|
-
storage=self.
|
|
88
|
+
storage=self._run_action(*args),
|
|
69
89
|
task_id=task.task_id,
|
|
70
|
-
workflow_id=task.workflow_id
|
|
90
|
+
workflow_id=task.workflow_id,
|
|
71
91
|
)
|
|
72
92
|
|
|
73
93
|
# No parameters
|
|
@@ -80,31 +100,41 @@ class Action(object):
|
|
|
80
100
|
|
|
81
101
|
if contexts:
|
|
82
102
|
return Context(
|
|
83
|
-
storage=self.
|
|
103
|
+
storage=self._run_action(*_args, **contexts),
|
|
84
104
|
task_id=task.task_id,
|
|
85
|
-
workflow_id=task.workflow_id
|
|
105
|
+
workflow_id=task.workflow_id,
|
|
86
106
|
)
|
|
87
107
|
|
|
88
108
|
return Context(
|
|
89
|
-
storage=self.
|
|
109
|
+
storage=self._run_action(*_args),
|
|
90
110
|
task_id=task.task_id,
|
|
91
|
-
workflow_id=task.workflow_id
|
|
111
|
+
workflow_id=task.workflow_id,
|
|
92
112
|
)
|
|
93
113
|
|
|
94
114
|
return action
|
|
95
115
|
|
|
96
|
-
def
|
|
97
|
-
attempt
|
|
98
|
-
exception = Exception()
|
|
99
|
-
|
|
100
|
-
while self.retry > attempt:
|
|
116
|
+
def _run_action(self, *args, **kwargs):
|
|
117
|
+
for attempt in range(1, self.retry + 1):
|
|
101
118
|
try:
|
|
119
|
+
if self.timeout:
|
|
120
|
+
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
121
|
+
future = executor.submit(self.func, *args, **kwargs)
|
|
122
|
+
return future.result(timeout=self.timeout)
|
|
123
|
+
|
|
102
124
|
return self.func(*args, **kwargs)
|
|
125
|
+
|
|
103
126
|
except Exception as error:
|
|
104
|
-
|
|
105
|
-
|
|
127
|
+
last_exception = error
|
|
128
|
+
|
|
129
|
+
if is_execution_with_class_internal_error(error=last_exception):
|
|
130
|
+
raise ExecutionWithClassError()
|
|
131
|
+
|
|
132
|
+
if attempt == self.retry:
|
|
133
|
+
raise last_exception
|
|
106
134
|
|
|
107
|
-
|
|
135
|
+
sleep(self.retry_delay)
|
|
136
|
+
if self.backoff:
|
|
137
|
+
self.retry_delay *= 2
|
|
108
138
|
|
|
109
139
|
def _set_params(self):
|
|
110
140
|
if isinstance(self.func, FunctionType):
|
|
@@ -113,7 +143,9 @@ class Action(object):
|
|
|
113
143
|
if type(self.func) is type:
|
|
114
144
|
if hasattr(self.func, "__init__"):
|
|
115
145
|
if hasattr(self.func.__init__, "__code__"):
|
|
116
|
-
self.params = [
|
|
146
|
+
self.params = [
|
|
147
|
+
param for param in self.func.__init__.__code__.co_varnames
|
|
148
|
+
]
|
|
117
149
|
|
|
118
150
|
def _get_context(self, kwargs: Dict):
|
|
119
151
|
context = {}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Config module"""
|
|
2
|
+
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
from dotflow.abc.log import Log
|
|
6
|
+
from dotflow.abc.storage import Storage
|
|
7
|
+
from dotflow.abc.notify import Notify
|
|
8
|
+
|
|
9
|
+
from dotflow.providers.log_default import LogDefault
|
|
10
|
+
from dotflow.providers.storage_default import StorageDefault
|
|
11
|
+
from dotflow.providers.notify_default import NotifyDefault
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Config:
|
|
15
|
+
"""
|
|
16
|
+
Import:
|
|
17
|
+
You can import the **Config** class with:
|
|
18
|
+
|
|
19
|
+
from dotflow import Config
|
|
20
|
+
from dotflow.providers import (
|
|
21
|
+
StorageDefault,
|
|
22
|
+
NotifyDefault,
|
|
23
|
+
LogDefault
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
Example:
|
|
27
|
+
`class` dotflow.core.config.Config
|
|
28
|
+
|
|
29
|
+
config = Config(
|
|
30
|
+
storage=StorageFile(path=".output"),
|
|
31
|
+
notify=NotifyDefault(),
|
|
32
|
+
log=LogDefault()
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
storage (Optional[Storage]): Type of the storage.
|
|
37
|
+
notify (Optional[Notify]): Type of the notify.
|
|
38
|
+
log (Optional[Log]): Type of the notify.
|
|
39
|
+
|
|
40
|
+
Attributes:
|
|
41
|
+
storage (Optional[Storage]):
|
|
42
|
+
notify (Optional[Notify]):
|
|
43
|
+
log (Optional[Log]):
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
storage: Optional[Storage] = None,
|
|
49
|
+
notify: Optional[Notify] = None,
|
|
50
|
+
log: Optional[Log] = None,
|
|
51
|
+
) -> None:
|
|
52
|
+
self.storage = storage if storage else StorageDefault()
|
|
53
|
+
self.notify = notify if notify else NotifyDefault()
|
|
54
|
+
self.log = log if log else LogDefault()
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from uuid import uuid4
|
|
4
4
|
from functools import partial
|
|
5
|
+
from typing import Optional
|
|
5
6
|
|
|
6
7
|
from dotflow.core.config import Config
|
|
7
8
|
from dotflow.core.workflow import Manager
|
|
@@ -26,7 +27,7 @@ class DotFlow:
|
|
|
26
27
|
workflow = DotFlow(config=config)
|
|
27
28
|
|
|
28
29
|
Args:
|
|
29
|
-
config (Config): Configuration class.
|
|
30
|
+
config (Optional[Config]): Configuration class.
|
|
30
31
|
|
|
31
32
|
Attributes:
|
|
32
33
|
workflow_id (UUID):
|
|
@@ -38,9 +39,10 @@ class DotFlow:
|
|
|
38
39
|
|
|
39
40
|
def __init__(
|
|
40
41
|
self,
|
|
41
|
-
config: Config =
|
|
42
|
+
config: Optional[Config] = None
|
|
42
43
|
) -> None:
|
|
43
44
|
self.workflow_id = uuid4()
|
|
45
|
+
config = config if config else Config()
|
|
44
46
|
|
|
45
47
|
self.task = TaskBuilder(
|
|
46
48
|
config=config,
|
|
@@ -75,4 +77,4 @@ class DotFlow:
|
|
|
75
77
|
return [task.current_context.storage for task in self.task.queue]
|
|
76
78
|
|
|
77
79
|
def result(self):
|
|
78
|
-
return self.task.
|
|
80
|
+
return self.task.result()
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Execution module"""
|
|
2
2
|
|
|
3
3
|
from uuid import UUID
|
|
4
|
+
from datetime import datetime
|
|
4
5
|
from typing import Callable, List, Tuple
|
|
5
6
|
from inspect import getsourcelines
|
|
6
7
|
from types import FunctionType
|
|
@@ -10,13 +11,13 @@ try:
|
|
|
10
11
|
except ImportError:
|
|
11
12
|
NoneType = type(None)
|
|
12
13
|
|
|
14
|
+
from dotflow.core.exception import ExecutionWithClassError
|
|
13
15
|
from dotflow.logging import logger
|
|
14
16
|
from dotflow.core.action import Action
|
|
15
17
|
from dotflow.core.context import Context
|
|
16
18
|
from dotflow.core.task import Task
|
|
17
|
-
from dotflow.core.types import
|
|
19
|
+
from dotflow.core.types import TypeStatus
|
|
18
20
|
|
|
19
|
-
from dotflow.core.decorators import time
|
|
20
21
|
from dotflow.utils import basic_callback
|
|
21
22
|
|
|
22
23
|
|
|
@@ -46,14 +47,14 @@ class Execution:
|
|
|
46
47
|
task: Task,
|
|
47
48
|
workflow_id: UUID,
|
|
48
49
|
previous_context: Context = None,
|
|
49
|
-
|
|
50
|
+
_flow_callback: Callable = basic_callback,
|
|
50
51
|
) -> None:
|
|
51
52
|
self.task = task
|
|
52
|
-
self.task.status =
|
|
53
|
+
self.task.status = TypeStatus.IN_PROGRESS
|
|
53
54
|
self.task.previous_context = previous_context
|
|
54
55
|
self.task.workflow_id = workflow_id
|
|
55
56
|
|
|
56
|
-
self._excution(
|
|
57
|
+
self._excution(_flow_callback)
|
|
57
58
|
|
|
58
59
|
def _is_action(self, class_instance: Callable, func: Callable):
|
|
59
60
|
try:
|
|
@@ -66,9 +67,7 @@ class Execution:
|
|
|
66
67
|
return False
|
|
67
68
|
|
|
68
69
|
def _execution_orderer(
|
|
69
|
-
|
|
70
|
-
callable_list: List[str],
|
|
71
|
-
class_instance: Callable
|
|
70
|
+
self, callable_list: List[str], class_instance: Callable
|
|
72
71
|
) -> Tuple[int, Callable]:
|
|
73
72
|
ordered_list = []
|
|
74
73
|
|
|
@@ -84,7 +83,10 @@ class Execution:
|
|
|
84
83
|
return ordered_list
|
|
85
84
|
|
|
86
85
|
except TypeError as err:
|
|
87
|
-
logger.error(
|
|
86
|
+
logger.error(
|
|
87
|
+
"Internal problem with ordering the class functions, but don't worry, it was executed.: %s",
|
|
88
|
+
str(err),
|
|
89
|
+
)
|
|
88
90
|
|
|
89
91
|
for index, callable_name in enumerate(callable_list):
|
|
90
92
|
ordered_list.append((index, callable_name))
|
|
@@ -110,17 +112,20 @@ class Execution:
|
|
|
110
112
|
subcontext = new_object(
|
|
111
113
|
initial_context=self.task.initial_context,
|
|
112
114
|
previous_context=previous_context,
|
|
113
|
-
task=self.task
|
|
115
|
+
task=self.task,
|
|
114
116
|
)
|
|
115
117
|
new_context.storage.append(subcontext)
|
|
116
118
|
previous_context = subcontext
|
|
117
119
|
|
|
118
|
-
except Exception:
|
|
120
|
+
except Exception as error:
|
|
121
|
+
if not isinstance(error, ExecutionWithClassError):
|
|
122
|
+
raise error
|
|
123
|
+
|
|
119
124
|
subcontext = new_object(
|
|
120
125
|
class_instance,
|
|
121
126
|
initial_context=self.task.initial_context,
|
|
122
127
|
previous_context=previous_context,
|
|
123
|
-
task=self.task
|
|
128
|
+
task=self.task,
|
|
124
129
|
)
|
|
125
130
|
new_context.storage.append(subcontext)
|
|
126
131
|
previous_context = subcontext
|
|
@@ -130,13 +135,13 @@ class Execution:
|
|
|
130
135
|
|
|
131
136
|
return new_context
|
|
132
137
|
|
|
133
|
-
|
|
134
|
-
def _excution(self, _internal_callback):
|
|
138
|
+
def _excution(self, _flow_callback):
|
|
135
139
|
try:
|
|
140
|
+
start = datetime.now()
|
|
136
141
|
current_context = self.task.step(
|
|
137
142
|
initial_context=self.task.initial_context,
|
|
138
143
|
previous_context=self.task.previous_context,
|
|
139
|
-
task=self.task
|
|
144
|
+
task=self.task,
|
|
140
145
|
)
|
|
141
146
|
|
|
142
147
|
if type(current_context.storage) not in self.VALID_OBJECTS:
|
|
@@ -144,19 +149,20 @@ class Execution:
|
|
|
144
149
|
class_instance=current_context.storage
|
|
145
150
|
)
|
|
146
151
|
|
|
147
|
-
self.task.status = TaskStatus.COMPLETED
|
|
148
152
|
self.task.current_context = current_context
|
|
153
|
+
self.task.duration = (datetime.now() - start).total_seconds()
|
|
154
|
+
self.task.status = TypeStatus.COMPLETED
|
|
149
155
|
|
|
150
156
|
except AssertionError as err:
|
|
151
157
|
raise err
|
|
152
158
|
|
|
153
159
|
except Exception as err:
|
|
154
|
-
self.task.status = TaskStatus.FAILED
|
|
155
|
-
self.task.current_context = None
|
|
156
160
|
self.task.error = err
|
|
161
|
+
self.task.current_context = None
|
|
162
|
+
self.task.status = TypeStatus.FAILED
|
|
157
163
|
|
|
158
164
|
finally:
|
|
159
165
|
self.task.callback(task=self.task)
|
|
160
|
-
|
|
166
|
+
_flow_callback(task=self.task)
|
|
161
167
|
|
|
162
168
|
return self.task
|
|
@@ -32,7 +32,7 @@ class SerializerTask(BaseModel):
|
|
|
32
32
|
def error_validator(cls, value: str) -> str:
|
|
33
33
|
if value:
|
|
34
34
|
return SerializerTaskError(**value.__dict__)
|
|
35
|
-
return
|
|
35
|
+
return None
|
|
36
36
|
|
|
37
37
|
@field_validator(
|
|
38
38
|
"initial_context", "current_context", "previous_context", mode="before"
|
|
@@ -5,9 +5,6 @@ import json
|
|
|
5
5
|
from uuid import UUID
|
|
6
6
|
from typing import Any, Callable, List
|
|
7
7
|
|
|
8
|
-
from rich.console import Console # type: ignore
|
|
9
|
-
|
|
10
|
-
from dotflow.logging import logger
|
|
11
8
|
from dotflow.core.config import Config
|
|
12
9
|
from dotflow.core.action import Action
|
|
13
10
|
from dotflow.core.context import Context
|
|
@@ -15,7 +12,7 @@ from dotflow.core.module import Module
|
|
|
15
12
|
from dotflow.core.serializers.task import SerializerTask
|
|
16
13
|
from dotflow.core.serializers.workflow import SerializerWorkflow
|
|
17
14
|
from dotflow.core.exception import MissingActionDecorator, NotCallableObject
|
|
18
|
-
from dotflow.core.types.status import
|
|
15
|
+
from dotflow.core.types.status import TypeStatus
|
|
19
16
|
from dotflow.utils import (
|
|
20
17
|
basic_callback,
|
|
21
18
|
traceback_error,
|
|
@@ -103,14 +100,14 @@ class Task(TaskInstance):
|
|
|
103
100
|
config,
|
|
104
101
|
group_name
|
|
105
102
|
)
|
|
103
|
+
self.config = config
|
|
104
|
+
self.group_name = group_name
|
|
106
105
|
self.task_id = task_id
|
|
107
106
|
self.workflow_id = workflow_id
|
|
108
107
|
self.step = step
|
|
109
108
|
self.callback = callback
|
|
110
109
|
self.initial_context = initial_context
|
|
111
|
-
self.status =
|
|
112
|
-
self.config = config
|
|
113
|
-
self.group_name = group_name
|
|
110
|
+
self.status = TypeStatus.NOT_STARTED
|
|
114
111
|
|
|
115
112
|
@property
|
|
116
113
|
def step(self):
|
|
@@ -206,33 +203,20 @@ class Task(TaskInstance):
|
|
|
206
203
|
task_error = TaskError(value)
|
|
207
204
|
self._error = task_error
|
|
208
205
|
|
|
209
|
-
|
|
210
|
-
"ID %s - %s - %s \n %s",
|
|
211
|
-
self.workflow_id,
|
|
212
|
-
self.task_id,
|
|
213
|
-
self.status,
|
|
214
|
-
task_error.traceback,
|
|
215
|
-
)
|
|
216
|
-
|
|
217
|
-
console = Console()
|
|
218
|
-
console.print_exception(show_locals=True)
|
|
206
|
+
self.config.log.error(task=self)
|
|
219
207
|
|
|
220
208
|
@property
|
|
221
209
|
def status(self):
|
|
222
210
|
if not self._status:
|
|
223
|
-
return
|
|
211
|
+
return TypeStatus.NOT_STARTED
|
|
224
212
|
return self._status
|
|
225
213
|
|
|
226
214
|
@status.setter
|
|
227
|
-
def status(self, value:
|
|
215
|
+
def status(self, value: TypeStatus) -> None:
|
|
228
216
|
self._status = value
|
|
229
217
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
self.workflow_id,
|
|
233
|
-
self.task_id,
|
|
234
|
-
self.status,
|
|
235
|
-
)
|
|
218
|
+
self.config.notify.send(task=self)
|
|
219
|
+
self.config.log.info(task=self)
|
|
236
220
|
|
|
237
221
|
@property
|
|
238
222
|
def config(self):
|
|
@@ -245,10 +229,12 @@ class Task(TaskInstance):
|
|
|
245
229
|
self._config = value
|
|
246
230
|
|
|
247
231
|
def schema(self) -> SerializerTask:
|
|
248
|
-
|
|
232
|
+
return SerializerTask(
|
|
249
233
|
**self.__dict__
|
|
250
|
-
)
|
|
234
|
+
)
|
|
251
235
|
|
|
236
|
+
def result(self) -> SerializerWorkflow:
|
|
237
|
+
item = self.schema().model_dump_json()
|
|
252
238
|
return json.loads(item)
|
|
253
239
|
|
|
254
240
|
|
|
@@ -353,9 +339,11 @@ class TaskBuilder:
|
|
|
353
339
|
self.queue.reverse()
|
|
354
340
|
|
|
355
341
|
def schema(self) -> SerializerWorkflow:
|
|
356
|
-
|
|
342
|
+
return SerializerWorkflow(
|
|
357
343
|
workflow_id=self.workflow_id,
|
|
358
344
|
tasks=[item.schema() for item in self.queue]
|
|
359
|
-
)
|
|
345
|
+
)
|
|
360
346
|
|
|
347
|
+
def result(self) -> SerializerWorkflow:
|
|
348
|
+
item = self.schema().model_dump_json()
|
|
361
349
|
return json.loads(item)
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
"""Types __init__ module."""
|
|
2
2
|
|
|
3
3
|
from dotflow.core.types.execution import TypeExecution
|
|
4
|
-
from dotflow.core.types.status import
|
|
4
|
+
from dotflow.core.types.status import TypeStatus
|
|
5
5
|
from dotflow.core.types.storage import TypeStorage
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
__all__ = [
|
|
9
9
|
"TypeExecution",
|
|
10
|
-
"
|
|
10
|
+
"TypeStatus",
|
|
11
11
|
"TypeStorage"
|
|
12
12
|
]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Type TypeStatus mode module"""
|
|
2
|
+
|
|
3
|
+
from typing_extensions import Annotated, Doc
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TypeStatus:
|
|
7
|
+
"""
|
|
8
|
+
Import:
|
|
9
|
+
You can import the **TypeStatus** class with:
|
|
10
|
+
|
|
11
|
+
from dotflow.core.types import TypeStatus
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
NOT_STARTED: Annotated[str, Doc("Status not started.")] = "Not started"
|
|
15
|
+
IN_PROGRESS: Annotated[str, Doc("Status in progress.")] = "In progress"
|
|
16
|
+
COMPLETED: Annotated[str, Doc("Status completed.")] = "Completed"
|
|
17
|
+
PAUSED: Annotated[str, Doc("Status paused.")] = "Paused"
|
|
18
|
+
RETRY: Annotated[str, Doc("Status retry.")] = "Retry"
|
|
19
|
+
FAILED: Annotated[str, Doc("Status failed.")] = "Failed"
|
|
20
|
+
|
|
21
|
+
@classmethod
|
|
22
|
+
def get_symbol(cls, value: str) -> str:
|
|
23
|
+
status = {
|
|
24
|
+
TypeStatus.NOT_STARTED: "⚪",
|
|
25
|
+
TypeStatus.IN_PROGRESS: "🔵",
|
|
26
|
+
TypeStatus.COMPLETED: "✅",
|
|
27
|
+
TypeStatus.PAUSED: "◼️",
|
|
28
|
+
TypeStatus.RETRY: "❗",
|
|
29
|
+
TypeStatus.FAILED: "❌"
|
|
30
|
+
}
|
|
31
|
+
return status.get(value)
|
|
@@ -14,7 +14,7 @@ from dotflow.abc.flow import Flow
|
|
|
14
14
|
from dotflow.core.context import Context
|
|
15
15
|
from dotflow.core.execution import Execution
|
|
16
16
|
from dotflow.core.exception import ExecutionModeNotExist
|
|
17
|
-
from dotflow.core.types import TypeExecution,
|
|
17
|
+
from dotflow.core.types import TypeExecution, TypeStatus
|
|
18
18
|
from dotflow.core.task import Task
|
|
19
19
|
from dotflow.utils import basic_callback
|
|
20
20
|
|
|
@@ -117,7 +117,7 @@ class Manager:
|
|
|
117
117
|
def _callback_workflow(self, tasks: List[Task]):
|
|
118
118
|
final_status = [task.status for task in tasks]
|
|
119
119
|
|
|
120
|
-
if
|
|
120
|
+
if TypeStatus.FAILED in final_status:
|
|
121
121
|
self.failure(tasks=tasks)
|
|
122
122
|
else:
|
|
123
123
|
self.success(tasks=tasks)
|
|
@@ -161,7 +161,7 @@ class Sequential(Flow):
|
|
|
161
161
|
def get_tasks(self) -> List[Task]:
|
|
162
162
|
return self.queue
|
|
163
163
|
|
|
164
|
-
def
|
|
164
|
+
def _flow_callback(self, task: Task) -> None:
|
|
165
165
|
self.queue.append(task)
|
|
166
166
|
|
|
167
167
|
def run(self) -> None:
|
|
@@ -172,14 +172,14 @@ class Sequential(Flow):
|
|
|
172
172
|
task=task,
|
|
173
173
|
workflow_id=self.workflow_id,
|
|
174
174
|
previous_context=previous_context,
|
|
175
|
-
|
|
175
|
+
_flow_callback=self._flow_callback,
|
|
176
176
|
)
|
|
177
177
|
|
|
178
178
|
previous_context = task.config.storage.get(
|
|
179
179
|
key=task.config.storage.key(task=task)
|
|
180
180
|
)
|
|
181
181
|
|
|
182
|
-
if not self.ignore and task.status ==
|
|
182
|
+
if not self.ignore and task.status == TypeStatus.FAILED:
|
|
183
183
|
break
|
|
184
184
|
|
|
185
185
|
|
|
@@ -204,7 +204,7 @@ class SequentialGroup(Flow):
|
|
|
204
204
|
|
|
205
205
|
return self.tasks
|
|
206
206
|
|
|
207
|
-
def
|
|
207
|
+
def _flow_callback(self, task: Task) -> None:
|
|
208
208
|
current_task = {
|
|
209
209
|
task.task_id: {
|
|
210
210
|
"current_context": task.current_context,
|
|
@@ -249,14 +249,14 @@ class SequentialGroup(Flow):
|
|
|
249
249
|
task=task,
|
|
250
250
|
workflow_id=self.workflow_id,
|
|
251
251
|
previous_context=previous_context,
|
|
252
|
-
|
|
252
|
+
_flow_callback=self._flow_callback,
|
|
253
253
|
)
|
|
254
254
|
|
|
255
255
|
previous_context = task.config.storage.get(
|
|
256
256
|
key=task.config.storage.key(task=task)
|
|
257
257
|
)
|
|
258
258
|
|
|
259
|
-
if not self.ignore and task.status ==
|
|
259
|
+
if not self.ignore and task.status == TypeStatus.FAILED:
|
|
260
260
|
break
|
|
261
261
|
|
|
262
262
|
|
|
@@ -269,7 +269,7 @@ class Background(Flow):
|
|
|
269
269
|
def get_tasks(self) -> List[Task]:
|
|
270
270
|
return self.tasks
|
|
271
271
|
|
|
272
|
-
def
|
|
272
|
+
def _flow_callback(self, task: Task) -> None:
|
|
273
273
|
pass
|
|
274
274
|
|
|
275
275
|
def run(self) -> None:
|
|
@@ -306,7 +306,7 @@ class Parallel(Flow):
|
|
|
306
306
|
|
|
307
307
|
return self.tasks
|
|
308
308
|
|
|
309
|
-
def
|
|
309
|
+
def _flow_callback(self, task: Task) -> None:
|
|
310
310
|
current_task = {
|
|
311
311
|
task.task_id: {
|
|
312
312
|
"current_context": task.current_context,
|
|
@@ -324,7 +324,7 @@ class Parallel(Flow):
|
|
|
324
324
|
for task in self.tasks:
|
|
325
325
|
process = Process(
|
|
326
326
|
target=Execution,
|
|
327
|
-
args=(task, self.workflow_id, previous_context, self.
|
|
327
|
+
args=(task, self.workflow_id, previous_context, self._flow_callback),
|
|
328
328
|
)
|
|
329
329
|
process.start()
|
|
330
330
|
processes.append(process)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Notify Default"""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from rich.console import Console # type: ignore
|
|
6
|
+
|
|
7
|
+
from dotflow.abc.log import Log
|
|
8
|
+
from dotflow.logging import logger
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class LogDefault(Log):
|
|
12
|
+
|
|
13
|
+
def info(self, task: Any) -> None:
|
|
14
|
+
logger.info(
|
|
15
|
+
"ID %s - %s - %s",
|
|
16
|
+
task.workflow_id,
|
|
17
|
+
task.task_id,
|
|
18
|
+
task.status,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
def error(self, task: Any) -> None:
|
|
22
|
+
logger.error(
|
|
23
|
+
"ID %s - %s - %s \n %s",
|
|
24
|
+
task.workflow_id,
|
|
25
|
+
task.task_id,
|
|
26
|
+
task.status,
|
|
27
|
+
task.error.traceback,
|
|
28
|
+
)
|
|
29
|
+
console = Console()
|
|
30
|
+
console.print_exception(show_locals=True)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Notify Default"""
|
|
2
|
+
|
|
3
|
+
from json import dumps
|
|
4
|
+
from typing import Any, Optional
|
|
5
|
+
|
|
6
|
+
from requests import post
|
|
7
|
+
|
|
8
|
+
from dotflow.core.types.status import TypeStatus
|
|
9
|
+
from dotflow.abc.notify import Notify
|
|
10
|
+
from dotflow.logging import logger
|
|
11
|
+
|
|
12
|
+
MESSAGE = "{symbol} {status}\n```json\n{task}```\n{workflow_id}-{task_id}"
|
|
13
|
+
API_TELEGRAM = "https://api.telegram.org/bot{token}/sendMessage"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class NotifyTelegram(Notify):
|
|
17
|
+
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
token: str,
|
|
21
|
+
chat_id: int,
|
|
22
|
+
notification_type: Optional[TypeStatus] = None
|
|
23
|
+
):
|
|
24
|
+
self.token = token
|
|
25
|
+
self.chat_id = chat_id
|
|
26
|
+
self.notification_type = notification_type
|
|
27
|
+
|
|
28
|
+
def send(self, task: Any) -> None:
|
|
29
|
+
if not self.notification_type or self.notification_type == task.status:
|
|
30
|
+
data = {
|
|
31
|
+
"chat_id": self.chat_id,
|
|
32
|
+
"text": self._get_text(task=task),
|
|
33
|
+
"parse_mode": "markdown",
|
|
34
|
+
}
|
|
35
|
+
try:
|
|
36
|
+
response = post(
|
|
37
|
+
url=API_TELEGRAM.format(token=self.token),
|
|
38
|
+
headers={"Content-Type": "application/json"},
|
|
39
|
+
data=dumps(data),
|
|
40
|
+
timeout=5
|
|
41
|
+
)
|
|
42
|
+
response.raise_for_status()
|
|
43
|
+
except Exception as error:
|
|
44
|
+
logger.error(
|
|
45
|
+
"Internal problem sending notification on Telegram: %s",
|
|
46
|
+
str(error),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
def _get_text(self, task: Any) -> str:
|
|
50
|
+
return MESSAGE.format(
|
|
51
|
+
symbol=TypeStatus.get_symbol(task.status),
|
|
52
|
+
status=task.status,
|
|
53
|
+
workflow_id=task.workflow_id,
|
|
54
|
+
task_id=task.task_id,
|
|
55
|
+
task=task.result(),
|
|
56
|
+
)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Providers module"""
|
|
2
|
+
|
|
3
|
+
from dotflow.providers.log_default import LogDefault
|
|
4
|
+
from dotflow.providers.notify_default import NotifyDefault
|
|
5
|
+
from dotflow.providers.notify_telegram import NotifyTelegram
|
|
6
|
+
from dotflow.providers.storage_default import StorageDefault
|
|
7
|
+
from dotflow.providers.storage_file import StorageFile
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"LogDefault",
|
|
11
|
+
"NotifyDefault",
|
|
12
|
+
"NotifyTelegram",
|
|
13
|
+
"StorageDefault",
|
|
14
|
+
"StorageFile"
|
|
15
|
+
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dotflow"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.13.0.dev1"
|
|
4
4
|
authors = [
|
|
5
5
|
{ name="Fernando Celmer", email="email@fernandocelmer.com" },
|
|
6
6
|
]
|
|
@@ -11,7 +11,8 @@ requires-python = ">=3.9"
|
|
|
11
11
|
dependencies = [
|
|
12
12
|
"rich",
|
|
13
13
|
"pydantic",
|
|
14
|
-
"typing-extensions"
|
|
14
|
+
"typing-extensions",
|
|
15
|
+
"requests"
|
|
15
16
|
]
|
|
16
17
|
classifiers = [
|
|
17
18
|
'Development Status :: 4 - Beta',
|
|
@@ -35,7 +36,7 @@ mongodb = ["dotflow-mongodb"]
|
|
|
35
36
|
|
|
36
37
|
[tool.poetry]
|
|
37
38
|
name = "dotflow"
|
|
38
|
-
version = "0.
|
|
39
|
+
version = "0.13.0.dev1"
|
|
39
40
|
description = "🎲 Dotflow turns an idea into flow!"
|
|
40
41
|
authors = ["Fernando Celmer <email@fernandocelmer.com>"]
|
|
41
42
|
readme = "README.md"
|
|
@@ -63,6 +64,8 @@ python = ">=3.9.0"
|
|
|
63
64
|
rich = "^13.9.4"
|
|
64
65
|
pydantic = "^2.10.6"
|
|
65
66
|
typing-extensions = "^4.12.2"
|
|
67
|
+
python-dotenv = "^1.1.0"
|
|
68
|
+
requests = "^2.32.3"
|
|
66
69
|
|
|
67
70
|
[tool.poetry.group.dev.dependencies]
|
|
68
71
|
build = "^1.2.2.post1"
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
"""Config module"""
|
|
2
|
-
|
|
3
|
-
from dotflow.abc.storage import Storage
|
|
4
|
-
from dotflow.providers.storage_default import StorageDefault
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class Config:
|
|
8
|
-
"""
|
|
9
|
-
Import:
|
|
10
|
-
You can import the **Config** class with:
|
|
11
|
-
|
|
12
|
-
from dotflow import Config
|
|
13
|
-
from dotflow.storage import StorageDefault
|
|
14
|
-
|
|
15
|
-
Example:
|
|
16
|
-
`class` dotflow.core.config.Config
|
|
17
|
-
|
|
18
|
-
config = Config(storage=StorageDefault)
|
|
19
|
-
|
|
20
|
-
Args:
|
|
21
|
-
storage (Storage): Type of the storage.
|
|
22
|
-
|
|
23
|
-
Attributes:
|
|
24
|
-
storage (Storage):
|
|
25
|
-
"""
|
|
26
|
-
|
|
27
|
-
def __init__(self, storage: Storage = StorageDefault()) -> None:
|
|
28
|
-
self.storage = storage
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
"""Type TaskStatus mode module"""
|
|
2
|
-
|
|
3
|
-
from typing_extensions import Annotated, Doc
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class TaskStatus:
|
|
7
|
-
"""
|
|
8
|
-
Import:
|
|
9
|
-
You can import the **TaskStatus** class with:
|
|
10
|
-
|
|
11
|
-
from dotflow.core.types import TaskStatus
|
|
12
|
-
"""
|
|
13
|
-
|
|
14
|
-
NOT_STARTED: Annotated[str, Doc("Status not started.")] = "Not started"
|
|
15
|
-
IN_PROGRESS: Annotated[str, Doc("Status in progress.")] = "In progress"
|
|
16
|
-
COMPLETED: Annotated[str, Doc("Status completed.")] = "Completed"
|
|
17
|
-
PAUSED: Annotated[str, Doc("Status paused.")] = "Paused"
|
|
18
|
-
RETRY: Annotated[str, Doc("Status retry.")] = "Retry"
|
|
19
|
-
FAILED: Annotated[str, Doc("Status failed.")] = "Failed"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|