dotflow 0.12.0.dev3__tar.gz → 0.13.0.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/PKG-INFO +49 -41
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/README.md +46 -40
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/__init__.py +1 -1
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/abc/flow.py +1 -1
- dotflow-0.13.0.dev2/dotflow/abc/log.py +17 -0
- dotflow-0.13.0.dev2/dotflow/abc/notify.py +13 -0
- dotflow-0.13.0.dev2/dotflow/core/action.py +185 -0
- dotflow-0.13.0.dev2/dotflow/core/config.py +55 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/dotflow.py +5 -3
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/exception.py +5 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/execution.py +21 -14
- dotflow-0.13.0.dev2/dotflow/core/serializers/task.py +83 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/task.py +13 -31
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/types/__init__.py +2 -2
- dotflow-0.13.0.dev2/dotflow/core/types/status.py +31 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/workflow.py +27 -24
- dotflow-0.13.0.dev2/dotflow/notify.py +6 -0
- dotflow-0.13.0.dev2/dotflow/providers/log_default.py +30 -0
- dotflow-0.13.0.dev2/dotflow/providers/notify_default.py +11 -0
- dotflow-0.13.0.dev2/dotflow/providers/notify_telegram.py +58 -0
- dotflow-0.13.0.dev2/dotflow/providers.py +15 -0
- dotflow-0.13.0.dev2/dotflow/types.py +9 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/pyproject.toml +11 -4
- dotflow-0.12.0.dev3/dotflow/core/action.py +0 -129
- dotflow-0.12.0.dev3/dotflow/core/config.py +0 -28
- dotflow-0.12.0.dev3/dotflow/core/serializers/task.py +0 -47
- dotflow-0.12.0.dev3/dotflow/core/types/status.py +0 -19
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/LICENSE +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/abc/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/abc/file.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/abc/http.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/abc/storage.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/abc/tcp.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/cli/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/cli/command.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/cli/commands/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/cli/commands/init.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/cli/commands/log.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/cli/commands/start.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/cli/setup.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/cli/validators/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/cli/validators/start.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/context.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/decorators/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/decorators/time.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/module.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/serializers/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/serializers/transport.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/serializers/workflow.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/types/execution.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/types/storage.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/core/types/worflow.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/logging.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/main.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/providers/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/providers/storage_default.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/providers/storage_file.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/settings.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/storage.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/utils/__init__.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/utils/basic_functions.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/utils/error_handler.py +0 -0
- {dotflow-0.12.0.dev3 → dotflow-0.13.0.dev2}/dotflow/utils/tools.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: dotflow
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0.dev2
|
|
4
4
|
Summary: 🎲 Dotflow turns an idea into flow!
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -34,9 +34,11 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
34
34
|
Classifier: Programming Language :: Python :: 3.10
|
|
35
35
|
Classifier: Programming Language :: Python :: 3.11
|
|
36
36
|
Classifier: Programming Language :: Python :: 3.12
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
37
38
|
Provides-Extra: mongodb
|
|
38
39
|
Requires-Dist: dotflow-mongodb ; extra == "mongodb"
|
|
39
40
|
Requires-Dist: pydantic
|
|
41
|
+
Requires-Dist: requests
|
|
40
42
|
Requires-Dist: rich
|
|
41
43
|
Requires-Dist: typing-extensions
|
|
42
44
|
Project-URL: Documentation, https://github.com/dotflow-io/dotflow/blob/master/README.md
|
|
@@ -143,7 +145,7 @@ def my_callback(*args, **kwargs):
|
|
|
143
145
|
def my_task_x():
|
|
144
146
|
print("task")
|
|
145
147
|
|
|
146
|
-
@action
|
|
148
|
+
@action
|
|
147
149
|
def my_task_y():
|
|
148
150
|
print("task")
|
|
149
151
|
|
|
@@ -157,7 +159,7 @@ workflow.start()
|
|
|
157
159
|
|
|
158
160
|
## First Steps
|
|
159
161
|
|
|
160
|
-
#### Import
|
|
162
|
+
#### 1. Import
|
|
161
163
|
|
|
162
164
|
Start with the basics, which is importing the necessary classes and methods. ([DotFlow](https://dotflow-io.github.io/dotflow/nav/reference/dotflow/), [action](https://dotflow-io.github.io/dotflow/nav/reference/action/))
|
|
163
165
|
|
|
@@ -165,7 +167,7 @@ Start with the basics, which is importing the necessary classes and methods. ([D
|
|
|
165
167
|
from dotflow import DotFlow, action
|
|
166
168
|
```
|
|
167
169
|
|
|
168
|
-
#### Callback function
|
|
170
|
+
#### 2. Callback function
|
|
169
171
|
|
|
170
172
|
Create a `my_callback` function to receive execution information of a task. `It is not necessary` to include this function, as you will still have a report at the end of the execution in the instantiated object of the `DotFlow` class. This `my_callback` function is only needed if you need to do something after the execution of the task, for example: sending a message to someone, making a phone call, or sending a letter. [More details](https://dotflow-io.github.io/dotflow/nav/reference/utils/#dotflow.utils.basic_functions.basic_callback)
|
|
171
173
|
|
|
@@ -174,17 +176,17 @@ def my_callback(*args, **kwargs):
|
|
|
174
176
|
print(args, kwargs)
|
|
175
177
|
```
|
|
176
178
|
|
|
177
|
-
#### Task function
|
|
179
|
+
#### 3. Task function
|
|
178
180
|
|
|
179
|
-
Now, create the function responsible for executing your task. It's very simple; just use the [action](https://dotflow-io.github.io/dotflow/nav/reference/action/) decorator above the function, and that's it—you've created a task.
|
|
181
|
+
Now, create the function responsible for executing your task. It's very simple; just use the [action](https://dotflow-io.github.io/dotflow/nav/reference/action/) decorator above the function, and that's it—you've created a task.
|
|
180
182
|
|
|
181
183
|
```python
|
|
182
|
-
@action
|
|
184
|
+
@action
|
|
183
185
|
def my_task_x():
|
|
184
186
|
print("task")
|
|
185
187
|
```
|
|
186
188
|
|
|
187
|
-
#### DotFlow Class
|
|
189
|
+
#### 4. DotFlow Class
|
|
188
190
|
|
|
189
191
|
Instantiate the DotFlow class in a `workflow` variable to be used in the following steps. [More details](https://dotflow-io.github.io/dotflow/nav/reference/dotflow/).
|
|
190
192
|
|
|
@@ -192,7 +194,7 @@ Instantiate the DotFlow class in a `workflow` variable to be used in the followi
|
|
|
192
194
|
workflow = DotFlow()
|
|
193
195
|
```
|
|
194
196
|
|
|
195
|
-
#### Add Task
|
|
197
|
+
#### 5. Add Task
|
|
196
198
|
|
|
197
199
|
Now, simply add the `my_task_x` and `my_callback` functions you created earlier to the workflow using the code below. This process is necessary to define which tasks will be executed and the order in which they will run. The execution order follows the sequence in which they were added to the workflow. [More details](https://dotflow-io.github.io/dotflow/nav/reference/task-builder/#dotflow.core.task.TaskBuilder.add)
|
|
198
200
|
|
|
@@ -215,7 +217,7 @@ workflow.task.add(step=[my_task_x, my_task_y], callback=my_callback)
|
|
|
215
217
|
workflow.task.add(step="module.task.my_task_x", callback=my_callback)
|
|
216
218
|
```
|
|
217
219
|
|
|
218
|
-
#### Start
|
|
220
|
+
#### 6. Start
|
|
219
221
|
|
|
220
222
|
Finally, just execute the workflow with the following code snippet. [More details](https://dotflow-io.github.io/dotflow/nav/reference/workflow/#dotflow.core.workflow.Manager)
|
|
221
223
|
|
|
@@ -357,37 +359,43 @@ flowchart TD
|
|
|
357
359
|
|
|
358
360
|
## More Examples
|
|
359
361
|
|
|
360
|
-
| Example
|
|
361
|
-
|
|
|
362
|
-
| [cli_with_callback](https://github.com/dotflow-io/examples/blob/master/cli_with_callback.py)
|
|
363
|
-
| [cli_with_initial_context](https://github.com/dotflow-io/examples/blob/master/cli_with_initial_context.py)
|
|
364
|
-
| [cli_with_mode](https://github.com/dotflow-io/examples/blob/master/cli_with_mode.py)
|
|
365
|
-
| [cli_with_output_context](https://github.com/dotflow-io/examples/blob/master/cli_with_output_context.py)
|
|
366
|
-
| [cli_with_path](https://github.com/dotflow-io/examples/blob/master/cli_with_path.py)
|
|
367
|
-
| [
|
|
368
|
-
| [simple_class_workflow](https://github.com/dotflow-io/examples/blob/master/simple_class_workflow.py)
|
|
369
|
-
| [
|
|
370
|
-
| [simple_function_workflow](https://github.com/dotflow-io/examples/blob/master/simple_function_workflow.py)
|
|
371
|
-
| [
|
|
372
|
-
| [
|
|
373
|
-
| [
|
|
374
|
-
| [
|
|
375
|
-
| [
|
|
376
|
-
| [
|
|
377
|
-
| [
|
|
378
|
-
| [
|
|
379
|
-
| [
|
|
380
|
-
| [
|
|
381
|
-
| [
|
|
382
|
-
| [
|
|
383
|
-
| [
|
|
384
|
-
| [
|
|
385
|
-
| [
|
|
386
|
-
| [
|
|
387
|
-
| [workflow_parallel_mode](https://github.com/dotflow-io/examples/blob/master/workflow_parallel_mode.py)
|
|
388
|
-
| [workflow_sequential_group_mode](https://github.com/dotflow-io/examples/blob/master/workflow_sequential_group_mode.py)
|
|
389
|
-
| [workflow_sequential_mode](https://github.com/dotflow-io/examples/blob/master/workflow_sequential_mode.py)
|
|
390
|
-
|
|
362
|
+
| Example | Command |
|
|
363
|
+
| ------- | ------- |
|
|
364
|
+
| [cli_with_callback](https://github.com/dotflow-io/examples/blob/master/cli_with_callback.py) | `dotflow start --step examples.cli_with_callback.simple_step --callback examples.cli_with_callback.callback` |
|
|
365
|
+
| [cli_with_initial_context](https://github.com/dotflow-io/examples/blob/master/cli_with_initial_context.py) | `dotflow start --step examples.cli_with_initial_context.simple_step --initial-context abc` |
|
|
366
|
+
| [cli_with_mode](https://github.com/dotflow-io/examples/blob/master/cli_with_mode.py) | `dotflow start --step examples.cli_with_mode.simple_step --mode sequential` |
|
|
367
|
+
| [cli_with_output_context](https://github.com/dotflow-io/examples/blob/master/cli_with_output_context.py) | `dotflow start --step examples.cli_with_output_context.simple_step --storage file` |
|
|
368
|
+
| [cli_with_path](https://github.com/dotflow-io/examples/blob/master/cli_with_path.py) | `dotflow start --step examples.cli_with_path.simple_step --path .storage --storage file` |
|
|
369
|
+
| [flow](https://github.com/dotflow-io/examples/blob/master/flow.py) | `python examples/flow.py` |
|
|
370
|
+
| [simple_class_workflow](https://github.com/dotflow-io/examples/blob/master/simple_class_workflow.py) | `python examples/simple_class_workflow.py` |
|
|
371
|
+
| [simple_cli](https://github.com/dotflow-io/examples/blob/master/simple_cli.py) | `dotflow start --step examples.simple_cli.simple_step` |
|
|
372
|
+
| [simple_function_workflow](https://github.com/dotflow-io/examples/blob/master/simple_function_workflow.py) | `python examples/simple_function_workflow.py` |
|
|
373
|
+
| [simple_function_workflow_with_error](https://github.com/dotflow-io/examples/blob/master/simple_function_workflow_with_error.py) | `python examples/simple_function_workflow_with_error.py` |
|
|
374
|
+
| [step_class_result_context](https://github.com/dotflow-io/examples/blob/master/step_class_result_context.py) | `python examples/step_class_result_context.py` |
|
|
375
|
+
| [step_class_result_storage](https://github.com/dotflow-io/examples/blob/master/step_class_result_storage.py) | `python examples/step_class_result_storage.py` |
|
|
376
|
+
| [step_class_result_task](https://github.com/dotflow-io/examples/blob/master/step_class_result_task.py) | `python examples/step_class_result_task.py` |
|
|
377
|
+
| [step_function_result_context](https://github.com/dotflow-io/examples/blob/master/step_function_result_context.py) | `python examples/step_function_result_context.py` |
|
|
378
|
+
| [step_function_result_storage](https://github.com/dotflow-io/examples/blob/master/step_function_result_storage.py) | `python examples/step_function_result_storage.py` |
|
|
379
|
+
| [step_function_result_task](https://github.com/dotflow-io/examples/blob/master/step_function_result_task.py) | `python examples/step_function_result_task.py` |
|
|
380
|
+
| [step_with_groups](https://github.com/dotflow-io/examples/blob/master/step_with_groups.py) | `python examples/step_with_groups.py` |
|
|
381
|
+
| [step_with_initial_context](https://github.com/dotflow-io/examples/blob/master/step_with_initial_context.py) | `python examples/step_with_initial_context.py` |
|
|
382
|
+
| [step_with_many_contexts](https://github.com/dotflow-io/examples/blob/master/step_with_many_contexts.py) | `python examples/step_with_many_contexts.py` |
|
|
383
|
+
| [step_with_notify_telegram](https://github.com/dotflow-io/examples/blob/master/step_with_notify_telegram.py) | `python examples/step_with_notify_telegram.py` |
|
|
384
|
+
| [step_with_previous_context](https://github.com/dotflow-io/examples/blob/master/step_with_previous_context.py) | `python examples/step_with_previous_context.py` |
|
|
385
|
+
| [step_with_storage_file](https://github.com/dotflow-io/examples/blob/master/step_with_storage_file.py) | `python examples/step_with_storage_file.py` |
|
|
386
|
+
| [step_with_storage_mongodb](https://github.com/dotflow-io/examples/blob/master/step_with_storage_mongodb.py) | `python examples/step_with_storage_mongodb.py` |
|
|
387
|
+
| [workflow_background_mode](https://github.com/dotflow-io/examples/blob/master/workflow_background_mode.py) | `python examples/workflow_background_mode.py` |
|
|
388
|
+
| [workflow_keep_going_true](https://github.com/dotflow-io/examples/blob/master/workflow_keep_going_true.py) | `python examples/workflow_keep_going_true.py` |
|
|
389
|
+
| [workflow_parallel_mode](https://github.com/dotflow-io/examples/blob/master/workflow_parallel_mode.py) | `python examples/workflow_parallel_mode.py` |
|
|
390
|
+
| [workflow_sequential_group_mode](https://github.com/dotflow-io/examples/blob/master/workflow_sequential_group_mode.py) | `python examples/workflow_sequential_group_mode.py` |
|
|
391
|
+
| [workflow_sequential_mode](https://github.com/dotflow-io/examples/blob/master/workflow_sequential_mode.py) | `python examples/workflow_sequential_mode.py` |
|
|
392
|
+
| [workflow_step_callback](https://github.com/dotflow-io/examples/blob/master/workflow_step_callback.py) | `python examples/workflow_step_callback.py` |
|
|
393
|
+
| [workflow_with_backoff](https://github.com/dotflow-io/examples/blob/master/workflow_with_backoff.py) | `python examples/workflow_with_backoff.py` |
|
|
394
|
+
| [workflow_with_callback_failure](https://github.com/dotflow-io/examples/blob/master/workflow_with_callback_failure.py) | `python examples/workflow_with_callback_failure.py` |
|
|
395
|
+
| [workflow_with_callback_success](https://github.com/dotflow-io/examples/blob/master/workflow_with_callback_success.py) | `python examples/workflow_with_callback_success.py` |
|
|
396
|
+
| [workflow_with_retry](https://github.com/dotflow-io/examples/blob/master/workflow_with_retry.py) | `python examples/workflow_with_retry.py` |
|
|
397
|
+
| [workflow_with_retry_delay](https://github.com/dotflow-io/examples/blob/master/workflow_with_retry_delay.py) | `python examples/workflow_with_retry_delay.py` |
|
|
398
|
+
| [workflow_with_timeout](https://github.com/dotflow-io/examples/blob/master/workflow_with_timeout.py) | `python examples/workflow_with_timeout.py` |
|
|
391
399
|
|
|
392
400
|
## Commit Style
|
|
393
401
|
|
|
@@ -96,7 +96,7 @@ def my_callback(*args, **kwargs):
|
|
|
96
96
|
def my_task_x():
|
|
97
97
|
print("task")
|
|
98
98
|
|
|
99
|
-
@action
|
|
99
|
+
@action
|
|
100
100
|
def my_task_y():
|
|
101
101
|
print("task")
|
|
102
102
|
|
|
@@ -110,7 +110,7 @@ workflow.start()
|
|
|
110
110
|
|
|
111
111
|
## First Steps
|
|
112
112
|
|
|
113
|
-
#### Import
|
|
113
|
+
#### 1. Import
|
|
114
114
|
|
|
115
115
|
Start with the basics, which is importing the necessary classes and methods. ([DotFlow](https://dotflow-io.github.io/dotflow/nav/reference/dotflow/), [action](https://dotflow-io.github.io/dotflow/nav/reference/action/))
|
|
116
116
|
|
|
@@ -118,7 +118,7 @@ Start with the basics, which is importing the necessary classes and methods. ([D
|
|
|
118
118
|
from dotflow import DotFlow, action
|
|
119
119
|
```
|
|
120
120
|
|
|
121
|
-
#### Callback function
|
|
121
|
+
#### 2. Callback function
|
|
122
122
|
|
|
123
123
|
Create a `my_callback` function to receive execution information of a task. `It is not necessary` to include this function, as you will still have a report at the end of the execution in the instantiated object of the `DotFlow` class. This `my_callback` function is only needed if you need to do something after the execution of the task, for example: sending a message to someone, making a phone call, or sending a letter. [More details](https://dotflow-io.github.io/dotflow/nav/reference/utils/#dotflow.utils.basic_functions.basic_callback)
|
|
124
124
|
|
|
@@ -127,17 +127,17 @@ def my_callback(*args, **kwargs):
|
|
|
127
127
|
print(args, kwargs)
|
|
128
128
|
```
|
|
129
129
|
|
|
130
|
-
#### Task function
|
|
130
|
+
#### 3. Task function
|
|
131
131
|
|
|
132
|
-
Now, create the function responsible for executing your task. It's very simple; just use the [action](https://dotflow-io.github.io/dotflow/nav/reference/action/) decorator above the function, and that's it—you've created a task.
|
|
132
|
+
Now, create the function responsible for executing your task. It's very simple; just use the [action](https://dotflow-io.github.io/dotflow/nav/reference/action/) decorator above the function, and that's it—you've created a task.
|
|
133
133
|
|
|
134
134
|
```python
|
|
135
|
-
@action
|
|
135
|
+
@action
|
|
136
136
|
def my_task_x():
|
|
137
137
|
print("task")
|
|
138
138
|
```
|
|
139
139
|
|
|
140
|
-
#### DotFlow Class
|
|
140
|
+
#### 4. DotFlow Class
|
|
141
141
|
|
|
142
142
|
Instantiate the DotFlow class in a `workflow` variable to be used in the following steps. [More details](https://dotflow-io.github.io/dotflow/nav/reference/dotflow/).
|
|
143
143
|
|
|
@@ -145,7 +145,7 @@ Instantiate the DotFlow class in a `workflow` variable to be used in the followi
|
|
|
145
145
|
workflow = DotFlow()
|
|
146
146
|
```
|
|
147
147
|
|
|
148
|
-
#### Add Task
|
|
148
|
+
#### 5. Add Task
|
|
149
149
|
|
|
150
150
|
Now, simply add the `my_task_x` and `my_callback` functions you created earlier to the workflow using the code below. This process is necessary to define which tasks will be executed and the order in which they will run. The execution order follows the sequence in which they were added to the workflow. [More details](https://dotflow-io.github.io/dotflow/nav/reference/task-builder/#dotflow.core.task.TaskBuilder.add)
|
|
151
151
|
|
|
@@ -168,7 +168,7 @@ workflow.task.add(step=[my_task_x, my_task_y], callback=my_callback)
|
|
|
168
168
|
workflow.task.add(step="module.task.my_task_x", callback=my_callback)
|
|
169
169
|
```
|
|
170
170
|
|
|
171
|
-
#### Start
|
|
171
|
+
#### 6. Start
|
|
172
172
|
|
|
173
173
|
Finally, just execute the workflow with the following code snippet. [More details](https://dotflow-io.github.io/dotflow/nav/reference/workflow/#dotflow.core.workflow.Manager)
|
|
174
174
|
|
|
@@ -310,37 +310,43 @@ flowchart TD
|
|
|
310
310
|
|
|
311
311
|
## More Examples
|
|
312
312
|
|
|
313
|
-
| Example
|
|
314
|
-
|
|
|
315
|
-
| [cli_with_callback](https://github.com/dotflow-io/examples/blob/master/cli_with_callback.py)
|
|
316
|
-
| [cli_with_initial_context](https://github.com/dotflow-io/examples/blob/master/cli_with_initial_context.py)
|
|
317
|
-
| [cli_with_mode](https://github.com/dotflow-io/examples/blob/master/cli_with_mode.py)
|
|
318
|
-
| [cli_with_output_context](https://github.com/dotflow-io/examples/blob/master/cli_with_output_context.py)
|
|
319
|
-
| [cli_with_path](https://github.com/dotflow-io/examples/blob/master/cli_with_path.py)
|
|
320
|
-
| [
|
|
321
|
-
| [simple_class_workflow](https://github.com/dotflow-io/examples/blob/master/simple_class_workflow.py)
|
|
322
|
-
| [
|
|
323
|
-
| [simple_function_workflow](https://github.com/dotflow-io/examples/blob/master/simple_function_workflow.py)
|
|
324
|
-
| [
|
|
325
|
-
| [
|
|
326
|
-
| [
|
|
327
|
-
| [
|
|
328
|
-
| [
|
|
329
|
-
| [
|
|
330
|
-
| [
|
|
331
|
-
| [
|
|
332
|
-
| [
|
|
333
|
-
| [
|
|
334
|
-
| [
|
|
335
|
-
| [
|
|
336
|
-
| [
|
|
337
|
-
| [
|
|
338
|
-
| [
|
|
339
|
-
| [
|
|
340
|
-
| [workflow_parallel_mode](https://github.com/dotflow-io/examples/blob/master/workflow_parallel_mode.py)
|
|
341
|
-
| [workflow_sequential_group_mode](https://github.com/dotflow-io/examples/blob/master/workflow_sequential_group_mode.py)
|
|
342
|
-
| [workflow_sequential_mode](https://github.com/dotflow-io/examples/blob/master/workflow_sequential_mode.py)
|
|
343
|
-
|
|
313
|
+
| Example | Command |
|
|
314
|
+
| ------- | ------- |
|
|
315
|
+
| [cli_with_callback](https://github.com/dotflow-io/examples/blob/master/cli_with_callback.py) | `dotflow start --step examples.cli_with_callback.simple_step --callback examples.cli_with_callback.callback` |
|
|
316
|
+
| [cli_with_initial_context](https://github.com/dotflow-io/examples/blob/master/cli_with_initial_context.py) | `dotflow start --step examples.cli_with_initial_context.simple_step --initial-context abc` |
|
|
317
|
+
| [cli_with_mode](https://github.com/dotflow-io/examples/blob/master/cli_with_mode.py) | `dotflow start --step examples.cli_with_mode.simple_step --mode sequential` |
|
|
318
|
+
| [cli_with_output_context](https://github.com/dotflow-io/examples/blob/master/cli_with_output_context.py) | `dotflow start --step examples.cli_with_output_context.simple_step --storage file` |
|
|
319
|
+
| [cli_with_path](https://github.com/dotflow-io/examples/blob/master/cli_with_path.py) | `dotflow start --step examples.cli_with_path.simple_step --path .storage --storage file` |
|
|
320
|
+
| [flow](https://github.com/dotflow-io/examples/blob/master/flow.py) | `python examples/flow.py` |
|
|
321
|
+
| [simple_class_workflow](https://github.com/dotflow-io/examples/blob/master/simple_class_workflow.py) | `python examples/simple_class_workflow.py` |
|
|
322
|
+
| [simple_cli](https://github.com/dotflow-io/examples/blob/master/simple_cli.py) | `dotflow start --step examples.simple_cli.simple_step` |
|
|
323
|
+
| [simple_function_workflow](https://github.com/dotflow-io/examples/blob/master/simple_function_workflow.py) | `python examples/simple_function_workflow.py` |
|
|
324
|
+
| [simple_function_workflow_with_error](https://github.com/dotflow-io/examples/blob/master/simple_function_workflow_with_error.py) | `python examples/simple_function_workflow_with_error.py` |
|
|
325
|
+
| [step_class_result_context](https://github.com/dotflow-io/examples/blob/master/step_class_result_context.py) | `python examples/step_class_result_context.py` |
|
|
326
|
+
| [step_class_result_storage](https://github.com/dotflow-io/examples/blob/master/step_class_result_storage.py) | `python examples/step_class_result_storage.py` |
|
|
327
|
+
| [step_class_result_task](https://github.com/dotflow-io/examples/blob/master/step_class_result_task.py) | `python examples/step_class_result_task.py` |
|
|
328
|
+
| [step_function_result_context](https://github.com/dotflow-io/examples/blob/master/step_function_result_context.py) | `python examples/step_function_result_context.py` |
|
|
329
|
+
| [step_function_result_storage](https://github.com/dotflow-io/examples/blob/master/step_function_result_storage.py) | `python examples/step_function_result_storage.py` |
|
|
330
|
+
| [step_function_result_task](https://github.com/dotflow-io/examples/blob/master/step_function_result_task.py) | `python examples/step_function_result_task.py` |
|
|
331
|
+
| [step_with_groups](https://github.com/dotflow-io/examples/blob/master/step_with_groups.py) | `python examples/step_with_groups.py` |
|
|
332
|
+
| [step_with_initial_context](https://github.com/dotflow-io/examples/blob/master/step_with_initial_context.py) | `python examples/step_with_initial_context.py` |
|
|
333
|
+
| [step_with_many_contexts](https://github.com/dotflow-io/examples/blob/master/step_with_many_contexts.py) | `python examples/step_with_many_contexts.py` |
|
|
334
|
+
| [step_with_notify_telegram](https://github.com/dotflow-io/examples/blob/master/step_with_notify_telegram.py) | `python examples/step_with_notify_telegram.py` |
|
|
335
|
+
| [step_with_previous_context](https://github.com/dotflow-io/examples/blob/master/step_with_previous_context.py) | `python examples/step_with_previous_context.py` |
|
|
336
|
+
| [step_with_storage_file](https://github.com/dotflow-io/examples/blob/master/step_with_storage_file.py) | `python examples/step_with_storage_file.py` |
|
|
337
|
+
| [step_with_storage_mongodb](https://github.com/dotflow-io/examples/blob/master/step_with_storage_mongodb.py) | `python examples/step_with_storage_mongodb.py` |
|
|
338
|
+
| [workflow_background_mode](https://github.com/dotflow-io/examples/blob/master/workflow_background_mode.py) | `python examples/workflow_background_mode.py` |
|
|
339
|
+
| [workflow_keep_going_true](https://github.com/dotflow-io/examples/blob/master/workflow_keep_going_true.py) | `python examples/workflow_keep_going_true.py` |
|
|
340
|
+
| [workflow_parallel_mode](https://github.com/dotflow-io/examples/blob/master/workflow_parallel_mode.py) | `python examples/workflow_parallel_mode.py` |
|
|
341
|
+
| [workflow_sequential_group_mode](https://github.com/dotflow-io/examples/blob/master/workflow_sequential_group_mode.py) | `python examples/workflow_sequential_group_mode.py` |
|
|
342
|
+
| [workflow_sequential_mode](https://github.com/dotflow-io/examples/blob/master/workflow_sequential_mode.py) | `python examples/workflow_sequential_mode.py` |
|
|
343
|
+
| [workflow_step_callback](https://github.com/dotflow-io/examples/blob/master/workflow_step_callback.py) | `python examples/workflow_step_callback.py` |
|
|
344
|
+
| [workflow_with_backoff](https://github.com/dotflow-io/examples/blob/master/workflow_with_backoff.py) | `python examples/workflow_with_backoff.py` |
|
|
345
|
+
| [workflow_with_callback_failure](https://github.com/dotflow-io/examples/blob/master/workflow_with_callback_failure.py) | `python examples/workflow_with_callback_failure.py` |
|
|
346
|
+
| [workflow_with_callback_success](https://github.com/dotflow-io/examples/blob/master/workflow_with_callback_success.py) | `python examples/workflow_with_callback_success.py` |
|
|
347
|
+
| [workflow_with_retry](https://github.com/dotflow-io/examples/blob/master/workflow_with_retry.py) | `python examples/workflow_with_retry.py` |
|
|
348
|
+
| [workflow_with_retry_delay](https://github.com/dotflow-io/examples/blob/master/workflow_with_retry_delay.py) | `python examples/workflow_with_retry_delay.py` |
|
|
349
|
+
| [workflow_with_timeout](https://github.com/dotflow-io/examples/blob/master/workflow_with_timeout.py) | `python examples/workflow_with_timeout.py` |
|
|
344
350
|
|
|
345
351
|
## Commit Style
|
|
346
352
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Notify ABC"""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Log(ABC):
|
|
9
|
+
"""Log"""
|
|
10
|
+
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def info(self, task: Any) -> None:
|
|
13
|
+
"""Info"""
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def error(self, task: Any) -> None:
|
|
17
|
+
"""Error"""
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""Action module"""
|
|
2
|
+
|
|
3
|
+
from time import sleep
|
|
4
|
+
|
|
5
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
6
|
+
from typing import Callable, Dict
|
|
7
|
+
from types import FunctionType
|
|
8
|
+
|
|
9
|
+
from dotflow.core.exception import ExecutionWithClassError
|
|
10
|
+
from dotflow.core.context import Context
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def is_execution_with_class_internal_error(error: Exception) -> bool:
|
|
14
|
+
message = str(error)
|
|
15
|
+
patterns = [
|
|
16
|
+
"initial_context",
|
|
17
|
+
"previous_context",
|
|
18
|
+
"missing 1 required positional argument: 'self'",
|
|
19
|
+
]
|
|
20
|
+
return any(pattern in message for pattern in patterns)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Action(object):
|
|
24
|
+
"""
|
|
25
|
+
Import:
|
|
26
|
+
You can import the **action** decorator directly from dotflow:
|
|
27
|
+
|
|
28
|
+
from dotflow import action
|
|
29
|
+
|
|
30
|
+
Example:
|
|
31
|
+
`class` dotflow.core.action.Action
|
|
32
|
+
|
|
33
|
+
Standard
|
|
34
|
+
|
|
35
|
+
@action
|
|
36
|
+
def my_task():
|
|
37
|
+
print("task")
|
|
38
|
+
|
|
39
|
+
With Retry
|
|
40
|
+
|
|
41
|
+
@action(retry=5)
|
|
42
|
+
def my_task():
|
|
43
|
+
print("task")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
With Timeout
|
|
47
|
+
|
|
48
|
+
@action(timeout=60)
|
|
49
|
+
def my_task():
|
|
50
|
+
print("task")
|
|
51
|
+
|
|
52
|
+
With Retry delay
|
|
53
|
+
|
|
54
|
+
@action(retry=5, retry_delay=5)
|
|
55
|
+
def my_task():
|
|
56
|
+
print("task")
|
|
57
|
+
|
|
58
|
+
With Backoff
|
|
59
|
+
|
|
60
|
+
@action(retry=5, backoff=True)
|
|
61
|
+
def my_task():
|
|
62
|
+
print("task")
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
func (Callable):
|
|
66
|
+
|
|
67
|
+
task (Callable):
|
|
68
|
+
|
|
69
|
+
retry (int): Number of task retries on on_failure.
|
|
70
|
+
|
|
71
|
+
timeout (int): Execution timeout for a task. Duration (in seconds)
|
|
72
|
+
|
|
73
|
+
retry_delay (int): Retry delay on task on_failure. Duration (in seconds)
|
|
74
|
+
|
|
75
|
+
backoff (int): Exponential backoff
|
|
76
|
+
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
def __init__(
|
|
80
|
+
self,
|
|
81
|
+
func: Callable = None,
|
|
82
|
+
task: Callable = None,
|
|
83
|
+
retry: int = 1,
|
|
84
|
+
timeout: int = 0,
|
|
85
|
+
retry_delay: int = 1,
|
|
86
|
+
backoff: bool = False,
|
|
87
|
+
) -> None:
|
|
88
|
+
self.func = func
|
|
89
|
+
self.task = task
|
|
90
|
+
self.retry = retry
|
|
91
|
+
self.timeout = timeout
|
|
92
|
+
self.retry_delay = retry_delay
|
|
93
|
+
self.backoff = backoff
|
|
94
|
+
self.params = []
|
|
95
|
+
|
|
96
|
+
def __call__(self, *args, **kwargs):
|
|
97
|
+
# With parameters
|
|
98
|
+
if self.func:
|
|
99
|
+
self._set_params()
|
|
100
|
+
|
|
101
|
+
task = self._get_task(kwargs=kwargs)
|
|
102
|
+
contexts = self._get_context(kwargs=kwargs)
|
|
103
|
+
|
|
104
|
+
if contexts:
|
|
105
|
+
return Context(
|
|
106
|
+
storage=self._run_action(*args, **contexts),
|
|
107
|
+
task_id=task.task_id,
|
|
108
|
+
workflow_id=task.workflow_id,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
return Context(
|
|
112
|
+
storage=self._run_action(*args),
|
|
113
|
+
task_id=task.task_id,
|
|
114
|
+
workflow_id=task.workflow_id,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
# No parameters
|
|
118
|
+
def action(*_args, **_kwargs):
|
|
119
|
+
self.func = args[0]
|
|
120
|
+
self._set_params()
|
|
121
|
+
|
|
122
|
+
task = self._get_task(kwargs=_kwargs)
|
|
123
|
+
contexts = self._get_context(kwargs=_kwargs)
|
|
124
|
+
|
|
125
|
+
if contexts:
|
|
126
|
+
return Context(
|
|
127
|
+
storage=self._run_action(*_args, **contexts),
|
|
128
|
+
task_id=task.task_id,
|
|
129
|
+
workflow_id=task.workflow_id,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
return Context(
|
|
133
|
+
storage=self._run_action(*_args),
|
|
134
|
+
task_id=task.task_id,
|
|
135
|
+
workflow_id=task.workflow_id,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
return action
|
|
139
|
+
|
|
140
|
+
def _run_action(self, *args, **kwargs):
|
|
141
|
+
for attempt in range(1, self.retry + 1):
|
|
142
|
+
try:
|
|
143
|
+
if self.timeout:
|
|
144
|
+
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
145
|
+
future = executor.submit(self.func, *args, **kwargs)
|
|
146
|
+
return future.result(timeout=self.timeout)
|
|
147
|
+
|
|
148
|
+
return self.func(*args, **kwargs)
|
|
149
|
+
|
|
150
|
+
except Exception as error:
|
|
151
|
+
last_exception = error
|
|
152
|
+
|
|
153
|
+
if is_execution_with_class_internal_error(error=last_exception):
|
|
154
|
+
raise ExecutionWithClassError()
|
|
155
|
+
|
|
156
|
+
if attempt == self.retry:
|
|
157
|
+
raise last_exception
|
|
158
|
+
|
|
159
|
+
sleep(self.retry_delay)
|
|
160
|
+
if self.backoff:
|
|
161
|
+
self.retry_delay *= 2
|
|
162
|
+
|
|
163
|
+
def _set_params(self):
|
|
164
|
+
if isinstance(self.func, FunctionType):
|
|
165
|
+
self.params = [param for param in self.func.__code__.co_varnames]
|
|
166
|
+
|
|
167
|
+
if type(self.func) is type:
|
|
168
|
+
if hasattr(self.func, "__init__"):
|
|
169
|
+
if hasattr(self.func.__init__, "__code__"):
|
|
170
|
+
self.params = [
|
|
171
|
+
param for param in self.func.__init__.__code__.co_varnames
|
|
172
|
+
]
|
|
173
|
+
|
|
174
|
+
def _get_context(self, kwargs: Dict):
|
|
175
|
+
context = {}
|
|
176
|
+
if "initial_context" in self.params:
|
|
177
|
+
context["initial_context"] = Context(kwargs.get("initial_context"))
|
|
178
|
+
|
|
179
|
+
if "previous_context" in self.params:
|
|
180
|
+
context["previous_context"] = Context(kwargs.get("previous_context"))
|
|
181
|
+
|
|
182
|
+
return context
|
|
183
|
+
|
|
184
|
+
def _get_task(self, kwargs: Dict):
|
|
185
|
+
return kwargs.get("task")
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Config module"""
|
|
2
|
+
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
from dotflow.abc.log import Log
|
|
6
|
+
from dotflow.abc.storage import Storage
|
|
7
|
+
from dotflow.abc.notify import Notify
|
|
8
|
+
|
|
9
|
+
from dotflow.providers.log_default import LogDefault
|
|
10
|
+
from dotflow.providers.storage_default import StorageDefault
|
|
11
|
+
from dotflow.providers.notify_default import NotifyDefault
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Config:
|
|
15
|
+
"""
|
|
16
|
+
Import:
|
|
17
|
+
You can import the **Config** class with:
|
|
18
|
+
|
|
19
|
+
from dotflow import Config
|
|
20
|
+
|
|
21
|
+
from dotflow.providers import (
|
|
22
|
+
StorageDefault,
|
|
23
|
+
NotifyDefault,
|
|
24
|
+
LogDefault
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
Example:
|
|
28
|
+
`class` dotflow.core.config.Config
|
|
29
|
+
|
|
30
|
+
config = Config(
|
|
31
|
+
storage=StorageFile(path=".output"),
|
|
32
|
+
notify=NotifyDefault(),
|
|
33
|
+
log=LogDefault()
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
storage (Optional[Storage]): Type of the storage.
|
|
38
|
+
notify (Optional[Notify]): Type of the notify.
|
|
39
|
+
log (Optional[Log]): Type of the notify.
|
|
40
|
+
|
|
41
|
+
Attributes:
|
|
42
|
+
storage (Optional[Storage]):
|
|
43
|
+
notify (Optional[Notify]):
|
|
44
|
+
log (Optional[Log]):
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
storage: Optional[Storage] = StorageDefault(),
|
|
50
|
+
notify: Optional[Notify] = NotifyDefault(),
|
|
51
|
+
log: Optional[Log] = LogDefault(),
|
|
52
|
+
) -> None:
|
|
53
|
+
self.storage = storage
|
|
54
|
+
self.notify = notify
|
|
55
|
+
self.log = log
|