ddeutil-workflow 0.0.64__tar.gz → 0.0.65__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/PKG-INFO +14 -12
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/README.md +12 -10
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/pyproject.toml +3 -1
- ddeutil_workflow-0.0.65/src/ddeutil/workflow/__about__.py +1 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/__init__.py +1 -1
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/api/routes/job.py +2 -2
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/conf.py +0 -4
- ddeutil_workflow-0.0.64/src/ddeutil/workflow/exceptions.py → ddeutil_workflow-0.0.65/src/ddeutil/workflow/errors.py +49 -11
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/job.py +249 -118
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/params.py +11 -11
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/result.py +86 -10
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/reusables.py +15 -17
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/stages.py +676 -450
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/utils.py +33 -0
- ddeutil_workflow-0.0.65/src/ddeutil/workflow/workflow.py +767 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil_workflow.egg-info/PKG-INFO +14 -12
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil_workflow.egg-info/SOURCES.txt +4 -6
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil_workflow.egg-info/requires.txt +1 -1
- ddeutil_workflow-0.0.65/tests/test_errors.py +10 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_job.py +21 -11
- ddeutil_workflow-0.0.65/tests/test_job_exec.py +506 -0
- ddeutil_workflow-0.0.65/tests/test_job_exec_strategy.py +182 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_params.py +17 -11
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_result.py +16 -10
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_reusables_call_tag.py +7 -6
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_reusables_template.py +12 -7
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_reusables_template_filter.py +10 -10
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_utils.py +2 -2
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_workflow.py +36 -13
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_workflow_exec.py +338 -197
- ddeutil_workflow-0.0.65/tests/test_workflow_exec_job.py +78 -0
- ddeutil_workflow-0.0.65/tests/test_workflow_release.py +100 -0
- ddeutil_workflow-0.0.64/src/ddeutil/workflow/__about__.py +0 -1
- ddeutil_workflow-0.0.64/src/ddeutil/workflow/workflow.py +0 -1268
- ddeutil_workflow-0.0.64/tests/test_job_exec.py +0 -315
- ddeutil_workflow-0.0.64/tests/test_job_exec_strategy.py +0 -143
- ddeutil_workflow-0.0.64/tests/test_release.py +0 -49
- ddeutil_workflow-0.0.64/tests/test_release_queue.py +0 -70
- ddeutil_workflow-0.0.64/tests/test_stage.py +0 -112
- ddeutil_workflow-0.0.64/tests/test_stage_handler_exec.py +0 -996
- ddeutil_workflow-0.0.64/tests/test_workflow_exec_job.py +0 -61
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/LICENSE +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/setup.cfg +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/__cron.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/__main__.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/__types.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/api/__init__.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/api/logs.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/api/routes/__init__.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/api/routes/logs.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/api/routes/workflows.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/event.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil/workflow/logs.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil_workflow.egg-info/dependency_links.txt +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil_workflow.egg-info/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/src/ddeutil_workflow.egg-info/top_level.txt +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test__cron.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test__regex.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_conf.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_event.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_logs_audit.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_logs_trace.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_reusables_func_model.py +0 -0
- {ddeutil_workflow-0.0.64 → ddeutil_workflow-0.0.65}/tests/test_strategy.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.65
|
4
4
|
Summary: Lightweight workflow orchestration
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -23,7 +23,7 @@ Requires-Python: >=3.9.13
|
|
23
23
|
Description-Content-Type: text/markdown
|
24
24
|
License-File: LICENSE
|
25
25
|
Requires-Dist: ddeutil[checksum]>=0.4.8
|
26
|
-
Requires-Dist: ddeutil-io[toml,yaml]>=0.2.
|
26
|
+
Requires-Dist: ddeutil-io[toml,yaml]>=0.2.14
|
27
27
|
Requires-Dist: pydantic==2.11.4
|
28
28
|
Requires-Dist: pydantic-extra-types==2.10.4
|
29
29
|
Requires-Dist: python-dotenv==1.1.0
|
@@ -222,32 +222,35 @@ value (This config can override by extra parameters with `registry_caller` key).
|
|
222
222
|
|
223
223
|
```python
|
224
224
|
from ddeutil.workflow import Result, CallerSecret, tag
|
225
|
-
from ddeutil.workflow.
|
225
|
+
from ddeutil.workflow.errors import StageError
|
226
226
|
from pydantic import BaseModel
|
227
227
|
|
228
|
+
|
228
229
|
class AwsCredential(BaseModel):
|
229
230
|
path: str
|
230
231
|
access_client_id: str
|
231
232
|
access_client_secret: CallerSecret
|
232
233
|
|
234
|
+
|
233
235
|
class RestAuth(BaseModel):
|
234
236
|
type: str
|
235
237
|
keys: CallerSecret
|
236
238
|
|
239
|
+
|
237
240
|
@tag("requests", alias="get-api-with-oauth-to-s3")
|
238
241
|
def get_api_with_oauth_to_s3(
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
242
|
+
method: str,
|
243
|
+
url: str,
|
244
|
+
body: dict[str, str],
|
245
|
+
auth: RestAuth,
|
246
|
+
writing_node: str,
|
247
|
+
aws: AwsCredential,
|
248
|
+
result: Result,
|
246
249
|
) -> dict[str, int]:
|
247
250
|
result.trace.info("[CALLER]: Start get data via RestAPI to S3.")
|
248
251
|
result.trace.info(f"... {method}: {url}")
|
249
252
|
if method != "post":
|
250
|
-
|
253
|
+
raise StageError(f"RestAPI does not support for {method} action.")
|
251
254
|
# NOTE: If you want to use secret, you can use `auth.keys.get_secret_value()`.
|
252
255
|
return {"records": 1000}
|
253
256
|
```
|
@@ -282,7 +285,6 @@ it will use default value and do not raise any error to you.
|
|
282
285
|
| **CONF_PATH** | Core | `./conf` | The config path that keep all template `.yaml` files. |
|
283
286
|
| **TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
284
287
|
| **STAGE_DEFAULT_ID** | Core | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
285
|
-
| **STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. |
|
286
288
|
| **MAX_CRON_PER_WORKFLOW** | Core | `5` | |
|
287
289
|
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16` | |
|
288
290
|
| **GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. |
|
@@ -171,32 +171,35 @@ value (This config can override by extra parameters with `registry_caller` key).
|
|
171
171
|
|
172
172
|
```python
|
173
173
|
from ddeutil.workflow import Result, CallerSecret, tag
|
174
|
-
from ddeutil.workflow.
|
174
|
+
from ddeutil.workflow.errors import StageError
|
175
175
|
from pydantic import BaseModel
|
176
176
|
|
177
|
+
|
177
178
|
class AwsCredential(BaseModel):
|
178
179
|
path: str
|
179
180
|
access_client_id: str
|
180
181
|
access_client_secret: CallerSecret
|
181
182
|
|
183
|
+
|
182
184
|
class RestAuth(BaseModel):
|
183
185
|
type: str
|
184
186
|
keys: CallerSecret
|
185
187
|
|
188
|
+
|
186
189
|
@tag("requests", alias="get-api-with-oauth-to-s3")
|
187
190
|
def get_api_with_oauth_to_s3(
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
191
|
+
method: str,
|
192
|
+
url: str,
|
193
|
+
body: dict[str, str],
|
194
|
+
auth: RestAuth,
|
195
|
+
writing_node: str,
|
196
|
+
aws: AwsCredential,
|
197
|
+
result: Result,
|
195
198
|
) -> dict[str, int]:
|
196
199
|
result.trace.info("[CALLER]: Start get data via RestAPI to S3.")
|
197
200
|
result.trace.info(f"... {method}: {url}")
|
198
201
|
if method != "post":
|
199
|
-
|
202
|
+
raise StageError(f"RestAPI does not support for {method} action.")
|
200
203
|
# NOTE: If you want to use secret, you can use `auth.keys.get_secret_value()`.
|
201
204
|
return {"records": 1000}
|
202
205
|
```
|
@@ -231,7 +234,6 @@ it will use default value and do not raise any error to you.
|
|
231
234
|
| **CONF_PATH** | Core | `./conf` | The config path that keep all template `.yaml` files. |
|
232
235
|
| **TIMEZONE** | Core | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
233
236
|
| **STAGE_DEFAULT_ID** | Core | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
234
|
-
| **STAGE_RAISE_ERROR** | Core | `false` | A flag that all stage raise StageException from stage execution. |
|
235
237
|
| **MAX_CRON_PER_WORKFLOW** | Core | `5` | |
|
236
238
|
| **MAX_QUEUE_COMPLETE_HIST** | Core | `16` | |
|
237
239
|
| **GENERATE_ID_SIMPLE_MODE** | Core | `true` | A flog that enable generating ID with `md5` algorithm. |
|
@@ -26,7 +26,7 @@ classifiers = [
|
|
26
26
|
requires-python = ">=3.9.13"
|
27
27
|
dependencies = [
|
28
28
|
"ddeutil[checksum]>=0.4.8",
|
29
|
-
"ddeutil-io[yaml,toml]>=0.2.
|
29
|
+
"ddeutil-io[yaml,toml]>=0.2.14",
|
30
30
|
"pydantic==2.11.4",
|
31
31
|
"pydantic-extra-types==2.10.4",
|
32
32
|
"python-dotenv==1.1.0",
|
@@ -121,6 +121,8 @@ console_output_style = "count"
|
|
121
121
|
addopts = [
|
122
122
|
"--strict-config",
|
123
123
|
"--strict-markers",
|
124
|
+
# "-p no:launch",
|
125
|
+
# "-p no:launch_ros",
|
124
126
|
]
|
125
127
|
filterwarnings = [
|
126
128
|
"error",
|
@@ -0,0 +1 @@
|
|
1
|
+
__version__: str = "0.0.65"
|
@@ -12,7 +12,7 @@ from fastapi.responses import UJSONResponse
|
|
12
12
|
from pydantic import BaseModel, Field
|
13
13
|
|
14
14
|
from ...__types import DictData
|
15
|
-
from ...
|
15
|
+
from ...errors import JobError
|
16
16
|
from ...job import Job
|
17
17
|
from ...logs import get_logger
|
18
18
|
from ...result import Result
|
@@ -59,7 +59,7 @@ async def job_execute(
|
|
59
59
|
).context,
|
60
60
|
to=context,
|
61
61
|
)
|
62
|
-
except
|
62
|
+
except JobError as err:
|
63
63
|
rs.trace.error(f"[JOB]: {err.__class__.__name__}: {err}")
|
64
64
|
|
65
65
|
return {
|
@@ -141,10 +141,6 @@ class Config: # pragma: no cov
|
|
141
141
|
def log_datetime_format(self) -> str:
|
142
142
|
return env("LOG_DATETIME_FORMAT", "%Y-%m-%d %H:%M:%S")
|
143
143
|
|
144
|
-
@property
|
145
|
-
def stage_raise_error(self) -> bool:
|
146
|
-
return str2bool(env("CORE_STAGE_RAISE_ERROR", "false"))
|
147
|
-
|
148
144
|
@property
|
149
145
|
def stage_default_id(self) -> bool:
|
150
146
|
return str2bool(env("CORE_STAGE_DEFAULT_ID", "false"))
|
@@ -11,6 +11,8 @@ from __future__ import annotations
|
|
11
11
|
|
12
12
|
from typing import Literal, Optional, TypedDict, Union, overload
|
13
13
|
|
14
|
+
from .__types import DictData, StrOrInt
|
15
|
+
|
14
16
|
|
15
17
|
class ErrorData(TypedDict):
|
16
18
|
"""Error data type dict for typing necessary keys of return of to_dict func
|
@@ -21,7 +23,7 @@ class ErrorData(TypedDict):
|
|
21
23
|
message: str
|
22
24
|
|
23
25
|
|
24
|
-
def to_dict(exception: Exception) -> ErrorData: # pragma: no cov
|
26
|
+
def to_dict(exception: Exception, **kwargs) -> ErrorData: # pragma: no cov
|
25
27
|
"""Create dict data from exception instance.
|
26
28
|
|
27
29
|
:param exception: An exception object.
|
@@ -31,17 +33,27 @@ def to_dict(exception: Exception) -> ErrorData: # pragma: no cov
|
|
31
33
|
return {
|
32
34
|
"name": exception.__class__.__name__,
|
33
35
|
"message": str(exception),
|
36
|
+
**kwargs,
|
34
37
|
}
|
35
38
|
|
36
39
|
|
37
|
-
class
|
40
|
+
class BaseError(Exception):
|
38
41
|
"""Base Workflow exception class will implement the `refs` argument for
|
39
42
|
making an error context to the result context.
|
40
43
|
"""
|
41
44
|
|
42
|
-
def __init__(
|
45
|
+
def __init__(
|
46
|
+
self,
|
47
|
+
message: str,
|
48
|
+
*,
|
49
|
+
refs: Optional[StrOrInt] = None,
|
50
|
+
context: Optional[DictData] = None,
|
51
|
+
params: Optional[DictData] = None,
|
52
|
+
) -> None:
|
43
53
|
super().__init__(message)
|
44
54
|
self.refs: Optional[str] = refs
|
55
|
+
self.context: DictData = context or {}
|
56
|
+
self.params: DictData = params or {}
|
45
57
|
|
46
58
|
@overload
|
47
59
|
def to_dict(
|
@@ -54,7 +66,9 @@ class BaseWorkflowException(Exception):
|
|
54
66
|
) -> ErrorData: ... # pragma: no cov
|
55
67
|
|
56
68
|
def to_dict(
|
57
|
-
self,
|
69
|
+
self,
|
70
|
+
with_refs: bool = False,
|
71
|
+
**kwargs,
|
58
72
|
) -> Union[ErrorData, dict[str, ErrorData]]:
|
59
73
|
"""Return ErrorData data from the current exception object. If with_refs
|
60
74
|
flag was set, it will return mapping of refs and itself data.
|
@@ -64,22 +78,46 @@ class BaseWorkflowException(Exception):
|
|
64
78
|
data: ErrorData = to_dict(self)
|
65
79
|
if with_refs and (self.refs is not None and self.refs != "EMPTY"):
|
66
80
|
return {self.refs: data}
|
67
|
-
return data
|
81
|
+
return data | kwargs
|
82
|
+
|
83
|
+
|
84
|
+
class UtilError(BaseError): ...
|
85
|
+
|
86
|
+
|
87
|
+
class ResultError(UtilError): ...
|
88
|
+
|
89
|
+
|
90
|
+
class StageError(BaseError): ...
|
91
|
+
|
92
|
+
|
93
|
+
class StageRetryError(StageError): ...
|
94
|
+
|
95
|
+
|
96
|
+
class StageCancelError(StageError): ...
|
97
|
+
|
98
|
+
|
99
|
+
class StageSkipError(StageError): ...
|
100
|
+
|
101
|
+
|
102
|
+
class JobError(BaseError): ...
|
103
|
+
|
104
|
+
|
105
|
+
class JobCancelError(JobError): ...
|
68
106
|
|
69
107
|
|
70
|
-
class
|
108
|
+
class JobSkipError(JobError): ...
|
71
109
|
|
72
110
|
|
73
|
-
class
|
111
|
+
class WorkflowError(BaseError): ...
|
74
112
|
|
75
113
|
|
76
|
-
class
|
114
|
+
class WorkflowCancelError(WorkflowError): ...
|
77
115
|
|
78
116
|
|
79
|
-
class
|
117
|
+
class WorkflowSkipError(WorkflowError): ...
|
80
118
|
|
81
119
|
|
82
|
-
class
|
120
|
+
class WorkflowTimeoutError(WorkflowError): ...
|
83
121
|
|
84
122
|
|
85
|
-
class
|
123
|
+
class ParamError(WorkflowError): ...
|