ddeutil-workflow 0.0.74__py3-none-any.whl → 0.0.76__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__cron.py +18 -7
- ddeutil/workflow/__init__.py +2 -1
- ddeutil/workflow/audits.py +15 -10
- ddeutil/workflow/cli.py +87 -21
- ddeutil/workflow/conf.py +10 -10
- ddeutil/workflow/errors.py +8 -10
- ddeutil/workflow/job.py +13 -10
- ddeutil/workflow/params.py +6 -4
- ddeutil/workflow/result.py +15 -14
- ddeutil/workflow/stages.py +11 -9
- ddeutil/workflow/traces.py +19 -14
- ddeutil/workflow/utils.py +9 -22
- ddeutil/workflow/workflow.py +67 -55
- {ddeutil_workflow-0.0.74.dist-info → ddeutil_workflow-0.0.76.dist-info}/METADATA +3 -3
- ddeutil_workflow-0.0.76.dist-info/RECORD +30 -0
- ddeutil_workflow-0.0.74.dist-info/RECORD +0 -30
- {ddeutil_workflow-0.0.74.dist-info → ddeutil_workflow-0.0.76.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.74.dist-info → ddeutil_workflow-0.0.76.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.74.dist-info → ddeutil_workflow-0.0.76.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.74.dist-info → ddeutil_workflow-0.0.76.dist-info}/top_level.txt +0 -0
ddeutil/workflow/__about__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__: str = "0.0.
|
1
|
+
__version__: str = "0.0.76"
|
ddeutil/workflow/__cron.py
CHANGED
@@ -793,10 +793,11 @@ class CronRunner:
|
|
793
793
|
"Invalid type of `tz` parameter, it should be str or "
|
794
794
|
"ZoneInfo instance."
|
795
795
|
)
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
796
|
+
else:
|
797
|
+
try:
|
798
|
+
self.tz = ZoneInfo(tz)
|
799
|
+
except ZoneInfoNotFoundError as err:
|
800
|
+
raise ValueError(f"Invalid timezone: {tz}") from err
|
800
801
|
|
801
802
|
# NOTE: Prepare date
|
802
803
|
if date:
|
@@ -807,6 +808,7 @@ class CronRunner:
|
|
807
808
|
if tz is not None:
|
808
809
|
self.date: datetime = date.astimezone(self.tz)
|
809
810
|
else:
|
811
|
+
self.tz = date.tzinfo
|
810
812
|
self.date: datetime = date
|
811
813
|
else:
|
812
814
|
self.date: datetime = datetime.now(tz=self.tz)
|
@@ -841,7 +843,11 @@ class CronRunner:
|
|
841
843
|
|
842
844
|
@property
|
843
845
|
def next(self) -> datetime:
|
844
|
-
"""Returns the next time of the schedule.
|
846
|
+
"""Returns the next time of the schedule.
|
847
|
+
|
848
|
+
Returns:
|
849
|
+
datetime: A next datetime from the current with shifting step.
|
850
|
+
"""
|
845
851
|
self.date = (
|
846
852
|
self.date
|
847
853
|
if self.reset_flag
|
@@ -858,7 +864,11 @@ class CronRunner:
|
|
858
864
|
def find_date(self, reverse: bool = False) -> datetime:
|
859
865
|
"""Returns the time the schedule would run by `next` or `prev` methods.
|
860
866
|
|
861
|
-
:
|
867
|
+
Args:
|
868
|
+
reverse: A reverse flag.
|
869
|
+
|
870
|
+
Returns:
|
871
|
+
datetime: A next datetime from shifting step.
|
862
872
|
"""
|
863
873
|
# NOTE: Set reset flag to false if start any action.
|
864
874
|
self.reset_flag: bool = False
|
@@ -868,7 +878,8 @@ class CronRunner:
|
|
868
878
|
max(self.shift_limit, 100) if self.is_year else self.shift_limit
|
869
879
|
):
|
870
880
|
|
871
|
-
# NOTE: Shift the date
|
881
|
+
# NOTE: Shift the date from year to minute.
|
882
|
+
mode: DatetimeMode # noqa: F842
|
872
883
|
if all(
|
873
884
|
not self.__shift_date(mode, reverse)
|
874
885
|
for mode in ("year", "month", "day", "hour", "minute")
|
ddeutil/workflow/__init__.py
CHANGED
@@ -113,6 +113,7 @@ from .result import (
|
|
113
113
|
WAIT,
|
114
114
|
Result,
|
115
115
|
Status,
|
116
|
+
get_status_from_error,
|
116
117
|
)
|
117
118
|
from .reusables import *
|
118
119
|
from .stages import (
|
@@ -131,7 +132,7 @@ from .stages import (
|
|
131
132
|
VirtualPyStage,
|
132
133
|
)
|
133
134
|
from .traces import (
|
134
|
-
|
135
|
+
BaseTrace,
|
135
136
|
FileTrace,
|
136
137
|
Trace,
|
137
138
|
TraceData,
|
ddeutil/workflow/audits.py
CHANGED
@@ -79,7 +79,10 @@ class BaseAudit(BaseModel, ABC):
|
|
79
79
|
default=None, description="A parent running ID."
|
80
80
|
)
|
81
81
|
run_id: str = Field(description="A running ID")
|
82
|
-
|
82
|
+
runs_metadata: DictData = Field(
|
83
|
+
default_factory=dict,
|
84
|
+
description="A runs metadata that will use to tracking this audit log.",
|
85
|
+
)
|
83
86
|
|
84
87
|
@model_validator(mode="after")
|
85
88
|
def __model_action(self) -> Self:
|
@@ -296,20 +299,22 @@ class FileAudit(BaseAudit):
|
|
296
299
|
|
297
300
|
|
298
301
|
class SQLiteAudit(BaseAudit): # pragma: no cov
|
299
|
-
"""SQLite Audit
|
302
|
+
"""SQLite Audit model."""
|
300
303
|
|
301
304
|
table_name: ClassVar[str] = "audits"
|
302
305
|
schemas: ClassVar[
|
303
306
|
str
|
304
307
|
] = """
|
305
|
-
workflow
|
306
|
-
release int
|
307
|
-
type str
|
308
|
-
context
|
309
|
-
parent_run_id int
|
310
|
-
run_id int
|
311
|
-
|
312
|
-
|
308
|
+
workflow str
|
309
|
+
, release int
|
310
|
+
, type str
|
311
|
+
, context JSON
|
312
|
+
, parent_run_id int
|
313
|
+
, run_id int
|
314
|
+
, metadata JSON
|
315
|
+
, created_at datetime
|
316
|
+
, updated_at datetime
|
317
|
+
primary key ( workflow, release )
|
313
318
|
"""
|
314
319
|
|
315
320
|
@classmethod
|
ddeutil/workflow/cli.py
CHANGED
@@ -8,6 +8,7 @@ from __future__ import annotations
|
|
8
8
|
import json
|
9
9
|
from pathlib import Path
|
10
10
|
from platform import python_version
|
11
|
+
from textwrap import dedent
|
11
12
|
from typing import Annotated, Any, Literal, Optional, Union
|
12
13
|
|
13
14
|
import typer
|
@@ -15,15 +16,13 @@ from pydantic import Field, TypeAdapter
|
|
15
16
|
|
16
17
|
from .__about__ import __version__
|
17
18
|
from .__types import DictData
|
19
|
+
from .conf import config
|
18
20
|
from .errors import JobError
|
19
21
|
from .job import Job
|
20
22
|
from .params import Param
|
21
|
-
from .result import Result
|
22
23
|
from .workflow import Workflow
|
23
24
|
|
24
|
-
app = typer.Typer(
|
25
|
-
pretty_exceptions_enable=True,
|
26
|
-
)
|
25
|
+
app = typer.Typer(pretty_exceptions_enable=True)
|
27
26
|
|
28
27
|
|
29
28
|
@app.callback()
|
@@ -41,12 +40,70 @@ def version() -> None:
|
|
41
40
|
typer.echo(f"python-version=={python_version()}")
|
42
41
|
|
43
42
|
|
43
|
+
@app.command()
|
44
|
+
def init() -> None:
|
45
|
+
"""Initialize a Workflow structure on the current context."""
|
46
|
+
config.conf_path.mkdir(exist_ok=True)
|
47
|
+
(config.conf_path / ".confignore").touch()
|
48
|
+
|
49
|
+
conf_example_path: Path = config.conf_path / "examples"
|
50
|
+
conf_example_path.mkdir(exist_ok=True)
|
51
|
+
|
52
|
+
example_template: Path = conf_example_path / "wf_examples.yml"
|
53
|
+
example_template.write_text(
|
54
|
+
dedent(
|
55
|
+
"""
|
56
|
+
# Example workflow template.
|
57
|
+
wf-example:
|
58
|
+
type: Workflow
|
59
|
+
desc: |
|
60
|
+
An example workflow template.
|
61
|
+
params:
|
62
|
+
name:
|
63
|
+
type: str
|
64
|
+
default: "World"
|
65
|
+
jobs:
|
66
|
+
first-job:
|
67
|
+
stages:
|
68
|
+
- name: "Call tasks"
|
69
|
+
uses: tasks/say-hello-func@example
|
70
|
+
with:
|
71
|
+
name: ${{ params.name }}
|
72
|
+
"""
|
73
|
+
).lstrip("\n")
|
74
|
+
)
|
75
|
+
|
76
|
+
if "." in config.registry_caller:
|
77
|
+
task_path = Path("./tasks")
|
78
|
+
task_path.mkdir(exist_ok=True)
|
79
|
+
|
80
|
+
dummy_tasks_path = task_path / "example.py"
|
81
|
+
dummy_tasks_path.write_text(
|
82
|
+
dedent(
|
83
|
+
"""
|
84
|
+
from ddeutil.workflow import Result, tag
|
85
|
+
|
86
|
+
@tag(name="example", alias="say-hello-func")
|
87
|
+
def hello_world_task(name: str, rs: Result) -> dict[str, str]:
|
88
|
+
\"\"\"Logging hello task function\"\"\"
|
89
|
+
rs.trace.info(f"Hello, {name}")
|
90
|
+
return {"name": name}
|
91
|
+
"""
|
92
|
+
).lstrip("\n")
|
93
|
+
)
|
94
|
+
|
95
|
+
init_path = task_path / "__init__.py"
|
96
|
+
init_path.write_text("from .example import hello_world_task\n")
|
97
|
+
typer.echo(
|
98
|
+
"Starter command: `workflow-cli workflows execute --name=wf-example`"
|
99
|
+
)
|
100
|
+
|
101
|
+
|
44
102
|
@app.command(name="job")
|
45
103
|
def execute_job(
|
46
104
|
params: Annotated[str, typer.Option(help="A job execute parameters")],
|
47
105
|
job: Annotated[str, typer.Option(help="A job model")],
|
48
|
-
|
49
|
-
run_id: Annotated[Optional[str], typer.Option(help="A running ID")] = None,
|
106
|
+
run_id: Annotated[str, typer.Option(help="A running ID")],
|
50
107
|
) -> None:
|
51
108
|
"""Job execution on the local.
|
52
109
|
|
@@ -62,26 +119,19 @@ def execute_job(
|
|
62
119
|
job_dict: dict[str, Any] = json.loads(job)
|
63
120
|
_job: Job = Job.model_validate(obj=job_dict)
|
64
121
|
except json.JSONDecodeError as e:
|
65
|
-
raise ValueError(f"
|
122
|
+
raise ValueError(f"Jobs does not support format: {job!r}.") from e
|
66
123
|
|
67
124
|
typer.echo(f"Job params: {params_dict}")
|
68
|
-
rs: Result = Result(
|
69
|
-
run_id=run_id,
|
70
|
-
parent_run_id=parent_run_id,
|
71
|
-
)
|
72
|
-
|
73
125
|
context: DictData = {}
|
74
126
|
try:
|
75
127
|
_job.set_outputs(
|
76
|
-
_job.execute(
|
77
|
-
params=params_dict,
|
78
|
-
run_id=rs.run_id,
|
79
|
-
parent_run_id=rs.parent_run_id,
|
80
|
-
).context,
|
128
|
+
_job.execute(params=params_dict, run_id=run_id).context,
|
81
129
|
to=context,
|
82
130
|
)
|
131
|
+
typer.echo("[JOB]: Context result:")
|
132
|
+
typer.echo(json.dumps(context, default=str, indent=0))
|
83
133
|
except JobError as err:
|
84
|
-
|
134
|
+
typer.echo(f"[JOB]: {err.__class__.__name__}: {err}")
|
85
135
|
|
86
136
|
|
87
137
|
@app.command()
|
@@ -136,8 +186,24 @@ def workflow_callback():
|
|
136
186
|
|
137
187
|
|
138
188
|
@workflow_app.command(name="execute")
|
139
|
-
def workflow_execute(
|
140
|
-
|
189
|
+
def workflow_execute(
|
190
|
+
name: Annotated[
|
191
|
+
str,
|
192
|
+
typer.Option(help="A name of workflow template."),
|
193
|
+
],
|
194
|
+
params: Annotated[
|
195
|
+
str,
|
196
|
+
typer.Option(help="A workflow execute parameters"),
|
197
|
+
] = "{}",
|
198
|
+
):
|
199
|
+
"""Execute workflow by passing a workflow template name."""
|
200
|
+
try:
|
201
|
+
params_dict: dict[str, Any] = json.loads(params)
|
202
|
+
except json.JSONDecodeError as e:
|
203
|
+
raise ValueError(f"Params does not support format: {params!r}.") from e
|
204
|
+
|
205
|
+
typer.echo(f"Start execute workflow template: {name}")
|
206
|
+
typer.echo(f"... with params: {params_dict}")
|
141
207
|
|
142
208
|
|
143
209
|
WORKFLOW_TYPE = Literal["Workflow"]
|
@@ -167,7 +233,7 @@ def workflow_json_schema(
|
|
167
233
|
template_schema: dict[str, str] = {
|
168
234
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
169
235
|
"title": "Workflow Configuration Schema",
|
170
|
-
"version":
|
236
|
+
"version": __version__,
|
171
237
|
}
|
172
238
|
with open(output, mode="w", encoding="utf-8") as f:
|
173
239
|
json.dump(template_schema | json_schema, f, indent=2)
|
ddeutil/workflow/conf.py
CHANGED
@@ -89,16 +89,6 @@ class Config: # pragma: no cov
|
|
89
89
|
"""
|
90
90
|
return Path(env("CORE_CONF_PATH", "./conf"))
|
91
91
|
|
92
|
-
@property
|
93
|
-
def tz(self) -> ZoneInfo:
|
94
|
-
"""Timezone value that return with the `ZoneInfo` object and use for all
|
95
|
-
datetime object in this workflow engine.
|
96
|
-
|
97
|
-
Returns:
|
98
|
-
ZoneInfo: The timezone configuration for the workflow engine.
|
99
|
-
"""
|
100
|
-
return ZoneInfo(env("CORE_TIMEZONE", "UTC"))
|
101
|
-
|
102
92
|
@property
|
103
93
|
def generate_id_simple_mode(self) -> bool:
|
104
94
|
"""Flag for generate running ID with simple mode. That does not use
|
@@ -143,6 +133,16 @@ class Config: # pragma: no cov
|
|
143
133
|
"""
|
144
134
|
return str2bool(env("LOG_DEBUG_MODE", "true"))
|
145
135
|
|
136
|
+
@property
|
137
|
+
def log_tz(self) -> ZoneInfo:
|
138
|
+
"""Timezone value that return with the `ZoneInfo` object and use for all
|
139
|
+
datetime object in this workflow engine.
|
140
|
+
|
141
|
+
Returns:
|
142
|
+
ZoneInfo: The timezone configuration for the workflow engine.
|
143
|
+
"""
|
144
|
+
return ZoneInfo(env("LOG_TIMEZONE", "UTC"))
|
145
|
+
|
146
146
|
@property
|
147
147
|
def log_format(self) -> str:
|
148
148
|
return env(
|
ddeutil/workflow/errors.py
CHANGED
@@ -136,16 +136,14 @@ class BaseError(Exception):
|
|
136
136
|
ErrorData or dict: Exception data, optionally mapped by reference ID
|
137
137
|
|
138
138
|
Example:
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
#
|
143
|
-
|
144
|
-
#
|
145
|
-
|
146
|
-
#
|
147
|
-
ref_data = error.to_dict(with_refs=True)
|
148
|
-
# Returns: {"stage-1": {"name": "BaseError", "message": "Something failed"}}
|
139
|
+
>>> error = BaseError("Something failed", refs="stage-1")
|
140
|
+
>>> # Simple format
|
141
|
+
>>> error.to_dict()
|
142
|
+
>>> # Returns: {"name": "BaseError", "message": "Something failed"}
|
143
|
+
|
144
|
+
>>> # With reference mapping
|
145
|
+
>>> error.to_dict(with_refs=True)
|
146
|
+
>>> # Returns: {"stage-1": {"name": "BaseError", "message": "Something failed"}}
|
149
147
|
```
|
150
148
|
"""
|
151
149
|
data: ErrorData = to_dict(self)
|
ddeutil/workflow/job.py
CHANGED
@@ -656,6 +656,7 @@ class Job(BaseModel):
|
|
656
656
|
to: DictData,
|
657
657
|
*,
|
658
658
|
job_id: StrOrNone = None,
|
659
|
+
**kwargs,
|
659
660
|
) -> DictData:
|
660
661
|
"""Set an outputs from execution result context to the received context
|
661
662
|
with a `to` input parameter. The result context from job strategy
|
@@ -693,12 +694,15 @@ class Job(BaseModel):
|
|
693
694
|
:raise JobError: If the job's ID does not set and the setting
|
694
695
|
default job ID flag does not set.
|
695
696
|
|
696
|
-
:
|
697
|
-
|
698
|
-
|
699
|
-
|
697
|
+
Args:
|
698
|
+
output: (DictData) A result data context that want to extract
|
699
|
+
and transfer to the `strategies` key in receive context.
|
700
|
+
to: (DictData) A received context data.
|
701
|
+
job_id: (StrOrNone) A job ID if the `id` field does not set.
|
702
|
+
kwargs: Any values that want to add to the target context.
|
700
703
|
|
701
|
-
:
|
704
|
+
Returns:
|
705
|
+
DictData: Return updated the target context with a result context.
|
702
706
|
"""
|
703
707
|
if "jobs" not in to:
|
704
708
|
to["jobs"] = {}
|
@@ -716,8 +720,9 @@ class Job(BaseModel):
|
|
716
720
|
status: dict[str, Status] = (
|
717
721
|
{"status": output.pop("status")} if "status" in output else {}
|
718
722
|
)
|
723
|
+
kwargs: DictData = kwargs or {}
|
719
724
|
if self.strategy.is_set():
|
720
|
-
to["jobs"][_id] = {"strategies": output} | errors | status
|
725
|
+
to["jobs"][_id] = {"strategies": output} | errors | status | kwargs
|
721
726
|
elif len(k := output.keys()) > 1: # pragma: no cov
|
722
727
|
raise JobError(
|
723
728
|
"Strategy output from execution return more than one ID while "
|
@@ -726,7 +731,7 @@ class Job(BaseModel):
|
|
726
731
|
else:
|
727
732
|
_output: DictData = {} if len(k) == 0 else output[list(k)[0]]
|
728
733
|
_output.pop("matrix", {})
|
729
|
-
to["jobs"][_id] = _output | errors | status
|
734
|
+
to["jobs"][_id] = _output | errors | status | kwargs
|
730
735
|
return to
|
731
736
|
|
732
737
|
def get_outputs(
|
@@ -800,8 +805,7 @@ class Job(BaseModel):
|
|
800
805
|
return docker_execution(
|
801
806
|
self,
|
802
807
|
params,
|
803
|
-
run_id=
|
804
|
-
parent_run_id=parent_run_id,
|
808
|
+
run_id=parent_run_id,
|
805
809
|
event=event,
|
806
810
|
).make_info({"execution_time": time.monotonic() - ts})
|
807
811
|
|
@@ -1294,7 +1298,6 @@ def docker_execution(
|
|
1294
1298
|
params: DictData,
|
1295
1299
|
*,
|
1296
1300
|
run_id: StrOrNone = None,
|
1297
|
-
parent_run_id: StrOrNone = None,
|
1298
1301
|
event: Optional[Event] = None,
|
1299
1302
|
): # pragma: no cov
|
1300
1303
|
"""Docker job execution.
|
ddeutil/workflow/params.py
CHANGED
@@ -52,7 +52,7 @@ from pydantic import BaseModel, Field
|
|
52
52
|
|
53
53
|
from .__types import StrOrInt
|
54
54
|
from .errors import ParamError
|
55
|
-
from .utils import get_d_now, get_dt_now
|
55
|
+
from .utils import UTC, get_d_now, get_dt_now
|
56
56
|
|
57
57
|
T = TypeVar("T")
|
58
58
|
|
@@ -169,16 +169,18 @@ class DatetimeParam(DefaultParam):
|
|
169
169
|
return self.default
|
170
170
|
|
171
171
|
if isinstance(value, datetime):
|
172
|
-
|
172
|
+
if value.tzinfo is None:
|
173
|
+
return value.replace(tzinfo=UTC)
|
174
|
+
return value.astimezone(UTC)
|
173
175
|
elif isinstance(value, date):
|
174
|
-
return datetime(value.year, value.month, value.day)
|
176
|
+
return datetime(value.year, value.month, value.day, tzinfo=UTC)
|
175
177
|
elif not isinstance(value, str):
|
176
178
|
raise ParamError(
|
177
179
|
f"Value that want to convert to datetime does not support for "
|
178
180
|
f"type: {type(value)}"
|
179
181
|
)
|
180
182
|
try:
|
181
|
-
return datetime.fromisoformat(value)
|
183
|
+
return datetime.fromisoformat(value).replace(tzinfo=UTC)
|
182
184
|
except ValueError:
|
183
185
|
raise ParamError(
|
184
186
|
f"Invalid the ISO format string for datetime: {value!r}"
|
ddeutil/workflow/result.py
CHANGED
@@ -16,7 +16,6 @@ Classes:
|
|
16
16
|
Functions:
|
17
17
|
validate_statuses: Determine final status from multiple status values
|
18
18
|
get_status_from_error: Convert exception types to appropriate status
|
19
|
-
get_dt_tznow: Get current datetime with timezone configuration
|
20
19
|
"""
|
21
20
|
from __future__ import annotations
|
22
21
|
|
@@ -43,7 +42,7 @@ from . import (
|
|
43
42
|
from .__types import DictData
|
44
43
|
from .audits import Trace, get_trace
|
45
44
|
from .errors import ResultError
|
46
|
-
from .utils import default_gen_id,
|
45
|
+
from .utils import default_gen_id, get_dt_now
|
47
46
|
|
48
47
|
|
49
48
|
class Status(str, Enum):
|
@@ -89,6 +88,7 @@ class Status(str, Enum):
|
|
89
88
|
return self.name
|
90
89
|
|
91
90
|
def is_result(self) -> bool:
|
91
|
+
"""Return True if this status is the status for result object."""
|
92
92
|
return self in ResultStatuses
|
93
93
|
|
94
94
|
|
@@ -115,15 +115,13 @@ def validate_statuses(statuses: list[Status]) -> Status:
|
|
115
115
|
Status: Final consolidated status based on workflow logic
|
116
116
|
|
117
117
|
Example:
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
# Returns: SUCCESS
|
126
|
-
```
|
118
|
+
>>> # Mixed statuses - FAILED takes priority
|
119
|
+
>>> validate_statuses([SUCCESS, FAILED, SUCCESS])
|
120
|
+
>>> # Returns: FAILED
|
121
|
+
|
122
|
+
>>> # All same status
|
123
|
+
>>> validate_statuses([SUCCESS, SUCCESS, SUCCESS])
|
124
|
+
>>> # Returns: SUCCESS
|
127
125
|
"""
|
128
126
|
if any(s == CANCEL for s in statuses):
|
129
127
|
return CANCEL
|
@@ -153,6 +151,9 @@ def get_status_from_error(
|
|
153
151
|
) -> Status:
|
154
152
|
"""Get the Status from the error object.
|
155
153
|
|
154
|
+
Args:
|
155
|
+
error: An error object.
|
156
|
+
|
156
157
|
Returns:
|
157
158
|
Status: The status from the specific exception class.
|
158
159
|
"""
|
@@ -189,8 +190,8 @@ class Result:
|
|
189
190
|
context: DictData = field(default_factory=default_context)
|
190
191
|
info: DictData = field(default_factory=dict)
|
191
192
|
run_id: Optional[str] = field(default_factory=default_gen_id)
|
192
|
-
parent_run_id: Optional[str] = field(default=None
|
193
|
-
ts: datetime = field(default_factory=
|
193
|
+
parent_run_id: Optional[str] = field(default=None)
|
194
|
+
ts: datetime = field(default_factory=get_dt_now, compare=False)
|
194
195
|
trace: Optional[Trace] = field(default=None, compare=False, repr=False)
|
195
196
|
extras: DictData = field(default_factory=dict, compare=False, repr=False)
|
196
197
|
|
@@ -266,7 +267,7 @@ class Result:
|
|
266
267
|
|
267
268
|
:rtype: float
|
268
269
|
"""
|
269
|
-
return (
|
270
|
+
return (get_dt_now() - self.ts).total_seconds()
|
270
271
|
|
271
272
|
|
272
273
|
def catch(
|
ddeutil/workflow/stages.py
CHANGED
@@ -295,7 +295,7 @@ class BaseStage(BaseModel, ABC):
|
|
295
295
|
ts: float = time.monotonic()
|
296
296
|
parent_run_id: str = run_id
|
297
297
|
run_id: str = run_id or gen_id(self.iden, unique=True)
|
298
|
-
context: DictData = {}
|
298
|
+
context: DictData = {"status": WAIT}
|
299
299
|
trace: Trace = get_trace(
|
300
300
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
301
301
|
)
|
@@ -413,7 +413,7 @@ class BaseStage(BaseModel, ABC):
|
|
413
413
|
self,
|
414
414
|
output: DictData,
|
415
415
|
to: DictData,
|
416
|
-
|
416
|
+
**kwargs,
|
417
417
|
) -> DictData:
|
418
418
|
"""Set an outputs from execution result context to the received context
|
419
419
|
with a `to` input parameter. The result context from stage execution
|
@@ -447,12 +447,14 @@ class BaseStage(BaseModel, ABC):
|
|
447
447
|
to the `to` argument. The result context was soft copied before set
|
448
448
|
output step.
|
449
449
|
|
450
|
-
:
|
451
|
-
|
452
|
-
|
453
|
-
|
450
|
+
Args:
|
451
|
+
output: (DictData) A result data context that want to extract
|
452
|
+
and transfer to the `outputs` key in receive context.
|
453
|
+
to: (DictData) A received context data.
|
454
|
+
kwargs: Any values that want to add to the target context.
|
454
455
|
|
455
|
-
:
|
456
|
+
Returns:
|
457
|
+
DictData: Return updated the target context with a result context.
|
456
458
|
"""
|
457
459
|
if "stages" not in to:
|
458
460
|
to["stages"] = {}
|
@@ -470,8 +472,8 @@ class BaseStage(BaseModel, ABC):
|
|
470
472
|
status: dict[str, Status] = (
|
471
473
|
{"status": output.pop("status")} if "status" in output else {}
|
472
474
|
)
|
473
|
-
|
474
|
-
to["stages"][_id] = {"outputs": output} | errors | status |
|
475
|
+
kwargs: DictData = kwargs or {}
|
476
|
+
to["stages"][_id] = {"outputs": output} | errors | status | kwargs
|
475
477
|
return to
|
476
478
|
|
477
479
|
def get_outputs(self, output: DictData) -> DictData:
|
ddeutil/workflow/traces.py
CHANGED
@@ -167,7 +167,9 @@ class TraceMeta(BaseModel): # pragma: no cov
|
|
167
167
|
|
168
168
|
mode: Literal["stdout", "stderr"] = Field(description="A meta mode.")
|
169
169
|
level: str = Field(description="A log level.")
|
170
|
-
datetime: str = Field(
|
170
|
+
datetime: str = Field(
|
171
|
+
description="A datetime string with the specific config format."
|
172
|
+
)
|
171
173
|
process: int = Field(description="A process ID.")
|
172
174
|
thread: int = Field(description="A thread ID.")
|
173
175
|
message: str = Field(description="A message log.")
|
@@ -234,9 +236,9 @@ class TraceMeta(BaseModel): # pragma: no cov
|
|
234
236
|
mode=mode,
|
235
237
|
level=level,
|
236
238
|
datetime=(
|
237
|
-
get_dt_now(
|
238
|
-
|
239
|
-
)
|
239
|
+
get_dt_now()
|
240
|
+
.astimezone(dynamic("log_tz", extras=extras))
|
241
|
+
.strftime(dynamic("log_datetime_format", extras=extras))
|
240
242
|
),
|
241
243
|
process=os.getpid(),
|
242
244
|
thread=get_ident(),
|
@@ -285,7 +287,7 @@ class TraceData(BaseModel): # pragma: no cov
|
|
285
287
|
return cls.model_validate(data)
|
286
288
|
|
287
289
|
|
288
|
-
class
|
290
|
+
class BaseEmitTrace(BaseModel, ABC): # pragma: no cov
|
289
291
|
"""Base Trace model with abstraction class property."""
|
290
292
|
|
291
293
|
model_config = ConfigDict(frozen=True)
|
@@ -472,7 +474,7 @@ class BaseTrace(BaseModel, ABC): # pragma: no cov
|
|
472
474
|
await self.amit(message, mode="exception", is_err=True)
|
473
475
|
|
474
476
|
|
475
|
-
class ConsoleTrace(
|
477
|
+
class ConsoleTrace(BaseEmitTrace): # pragma: no cov
|
476
478
|
"""Console Trace log model."""
|
477
479
|
|
478
480
|
def writer(
|
@@ -564,7 +566,11 @@ class ConsoleTrace(BaseTrace): # pragma: no cov
|
|
564
566
|
getattr(logger, mode)(msg, stacklevel=3, extra={"cut_id": self.cut_id})
|
565
567
|
|
566
568
|
|
567
|
-
class
|
569
|
+
class BaseTrace(ConsoleTrace, ABC):
|
570
|
+
"""A Base Trace model that will use for override writing or sending trace
|
571
|
+
log to any service type.
|
572
|
+
"""
|
573
|
+
|
568
574
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
569
575
|
|
570
576
|
url: ParseResult = Field(description="An URL for create pointer.")
|
@@ -573,9 +579,8 @@ class OutsideTrace(ConsoleTrace, ABC):
|
|
573
579
|
"url", mode="before", json_schema_input_type=Union[ParseResult, str]
|
574
580
|
)
|
575
581
|
def __parse_url(cls, value: Union[ParseResult, str]) -> ParseResult:
|
576
|
-
|
577
|
-
|
578
|
-
return value
|
582
|
+
"""Parsing an URL value."""
|
583
|
+
return urlparse(value) if isinstance(value, str) else value
|
579
584
|
|
580
585
|
@field_serializer("url")
|
581
586
|
def __serialize_url(self, value: ParseResult) -> str:
|
@@ -619,7 +624,7 @@ class OutsideTrace(ConsoleTrace, ABC):
|
|
619
624
|
)
|
620
625
|
|
621
626
|
|
622
|
-
class FileTrace(
|
627
|
+
class FileTrace(BaseTrace): # pragma: no cov
|
623
628
|
"""File Trace dataclass that write file to the local storage."""
|
624
629
|
|
625
630
|
@classmethod
|
@@ -763,7 +768,7 @@ class FileTrace(OutsideTrace): # pragma: no cov
|
|
763
768
|
await f.write(trace_meta.model_dump_json() + "\n")
|
764
769
|
|
765
770
|
|
766
|
-
class SQLiteTrace(
|
771
|
+
class SQLiteTrace(BaseTrace): # pragma: no cov
|
767
772
|
"""SQLite Trace dataclass that write trace log to the SQLite database file."""
|
768
773
|
|
769
774
|
table_name: ClassVar[str] = "audits"
|
@@ -777,7 +782,7 @@ class SQLiteTrace(OutsideTrace): # pragma: no cov
|
|
777
782
|
, metadata JSON
|
778
783
|
, created_at datetime
|
779
784
|
, updated_at datetime
|
780
|
-
primary key (
|
785
|
+
primary key ( parent_run_id )
|
781
786
|
"""
|
782
787
|
|
783
788
|
@classmethod
|
@@ -822,7 +827,7 @@ class SQLiteTrace(OutsideTrace): # pragma: no cov
|
|
822
827
|
Trace = Union[
|
823
828
|
FileTrace,
|
824
829
|
SQLiteTrace,
|
825
|
-
|
830
|
+
BaseTrace,
|
826
831
|
]
|
827
832
|
|
828
833
|
|
ddeutil/workflow/utils.py
CHANGED
@@ -109,45 +109,32 @@ def replace_sec(dt: datetime) -> datetime:
|
|
109
109
|
|
110
110
|
|
111
111
|
def clear_tz(dt: datetime) -> datetime:
|
112
|
-
"""Replace timezone info on an input datetime object to
|
113
|
-
return dt.replace(tzinfo=
|
112
|
+
"""Replace timezone info on an input datetime object to UTC."""
|
113
|
+
return dt.replace(tzinfo=UTC)
|
114
114
|
|
115
115
|
|
116
|
-
def get_dt_now(
|
116
|
+
def get_dt_now(offset: float = 0.0) -> datetime:
|
117
117
|
"""Return the current datetime object.
|
118
118
|
|
119
|
-
:param tz: A ZoneInfo object for replace timezone of return datetime object.
|
120
119
|
:param offset: An offset second value.
|
121
120
|
|
122
121
|
:rtype: datetime
|
123
122
|
:return: The current datetime object that use an input timezone or UTC.
|
124
123
|
"""
|
125
|
-
return datetime.now(
|
124
|
+
return datetime.now().replace(tzinfo=UTC) - timedelta(seconds=offset)
|
126
125
|
|
127
126
|
|
128
|
-
def
|
129
|
-
"""Get current datetime with no timezone.
|
130
|
-
|
131
|
-
Returns the current datetime object using the None timezone.
|
132
|
-
|
133
|
-
Returns:
|
134
|
-
datetime: Current datetime with no timezone
|
135
|
-
"""
|
136
|
-
return get_dt_now(tz=None)
|
137
|
-
|
138
|
-
|
139
|
-
def get_d_now(
|
140
|
-
tz: Optional[ZoneInfo] = None, offset: float = 0.0
|
141
|
-
) -> date: # pragma: no cov
|
127
|
+
def get_d_now(offset: float = 0.0) -> date: # pragma: no cov
|
142
128
|
"""Return the current date object.
|
143
129
|
|
144
|
-
:param tz: A ZoneInfo object for replace timezone of return date object.
|
145
130
|
:param offset: An offset second value.
|
146
131
|
|
147
132
|
:rtype: date
|
148
133
|
:return: The current date object that use an input timezone or UTC.
|
149
134
|
"""
|
150
|
-
return (
|
135
|
+
return (
|
136
|
+
datetime.now().replace(tzinfo=UTC) - timedelta(seconds=offset)
|
137
|
+
).date()
|
151
138
|
|
152
139
|
|
153
140
|
def get_diff_sec(dt: datetime, offset: float = 0.0) -> int:
|
@@ -240,7 +227,7 @@ def gen_id(
|
|
240
227
|
if not isinstance(value, str):
|
241
228
|
value: str = str(value)
|
242
229
|
|
243
|
-
dt: datetime = datetime.now(tz=
|
230
|
+
dt: datetime = datetime.now(tz=UTC)
|
244
231
|
if dynamic("generate_id_simple_mode", f=simple_mode, extras=extras):
|
245
232
|
return (f"{dt:%Y%m%d%H%M%S%f}T" if unique else "") + hash_str(
|
246
233
|
f"{(value if sensitive else value.lower())}", n=10
|
ddeutil/workflow/workflow.py
CHANGED
@@ -36,7 +36,6 @@ from queue import Queue
|
|
36
36
|
from textwrap import dedent
|
37
37
|
from threading import Event as ThreadEvent
|
38
38
|
from typing import Any, Optional, Union
|
39
|
-
from zoneinfo import ZoneInfo
|
40
39
|
|
41
40
|
from pydantic import BaseModel, Field
|
42
41
|
from pydantic.functional_validators import field_validator, model_validator
|
@@ -64,8 +63,9 @@ from .result import (
|
|
64
63
|
from .reusables import has_template, param2template
|
65
64
|
from .traces import Trace, get_trace
|
66
65
|
from .utils import (
|
66
|
+
UTC,
|
67
67
|
gen_id,
|
68
|
-
|
68
|
+
get_dt_now,
|
69
69
|
replace_sec,
|
70
70
|
)
|
71
71
|
|
@@ -153,14 +153,14 @@ class Workflow(BaseModel):
|
|
153
153
|
description="A mapping of job ID and job model that already loaded.",
|
154
154
|
)
|
155
155
|
created_at: datetime = Field(
|
156
|
-
default_factory=
|
156
|
+
default_factory=get_dt_now,
|
157
157
|
description=(
|
158
158
|
"A created datetime of this workflow template when loading from "
|
159
159
|
"file."
|
160
160
|
),
|
161
161
|
)
|
162
162
|
updated_dt: datetime = Field(
|
163
|
-
default_factory=
|
163
|
+
default_factory=get_dt_now,
|
164
164
|
description=(
|
165
165
|
"A updated datetime of this workflow template when loading from "
|
166
166
|
"file."
|
@@ -369,12 +369,15 @@ class Workflow(BaseModel):
|
|
369
369
|
Returns:
|
370
370
|
datetime: The validated release datetime.
|
371
371
|
"""
|
372
|
-
|
372
|
+
if dt.tzinfo is None:
|
373
|
+
dt = dt.replace(tzinfo=UTC)
|
374
|
+
|
375
|
+
release: datetime = replace_sec(dt.astimezone(UTC))
|
373
376
|
if not self.on:
|
374
377
|
return release
|
375
378
|
|
376
379
|
for on in self.on.schedule:
|
377
|
-
if release == on.cronjob.schedule(release).next:
|
380
|
+
if release == on.cronjob.schedule(release, tz=UTC).next:
|
378
381
|
return release
|
379
382
|
raise WorkflowError(
|
380
383
|
"Release datetime does not support for this workflow"
|
@@ -385,12 +388,13 @@ class Workflow(BaseModel):
|
|
385
388
|
release: datetime,
|
386
389
|
params: DictData,
|
387
390
|
*,
|
388
|
-
release_type: ReleaseType = NORMAL,
|
389
391
|
run_id: Optional[str] = None,
|
390
|
-
|
392
|
+
runs_metadata: Optional[DictData] = None,
|
393
|
+
release_type: ReleaseType = NORMAL,
|
391
394
|
override_log_name: Optional[str] = None,
|
392
395
|
timeout: int = 600,
|
393
|
-
|
396
|
+
audit_excluded: Optional[list[str]] = None,
|
397
|
+
audit: type[Audit] = None,
|
394
398
|
) -> Result:
|
395
399
|
"""Release the workflow which is executes workflow with writing audit
|
396
400
|
log tracking. The method is overriding parameter with the release
|
@@ -406,40 +410,47 @@ class Workflow(BaseModel):
|
|
406
410
|
- Execute this workflow with mapping release data to its parameters.
|
407
411
|
- Writing result audit
|
408
412
|
|
409
|
-
:
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
413
|
+
Args:
|
414
|
+
release: (datetime) A release datetime.
|
415
|
+
params: A workflow parameter that pass to execute method.
|
416
|
+
release_type:
|
417
|
+
run_id: (str) A workflow running ID.
|
418
|
+
runs_metadata: (DictData)
|
419
|
+
audit: An audit class that want to save the execution result.
|
420
|
+
override_log_name: (str) An override logging name that use
|
421
|
+
instead the workflow name.
|
422
|
+
timeout: (int) A workflow execution time out in second unit.
|
423
|
+
audit_excluded: (list[str]) A list of key that want to exclude
|
424
|
+
from the audit data.
|
419
425
|
|
420
|
-
:
|
426
|
+
Returns:
|
427
|
+
Result: return result object that pass context data from the execute
|
428
|
+
method.
|
421
429
|
"""
|
422
430
|
name: str = override_log_name or self.name
|
431
|
+
|
432
|
+
# NOTE: Generate the parent running ID with not None value.
|
423
433
|
if run_id:
|
424
434
|
parent_run_id: str = run_id
|
425
435
|
run_id: str = gen_id(name, unique=True)
|
426
436
|
else:
|
427
437
|
run_id: str = gen_id(name, unique=True)
|
428
438
|
parent_run_id: str = run_id
|
429
|
-
|
439
|
+
|
440
|
+
context: DictData = {"status": WAIT}
|
430
441
|
trace: Trace = get_trace(
|
431
442
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
432
443
|
)
|
433
444
|
release: datetime = self.validate_release(dt=release)
|
434
445
|
trace.info(f"[RELEASE]: Start {name!r} : {release:%Y-%m-%d %H:%M:%S}")
|
435
|
-
tz: ZoneInfo = dynamic("tz", extras=self.extras)
|
436
446
|
values: DictData = param2template(
|
437
447
|
params,
|
438
448
|
params={
|
439
449
|
"release": {
|
440
450
|
"logical_date": release,
|
441
|
-
"execute_date":
|
451
|
+
"execute_date": get_dt_now(),
|
442
452
|
"run_id": run_id,
|
453
|
+
"runs_metadata": runs_metadata or {},
|
443
454
|
}
|
444
455
|
},
|
445
456
|
extras=self.extras,
|
@@ -460,9 +471,17 @@ class Workflow(BaseModel):
|
|
460
471
|
context=context,
|
461
472
|
parent_run_id=parent_run_id,
|
462
473
|
run_id=run_id,
|
463
|
-
execution_time=rs.info.get("execution_time", 0),
|
464
474
|
extras=self.extras,
|
465
|
-
|
475
|
+
runs_metadata=(
|
476
|
+
(runs_metadata or {})
|
477
|
+
| rs.info
|
478
|
+
| {
|
479
|
+
"timeout": timeout,
|
480
|
+
"original_name": self.name,
|
481
|
+
"audit_excluded": audit_excluded,
|
482
|
+
}
|
483
|
+
),
|
484
|
+
).save(excluded=audit_excluded)
|
466
485
|
)
|
467
486
|
return Result(
|
468
487
|
run_id=run_id,
|
@@ -487,7 +506,6 @@ class Workflow(BaseModel):
|
|
487
506
|
def execute_job(
|
488
507
|
self,
|
489
508
|
job: Job,
|
490
|
-
params: DictData,
|
491
509
|
run_id: str,
|
492
510
|
context: DictData,
|
493
511
|
*,
|
@@ -506,7 +524,6 @@ class Workflow(BaseModel):
|
|
506
524
|
|
507
525
|
Args:
|
508
526
|
job: (Job) A job model that want to execute.
|
509
|
-
params: (DictData) A parameter data.
|
510
527
|
run_id: A running stage ID.
|
511
528
|
context: A context data.
|
512
529
|
parent_run_id: A parent running ID. (Default is None)
|
@@ -533,25 +550,24 @@ class Workflow(BaseModel):
|
|
533
550
|
)
|
534
551
|
|
535
552
|
trace.info(f"[WORKFLOW]: Execute Job: {job.id!r}")
|
536
|
-
|
537
|
-
params=
|
553
|
+
result: Result = job.execute(
|
554
|
+
params=context,
|
538
555
|
run_id=parent_run_id,
|
539
556
|
event=event,
|
540
557
|
)
|
541
|
-
job.set_outputs(
|
558
|
+
job.set_outputs(result.context, to=context)
|
542
559
|
|
543
|
-
if
|
560
|
+
if result.status == FAILED:
|
544
561
|
error_msg: str = f"Job execution, {job.id!r}, was failed."
|
545
562
|
return FAILED, catch(
|
546
563
|
context=context,
|
547
564
|
status=FAILED,
|
548
565
|
updated={
|
549
566
|
"errors": WorkflowError(error_msg).to_dict(),
|
550
|
-
**params,
|
551
567
|
},
|
552
568
|
)
|
553
569
|
|
554
|
-
elif
|
570
|
+
elif result.status == CANCEL:
|
555
571
|
error_msg: str = (
|
556
572
|
f"Job execution, {job.id!r}, was canceled from the event after "
|
557
573
|
f"end job execution."
|
@@ -561,13 +577,10 @@ class Workflow(BaseModel):
|
|
561
577
|
status=CANCEL,
|
562
578
|
updated={
|
563
579
|
"errors": WorkflowCancelError(error_msg).to_dict(),
|
564
|
-
**params,
|
565
580
|
},
|
566
581
|
)
|
567
582
|
|
568
|
-
return
|
569
|
-
context=context, status=rs.status, updated=params
|
570
|
-
)
|
583
|
+
return result.status, catch(context, status=result.status)
|
571
584
|
|
572
585
|
def execute(
|
573
586
|
self,
|
@@ -748,7 +761,6 @@ class Workflow(BaseModel):
|
|
748
761
|
executor.submit(
|
749
762
|
self.execute_job,
|
750
763
|
job=job,
|
751
|
-
params=context,
|
752
764
|
run_id=run_id,
|
753
765
|
context=context,
|
754
766
|
parent_run_id=parent_run_id,
|
@@ -763,7 +775,6 @@ class Workflow(BaseModel):
|
|
763
775
|
executor.submit(
|
764
776
|
self.execute_job,
|
765
777
|
job=job,
|
766
|
-
params=context,
|
767
778
|
run_id=run_id,
|
768
779
|
context=context,
|
769
780
|
parent_run_id=parent_run_id,
|
@@ -893,7 +904,7 @@ class Workflow(BaseModel):
|
|
893
904
|
extras=self.extras,
|
894
905
|
)
|
895
906
|
|
896
|
-
err = context
|
907
|
+
err: dict[str, str] = context.get("errors", {})
|
897
908
|
trace.info(f"[WORKFLOW]: Previous error: {err}")
|
898
909
|
|
899
910
|
event: ThreadEvent = event or ThreadEvent()
|
@@ -914,9 +925,9 @@ class Workflow(BaseModel):
|
|
914
925
|
extras=self.extras,
|
915
926
|
)
|
916
927
|
|
917
|
-
# NOTE: Prepare the new context for rerun process.
|
928
|
+
# NOTE: Prepare the new context variable for rerun process.
|
918
929
|
jobs: DictData = context.get("jobs")
|
919
|
-
|
930
|
+
context: DictData = {
|
920
931
|
"params": context["params"].copy(),
|
921
932
|
"jobs": {j: jobs[j] for j in jobs if jobs[j]["status"] == SUCCESS},
|
922
933
|
}
|
@@ -925,19 +936,22 @@ class Workflow(BaseModel):
|
|
925
936
|
job_queue: Queue = Queue()
|
926
937
|
for job_id in self.jobs:
|
927
938
|
|
928
|
-
if job_id in
|
939
|
+
if job_id in context["jobs"]:
|
929
940
|
continue
|
930
941
|
|
931
942
|
job_queue.put(job_id)
|
932
943
|
total_job += 1
|
933
944
|
|
934
945
|
if total_job == 0:
|
935
|
-
trace.warning(
|
946
|
+
trace.warning(
|
947
|
+
"[WORKFLOW]: It does not have job to rerun. it will change "
|
948
|
+
"status to skip."
|
949
|
+
)
|
936
950
|
return Result(
|
937
951
|
run_id=run_id,
|
938
952
|
parent_run_id=parent_run_id,
|
939
|
-
status=
|
940
|
-
context=catch(context=context, status=
|
953
|
+
status=SKIP,
|
954
|
+
context=catch(context=context, status=SKIP),
|
941
955
|
extras=self.extras,
|
942
956
|
)
|
943
957
|
|
@@ -949,14 +963,14 @@ class Workflow(BaseModel):
|
|
949
963
|
"max_job_exec_timeout", f=timeout, extras=self.extras
|
950
964
|
)
|
951
965
|
|
952
|
-
catch(
|
966
|
+
catch(context, status=WAIT)
|
953
967
|
if event and event.is_set():
|
954
968
|
return Result(
|
955
969
|
run_id=run_id,
|
956
970
|
parent_run_id=parent_run_id,
|
957
971
|
status=CANCEL,
|
958
972
|
context=catch(
|
959
|
-
|
973
|
+
context,
|
960
974
|
status=CANCEL,
|
961
975
|
updated={
|
962
976
|
"errors": WorkflowCancelError(
|
@@ -978,7 +992,7 @@ class Workflow(BaseModel):
|
|
978
992
|
):
|
979
993
|
job_id: str = job_queue.get()
|
980
994
|
job: Job = self.job(name=job_id)
|
981
|
-
if (check := job.check_needs(
|
995
|
+
if (check := job.check_needs(context["jobs"])) == WAIT:
|
982
996
|
job_queue.task_done()
|
983
997
|
job_queue.put(job_id)
|
984
998
|
consecutive_waits += 1
|
@@ -998,7 +1012,7 @@ class Workflow(BaseModel):
|
|
998
1012
|
parent_run_id=parent_run_id,
|
999
1013
|
status=FAILED,
|
1000
1014
|
context=catch(
|
1001
|
-
|
1015
|
+
context,
|
1002
1016
|
status=FAILED,
|
1003
1017
|
updated={
|
1004
1018
|
"status": FAILED,
|
@@ -1014,7 +1028,7 @@ class Workflow(BaseModel):
|
|
1014
1028
|
trace.info(
|
1015
1029
|
f"[JOB]: Skip job: {job_id!r} from trigger rule."
|
1016
1030
|
)
|
1017
|
-
job.set_outputs(output={"status": SKIP}, to=
|
1031
|
+
job.set_outputs(output={"status": SKIP}, to=context)
|
1018
1032
|
job_queue.task_done()
|
1019
1033
|
skip_count += 1
|
1020
1034
|
continue
|
@@ -1024,7 +1038,6 @@ class Workflow(BaseModel):
|
|
1024
1038
|
executor.submit(
|
1025
1039
|
self.execute_job,
|
1026
1040
|
job=job,
|
1027
|
-
params=new_context,
|
1028
1041
|
run_id=run_id,
|
1029
1042
|
context=context,
|
1030
1043
|
parent_run_id=parent_run_id,
|
@@ -1039,7 +1052,6 @@ class Workflow(BaseModel):
|
|
1039
1052
|
executor.submit(
|
1040
1053
|
self.execute_job,
|
1041
1054
|
job=job,
|
1042
|
-
params=new_context,
|
1043
1055
|
run_id=run_id,
|
1044
1056
|
context=context,
|
1045
1057
|
parent_run_id=parent_run_id,
|
@@ -1090,7 +1102,7 @@ class Workflow(BaseModel):
|
|
1090
1102
|
run_id=run_id,
|
1091
1103
|
parent_run_id=parent_run_id,
|
1092
1104
|
status=st,
|
1093
|
-
context=catch(
|
1105
|
+
context=catch(context, status=st),
|
1094
1106
|
extras=self.extras,
|
1095
1107
|
)
|
1096
1108
|
|
@@ -1110,7 +1122,7 @@ class Workflow(BaseModel):
|
|
1110
1122
|
parent_run_id=parent_run_id,
|
1111
1123
|
status=FAILED,
|
1112
1124
|
context=catch(
|
1113
|
-
|
1125
|
+
context,
|
1114
1126
|
status=FAILED,
|
1115
1127
|
updated={
|
1116
1128
|
"errors": WorkflowTimeoutError(
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.76
|
4
4
|
Summary: Lightweight workflow orchestration with YAML template
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -68,7 +68,7 @@ by a `.yaml` template.
|
|
68
68
|
3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
|
69
69
|
(Python 3.13 unlock GIL 🐍🔓)
|
70
70
|
4. Recommend to pass a **Secret Value** with environment variable in YAML template 🔐
|
71
|
-
5. Any datatime value convert to **
|
71
|
+
5. Any datatime value convert to **UTC Timezone** 🌐
|
72
72
|
|
73
73
|
---
|
74
74
|
|
@@ -288,10 +288,10 @@ it will use default value and do not raise any error to you.
|
|
288
288
|
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
289
289
|
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
290
290
|
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
291
|
-
| **TIMEZONE** | CORE | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
292
291
|
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
293
292
|
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
294
293
|
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
294
|
+
| **TIMEZONE** | LOG | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
295
295
|
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
296
296
|
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
297
297
|
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
@@ -0,0 +1,30 @@
|
|
1
|
+
ddeutil/workflow/__about__.py,sha256=ZWCr4NXnXaxhYuhduVXp9CkXr2m6FgZDruUsw8AdJBM,28
|
2
|
+
ddeutil/workflow/__cron.py,sha256=avOagaHl9xXOmizeRWm13cOrty9Tw0vRjFq-xoEgpAY,29167
|
3
|
+
ddeutil/workflow/__init__.py,sha256=_8sP-CTPOfwsFFhmdwQ2Gp7yY7qJemP7TYsIWgd5jc0,3300
|
4
|
+
ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
|
5
|
+
ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
|
6
|
+
ddeutil/workflow/audits.py,sha256=wANG0jEQ7slUSgVZG4JbjlR5PtmF8mHpM9RH-zpYM_g,12679
|
7
|
+
ddeutil/workflow/cli.py,sha256=eAwRZMSEJu-NONc_un0D_1swFlENMjl3C-iXYnyTTPY,7411
|
8
|
+
ddeutil/workflow/conf.py,sha256=UCw6v2GFD3tA2LRbp7vLifXniey0P5Ef0U9eBPknrWk,16267
|
9
|
+
ddeutil/workflow/errors.py,sha256=tHS6ekxBmZ6sIeLaxHHSaMfhVvlWnndfb2-Aq-bL2So,5509
|
10
|
+
ddeutil/workflow/event.py,sha256=siChcBhsu4ejzW1fK0tjHPXQVaSUCSxPYDgDrh6duwo,13676
|
11
|
+
ddeutil/workflow/job.py,sha256=_NOPWPs2FuiMvNE-L6c9mpXEChXmgQ8zmD33ZzqVi0A,44146
|
12
|
+
ddeutil/workflow/params.py,sha256=Cyz142OcvENIZrM7Efc2xuGPmmFBhROifP5ojoaCezg,13658
|
13
|
+
ddeutil/workflow/result.py,sha256=Fz6y6apivLW-94gAxcT42z-mGqWMk6-O3RJ2GGSNUHM,9146
|
14
|
+
ddeutil/workflow/reusables.py,sha256=q_OA-oifCGIhW_5j6hTZXZk7FBOmDt0xVrtNnscJfNg,23294
|
15
|
+
ddeutil/workflow/stages.py,sha256=Z6quvhJ5WZPnItd4xOoQyR_KWE2Z6LYWa5d49N0R5D8,121936
|
16
|
+
ddeutil/workflow/traces.py,sha256=0n6Mytp6oeNjOV8lIsFitzZ6TrtuSNVFkUmodBiE_vA,28466
|
17
|
+
ddeutil/workflow/utils.py,sha256=EXhIuWzOJHvlcoAdyvuDUomGtMTIB59HxOLpj2VJ1bI,10857
|
18
|
+
ddeutil/workflow/workflow.py,sha256=Yw7xuEIwQ61qhGfElky9AZY_1o_1Gqta4B1x1nwQNJs,41475
|
19
|
+
ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
|
20
|
+
ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
|
21
|
+
ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
|
22
|
+
ddeutil/workflow/api/routes/job.py,sha256=-lbZ_hS9pEdSy6zeke5qrXEgdNxtQ2w9in7cHuM2Jzs,2536
|
23
|
+
ddeutil/workflow/api/routes/logs.py,sha256=RiZ62eQVMWArPHE3lpan955U4DdLLkethlvSMlwF7Mg,5312
|
24
|
+
ddeutil/workflow/api/routes/workflows.py,sha256=1Mqx4Hft4uJglgJI-Wcw-JzkhomFYZrtP0DnQDBkAFQ,4410
|
25
|
+
ddeutil_workflow-0.0.76.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
26
|
+
ddeutil_workflow-0.0.76.dist-info/METADATA,sha256=zq63qmHeFG1DpJJWKyO5IjA6tMbYQnHNWa8E5OESD5w,15781
|
27
|
+
ddeutil_workflow-0.0.76.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
28
|
+
ddeutil_workflow-0.0.76.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
|
29
|
+
ddeutil_workflow-0.0.76.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
30
|
+
ddeutil_workflow-0.0.76.dist-info/RECORD,,
|
@@ -1,30 +0,0 @@
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=ovscLwcOTC_clQ-KeqVwtDeutK0Mcfq20RBXHRUIFew,28
|
2
|
-
ddeutil/workflow/__cron.py,sha256=9I_gAXgZu1agInPoDi9r1lH7LUWq7gnDOmUrLxXmkWE,28840
|
3
|
-
ddeutil/workflow/__init__.py,sha256=kjKFdRNOh19IiLdmSFPKesU8BqSijEWHRZ8_fKGXFUk,3276
|
4
|
-
ddeutil/workflow/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
|
5
|
-
ddeutil/workflow/__types.py,sha256=tA2vsr6mzTSzbWB1sb62c5GgxODlfVRz6FvgLNJtQao,4788
|
6
|
-
ddeutil/workflow/audits.py,sha256=PZ0dDBBANpXoLHlDrmlEoIXr0iYK9PCw4-9tPJX9UXE,12528
|
7
|
-
ddeutil/workflow/cli.py,sha256=V42C3lmtzRUiIjgMjLlr7v600PX0A3Aqbp8ccUSgQew,5244
|
8
|
-
ddeutil/workflow/conf.py,sha256=aXclEcxQKYYDb09O5EBQxmJmjPXNT73NMxeq238xO3s,16264
|
9
|
-
ddeutil/workflow/errors.py,sha256=n10YjXptY4iiY57FFVq22aedlUojpMuB5cM7aQ0KNpo,5522
|
10
|
-
ddeutil/workflow/event.py,sha256=siChcBhsu4ejzW1fK0tjHPXQVaSUCSxPYDgDrh6duwo,13676
|
11
|
-
ddeutil/workflow/job.py,sha256=oCjxG1Zz2D9Yc2SuZth3k1geIQ-AJ3oLs_38g5NMXLE,43994
|
12
|
-
ddeutil/workflow/params.py,sha256=5DMDp7ZJwa1pM-oATV4kEPDQsaL7Ol8NavR9FZk_uqg,13519
|
13
|
-
ddeutil/workflow/result.py,sha256=XBRZtD6b_5kJCFKvyk0ECWG1IjKIHc1Ad1mFvzpoPpY,9147
|
14
|
-
ddeutil/workflow/reusables.py,sha256=q_OA-oifCGIhW_5j6hTZXZk7FBOmDt0xVrtNnscJfNg,23294
|
15
|
-
ddeutil/workflow/stages.py,sha256=2Rz9NGVOftIqiYaSgasHDS8MvgvLp3jtAjBHEwdtx7U,121837
|
16
|
-
ddeutil/workflow/traces.py,sha256=nSHOF7DPolLjNEKCu1OwiFg1z2xLvQp_eu-mY5BZCOA,28288
|
17
|
-
ddeutil/workflow/utils.py,sha256=NqekWLsU2YK2ejvYk83N-8mjSXjxmBxGN7gPXRKONko,11337
|
18
|
-
ddeutil/workflow/workflow.py,sha256=lcW2sYRWaYpq2dKoeKS5sQSV5lF2sF6loeUU-AGJ5kE,41159
|
19
|
-
ddeutil/workflow/api/__init__.py,sha256=5DzYL3ngceoRshh5HYCSVWChqNJSiP01E1bEd8XxPi0,4799
|
20
|
-
ddeutil/workflow/api/log_conf.py,sha256=WfS3udDLSyrP-C80lWOvxxmhd_XWKvQPkwDqKblcH3E,1834
|
21
|
-
ddeutil/workflow/api/routes/__init__.py,sha256=JRaJZB0D6mgR17MbZo8yLtdYDtD62AA8MdKlFqhG84M,420
|
22
|
-
ddeutil/workflow/api/routes/job.py,sha256=-lbZ_hS9pEdSy6zeke5qrXEgdNxtQ2w9in7cHuM2Jzs,2536
|
23
|
-
ddeutil/workflow/api/routes/logs.py,sha256=RiZ62eQVMWArPHE3lpan955U4DdLLkethlvSMlwF7Mg,5312
|
24
|
-
ddeutil/workflow/api/routes/workflows.py,sha256=1Mqx4Hft4uJglgJI-Wcw-JzkhomFYZrtP0DnQDBkAFQ,4410
|
25
|
-
ddeutil_workflow-0.0.74.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
26
|
-
ddeutil_workflow-0.0.74.dist-info/METADATA,sha256=ZLpNyhuTVBCxVMLFegOvMfCLBkQXjrULcbaCwvY6F0I,15775
|
27
|
-
ddeutil_workflow-0.0.74.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
28
|
-
ddeutil_workflow-0.0.74.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
|
29
|
-
ddeutil_workflow-0.0.74.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
30
|
-
ddeutil_workflow-0.0.74.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|