ddeutil-workflow 0.0.79__py3-none-any.whl → 0.0.81__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +1 -1
- ddeutil/workflow/api/routes/logs.py +5 -5
- ddeutil/workflow/api/routes/workflows.py +3 -3
- ddeutil/workflow/audits.py +93 -164
- ddeutil/workflow/cli.py +39 -29
- ddeutil/workflow/conf.py +24 -22
- ddeutil/workflow/errors.py +3 -2
- ddeutil/workflow/job.py +0 -1
- ddeutil/workflow/reusables.py +16 -17
- ddeutil/workflow/traces.py +50 -32
- ddeutil/workflow/utils.py +0 -11
- ddeutil/workflow/workflow.py +31 -30
- {ddeutil_workflow-0.0.79.dist-info → ddeutil_workflow-0.0.81.dist-info}/METADATA +16 -29
- {ddeutil_workflow-0.0.79.dist-info → ddeutil_workflow-0.0.81.dist-info}/RECORD +19 -19
- {ddeutil_workflow-0.0.79.dist-info → ddeutil_workflow-0.0.81.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.79.dist-info → ddeutil_workflow-0.0.81.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.79.dist-info → ddeutil_workflow-0.0.81.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.79.dist-info → ddeutil_workflow-0.0.81.dist-info}/top_level.txt +0 -0
ddeutil/workflow/cli.py
CHANGED
@@ -54,29 +54,30 @@ def init() -> None:
|
|
54
54
|
dedent(
|
55
55
|
"""
|
56
56
|
# Example workflow template.
|
57
|
-
wf-example:
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
57
|
+
name: wf-example:
|
58
|
+
type: Workflow
|
59
|
+
desc: |
|
60
|
+
An example workflow template that provide the demo of workflow.
|
61
|
+
params:
|
62
|
+
name:
|
63
|
+
type: str
|
64
|
+
default: "World"
|
65
|
+
jobs:
|
66
|
+
first-job:
|
67
|
+
stages:
|
68
|
+
|
69
|
+
- name: "Hello Stage"
|
70
|
+
echo: "Start say hi to the console"
|
71
|
+
|
72
|
+
- name: "Call tasks"
|
73
|
+
uses: tasks/say-hello-func@example
|
74
|
+
with:
|
75
|
+
name: ${{ params.name }}
|
76
|
+
|
77
|
+
second-job:
|
78
|
+
|
79
|
+
- name: "Hello Env"
|
80
|
+
echo: "Start say hi with ${ WORKFLOW_DEMO_HELLO }"
|
80
81
|
"""
|
81
82
|
).lstrip("\n")
|
82
83
|
)
|
@@ -89,12 +90,20 @@ def init() -> None:
|
|
89
90
|
dummy_tasks_path.write_text(
|
90
91
|
dedent(
|
91
92
|
"""
|
93
|
+
from typing import Any, Optional
|
94
|
+
|
92
95
|
from ddeutil.workflow import Result, tag
|
93
96
|
|
94
97
|
@tag(name="example", alias="say-hello-func")
|
95
|
-
def hello_world_task(name: str, rs: Result) -> dict[str, str]:
|
98
|
+
def hello_world_task(name: str, rs: Result, extras: Optional[dict[str, Any]] = None) -> dict[str, str]:
|
96
99
|
\"\"\"Logging hello task function\"\"\"
|
97
|
-
|
100
|
+
_extras = extras or {}
|
101
|
+
# NOTE: I will use custom newline logging if you pass `||`.
|
102
|
+
rs.trace.info(
|
103
|
+
f"Hello, {name}||"
|
104
|
+
f"> running ID: {rs.run_id}"
|
105
|
+
f"> extras: {_extras}"
|
106
|
+
)
|
98
107
|
return {"name": name}
|
99
108
|
"""
|
100
109
|
).lstrip("\n")
|
@@ -106,18 +115,19 @@ def init() -> None:
|
|
106
115
|
dotenv_file = Path(".env")
|
107
116
|
mode: str = "a" if dotenv_file.exists() else "w"
|
108
117
|
with dotenv_file.open(mode=mode) as f:
|
109
|
-
f.write("\n# Workflow
|
118
|
+
f.write("\n# Workflow Environment Variables\n")
|
110
119
|
f.write(
|
111
120
|
"WORKFLOW_DEMO_HELLO=foo\n"
|
112
121
|
"WORKFLOW_CORE_DEBUG_MODE=true\n"
|
113
122
|
"WORKFLOW_LOG_TIMEZONE=Asia/Bangkok\n"
|
114
|
-
"
|
123
|
+
'WORKFLOW_LOG_TRACE_HANDLERS=\'[{"type": "console"}]\'\n'
|
124
|
+
'WORKFLOW_LOG_AUDIT_CONF=\'{"type": "file", "path": "./audits"}\''
|
115
125
|
"WORKFLOW_LOG_AUDIT_ENABLE_WRITE=true\n"
|
116
126
|
)
|
117
127
|
|
118
128
|
typer.echo("Starter command:")
|
119
129
|
typer.echo(
|
120
|
-
"
|
130
|
+
">>> `source .env && workflow-cli workflows execute --name=wf-example`"
|
121
131
|
)
|
122
132
|
|
123
133
|
|
@@ -163,7 +173,7 @@ def api(
|
|
163
173
|
debug: Annotated[bool, typer.Option(help="A debug mode flag")] = True,
|
164
174
|
workers: Annotated[int, typer.Option(help="A worker number")] = None,
|
165
175
|
reload: Annotated[bool, typer.Option(help="A reload flag")] = False,
|
166
|
-
):
|
176
|
+
) -> None:
|
167
177
|
"""
|
168
178
|
Provision API application from the FastAPI.
|
169
179
|
"""
|
ddeutil/workflow/conf.py
CHANGED
@@ -22,19 +22,6 @@ Functions:
|
|
22
22
|
pass_env: Process environment variable substitution
|
23
23
|
api_config: Get API-specific configuration settings
|
24
24
|
|
25
|
-
Example:
|
26
|
-
```python
|
27
|
-
from ddeutil.workflow.conf import Config, YamlParser
|
28
|
-
|
29
|
-
# Load workflow configuration
|
30
|
-
parser = YamlParser("my-workflow")
|
31
|
-
workflow_config = parser.data
|
32
|
-
|
33
|
-
# Access dynamic configuration
|
34
|
-
from ddeutil.workflow.conf import dynamic
|
35
|
-
log_level = dynamic("log_level", default="INFO")
|
36
|
-
```
|
37
|
-
|
38
25
|
Note:
|
39
26
|
Configuration files support environment variable substitution using
|
40
27
|
${VAR_NAME} syntax and provide extensive validation capabilities.
|
@@ -155,8 +142,10 @@ class Config: # pragma: no cov
|
|
155
142
|
)
|
156
143
|
|
157
144
|
@property
|
158
|
-
def
|
159
|
-
return
|
145
|
+
def audit_conf(self) -> dict[str, Any]:
|
146
|
+
return json.loads(
|
147
|
+
env("LOG_AUDIT_URL", '{"type": "file", "path": "./audits"}')
|
148
|
+
)
|
160
149
|
|
161
150
|
@property
|
162
151
|
def enable_write_audit(self) -> bool:
|
@@ -286,9 +275,12 @@ class YamlParser:
|
|
286
275
|
continue
|
287
276
|
|
288
277
|
if data := cls.filter_yaml(file, name=name):
|
278
|
+
|
279
|
+
# NOTE: Start adding file metadata.
|
289
280
|
file_stat: os.stat_result = file.lstat()
|
290
281
|
data["created_at"] = file_stat.st_ctime
|
291
282
|
data["updated_at"] = file_stat.st_mtime
|
283
|
+
|
292
284
|
if not obj_type:
|
293
285
|
all_data.append((file_stat.st_mtime, data))
|
294
286
|
elif (t := data.get("type")) and t == obj_type:
|
@@ -322,9 +314,8 @@ class YamlParser:
|
|
322
314
|
extras: (DictData) An extra parameter that use to override core
|
323
315
|
config values.
|
324
316
|
ignore_filename: (str) An ignore filename. Default is
|
325
|
-
|
326
|
-
tags
|
327
|
-
A list of tag that want to filter.
|
317
|
+
``.confignore`` filename.
|
318
|
+
tags (list[str]): A list of tag that want to filter.
|
328
319
|
|
329
320
|
:rtype: Iterator[tuple[str, DictData]]
|
330
321
|
"""
|
@@ -363,6 +354,8 @@ class YamlParser:
|
|
363
354
|
continue
|
364
355
|
|
365
356
|
if (t := data.get("type")) and t == obj_type:
|
357
|
+
|
358
|
+
# NOTE: Start adding file metadata.
|
366
359
|
file_stat: os.stat_result = file.lstat()
|
367
360
|
data["created_at"] = file_stat.st_ctime
|
368
361
|
data["updated_at"] = file_stat.st_mtime
|
@@ -370,6 +363,7 @@ class YamlParser:
|
|
370
363
|
file.lstat().st_mtime,
|
371
364
|
data,
|
372
365
|
)
|
366
|
+
|
373
367
|
if key in all_data:
|
374
368
|
all_data[key].append(marking)
|
375
369
|
else:
|
@@ -403,15 +397,23 @@ class YamlParser:
|
|
403
397
|
def filter_yaml(cls, file: Path, name: Optional[str] = None) -> DictData:
|
404
398
|
"""Read a YAML file context from an input file path and specific name.
|
405
399
|
|
406
|
-
:
|
407
|
-
|
400
|
+
Args:
|
401
|
+
file (Path): A file path that want to extract YAML context.
|
402
|
+
name (str): A key name that search on a YAML context.
|
408
403
|
|
409
|
-
:
|
404
|
+
Returns:
|
405
|
+
DictData: A data that read from this file if it is YAML format.
|
410
406
|
"""
|
411
407
|
if any(file.suffix.endswith(s) for s in (".yml", ".yaml")):
|
412
408
|
values: DictData = YamlFlResolve(file).read()
|
413
409
|
if values is not None:
|
414
|
-
|
410
|
+
if name:
|
411
|
+
if "name" in values and values.get("name") == name:
|
412
|
+
return values
|
413
|
+
return (
|
414
|
+
values[name] | {"name": name} if name in values else {}
|
415
|
+
)
|
416
|
+
return values
|
415
417
|
return {}
|
416
418
|
|
417
419
|
@cached_property
|
ddeutil/workflow/errors.py
CHANGED
@@ -135,11 +135,12 @@ class BaseError(Exception):
|
|
135
135
|
|
136
136
|
Example:
|
137
137
|
>>> error = BaseError("Something failed", refs="stage-1")
|
138
|
-
|
138
|
+
|
139
|
+
Simple format
|
139
140
|
>>> error.to_dict()
|
140
141
|
>>> # Returns: {"name": "BaseError", "message": "Something failed"}
|
141
142
|
|
142
|
-
|
143
|
+
With reference mapping
|
143
144
|
>>> error.to_dict(with_refs=True)
|
144
145
|
>>> # Returns: {"stage-1": {"name": "BaseError", "message": "Something failed"}}
|
145
146
|
```
|
ddeutil/workflow/job.py
CHANGED
ddeutil/workflow/reusables.py
CHANGED
@@ -25,22 +25,20 @@ Functions:
|
|
25
25
|
create_model_from_caller: Generate Pydantic models from function signatures
|
26
26
|
|
27
27
|
Example:
|
28
|
-
|
29
|
-
from ddeutil.workflow.reusables import tag
|
30
|
-
|
31
|
-
@tag("data-processing", alias="process-csv")
|
32
|
-
def process_csv_file(input_path: str, output_path: str) -> dict:
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
#
|
37
|
-
#
|
38
|
-
#
|
39
|
-
#
|
40
|
-
#
|
41
|
-
#
|
42
|
-
# output_path: "/data/output.csv"
|
43
|
-
```
|
28
|
+
|
29
|
+
>>> from ddeutil.workflow.reusables import tag
|
30
|
+
>>>
|
31
|
+
>>> @tag("data-processing", alias="process-csv")
|
32
|
+
>>> def process_csv_file(input_path: str, output_path: str) -> dict:
|
33
|
+
>>> return {"status": "completed", "rows_processed": 1000}
|
34
|
+
|
35
|
+
>>> # Use in workflow YAML:
|
36
|
+
>>> # stages:
|
37
|
+
>>> # - name: "Process data"
|
38
|
+
>>> # uses: "data-processing/process-csv@latest"
|
39
|
+
>>> # args:
|
40
|
+
>>> # input_path: "/data/input.csv"
|
41
|
+
>>> # output_path: "/data/output.csv"
|
44
42
|
|
45
43
|
Note:
|
46
44
|
The registry system supports versioning and aliasing for better function
|
@@ -64,6 +62,7 @@ from typing import (
|
|
64
62
|
Protocol,
|
65
63
|
TypeVar,
|
66
64
|
Union,
|
65
|
+
cast,
|
67
66
|
get_type_hints,
|
68
67
|
)
|
69
68
|
|
@@ -201,7 +200,7 @@ def get_args_const(
|
|
201
200
|
f"Post-filter: {expr} does not valid because it raise syntax error."
|
202
201
|
) from None
|
203
202
|
|
204
|
-
body: list[Expr] = mod.body
|
203
|
+
body: list[Expr] = cast(list[Expr], mod.body)
|
205
204
|
if len(body) > 1:
|
206
205
|
raise UtilError(
|
207
206
|
"Post-filter function should be only one calling per workflow."
|
ddeutil/workflow/traces.py
CHANGED
@@ -107,7 +107,9 @@ class Message(BaseModel):
|
|
107
107
|
with emoji support and categorization.
|
108
108
|
"""
|
109
109
|
|
110
|
-
name: Optional[str] = Field(
|
110
|
+
name: Optional[str] = Field(
|
111
|
+
default=None, description="A prefix name of message."
|
112
|
+
)
|
111
113
|
message: Optional[str] = Field(default=None, description="A message.")
|
112
114
|
|
113
115
|
@classmethod
|
@@ -952,9 +954,8 @@ class SQLiteHandler(BaseHandler): # pragma: no cov
|
|
952
954
|
except Exception as e:
|
953
955
|
logger.error(f"Failed to read from SQLite database: {e}")
|
954
956
|
|
955
|
-
@classmethod
|
956
957
|
def find_trace_with_id(
|
957
|
-
|
958
|
+
self,
|
958
959
|
run_id: str,
|
959
960
|
force_raise: bool = True,
|
960
961
|
*,
|
@@ -962,17 +963,7 @@ class SQLiteHandler(BaseHandler): # pragma: no cov
|
|
962
963
|
extras: Optional[DictData] = None,
|
963
964
|
) -> TraceData:
|
964
965
|
"""Find trace log with specific run ID from SQLite database."""
|
965
|
-
|
966
|
-
url = dynamic("trace_url", extras=extras)
|
967
|
-
if (
|
968
|
-
url is not None
|
969
|
-
and hasattr(url, "path")
|
970
|
-
and getattr(url, "path", None)
|
971
|
-
):
|
972
|
-
path = Path(url.path)
|
973
|
-
else:
|
974
|
-
path = Path("./logs/workflow_traces.db")
|
975
|
-
|
966
|
+
path = path or Path(self.path)
|
976
967
|
if not path.exists():
|
977
968
|
if force_raise:
|
978
969
|
raise FileNotFoundError(f"SQLite database not found: {path}")
|
@@ -1737,7 +1728,9 @@ TraceHandler = Annotated[
|
|
1737
1728
|
Union[
|
1738
1729
|
ConsoleHandler,
|
1739
1730
|
FileHandler,
|
1740
|
-
SQLiteHandler,
|
1731
|
+
# SQLiteHandler,
|
1732
|
+
# RestAPIHandler,
|
1733
|
+
# ElasticHandler
|
1741
1734
|
],
|
1742
1735
|
Field(discriminator="type"),
|
1743
1736
|
]
|
@@ -1874,7 +1867,9 @@ class BaseAsyncEmit(ABC):
|
|
1874
1867
|
|
1875
1868
|
|
1876
1869
|
class TraceManager(BaseModel, BaseEmit, BaseAsyncEmit):
|
1877
|
-
"""Trace
|
1870
|
+
"""Trace Manager model that keep all trance handler and emit log to its
|
1871
|
+
handler.
|
1872
|
+
"""
|
1878
1873
|
|
1879
1874
|
extras: DictData = Field(
|
1880
1875
|
default_factory=dict,
|
@@ -1892,7 +1887,7 @@ class TraceManager(BaseModel, BaseEmit, BaseAsyncEmit):
|
|
1892
1887
|
description="A list of Trace handler model."
|
1893
1888
|
)
|
1894
1889
|
buffer_size: int = Field(
|
1895
|
-
default=
|
1890
|
+
default=10,
|
1896
1891
|
description="A buffer size to trigger flush trace log",
|
1897
1892
|
)
|
1898
1893
|
|
@@ -1929,8 +1924,8 @@ class TraceManager(BaseModel, BaseEmit, BaseAsyncEmit):
|
|
1929
1924
|
"""Emit a trace log to all handler. This will use synchronise process.
|
1930
1925
|
|
1931
1926
|
Args:
|
1932
|
-
msg: A message.
|
1933
|
-
level: A tracing level.
|
1927
|
+
msg (str): A message.
|
1928
|
+
level (Level): A tracing level.
|
1934
1929
|
"""
|
1935
1930
|
_msg: str = self.make_message(msg)
|
1936
1931
|
metadata: Metadata = Metadata.make(
|
@@ -1942,16 +1937,18 @@ class TraceManager(BaseModel, BaseEmit, BaseAsyncEmit):
|
|
1942
1937
|
parent_run_id=self.parent_run_id,
|
1943
1938
|
extras=self.extras,
|
1944
1939
|
)
|
1945
|
-
if self._enable_buffer:
|
1946
|
-
self._buffer.append(metadata)
|
1947
|
-
|
1948
|
-
if len(self._buffer) >= self.buffer_size:
|
1949
|
-
for handler in self.handlers:
|
1950
|
-
handler.flush(self._buffer, extra=self.extras)
|
1951
|
-
self._buffer.clear()
|
1952
|
-
else:
|
1940
|
+
if not self._enable_buffer:
|
1953
1941
|
for handler in self.handlers:
|
1954
1942
|
handler.emit(metadata, extra=self.extras)
|
1943
|
+
return
|
1944
|
+
|
1945
|
+
# NOTE: Update metadata to the buffer.
|
1946
|
+
self._buffer.append(metadata)
|
1947
|
+
|
1948
|
+
if len(self._buffer) >= self.buffer_size: # pragma: no cov
|
1949
|
+
for handler in self.handlers:
|
1950
|
+
handler.flush(self._buffer, extra=self.extras)
|
1951
|
+
self._buffer.clear()
|
1955
1952
|
|
1956
1953
|
async def amit(self, msg: str, level: Level) -> None:
|
1957
1954
|
"""Async write trace log with append mode and logging this message with
|
@@ -1974,22 +1971,43 @@ class TraceManager(BaseModel, BaseEmit, BaseAsyncEmit):
|
|
1974
1971
|
for handler in self.handlers:
|
1975
1972
|
await handler.amit(metadata, extra=self.extras)
|
1976
1973
|
|
1977
|
-
def __enter__(self):
|
1974
|
+
def __enter__(self):
|
1975
|
+
"""Enter the trace for catching the logs that run so fast. It will use
|
1976
|
+
buffer strategy to flush the logs instead emit.
|
1977
|
+
"""
|
1978
1978
|
self._enable_buffer = True
|
1979
|
+
return self
|
1980
|
+
|
1981
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
1982
|
+
"""Exit the trace that will clear all log in the buffer."""
|
1983
|
+
if exc_type:
|
1984
|
+
_msg: str = self.make_message(str(exc_val))
|
1985
|
+
metadata: Metadata = Metadata.make(
|
1986
|
+
error_flag=True,
|
1987
|
+
level="error",
|
1988
|
+
message=_msg,
|
1989
|
+
cutting_id=self.cut_id,
|
1990
|
+
run_id=self.run_id,
|
1991
|
+
parent_run_id=self.parent_run_id,
|
1992
|
+
extras=self.extras,
|
1993
|
+
)
|
1994
|
+
self._buffer.append(metadata)
|
1979
1995
|
|
1980
|
-
def __exit__(self, exc_type, exc_val, exc_tb): # pragma: no cov
|
1981
1996
|
if self._buffer:
|
1982
1997
|
for handler in self.handlers:
|
1983
1998
|
handler.flush(self._buffer, extra=self.extras)
|
1984
1999
|
self._buffer.clear()
|
1985
2000
|
|
2001
|
+
# NOTE: Re-raise the exception if one occurred
|
2002
|
+
return False
|
2003
|
+
|
1986
2004
|
|
1987
2005
|
def get_trace(
|
1988
2006
|
run_id: str,
|
1989
2007
|
*,
|
1990
2008
|
parent_run_id: Optional[str] = None,
|
1991
2009
|
extras: Optional[DictData] = None,
|
1992
|
-
) -> TraceManager:
|
2010
|
+
) -> TraceManager:
|
1993
2011
|
"""Get dynamic TraceManager instance from the core config.
|
1994
2012
|
|
1995
2013
|
This factory function returns the appropriate trace implementation based on
|
@@ -1997,8 +2015,8 @@ def get_trace(
|
|
1997
2015
|
and parent running ID.
|
1998
2016
|
|
1999
2017
|
Args:
|
2000
|
-
run_id: A running ID.
|
2001
|
-
parent_run_id: A parent running ID.
|
2018
|
+
run_id (str): A running ID.
|
2019
|
+
parent_run_id (str | None, default None): A parent running ID.
|
2002
2020
|
extras: An extra parameter that want to override the core
|
2003
2021
|
config values.
|
2004
2022
|
|
ddeutil/workflow/utils.py
CHANGED
@@ -28,17 +28,6 @@ Functions:
|
|
28
28
|
cut_id: Cut running ID to specified length
|
29
29
|
dump_all: Serialize nested BaseModel objects to dictionaries
|
30
30
|
obj_name: Get object name or class name
|
31
|
-
|
32
|
-
Example:
|
33
|
-
```python
|
34
|
-
from ddeutil.workflow.utils import gen_id, get_dt_now
|
35
|
-
|
36
|
-
# Generate unique ID
|
37
|
-
run_id = gen_id("workflow")
|
38
|
-
|
39
|
-
# Get current datetime
|
40
|
-
now = get_dt_now()
|
41
|
-
```
|
42
31
|
"""
|
43
32
|
from __future__ import annotations
|
44
33
|
|
ddeutil/workflow/workflow.py
CHANGED
@@ -42,7 +42,7 @@ from pydantic.functional_validators import field_validator, model_validator
|
|
42
42
|
from typing_extensions import Self
|
43
43
|
|
44
44
|
from .__types import DictData
|
45
|
-
from .audits import Audit,
|
45
|
+
from .audits import Audit, get_audit
|
46
46
|
from .conf import YamlParser, dynamic
|
47
47
|
from .errors import WorkflowCancelError, WorkflowError, WorkflowTimeoutError
|
48
48
|
from .event import Event
|
@@ -198,17 +198,15 @@ class Workflow(BaseModel):
|
|
198
198
|
FileNotFoundError: If workflow configuration file not found
|
199
199
|
|
200
200
|
Example:
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
)
|
211
|
-
```
|
201
|
+
>>> # Load from default config path
|
202
|
+
>>> workflow = Workflow.from_conf('data-pipeline')
|
203
|
+
|
204
|
+
>>> # Load with custom path and extras
|
205
|
+
>>> workflow = Workflow.from_conf(
|
206
|
+
... 'data-pipeline',
|
207
|
+
... path=Path('./custom-configs'),
|
208
|
+
... extras={'env': 'prod'}
|
209
|
+
... )
|
212
210
|
"""
|
213
211
|
load: YamlParser = YamlParser(name, path=path, extras=extras, obj=cls)
|
214
212
|
data: DictData = copy.deepcopy(load.data)
|
@@ -509,24 +507,27 @@ class Workflow(BaseModel):
|
|
509
507
|
trace.info(f"[RELEASE]: End {name!r} : {release:%Y-%m-%d %H:%M:%S}")
|
510
508
|
trace.debug(f"[RELEASE]: Writing audit: {name!r}.")
|
511
509
|
(
|
512
|
-
(audit or
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
510
|
+
(audit or get_audit(extras=self.extras)).save(
|
511
|
+
data={
|
512
|
+
"name": name,
|
513
|
+
"release": release,
|
514
|
+
"type": release_type,
|
515
|
+
"context": context,
|
516
|
+
"parent_run_id": parent_run_id,
|
517
|
+
"run_id": run_id,
|
518
|
+
"extras": self.extras,
|
519
|
+
"runs_metadata": (
|
520
|
+
(runs_metadata or {})
|
521
|
+
| rs.info
|
522
|
+
| {
|
523
|
+
"timeout": timeout,
|
524
|
+
"original_name": self.name,
|
525
|
+
"audit_excluded": audit_excluded,
|
526
|
+
}
|
527
|
+
),
|
528
|
+
},
|
529
|
+
excluded=audit_excluded,
|
530
|
+
)
|
530
531
|
)
|
531
532
|
return Result(
|
532
533
|
run_id=run_id,
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.81
|
4
4
|
Summary: Lightweight workflow orchestration with YAML template
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -168,19 +168,6 @@ For comprehensive API documentation, examples, and best practices:
|
|
168
168
|
- **[Full Documentation](https://ddeutils.github.io/ddeutil-workflow/)** - Complete user guide and API reference
|
169
169
|
- **[Getting Started](https://ddeutils.github.io/ddeutil-workflow/getting-started/)** - Quick start guide
|
170
170
|
- **[API Reference](https://ddeutils.github.io/ddeutil-workflow/api/workflow/)** - Detailed API documentation
|
171
|
-
- **[Optimized Tracing](docs/optimized-tracing.md)** - High-performance logging system (2-5x faster)
|
172
|
-
|
173
|
-
## ⚡ Performance Improvements
|
174
|
-
|
175
|
-
The workflow system now includes an optimized tracing system that provides significant performance improvements:
|
176
|
-
|
177
|
-
- **🚀 2-5x faster logging** with buffered I/O operations
|
178
|
-
- **💾 60-80% reduction** in disk I/O operations
|
179
|
-
- **🛡️ Built-in thread safety** with minimal overhead
|
180
|
-
- **🔄 Backward compatible** - existing code automatically benefits
|
181
|
-
- **📊 Lower memory footprint** for high-volume logging
|
182
|
-
|
183
|
-
See [Optimized Tracing Documentation](docs/optimized-tracing.md) for details and performance benchmarks.
|
184
171
|
|
185
172
|
## 🎯 Usage
|
186
173
|
|
@@ -316,21 +303,21 @@ it will use default value and do not raise any error to you.
|
|
316
303
|
> The config value that you will set on the environment should combine with
|
317
304
|
> prefix, component, and name which is `WORKFLOW_{component}_{name}` (Upper case).
|
318
305
|
|
319
|
-
| Name
|
320
|
-
|
321
|
-
| **REGISTRY_CALLER**
|
322
|
-
| **REGISTRY_FILTER**
|
323
|
-
| **CONF_PATH**
|
324
|
-
| **STAGE_DEFAULT_ID**
|
325
|
-
| **GENERATE_ID_SIMPLE_MODE**
|
326
|
-
| **DEBUG_MODE**
|
327
|
-
| **TIMEZONE**
|
328
|
-
| **FORMAT**
|
329
|
-
| **FORMAT_FILE**
|
330
|
-
| **DATETIME_FORMAT**
|
331
|
-
| **TRACE_HANDLERS**
|
332
|
-
| **
|
333
|
-
| **AUDIT_ENABLE_WRITE**
|
306
|
+
| Name | Component | Default | Description |
|
307
|
+
|:----------------------------|:---------:|:--------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------|
|
308
|
+
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
309
|
+
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
310
|
+
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
311
|
+
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
312
|
+
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
313
|
+
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
314
|
+
| **TIMEZONE** | LOG | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
315
|
+
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
316
|
+
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
317
|
+
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
318
|
+
| **TRACE_HANDLERS** | LOG | `[{"type": "console"}]` | A pointer URL of trace log that use to emit log message. Now uses optimized handler by default. |
|
319
|
+
| **AUDIT_CONF** | LOG | `{"type": "file", "path": "./audits"}` | A pointer URL of audit log that use to write audit metrix. |
|
320
|
+
| **AUDIT_ENABLE_WRITE** | LOG | `true` | A flag that enable writing audit log after end execution in the workflow release step. |
|
334
321
|
|
335
322
|
## :rocket: Deployment
|
336
323
|
|