ddeutil-workflow 0.0.73__py3-none-any.whl → 0.0.75__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__cron.py +20 -12
- ddeutil/workflow/__init__.py +119 -10
- ddeutil/workflow/__types.py +53 -41
- ddeutil/workflow/api/__init__.py +74 -3
- ddeutil/workflow/api/routes/job.py +15 -29
- ddeutil/workflow/api/routes/logs.py +9 -9
- ddeutil/workflow/api/routes/workflows.py +3 -3
- ddeutil/workflow/audits.py +70 -55
- ddeutil/workflow/cli.py +1 -15
- ddeutil/workflow/conf.py +71 -26
- ddeutil/workflow/errors.py +86 -19
- ddeutil/workflow/event.py +268 -169
- ddeutil/workflow/job.py +331 -192
- ddeutil/workflow/params.py +43 -11
- ddeutil/workflow/result.py +96 -70
- ddeutil/workflow/reusables.py +56 -6
- ddeutil/workflow/stages.py +1059 -572
- ddeutil/workflow/traces.py +205 -124
- ddeutil/workflow/utils.py +58 -19
- ddeutil/workflow/workflow.py +435 -296
- {ddeutil_workflow-0.0.73.dist-info → ddeutil_workflow-0.0.75.dist-info}/METADATA +27 -17
- ddeutil_workflow-0.0.75.dist-info/RECORD +30 -0
- ddeutil_workflow-0.0.73.dist-info/RECORD +0 -30
- {ddeutil_workflow-0.0.73.dist-info → ddeutil_workflow-0.0.75.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.73.dist-info → ddeutil_workflow-0.0.75.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.73.dist-info → ddeutil_workflow-0.0.75.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.73.dist-info → ddeutil_workflow-0.0.75.dist-info}/top_level.txt +0 -0
ddeutil/workflow/utils.py
CHANGED
@@ -3,7 +3,35 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
-
"""Utility
|
6
|
+
"""Utility Functions for Workflow Operations.
|
7
|
+
|
8
|
+
This module provides essential utility functions used throughout the workflow
|
9
|
+
system for ID generation, datetime handling, string processing, template
|
10
|
+
operations, and other common tasks.
|
11
|
+
|
12
|
+
Functions:
|
13
|
+
gen_id: Generate unique identifiers for workflow components
|
14
|
+
make_exec: Create executable strings for shell commands
|
15
|
+
filter_func: Filter functions based on criteria
|
16
|
+
dump_all: Serialize data to various formats
|
17
|
+
delay: Create delays in execution
|
18
|
+
to_train: Convert strings to train-case format
|
19
|
+
get_dt_now: Get current datetime with timezone
|
20
|
+
get_d_now: Get current date
|
21
|
+
cross_product: Generate cross product of matrix values
|
22
|
+
replace_sec: Replace template variables in strings
|
23
|
+
|
24
|
+
Example:
|
25
|
+
```python
|
26
|
+
from ddeutil.workflow.utils import gen_id, get_dt_now
|
27
|
+
|
28
|
+
# Generate unique ID
|
29
|
+
run_id = gen_id("workflow")
|
30
|
+
|
31
|
+
# Get current datetime
|
32
|
+
now = get_dt_now()
|
33
|
+
```
|
34
|
+
"""
|
7
35
|
from __future__ import annotations
|
8
36
|
|
9
37
|
import stat
|
@@ -27,13 +55,19 @@ T = TypeVar("T")
|
|
27
55
|
UTC: Final[ZoneInfo] = ZoneInfo("UTC")
|
28
56
|
MARK_NEWLINE: Final[str] = "||"
|
29
57
|
|
58
|
+
# Cache for random delay values to avoid repeated randrange calls
|
59
|
+
_CACHED_DELAYS = [randrange(0, 99, step=10) / 100 for _ in range(100)]
|
60
|
+
_DELAY_INDEX = 0
|
61
|
+
|
30
62
|
|
31
63
|
def to_train(camel: str) -> str:
|
32
64
|
"""Convert camel case string to train case.
|
33
65
|
|
34
|
-
:
|
66
|
+
Args:
|
67
|
+
camel: A camel case string that want to convert.
|
35
68
|
|
36
|
-
:
|
69
|
+
Returns:
|
70
|
+
str: The converted train-case string.
|
37
71
|
"""
|
38
72
|
return "".join("-" + i if i.isupper() else i for i in camel).lstrip("-")
|
39
73
|
|
@@ -41,9 +75,11 @@ def to_train(camel: str) -> str:
|
|
41
75
|
def prepare_newline(msg: str) -> str:
|
42
76
|
"""Prepare message that has multiple newline char.
|
43
77
|
|
44
|
-
:
|
78
|
+
Args:
|
79
|
+
msg: A message that want to prepare.
|
45
80
|
|
46
|
-
:
|
81
|
+
Returns:
|
82
|
+
str: The prepared message with formatted newlines.
|
47
83
|
"""
|
48
84
|
# NOTE: Remove ending with "\n" and replace "\n" with the "||" value.
|
49
85
|
msg: str = msg.strip("\n").replace("\n", MARK_NEWLINE)
|
@@ -63,42 +99,42 @@ def prepare_newline(msg: str) -> str:
|
|
63
99
|
def replace_sec(dt: datetime) -> datetime:
|
64
100
|
"""Replace second and microsecond values to 0.
|
65
101
|
|
66
|
-
:
|
102
|
+
Args:
|
103
|
+
dt: A datetime object that want to replace.
|
67
104
|
|
68
|
-
:
|
105
|
+
Returns:
|
106
|
+
datetime: The datetime with seconds and microseconds set to 0.
|
69
107
|
"""
|
70
108
|
return dt.replace(second=0, microsecond=0)
|
71
109
|
|
72
110
|
|
73
111
|
def clear_tz(dt: datetime) -> datetime:
|
74
|
-
"""Replace timezone info on an input datetime object to
|
75
|
-
return dt.replace(tzinfo=
|
112
|
+
"""Replace timezone info on an input datetime object to UTC."""
|
113
|
+
return dt.replace(tzinfo=UTC)
|
76
114
|
|
77
115
|
|
78
|
-
def get_dt_now(
|
116
|
+
def get_dt_now(offset: float = 0.0) -> datetime:
|
79
117
|
"""Return the current datetime object.
|
80
118
|
|
81
|
-
:param tz: A ZoneInfo object for replace timezone of return datetime object.
|
82
119
|
:param offset: An offset second value.
|
83
120
|
|
84
121
|
:rtype: datetime
|
85
122
|
:return: The current datetime object that use an input timezone or UTC.
|
86
123
|
"""
|
87
|
-
return datetime.now(
|
124
|
+
return datetime.now().replace(tzinfo=UTC) - timedelta(seconds=offset)
|
88
125
|
|
89
126
|
|
90
|
-
def get_d_now(
|
91
|
-
tz: Optional[ZoneInfo] = None, offset: float = 0.0
|
92
|
-
) -> date: # pragma: no cov
|
127
|
+
def get_d_now(offset: float = 0.0) -> date: # pragma: no cov
|
93
128
|
"""Return the current date object.
|
94
129
|
|
95
|
-
:param tz: A ZoneInfo object for replace timezone of return date object.
|
96
130
|
:param offset: An offset second value.
|
97
131
|
|
98
132
|
:rtype: date
|
99
133
|
:return: The current date object that use an input timezone or UTC.
|
100
134
|
"""
|
101
|
-
return (
|
135
|
+
return (
|
136
|
+
datetime.now().replace(tzinfo=UTC) - timedelta(seconds=offset)
|
137
|
+
).date()
|
102
138
|
|
103
139
|
|
104
140
|
def get_diff_sec(dt: datetime, offset: float = 0.0) -> int:
|
@@ -151,7 +187,10 @@ def delay(second: float = 0) -> None: # pragma: no cov
|
|
151
187
|
|
152
188
|
:param second: (float) A second number that want to adds-on random value.
|
153
189
|
"""
|
154
|
-
|
190
|
+
global _DELAY_INDEX
|
191
|
+
cached_random = _CACHED_DELAYS[_DELAY_INDEX % len(_CACHED_DELAYS)]
|
192
|
+
_DELAY_INDEX = (_DELAY_INDEX + 1) % len(_CACHED_DELAYS)
|
193
|
+
time.sleep(second + cached_random)
|
155
194
|
|
156
195
|
|
157
196
|
def gen_id(
|
@@ -188,7 +227,7 @@ def gen_id(
|
|
188
227
|
if not isinstance(value, str):
|
189
228
|
value: str = str(value)
|
190
229
|
|
191
|
-
dt: datetime = datetime.now(tz=
|
230
|
+
dt: datetime = datetime.now(tz=UTC)
|
192
231
|
if dynamic("generate_id_simple_mode", f=simple_mode, extras=extras):
|
193
232
|
return (f"{dt:%Y%m%d%H%M%S%f}T" if unique else "") + hash_str(
|
194
233
|
f"{(value if sensitive else value.lower())}", n=10
|