ddeutil-workflow 0.0.74__tar.gz → 0.0.75__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/PKG-INFO +3 -3
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/README.md +2 -2
- ddeutil_workflow-0.0.75/src/ddeutil/workflow/__about__.py +1 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/__cron.py +6 -4
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/conf.py +10 -10
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/params.py +6 -4
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/result.py +3 -3
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/traces.py +6 -4
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/utils.py +9 -22
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/workflow.py +14 -9
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil_workflow.egg-info/PKG-INFO +3 -3
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_conf.py +6 -6
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_event.py +10 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_params.py +11 -4
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_reusables_template.py +1 -1
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_utils.py +3 -6
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_workflow_exec.py +9 -6
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_workflow_release.py +11 -7
- ddeutil_workflow-0.0.74/src/ddeutil/workflow/__about__.py +0 -1
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/LICENSE +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/pyproject.toml +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/setup.cfg +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/__init__.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/__main__.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/__types.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/api/__init__.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/api/log_conf.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/api/routes/__init__.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/api/routes/job.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/api/routes/logs.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/api/routes/workflows.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/audits.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/cli.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/errors.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/event.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/job.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/reusables.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/stages.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil_workflow.egg-info/SOURCES.txt +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil_workflow.egg-info/dependency_links.txt +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil_workflow.egg-info/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil_workflow.egg-info/requires.txt +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil_workflow.egg-info/top_level.txt +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test__cron.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test__regex.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_audits.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_cli.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_errors.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_job.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_job_exec.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_job_exec_strategy.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_result.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_reusables_call_tag.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_reusables_func_model.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_reusables_template_filter.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_strategy.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_traces.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_workflow.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_workflow_exec_job.py +0 -0
- {ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/tests/test_workflow_rerun.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.75
|
4
4
|
Summary: Lightweight workflow orchestration with YAML template
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -68,7 +68,7 @@ by a `.yaml` template.
|
|
68
68
|
3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
|
69
69
|
(Python 3.13 unlock GIL 🐍🔓)
|
70
70
|
4. Recommend to pass a **Secret Value** with environment variable in YAML template 🔐
|
71
|
-
5. Any datatime value convert to **
|
71
|
+
5. Any datatime value convert to **UTC Timezone** 🌐
|
72
72
|
|
73
73
|
---
|
74
74
|
|
@@ -288,10 +288,10 @@ it will use default value and do not raise any error to you.
|
|
288
288
|
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
289
289
|
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
290
290
|
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
291
|
-
| **TIMEZONE** | CORE | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
292
291
|
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
293
292
|
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
294
293
|
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
294
|
+
| **TIMEZONE** | LOG | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
295
295
|
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
296
296
|
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
297
297
|
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
@@ -26,7 +26,7 @@ by a `.yaml` template.
|
|
26
26
|
3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
|
27
27
|
(Python 3.13 unlock GIL 🐍🔓)
|
28
28
|
4. Recommend to pass a **Secret Value** with environment variable in YAML template 🔐
|
29
|
-
5. Any datatime value convert to **
|
29
|
+
5. Any datatime value convert to **UTC Timezone** 🌐
|
30
30
|
|
31
31
|
---
|
32
32
|
|
@@ -246,10 +246,10 @@ it will use default value and do not raise any error to you.
|
|
246
246
|
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
247
247
|
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
248
248
|
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
249
|
-
| **TIMEZONE** | CORE | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
250
249
|
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
251
250
|
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
252
251
|
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
252
|
+
| **TIMEZONE** | LOG | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
253
253
|
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
254
254
|
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
255
255
|
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
@@ -0,0 +1 @@
|
|
1
|
+
__version__: str = "0.0.75"
|
@@ -793,10 +793,11 @@ class CronRunner:
|
|
793
793
|
"Invalid type of `tz` parameter, it should be str or "
|
794
794
|
"ZoneInfo instance."
|
795
795
|
)
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
796
|
+
else:
|
797
|
+
try:
|
798
|
+
self.tz = ZoneInfo(tz)
|
799
|
+
except ZoneInfoNotFoundError as err:
|
800
|
+
raise ValueError(f"Invalid timezone: {tz}") from err
|
800
801
|
|
801
802
|
# NOTE: Prepare date
|
802
803
|
if date:
|
@@ -807,6 +808,7 @@ class CronRunner:
|
|
807
808
|
if tz is not None:
|
808
809
|
self.date: datetime = date.astimezone(self.tz)
|
809
810
|
else:
|
811
|
+
self.tz = date.tzinfo
|
810
812
|
self.date: datetime = date
|
811
813
|
else:
|
812
814
|
self.date: datetime = datetime.now(tz=self.tz)
|
@@ -89,16 +89,6 @@ class Config: # pragma: no cov
|
|
89
89
|
"""
|
90
90
|
return Path(env("CORE_CONF_PATH", "./conf"))
|
91
91
|
|
92
|
-
@property
|
93
|
-
def tz(self) -> ZoneInfo:
|
94
|
-
"""Timezone value that return with the `ZoneInfo` object and use for all
|
95
|
-
datetime object in this workflow engine.
|
96
|
-
|
97
|
-
Returns:
|
98
|
-
ZoneInfo: The timezone configuration for the workflow engine.
|
99
|
-
"""
|
100
|
-
return ZoneInfo(env("CORE_TIMEZONE", "UTC"))
|
101
|
-
|
102
92
|
@property
|
103
93
|
def generate_id_simple_mode(self) -> bool:
|
104
94
|
"""Flag for generate running ID with simple mode. That does not use
|
@@ -143,6 +133,16 @@ class Config: # pragma: no cov
|
|
143
133
|
"""
|
144
134
|
return str2bool(env("LOG_DEBUG_MODE", "true"))
|
145
135
|
|
136
|
+
@property
|
137
|
+
def log_tz(self) -> ZoneInfo:
|
138
|
+
"""Timezone value that return with the `ZoneInfo` object and use for all
|
139
|
+
datetime object in this workflow engine.
|
140
|
+
|
141
|
+
Returns:
|
142
|
+
ZoneInfo: The timezone configuration for the workflow engine.
|
143
|
+
"""
|
144
|
+
return ZoneInfo(env("LOG_TIMEZONE", "UTC"))
|
145
|
+
|
146
146
|
@property
|
147
147
|
def log_format(self) -> str:
|
148
148
|
return env(
|
@@ -52,7 +52,7 @@ from pydantic import BaseModel, Field
|
|
52
52
|
|
53
53
|
from .__types import StrOrInt
|
54
54
|
from .errors import ParamError
|
55
|
-
from .utils import get_d_now, get_dt_now
|
55
|
+
from .utils import UTC, get_d_now, get_dt_now
|
56
56
|
|
57
57
|
T = TypeVar("T")
|
58
58
|
|
@@ -169,16 +169,18 @@ class DatetimeParam(DefaultParam):
|
|
169
169
|
return self.default
|
170
170
|
|
171
171
|
if isinstance(value, datetime):
|
172
|
-
|
172
|
+
if value.tzinfo is None:
|
173
|
+
return value.replace(tzinfo=UTC)
|
174
|
+
return value.astimezone(UTC)
|
173
175
|
elif isinstance(value, date):
|
174
|
-
return datetime(value.year, value.month, value.day)
|
176
|
+
return datetime(value.year, value.month, value.day, tzinfo=UTC)
|
175
177
|
elif not isinstance(value, str):
|
176
178
|
raise ParamError(
|
177
179
|
f"Value that want to convert to datetime does not support for "
|
178
180
|
f"type: {type(value)}"
|
179
181
|
)
|
180
182
|
try:
|
181
|
-
return datetime.fromisoformat(value)
|
183
|
+
return datetime.fromisoformat(value).replace(tzinfo=UTC)
|
182
184
|
except ValueError:
|
183
185
|
raise ParamError(
|
184
186
|
f"Invalid the ISO format string for datetime: {value!r}"
|
@@ -43,7 +43,7 @@ from . import (
|
|
43
43
|
from .__types import DictData
|
44
44
|
from .audits import Trace, get_trace
|
45
45
|
from .errors import ResultError
|
46
|
-
from .utils import default_gen_id,
|
46
|
+
from .utils import default_gen_id, get_dt_now
|
47
47
|
|
48
48
|
|
49
49
|
class Status(str, Enum):
|
@@ -190,7 +190,7 @@ class Result:
|
|
190
190
|
info: DictData = field(default_factory=dict)
|
191
191
|
run_id: Optional[str] = field(default_factory=default_gen_id)
|
192
192
|
parent_run_id: Optional[str] = field(default=None, compare=False)
|
193
|
-
ts: datetime = field(default_factory=
|
193
|
+
ts: datetime = field(default_factory=get_dt_now, compare=False)
|
194
194
|
trace: Optional[Trace] = field(default=None, compare=False, repr=False)
|
195
195
|
extras: DictData = field(default_factory=dict, compare=False, repr=False)
|
196
196
|
|
@@ -266,7 +266,7 @@ class Result:
|
|
266
266
|
|
267
267
|
:rtype: float
|
268
268
|
"""
|
269
|
-
return (
|
269
|
+
return (get_dt_now() - self.ts).total_seconds()
|
270
270
|
|
271
271
|
|
272
272
|
def catch(
|
@@ -167,7 +167,9 @@ class TraceMeta(BaseModel): # pragma: no cov
|
|
167
167
|
|
168
168
|
mode: Literal["stdout", "stderr"] = Field(description="A meta mode.")
|
169
169
|
level: str = Field(description="A log level.")
|
170
|
-
datetime: str = Field(
|
170
|
+
datetime: str = Field(
|
171
|
+
description="A datetime string with the specific config format."
|
172
|
+
)
|
171
173
|
process: int = Field(description="A process ID.")
|
172
174
|
thread: int = Field(description="A thread ID.")
|
173
175
|
message: str = Field(description="A message log.")
|
@@ -234,9 +236,9 @@ class TraceMeta(BaseModel): # pragma: no cov
|
|
234
236
|
mode=mode,
|
235
237
|
level=level,
|
236
238
|
datetime=(
|
237
|
-
get_dt_now(
|
238
|
-
|
239
|
-
)
|
239
|
+
get_dt_now()
|
240
|
+
.astimezone(dynamic("log_tz", extras=extras))
|
241
|
+
.strftime(dynamic("log_datetime_format", extras=extras))
|
240
242
|
),
|
241
243
|
process=os.getpid(),
|
242
244
|
thread=get_ident(),
|
@@ -109,45 +109,32 @@ def replace_sec(dt: datetime) -> datetime:
|
|
109
109
|
|
110
110
|
|
111
111
|
def clear_tz(dt: datetime) -> datetime:
|
112
|
-
"""Replace timezone info on an input datetime object to
|
113
|
-
return dt.replace(tzinfo=
|
112
|
+
"""Replace timezone info on an input datetime object to UTC."""
|
113
|
+
return dt.replace(tzinfo=UTC)
|
114
114
|
|
115
115
|
|
116
|
-
def get_dt_now(
|
116
|
+
def get_dt_now(offset: float = 0.0) -> datetime:
|
117
117
|
"""Return the current datetime object.
|
118
118
|
|
119
|
-
:param tz: A ZoneInfo object for replace timezone of return datetime object.
|
120
119
|
:param offset: An offset second value.
|
121
120
|
|
122
121
|
:rtype: datetime
|
123
122
|
:return: The current datetime object that use an input timezone or UTC.
|
124
123
|
"""
|
125
|
-
return datetime.now(
|
124
|
+
return datetime.now().replace(tzinfo=UTC) - timedelta(seconds=offset)
|
126
125
|
|
127
126
|
|
128
|
-
def
|
129
|
-
"""Get current datetime with no timezone.
|
130
|
-
|
131
|
-
Returns the current datetime object using the None timezone.
|
132
|
-
|
133
|
-
Returns:
|
134
|
-
datetime: Current datetime with no timezone
|
135
|
-
"""
|
136
|
-
return get_dt_now(tz=None)
|
137
|
-
|
138
|
-
|
139
|
-
def get_d_now(
|
140
|
-
tz: Optional[ZoneInfo] = None, offset: float = 0.0
|
141
|
-
) -> date: # pragma: no cov
|
127
|
+
def get_d_now(offset: float = 0.0) -> date: # pragma: no cov
|
142
128
|
"""Return the current date object.
|
143
129
|
|
144
|
-
:param tz: A ZoneInfo object for replace timezone of return date object.
|
145
130
|
:param offset: An offset second value.
|
146
131
|
|
147
132
|
:rtype: date
|
148
133
|
:return: The current date object that use an input timezone or UTC.
|
149
134
|
"""
|
150
|
-
return (
|
135
|
+
return (
|
136
|
+
datetime.now().replace(tzinfo=UTC) - timedelta(seconds=offset)
|
137
|
+
).date()
|
151
138
|
|
152
139
|
|
153
140
|
def get_diff_sec(dt: datetime, offset: float = 0.0) -> int:
|
@@ -240,7 +227,7 @@ def gen_id(
|
|
240
227
|
if not isinstance(value, str):
|
241
228
|
value: str = str(value)
|
242
229
|
|
243
|
-
dt: datetime = datetime.now(tz=
|
230
|
+
dt: datetime = datetime.now(tz=UTC)
|
244
231
|
if dynamic("generate_id_simple_mode", f=simple_mode, extras=extras):
|
245
232
|
return (f"{dt:%Y%m%d%H%M%S%f}T" if unique else "") + hash_str(
|
246
233
|
f"{(value if sensitive else value.lower())}", n=10
|
@@ -36,7 +36,6 @@ from queue import Queue
|
|
36
36
|
from textwrap import dedent
|
37
37
|
from threading import Event as ThreadEvent
|
38
38
|
from typing import Any, Optional, Union
|
39
|
-
from zoneinfo import ZoneInfo
|
40
39
|
|
41
40
|
from pydantic import BaseModel, Field
|
42
41
|
from pydantic.functional_validators import field_validator, model_validator
|
@@ -64,8 +63,9 @@ from .result import (
|
|
64
63
|
from .reusables import has_template, param2template
|
65
64
|
from .traces import Trace, get_trace
|
66
65
|
from .utils import (
|
66
|
+
UTC,
|
67
67
|
gen_id,
|
68
|
-
|
68
|
+
get_dt_now,
|
69
69
|
replace_sec,
|
70
70
|
)
|
71
71
|
|
@@ -153,14 +153,14 @@ class Workflow(BaseModel):
|
|
153
153
|
description="A mapping of job ID and job model that already loaded.",
|
154
154
|
)
|
155
155
|
created_at: datetime = Field(
|
156
|
-
default_factory=
|
156
|
+
default_factory=get_dt_now,
|
157
157
|
description=(
|
158
158
|
"A created datetime of this workflow template when loading from "
|
159
159
|
"file."
|
160
160
|
),
|
161
161
|
)
|
162
162
|
updated_dt: datetime = Field(
|
163
|
-
default_factory=
|
163
|
+
default_factory=get_dt_now,
|
164
164
|
description=(
|
165
165
|
"A updated datetime of this workflow template when loading from "
|
166
166
|
"file."
|
@@ -369,12 +369,15 @@ class Workflow(BaseModel):
|
|
369
369
|
Returns:
|
370
370
|
datetime: The validated release datetime.
|
371
371
|
"""
|
372
|
-
|
372
|
+
if dt.tzinfo is None:
|
373
|
+
dt = dt.replace(tzinfo=UTC)
|
374
|
+
|
375
|
+
release: datetime = replace_sec(dt.astimezone(UTC))
|
373
376
|
if not self.on:
|
374
377
|
return release
|
375
378
|
|
376
379
|
for on in self.on.schedule:
|
377
|
-
if release == on.cronjob.schedule(release).next:
|
380
|
+
if release == on.cronjob.schedule(release, tz=UTC).next:
|
378
381
|
return release
|
379
382
|
raise WorkflowError(
|
380
383
|
"Release datetime does not support for this workflow"
|
@@ -385,8 +388,8 @@ class Workflow(BaseModel):
|
|
385
388
|
release: datetime,
|
386
389
|
params: DictData,
|
387
390
|
*,
|
388
|
-
release_type: ReleaseType = NORMAL,
|
389
391
|
run_id: Optional[str] = None,
|
392
|
+
release_type: ReleaseType = NORMAL,
|
390
393
|
audit: type[Audit] = None,
|
391
394
|
override_log_name: Optional[str] = None,
|
392
395
|
timeout: int = 600,
|
@@ -420,25 +423,27 @@ class Workflow(BaseModel):
|
|
420
423
|
:rtype: Result
|
421
424
|
"""
|
422
425
|
name: str = override_log_name or self.name
|
426
|
+
|
427
|
+
# NOTE: Generate the parent running ID with not None value.
|
423
428
|
if run_id:
|
424
429
|
parent_run_id: str = run_id
|
425
430
|
run_id: str = gen_id(name, unique=True)
|
426
431
|
else:
|
427
432
|
run_id: str = gen_id(name, unique=True)
|
428
433
|
parent_run_id: str = run_id
|
434
|
+
|
429
435
|
context: DictData = {}
|
430
436
|
trace: Trace = get_trace(
|
431
437
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
432
438
|
)
|
433
439
|
release: datetime = self.validate_release(dt=release)
|
434
440
|
trace.info(f"[RELEASE]: Start {name!r} : {release:%Y-%m-%d %H:%M:%S}")
|
435
|
-
tz: ZoneInfo = dynamic("tz", extras=self.extras)
|
436
441
|
values: DictData = param2template(
|
437
442
|
params,
|
438
443
|
params={
|
439
444
|
"release": {
|
440
445
|
"logical_date": release,
|
441
|
-
"execute_date":
|
446
|
+
"execute_date": get_dt_now(),
|
442
447
|
"run_id": run_id,
|
443
448
|
}
|
444
449
|
},
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.75
|
4
4
|
Summary: Lightweight workflow orchestration with YAML template
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -68,7 +68,7 @@ by a `.yaml` template.
|
|
68
68
|
3. All parallel tasks inside workflow core engine use **Multi-Threading** pool
|
69
69
|
(Python 3.13 unlock GIL 🐍🔓)
|
70
70
|
4. Recommend to pass a **Secret Value** with environment variable in YAML template 🔐
|
71
|
-
5. Any datatime value convert to **
|
71
|
+
5. Any datatime value convert to **UTC Timezone** 🌐
|
72
72
|
|
73
73
|
---
|
74
74
|
|
@@ -288,10 +288,10 @@ it will use default value and do not raise any error to you.
|
|
288
288
|
| **REGISTRY_CALLER** | CORE | `.` | List of importable string for the call stage. |
|
289
289
|
| **REGISTRY_FILTER** | CORE | `ddeutil.workflow.templates` | List of importable string for the filter template. |
|
290
290
|
| **CONF_PATH** | CORE | `./conf` | The config path that keep all template `.yaml` files. |
|
291
|
-
| **TIMEZONE** | CORE | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
292
291
|
| **STAGE_DEFAULT_ID** | CORE | `false` | A flag that enable default stage ID that use for catch an execution output. |
|
293
292
|
| **GENERATE_ID_SIMPLE_MODE** | CORE | `true` | A flog that enable generating ID with `md5` algorithm. |
|
294
293
|
| **DEBUG_MODE** | LOG | `true` | A flag that enable logging with debug level mode. |
|
294
|
+
| **TIMEZONE** | LOG | `Asia/Bangkok` | A Timezone string value that will pass to `ZoneInfo` object. |
|
295
295
|
| **FORMAT** | LOG | `%(asctime)s.%(msecs)03d (%(name)-10s, %(process)-5d,%(thread)-5d) [%(levelname)-7s] %(message)-120s (%(filename)s:%(lineno)s)` | A trace message console format. |
|
296
296
|
| **FORMAT_FILE** | LOG | `{datetime} ({process:5d}, {thread:5d}) {message:120s} ({filename}:{lineno})` | A trace message format that use to write to target pointer. |
|
297
297
|
| **DATETIME_FORMAT** | LOG | `%Y-%m-%d %H:%M:%S` | A datetime format of the trace log. |
|
@@ -22,8 +22,8 @@ from .utils import exclude_created_and_updated
|
|
22
22
|
|
23
23
|
def test_config():
|
24
24
|
conf = Config()
|
25
|
-
os.environ["
|
26
|
-
assert conf.
|
25
|
+
os.environ["WORKFLOW_LOG_TIMEZONE"] = "Asia/Bangkok"
|
26
|
+
assert conf.log_tz == ZoneInfo("Asia/Bangkok")
|
27
27
|
|
28
28
|
|
29
29
|
@pytest.fixture(scope="module")
|
@@ -35,7 +35,7 @@ def target_path(test_path):
|
|
35
35
|
json.dump({"foo": "bar"}, f)
|
36
36
|
|
37
37
|
with (target_p / "test_simple_file.toml").open(mode="w") as f:
|
38
|
-
rtoml.dump({"foo": "bar", "env": "${
|
38
|
+
rtoml.dump({"foo": "bar", "env": "${ WORKFLOW_LOG_TIMEZONE }"}, f)
|
39
39
|
|
40
40
|
yield target_p
|
41
41
|
|
@@ -60,7 +60,7 @@ def test_load_file(target_path: Path):
|
|
60
60
|
"test_load_file": {
|
61
61
|
"type": "Workflow",
|
62
62
|
"desc": "Test multi config path",
|
63
|
-
"env": "${
|
63
|
+
"env": "${WORKFLOW_LOG_TIMEZONE}",
|
64
64
|
}
|
65
65
|
},
|
66
66
|
f,
|
@@ -70,7 +70,7 @@ def test_load_file(target_path: Path):
|
|
70
70
|
assert exclude_created_and_updated(load.data) == {
|
71
71
|
"type": "Workflow",
|
72
72
|
"desc": "Test multi config path",
|
73
|
-
"env": "${
|
73
|
+
"env": "${WORKFLOW_LOG_TIMEZONE}",
|
74
74
|
}
|
75
75
|
assert pass_env(load.data["env"]) == "Asia/Bangkok"
|
76
76
|
assert exclude_created_and_updated(pass_env(load.data)) == {
|
@@ -85,7 +85,7 @@ def test_load_file(target_path: Path):
|
|
85
85
|
assert exclude_created_and_updated(load.data) == {
|
86
86
|
"type": "Workflow",
|
87
87
|
"desc": "Test multi config path",
|
88
|
-
"env": "${
|
88
|
+
"env": "${WORKFLOW_LOG_TIMEZONE}",
|
89
89
|
}
|
90
90
|
|
91
91
|
# NOTE: Raise because passing `conf_paths` invalid type.
|
@@ -34,6 +34,16 @@ def test_localize_timezone():
|
|
34
34
|
assert bkk_dt.replace(tzinfo=None) == datetime(2024, 1, 1, 19)
|
35
35
|
|
36
36
|
|
37
|
+
def test_convert_timezone():
|
38
|
+
ntz_dt = datetime(2025, 6, 18)
|
39
|
+
print(ntz_dt)
|
40
|
+
print(ntz_dt.astimezone(ZoneInfo("UTC")))
|
41
|
+
|
42
|
+
utc_dt = ntz_dt.replace(tzinfo=ZoneInfo("UTC"))
|
43
|
+
print(utc_dt)
|
44
|
+
print(utc_dt.astimezone(ZoneInfo("UTC")))
|
45
|
+
|
46
|
+
|
37
47
|
def test_interval2crontab():
|
38
48
|
assert interval2crontab(interval="daily", time="01:30") == "1 30 * * *"
|
39
49
|
assert (
|
@@ -2,6 +2,7 @@ from datetime import date, datetime
|
|
2
2
|
from decimal import Decimal
|
3
3
|
|
4
4
|
import pytest
|
5
|
+
from ddeutil.workflow import UTC
|
5
6
|
from ddeutil.workflow.errors import ParamError
|
6
7
|
from ddeutil.workflow.params import (
|
7
8
|
ArrayParam,
|
@@ -64,9 +65,15 @@ def test_param_date_default():
|
|
64
65
|
|
65
66
|
|
66
67
|
def test_param_datetime():
|
67
|
-
assert DatetimeParam().receive("2024-01-01") == datetime(
|
68
|
-
|
69
|
-
|
68
|
+
assert DatetimeParam().receive("2024-01-01") == datetime(
|
69
|
+
2024, 1, 1, tzinfo=UTC
|
70
|
+
)
|
71
|
+
assert DatetimeParam().receive(date(2024, 1, 1)) == datetime(
|
72
|
+
2024, 1, 1, tzinfo=UTC
|
73
|
+
)
|
74
|
+
assert DatetimeParam().receive(datetime(2024, 1, 1)) == datetime(
|
75
|
+
2024, 1, 1, tzinfo=UTC
|
76
|
+
)
|
70
77
|
|
71
78
|
with pytest.raises(ParamError):
|
72
79
|
DatetimeParam().receive(2024)
|
@@ -77,7 +84,7 @@ def test_param_datetime():
|
|
77
84
|
|
78
85
|
@freeze_time("2024-01-01 00:00:00")
|
79
86
|
def test_param_datetime_default():
|
80
|
-
assert DatetimeParam().receive() == datetime(2024, 1, 1)
|
87
|
+
assert DatetimeParam().receive() == datetime(2024, 1, 1, tzinfo=UTC)
|
81
88
|
|
82
89
|
|
83
90
|
def test_param_int():
|
@@ -32,7 +32,7 @@ def test_param2template():
|
|
32
32
|
"int_but_str": "value is ${{ params.value | abs}}",
|
33
33
|
"list": ["${{ params.src }}", "${{ params.value }}"],
|
34
34
|
"str_env": (
|
35
|
-
"${{ params.src }}-${
|
35
|
+
"${{ params.src }}-${WORKFLOW_LOG_TIMEZONE:-}"
|
36
36
|
"${WORKFLOW_DUMMY:-}"
|
37
37
|
),
|
38
38
|
},
|
@@ -34,13 +34,10 @@ def adjust_config_gen_id():
|
|
34
34
|
@freeze_time("2024-01-01 01:13:30")
|
35
35
|
def test_get_dt_now():
|
36
36
|
rs = get_dt_now()
|
37
|
-
assert rs == datetime(2024, 1, 1, 1, 13, 30)
|
38
|
-
|
39
|
-
rs = get_dt_now(tz=ZoneInfo("UTC"))
|
40
37
|
assert rs == datetime(2024, 1, 1, 1, 13, 30, tzinfo=ZoneInfo("UTC"))
|
41
38
|
|
42
39
|
rs = get_dt_now(offset=30)
|
43
|
-
assert rs == datetime(2024, 1, 1, 1, 13, 00)
|
40
|
+
assert rs == datetime(2024, 1, 1, 1, 13, 00, tzinfo=ZoneInfo("UTC"))
|
44
41
|
|
45
42
|
rs = get_d_now()
|
46
43
|
assert rs == date(2024, 1, 1)
|
@@ -53,8 +50,8 @@ def test_gen_id():
|
|
53
50
|
|
54
51
|
@freeze_time("2024-01-01 01:13:30")
|
55
52
|
def test_gen_id_unique():
|
56
|
-
assert "
|
57
|
-
assert "
|
53
|
+
assert "20240101011330000000T1354680202" == gen_id("{}", unique=True)
|
54
|
+
assert "20240101011330000000T1354680202" == gen_id(
|
58
55
|
"{}", unique=True, sensitive=False
|
59
56
|
)
|
60
57
|
|
@@ -8,6 +8,7 @@ from ddeutil.workflow import (
|
|
8
8
|
FAILED,
|
9
9
|
SKIP,
|
10
10
|
SUCCESS,
|
11
|
+
UTC,
|
11
12
|
Job,
|
12
13
|
Result,
|
13
14
|
Workflow,
|
@@ -124,7 +125,7 @@ def test_workflow_exec_py():
|
|
124
125
|
"status": SUCCESS,
|
125
126
|
"params": {
|
126
127
|
"author-run": "Local Workflow",
|
127
|
-
"run-date": datetime(2024, 1, 1, 0, 0),
|
128
|
+
"run-date": datetime(2024, 1, 1, 0, 0, tzinfo=UTC),
|
128
129
|
},
|
129
130
|
"jobs": {
|
130
131
|
"first-job": {
|
@@ -259,7 +260,7 @@ def test_workflow_exec_py_with_parallel():
|
|
259
260
|
"status": SUCCESS,
|
260
261
|
"params": {
|
261
262
|
"author-run": "Local Workflow",
|
262
|
-
"run-date": datetime(2024, 1, 1, 0, 0),
|
263
|
+
"run-date": datetime(2024, 1, 1, 0, 0, tzinfo=UTC),
|
263
264
|
},
|
264
265
|
"jobs": {
|
265
266
|
"first-job": {
|
@@ -586,7 +587,7 @@ def test_workflow_exec_call(test_path):
|
|
586
587
|
assert rs.context == {
|
587
588
|
"status": SUCCESS,
|
588
589
|
"params": {
|
589
|
-
"run-date": datetime(2024, 1, 1, 0, 0),
|
590
|
+
"run-date": datetime(2024, 1, 1, 0, 0, tzinfo=UTC),
|
590
591
|
"source": "ds_csv_local_file",
|
591
592
|
"sink": "ds_parquet_local_file_dir",
|
592
593
|
},
|
@@ -705,7 +706,7 @@ def test_workflow_exec_call_with_prefix(test_path):
|
|
705
706
|
assert rs.context == {
|
706
707
|
"status": SUCCESS,
|
707
708
|
"params": {
|
708
|
-
"run_date": datetime(2024, 1, 1, 0, 0),
|
709
|
+
"run_date": datetime(2024, 1, 1, 0, 0, tzinfo=UTC),
|
709
710
|
"sp_name": "proc-name",
|
710
711
|
"source_name": "src",
|
711
712
|
"target_name": "tgt",
|
@@ -719,7 +720,9 @@ def test_workflow_exec_call_with_prefix(test_path):
|
|
719
720
|
"exec": "proc-name",
|
720
721
|
"params": {
|
721
722
|
"run_mode": "T",
|
722
|
-
"run_date": datetime(
|
723
|
+
"run_date": datetime(
|
724
|
+
2024, 1, 1, 0, 0, tzinfo=UTC
|
725
|
+
),
|
723
726
|
"source": "src",
|
724
727
|
"target": "tgt",
|
725
728
|
},
|
@@ -738,7 +741,7 @@ def test_workflow_exec_trigger():
|
|
738
741
|
rs = job.set_outputs(job.execute(params={}).context, to={})
|
739
742
|
assert {
|
740
743
|
"author-run": "Trigger Runner",
|
741
|
-
"run-date": datetime(2024, 8, 1),
|
744
|
+
"run-date": datetime(2024, 8, 1, tzinfo=UTC),
|
742
745
|
} == getdot("jobs.trigger-job.stages.trigger-stage.outputs.params", rs)
|
743
746
|
|
744
747
|
|
@@ -1,12 +1,14 @@
|
|
1
1
|
from datetime import datetime
|
2
|
+
from zoneinfo import ZoneInfo
|
2
3
|
|
3
4
|
import pytest
|
4
|
-
from ddeutil.workflow import
|
5
|
-
from ddeutil.workflow.conf import config
|
6
|
-
from ddeutil.workflow.result import SUCCESS, Result
|
7
|
-
from ddeutil.workflow.workflow import (
|
5
|
+
from ddeutil.workflow import (
|
8
6
|
NORMAL,
|
7
|
+
SUCCESS,
|
8
|
+
UTC,
|
9
|
+
Result,
|
9
10
|
Workflow,
|
11
|
+
WorkflowError,
|
10
12
|
)
|
11
13
|
|
12
14
|
|
@@ -84,7 +86,7 @@ def test_workflow_release():
|
|
84
86
|
"params": {"asat-dt": datetime(2024, 10, 1, 0, 0)},
|
85
87
|
"release": {
|
86
88
|
"type": NORMAL,
|
87
|
-
"logical_date": release,
|
89
|
+
"logical_date": release.replace(tzinfo=UTC),
|
88
90
|
},
|
89
91
|
"jobs": {
|
90
92
|
"first-job": {
|
@@ -119,18 +121,20 @@ def test_workflow_release_with_datetime():
|
|
119
121
|
"extra": {"enable_write_audit": False},
|
120
122
|
}
|
121
123
|
)
|
122
|
-
dt: datetime = datetime
|
124
|
+
dt: datetime = datetime(2025, 1, 18, tzinfo=ZoneInfo("Asia/Bangkok"))
|
123
125
|
rs: Result = workflow.release(
|
124
126
|
release=dt,
|
125
127
|
params={"asat-dt": datetime(2024, 10, 1)},
|
126
128
|
)
|
129
|
+
assert dt == datetime(2025, 1, 18, tzinfo=ZoneInfo("Asia/Bangkok"))
|
127
130
|
assert rs.status == SUCCESS
|
128
131
|
assert rs.context == {
|
129
132
|
"status": SUCCESS,
|
130
133
|
"params": {"asat-dt": datetime(2024, 10, 1, 0, 0)},
|
131
134
|
"release": {
|
132
135
|
"type": NORMAL,
|
133
|
-
|
136
|
+
# NOTE: The date that pass to release method will convert to UTC.
|
137
|
+
"logical_date": datetime(2025, 1, 17, 17, tzinfo=UTC),
|
134
138
|
},
|
135
139
|
"jobs": {
|
136
140
|
"first-job": {
|
@@ -1 +0,0 @@
|
|
1
|
-
__version__: str = "0.0.74"
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/api/routes/__init__.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
{ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil/workflow/api/routes/workflows.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil_workflow.egg-info/SOURCES.txt
RENAMED
File without changes
|
File without changes
|
{ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil_workflow.egg-info/entry_points.txt
RENAMED
File without changes
|
{ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil_workflow.egg-info/requires.txt
RENAMED
File without changes
|
{ddeutil_workflow-0.0.74 → ddeutil_workflow-0.0.75}/src/ddeutil_workflow.egg-info/top_level.txt
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|