ddeutil-workflow 0.0.14__py3-none-any.whl → 0.0.16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__types.py +58 -13
- ddeutil/workflow/api.py +3 -4
- ddeutil/workflow/cli.py +2 -5
- ddeutil/workflow/conf.py +280 -3
- ddeutil/workflow/job.py +44 -25
- ddeutil/workflow/log.py +5 -8
- ddeutil/workflow/on.py +1 -1
- ddeutil/workflow/repeat.py +2 -5
- ddeutil/workflow/route.py +4 -11
- ddeutil/workflow/scheduler.py +97 -64
- ddeutil/workflow/stage.py +44 -30
- ddeutil/workflow/utils.py +91 -266
- {ddeutil_workflow-0.0.14.dist-info → ddeutil_workflow-0.0.16.dist-info}/METADATA +39 -23
- ddeutil_workflow-0.0.16.dist-info/RECORD +22 -0
- {ddeutil_workflow-0.0.14.dist-info → ddeutil_workflow-0.0.16.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.14.dist-info/RECORD +0 -22
- {ddeutil_workflow-0.0.14.dist-info → ddeutil_workflow-0.0.16.dist-info}/LICENSE +0 -0
- {ddeutil_workflow-0.0.14.dist-info → ddeutil_workflow-0.0.16.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.14.dist-info → ddeutil_workflow-0.0.16.dist-info}/top_level.txt +0 -0
ddeutil/workflow/utils.py
CHANGED
@@ -7,7 +7,6 @@ from __future__ import annotations
|
|
7
7
|
|
8
8
|
import inspect
|
9
9
|
import logging
|
10
|
-
import os
|
11
10
|
import stat
|
12
11
|
import time
|
13
12
|
from abc import ABC, abstractmethod
|
@@ -15,7 +14,7 @@ from ast import Call, Constant, Expr, Module, Name, parse
|
|
15
14
|
from collections.abc import Iterator
|
16
15
|
from dataclasses import field
|
17
16
|
from datetime import date, datetime
|
18
|
-
from functools import
|
17
|
+
from functools import wraps
|
19
18
|
from hashlib import md5
|
20
19
|
from importlib import import_module
|
21
20
|
from inspect import isfunction
|
@@ -30,17 +29,15 @@ try:
|
|
30
29
|
except ImportError:
|
31
30
|
from typing_extensions import ParamSpec
|
32
31
|
|
33
|
-
from ddeutil.core import getdot, hasdot, hash_str, import_string, lazy
|
34
|
-
from ddeutil.io import
|
35
|
-
from
|
36
|
-
from pydantic import BaseModel, ConfigDict, Field
|
32
|
+
from ddeutil.core import getdot, hasdot, hash_str, import_string, lazy
|
33
|
+
from ddeutil.io import search_env_replace
|
34
|
+
from pydantic import BaseModel, Field
|
37
35
|
from pydantic.dataclasses import dataclass
|
38
|
-
from pydantic.functional_serializers import field_serializer
|
39
36
|
from pydantic.functional_validators import model_validator
|
40
37
|
from typing_extensions import Self
|
41
38
|
|
42
39
|
from .__types import DictData, Matrix, Re
|
43
|
-
from .conf import config
|
40
|
+
from .conf import config, load_config
|
44
41
|
from .exceptions import ParamValueException, UtilException
|
45
42
|
|
46
43
|
P = ParamSpec("P")
|
@@ -50,16 +47,30 @@ AnyModelType = type[AnyModel]
|
|
50
47
|
logger = logging.getLogger("ddeutil.workflow")
|
51
48
|
|
52
49
|
|
53
|
-
def
|
50
|
+
def get_dt_now(tz: ZoneInfo | None = None) -> datetime: # pragma: no cov
|
51
|
+
"""Return the current datetime object.
|
52
|
+
|
53
|
+
:param tz:
|
54
|
+
:return: The current datetime object that use an input timezone or UTC.
|
55
|
+
"""
|
56
|
+
return datetime.now(tz=(tz or ZoneInfo("UTC")))
|
57
|
+
|
58
|
+
|
59
|
+
def get_diff_sec(
|
60
|
+
dt: datetime, tz: ZoneInfo | None = None
|
61
|
+
) -> int: # pragma: no cov
|
54
62
|
"""Return second value that come from diff of an input datetime and the
|
55
63
|
current datetime with specific timezone.
|
64
|
+
|
65
|
+
:param dt:
|
66
|
+
:param tz:
|
56
67
|
"""
|
57
68
|
return round(
|
58
69
|
(dt - datetime.now(tz=(tz or ZoneInfo("UTC")))).total_seconds()
|
59
70
|
)
|
60
71
|
|
61
72
|
|
62
|
-
def delay(second: float = 0) -> None:
|
73
|
+
def delay(second: float = 0) -> None: # pragma: no cov
|
63
74
|
"""Delay time that use time.sleep with random second value between
|
64
75
|
0.00 - 0.99 seconds.
|
65
76
|
|
@@ -68,197 +79,6 @@ def delay(second: float = 0) -> None:
|
|
68
79
|
time.sleep(second + randrange(0, 99, step=10) / 100)
|
69
80
|
|
70
81
|
|
71
|
-
class Engine(BaseModel):
|
72
|
-
"""Engine Model"""
|
73
|
-
|
74
|
-
paths: PathData = Field(default_factory=PathData)
|
75
|
-
registry: list[str] = Field(
|
76
|
-
default_factory=lambda: ["ddeutil.workflow"], # pragma: no cover
|
77
|
-
)
|
78
|
-
registry_filter: list[str] = Field(
|
79
|
-
default_factory=lambda: ["ddeutil.workflow.utils"], # pragma: no cover
|
80
|
-
)
|
81
|
-
|
82
|
-
@model_validator(mode="before")
|
83
|
-
def __prepare_registry(cls, values: DictData) -> DictData:
|
84
|
-
"""Prepare registry value that passing with string type. It convert the
|
85
|
-
string type to list of string.
|
86
|
-
"""
|
87
|
-
if (_regis := values.get("registry")) and isinstance(_regis, str):
|
88
|
-
values["registry"] = [_regis]
|
89
|
-
if (_regis_filter := values.get("registry_filter")) and isinstance(
|
90
|
-
_regis, str
|
91
|
-
):
|
92
|
-
values["registry_filter"] = [_regis_filter]
|
93
|
-
return values
|
94
|
-
|
95
|
-
|
96
|
-
class CoreConf(BaseModel):
|
97
|
-
"""Core Config Model"""
|
98
|
-
|
99
|
-
model_config = ConfigDict(arbitrary_types_allowed=True)
|
100
|
-
|
101
|
-
tz: ZoneInfo = Field(default_factory=lambda: ZoneInfo("UTC"))
|
102
|
-
|
103
|
-
|
104
|
-
class ConfParams(BaseModel):
|
105
|
-
"""Params Model"""
|
106
|
-
|
107
|
-
engine: Engine = Field(
|
108
|
-
default_factory=Engine,
|
109
|
-
description="A engine mapping values.",
|
110
|
-
)
|
111
|
-
core: CoreConf = Field(
|
112
|
-
default_factory=CoreConf,
|
113
|
-
description="A core config value",
|
114
|
-
)
|
115
|
-
|
116
|
-
|
117
|
-
def load_config() -> ConfParams:
|
118
|
-
"""Load Config data from ``workflows-conf.yaml`` file.
|
119
|
-
|
120
|
-
Configuration Docs:
|
121
|
-
---
|
122
|
-
:var engine.registry:
|
123
|
-
:var engine.registry_filter:
|
124
|
-
:var paths.root:
|
125
|
-
:var paths.conf:
|
126
|
-
"""
|
127
|
-
root_path: str = os.getenv("WORKFLOW_ROOT_PATH", ".")
|
128
|
-
|
129
|
-
regis: list[str] = ["ddeutil.workflow"]
|
130
|
-
if regis_env := os.getenv("WORKFLOW_CORE_REGISTRY"):
|
131
|
-
regis = [r.strip() for r in regis_env.split(",")]
|
132
|
-
|
133
|
-
regis_filter: list[str] = ["ddeutil.workflow.utils"]
|
134
|
-
if regis_filter_env := os.getenv("WORKFLOW_CORE_REGISTRY_FILTER"):
|
135
|
-
regis_filter = [r.strip() for r in regis_filter_env.split(",")]
|
136
|
-
|
137
|
-
conf_path: str = (
|
138
|
-
f"{root_path}/{conf_env}"
|
139
|
-
if (conf_env := os.getenv("WORKFLOW_CORE_PATH_CONF"))
|
140
|
-
else None
|
141
|
-
)
|
142
|
-
return ConfParams.model_validate(
|
143
|
-
obj={
|
144
|
-
"engine": {
|
145
|
-
"registry": regis,
|
146
|
-
"registry_filter": regis_filter,
|
147
|
-
"paths": {
|
148
|
-
"root": root_path,
|
149
|
-
"conf": conf_path,
|
150
|
-
},
|
151
|
-
},
|
152
|
-
}
|
153
|
-
)
|
154
|
-
|
155
|
-
|
156
|
-
class SimLoad:
|
157
|
-
"""Simple Load Object that will search config data by given some identity
|
158
|
-
value like name of workflow or on.
|
159
|
-
|
160
|
-
:param name: A name of config data that will read by Yaml Loader object.
|
161
|
-
:param params: A Params model object.
|
162
|
-
:param externals: An external parameters
|
163
|
-
|
164
|
-
Noted:
|
165
|
-
|
166
|
-
The config data should have ``type`` key for modeling validation that
|
167
|
-
make this loader know what is config should to do pass to.
|
168
|
-
|
169
|
-
... <identity-key>:
|
170
|
-
... type: <importable-object>
|
171
|
-
... <key-data>: <value-data>
|
172
|
-
... ...
|
173
|
-
|
174
|
-
"""
|
175
|
-
|
176
|
-
def __init__(
|
177
|
-
self,
|
178
|
-
name: str,
|
179
|
-
params: ConfParams,
|
180
|
-
externals: DictData | None = None,
|
181
|
-
) -> None:
|
182
|
-
self.data: DictData = {}
|
183
|
-
for file in PathSearch(params.engine.paths.conf).files:
|
184
|
-
if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
|
185
|
-
data := YamlFlResolve(file).read().get(name, {})
|
186
|
-
):
|
187
|
-
self.data = data
|
188
|
-
|
189
|
-
# VALIDATE: check the data that reading should not empty.
|
190
|
-
if not self.data:
|
191
|
-
raise ValueError(f"Config {name!r} does not found on conf path")
|
192
|
-
|
193
|
-
self.conf_params: ConfParams = params
|
194
|
-
self.externals: DictData = externals or {}
|
195
|
-
self.data.update(self.externals)
|
196
|
-
|
197
|
-
@classmethod
|
198
|
-
def finds(
|
199
|
-
cls,
|
200
|
-
obj: object,
|
201
|
-
params: ConfParams,
|
202
|
-
*,
|
203
|
-
include: list[str] | None = None,
|
204
|
-
exclude: list[str] | None = None,
|
205
|
-
) -> Iterator[tuple[str, DictData]]:
|
206
|
-
"""Find all data that match with object type in config path. This class
|
207
|
-
method can use include and exclude list of identity name for filter and
|
208
|
-
adds-on.
|
209
|
-
"""
|
210
|
-
exclude: list[str] = exclude or []
|
211
|
-
for file in PathSearch(params.engine.paths.conf).files:
|
212
|
-
if any(file.suffix.endswith(s) for s in (".yml", ".yaml")) and (
|
213
|
-
values := YamlFlResolve(file).read()
|
214
|
-
):
|
215
|
-
for key, data in values.items():
|
216
|
-
if key in exclude:
|
217
|
-
continue
|
218
|
-
if issubclass(get_type(data["type"], params), obj) and (
|
219
|
-
include is None or all(i in data for i in include)
|
220
|
-
):
|
221
|
-
yield key, data
|
222
|
-
|
223
|
-
@cached_property
|
224
|
-
def type(self) -> AnyModelType:
|
225
|
-
"""Return object of string type which implement on any registry. The
|
226
|
-
object type.
|
227
|
-
|
228
|
-
:rtype: AnyModelType
|
229
|
-
"""
|
230
|
-
if not (_typ := self.data.get("type")):
|
231
|
-
raise ValueError(
|
232
|
-
f"the 'type' value: {_typ} does not exists in config data."
|
233
|
-
)
|
234
|
-
return get_type(_typ, self.conf_params)
|
235
|
-
|
236
|
-
|
237
|
-
class Loader(SimLoad):
|
238
|
-
"""Loader Object that get the config `yaml` file from current path.
|
239
|
-
|
240
|
-
:param name: A name of config data that will read by Yaml Loader object.
|
241
|
-
:param externals: An external parameters
|
242
|
-
"""
|
243
|
-
|
244
|
-
@classmethod
|
245
|
-
def finds(
|
246
|
-
cls,
|
247
|
-
obj: object,
|
248
|
-
*,
|
249
|
-
include: list[str] | None = None,
|
250
|
-
exclude: list[str] | None = None,
|
251
|
-
**kwargs,
|
252
|
-
) -> DictData:
|
253
|
-
"""Override the find class method from the Simple Loader object."""
|
254
|
-
return super().finds(
|
255
|
-
obj=obj, params=load_config(), include=include, exclude=exclude
|
256
|
-
)
|
257
|
-
|
258
|
-
def __init__(self, name: str, externals: DictData) -> None:
|
259
|
-
super().__init__(name, load_config(), externals)
|
260
|
-
|
261
|
-
|
262
82
|
def gen_id(
|
263
83
|
value: Any,
|
264
84
|
*,
|
@@ -266,9 +86,9 @@ def gen_id(
|
|
266
86
|
unique: bool = False,
|
267
87
|
) -> str:
|
268
88
|
"""Generate running ID for able to tracking. This generate process use `md5`
|
269
|
-
algorithm function if ``
|
270
|
-
But it will cut this hashing value length to 10 it the setting value
|
271
|
-
true.
|
89
|
+
algorithm function if ``WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE`` set to
|
90
|
+
false. But it will cut this hashing value length to 10 it the setting value
|
91
|
+
set to true.
|
272
92
|
|
273
93
|
:param value: A value that want to add to prefix before hashing with md5.
|
274
94
|
:param sensitive: A flag that convert the value to lower case before hashing
|
@@ -279,7 +99,7 @@ def gen_id(
|
|
279
99
|
if not isinstance(value, str):
|
280
100
|
value: str = str(value)
|
281
101
|
|
282
|
-
if
|
102
|
+
if config.workflow_id_simple_mode:
|
283
103
|
return hash_str(f"{(value if sensitive else value.lower())}", n=10) + (
|
284
104
|
f"{datetime.now(tz=config.tz):%Y%m%d%H%M%S%f}" if unique else ""
|
285
105
|
)
|
@@ -291,36 +111,22 @@ def gen_id(
|
|
291
111
|
).hexdigest()
|
292
112
|
|
293
113
|
|
294
|
-
def get_type(t: str, params: ConfParams) -> AnyModelType:
|
295
|
-
"""Return import type from string importable value in the type key.
|
296
|
-
|
297
|
-
:param t: A importable type string.
|
298
|
-
:param params: A config parameters that use registry to search this
|
299
|
-
type.
|
300
|
-
:rtype: AnyModelType
|
301
|
-
"""
|
302
|
-
try:
|
303
|
-
# NOTE: Auto adding module prefix if it does not set
|
304
|
-
return import_string(f"ddeutil.workflow.{t}")
|
305
|
-
except ModuleNotFoundError:
|
306
|
-
for registry in params.engine.registry:
|
307
|
-
try:
|
308
|
-
return import_string(f"{registry}.{t}")
|
309
|
-
except ModuleNotFoundError:
|
310
|
-
continue
|
311
|
-
return import_string(f"{t}")
|
312
|
-
|
313
|
-
|
314
114
|
class TagFunc(Protocol):
|
315
115
|
"""Tag Function Protocol"""
|
316
116
|
|
317
117
|
name: str
|
318
118
|
tag: str
|
319
119
|
|
320
|
-
def __call__(self, *args, **kwargs): ...
|
120
|
+
def __call__(self, *args, **kwargs): ... # pragma: no cov
|
321
121
|
|
322
122
|
|
323
|
-
|
123
|
+
ReturnTagFunc = Callable[P, TagFunc]
|
124
|
+
DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
|
125
|
+
|
126
|
+
|
127
|
+
def tag(
|
128
|
+
name: str, alias: str | None = None
|
129
|
+
) -> DecoratorTagFunc: # pragma: no cov
|
324
130
|
"""Tag decorator function that set function attributes, ``tag`` and ``name``
|
325
131
|
for making registries variable.
|
326
132
|
|
@@ -330,7 +136,7 @@ def tag(name: str, alias: str | None = None) -> Callable[P, TagFunc]:
|
|
330
136
|
:rtype: Callable[P, TagFunc]
|
331
137
|
"""
|
332
138
|
|
333
|
-
def func_internal(func: Callable[[...], Any]) ->
|
139
|
+
def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
|
334
140
|
func.tag = name
|
335
141
|
func.name = alias or func.__name__.replace("_", "-")
|
336
142
|
|
@@ -386,9 +192,14 @@ def make_registry(submodule: str) -> dict[str, Registry]:
|
|
386
192
|
class BaseParam(BaseModel, ABC):
|
387
193
|
"""Base Parameter that use to make Params Model."""
|
388
194
|
|
389
|
-
desc: Optional[str] =
|
390
|
-
|
391
|
-
|
195
|
+
desc: Optional[str] = Field(
|
196
|
+
default=None, description="A description of parameter providing."
|
197
|
+
)
|
198
|
+
required: bool = Field(
|
199
|
+
default=True,
|
200
|
+
description="A require flag that force to pass this parameter value.",
|
201
|
+
)
|
202
|
+
type: str = Field(description="A type of parameter.")
|
392
203
|
|
393
204
|
@abstractmethod
|
394
205
|
def receive(self, value: Optional[Any] = None) -> Any:
|
@@ -396,15 +207,20 @@ class BaseParam(BaseModel, ABC):
|
|
396
207
|
"Receive value and validate typing before return valid value."
|
397
208
|
)
|
398
209
|
|
399
|
-
@field_serializer("type")
|
400
|
-
def __serializer_type(self, value: str) -> str:
|
401
|
-
return value
|
402
|
-
|
403
210
|
|
404
211
|
class DefaultParam(BaseParam):
|
405
|
-
"""Default Parameter that will check default if it required
|
212
|
+
"""Default Parameter that will check default if it required. This model do
|
213
|
+
not implement the receive method.
|
214
|
+
"""
|
406
215
|
|
407
|
-
|
216
|
+
required: bool = Field(
|
217
|
+
default=False,
|
218
|
+
description="A require flag for the default-able parameter value.",
|
219
|
+
)
|
220
|
+
default: Optional[str] = Field(
|
221
|
+
default=None,
|
222
|
+
description="A default value if parameter does not pass.",
|
223
|
+
)
|
408
224
|
|
409
225
|
@abstractmethod
|
410
226
|
def receive(self, value: Optional[Any] = None) -> Any:
|
@@ -415,9 +231,9 @@ class DefaultParam(BaseParam):
|
|
415
231
|
@model_validator(mode="after")
|
416
232
|
def __check_default(self) -> Self:
|
417
233
|
"""Check default value should pass when it set required."""
|
418
|
-
if
|
234
|
+
if self.required and self.default is None:
|
419
235
|
raise ParamValueException(
|
420
|
-
"Default should set when this parameter
|
236
|
+
"Default should be set when this parameter was required."
|
421
237
|
)
|
422
238
|
return self
|
423
239
|
|
@@ -426,8 +242,7 @@ class DatetimeParam(DefaultParam):
|
|
426
242
|
"""Datetime parameter."""
|
427
243
|
|
428
244
|
type: Literal["datetime"] = "datetime"
|
429
|
-
|
430
|
-
default: datetime = Field(default_factory=dt_now)
|
245
|
+
default: datetime = Field(default_factory=get_dt_now)
|
431
246
|
|
432
247
|
def receive(self, value: str | datetime | date | None = None) -> datetime:
|
433
248
|
"""Receive value that match with datetime. If a input value pass with
|
@@ -449,7 +264,12 @@ class DatetimeParam(DefaultParam):
|
|
449
264
|
f"Value that want to convert to datetime does not support for "
|
450
265
|
f"type: {type(value)}"
|
451
266
|
)
|
452
|
-
|
267
|
+
try:
|
268
|
+
return datetime.fromisoformat(value)
|
269
|
+
except ValueError:
|
270
|
+
raise ParamValueException(
|
271
|
+
f"Invalid isoformat string: {value!r}"
|
272
|
+
) from None
|
453
273
|
|
454
274
|
|
455
275
|
class StrParam(DefaultParam):
|
@@ -457,7 +277,7 @@ class StrParam(DefaultParam):
|
|
457
277
|
|
458
278
|
type: Literal["str"] = "str"
|
459
279
|
|
460
|
-
def receive(self, value:
|
280
|
+
def receive(self, value: str | None = None) -> str | None:
|
461
281
|
"""Receive value that match with str.
|
462
282
|
|
463
283
|
:param value: A value that want to validate with string parameter type.
|
@@ -472,8 +292,12 @@ class IntParam(DefaultParam):
|
|
472
292
|
"""Integer parameter."""
|
473
293
|
|
474
294
|
type: Literal["int"] = "int"
|
295
|
+
default: Optional[int] = Field(
|
296
|
+
default=None,
|
297
|
+
description="A default value if parameter does not pass.",
|
298
|
+
)
|
475
299
|
|
476
|
-
def receive(self, value:
|
300
|
+
def receive(self, value: int | None = None) -> int | None:
|
477
301
|
"""Receive value that match with int.
|
478
302
|
|
479
303
|
:param value: A value that want to validate with integer parameter type.
|
@@ -484,10 +308,9 @@ class IntParam(DefaultParam):
|
|
484
308
|
if not isinstance(value, int):
|
485
309
|
try:
|
486
310
|
return int(str(value))
|
487
|
-
except
|
311
|
+
except ValueError as err:
|
488
312
|
raise ParamValueException(
|
489
|
-
f"Value
|
490
|
-
f"for type: {type(value)}"
|
313
|
+
f"Value can not convert to int, {value}, with base 10"
|
491
314
|
) from err
|
492
315
|
return value
|
493
316
|
|
@@ -496,15 +319,19 @@ class ChoiceParam(BaseParam):
|
|
496
319
|
"""Choice parameter."""
|
497
320
|
|
498
321
|
type: Literal["choice"] = "choice"
|
499
|
-
options: list[str]
|
322
|
+
options: list[str] = Field(description="A list of choice parameters.")
|
323
|
+
|
324
|
+
def receive(self, value: str | None = None) -> str:
|
325
|
+
"""Receive value that match with options.
|
500
326
|
|
501
|
-
|
502
|
-
|
327
|
+
:param value: A value that want to select from the options field.
|
328
|
+
:rtype: str
|
329
|
+
"""
|
503
330
|
# NOTE:
|
504
331
|
# Return the first value in options if does not pass any input value
|
505
332
|
if value is None:
|
506
333
|
return self.options[0]
|
507
|
-
if
|
334
|
+
if value not in self.options:
|
508
335
|
raise ParamValueException(
|
509
336
|
f"{value!r} does not match any value in choice options."
|
510
337
|
)
|
@@ -531,7 +358,7 @@ class Result:
|
|
531
358
|
|
532
359
|
status: int = field(default=2)
|
533
360
|
context: DictData = field(default_factory=dict)
|
534
|
-
start_at: datetime = field(default_factory=
|
361
|
+
start_at: datetime = field(default_factory=get_dt_now, compare=False)
|
535
362
|
end_at: Optional[datetime] = field(default=None, compare=False)
|
536
363
|
|
537
364
|
# NOTE: Ignore this field to compare another result model with __eq__.
|
@@ -563,15 +390,15 @@ class Result:
|
|
563
390
|
:param running_id: A running ID that want to update on this model.
|
564
391
|
:rtype: Self
|
565
392
|
"""
|
566
|
-
self._parent_run_id = running_id
|
393
|
+
self._parent_run_id: str = running_id
|
567
394
|
return self
|
568
395
|
|
569
396
|
@property
|
570
|
-
def parent_run_id(self):
|
397
|
+
def parent_run_id(self) -> str:
|
571
398
|
return self._parent_run_id
|
572
399
|
|
573
400
|
@property
|
574
|
-
def run_id(self):
|
401
|
+
def run_id(self) -> str:
|
575
402
|
return self._run_id
|
576
403
|
|
577
404
|
def catch(self, status: int, context: DictData) -> Self:
|
@@ -608,8 +435,8 @@ class Result:
|
|
608
435
|
self.__dict__["context"]["jobs"].update(result.context)
|
609
436
|
|
610
437
|
# NOTE: Update running ID from an incoming result.
|
611
|
-
self._parent_run_id = result.parent_run_id
|
612
|
-
self._run_id = result.run_id
|
438
|
+
self._parent_run_id: str = result.parent_run_id
|
439
|
+
self._run_id: str = result.run_id
|
613
440
|
return self
|
614
441
|
|
615
442
|
|
@@ -793,8 +620,8 @@ def not_in_template(value: Any, *, not_in: str = "matrix.") -> bool:
|
|
793
620
|
elif not isinstance(value, str):
|
794
621
|
return False
|
795
622
|
return any(
|
796
|
-
(not found.
|
797
|
-
for found in Re.
|
623
|
+
(not found.caller.strip().startswith(not_in))
|
624
|
+
for found in Re.finditer_caller(value.strip())
|
798
625
|
)
|
799
626
|
|
800
627
|
|
@@ -835,18 +662,16 @@ def str2template(
|
|
835
662
|
|
836
663
|
# NOTE: remove space before and after this string value.
|
837
664
|
value: str = value.strip()
|
838
|
-
for found in Re.
|
665
|
+
for found in Re.finditer_caller(value):
|
839
666
|
# NOTE:
|
840
667
|
# Get caller and filter values that setting inside;
|
841
668
|
#
|
842
669
|
# ... ``${{ <caller-value> [ | <filter-value>] ... }}``
|
843
670
|
#
|
844
|
-
caller: str = found.
|
671
|
+
caller: str = found.caller
|
845
672
|
pfilter: list[str] = [
|
846
673
|
i.strip()
|
847
|
-
for i in (
|
848
|
-
found.group("post_filters").strip().removeprefix("|").split("|")
|
849
|
-
)
|
674
|
+
for i in (found.post_filters.strip().removeprefix("|").split("|"))
|
850
675
|
if i != ""
|
851
676
|
]
|
852
677
|
if not hasdot(caller, params):
|
@@ -859,7 +684,7 @@ def str2template(
|
|
859
684
|
# If type of getter caller is not string type and it does not use to
|
860
685
|
# concat other string value, it will return origin value from the
|
861
686
|
# ``getdot`` function.
|
862
|
-
if value.replace(found.
|
687
|
+
if value.replace(found.full, "", 1) == "":
|
863
688
|
return map_post_filter(getter, pfilter, filters=filters)
|
864
689
|
|
865
690
|
# NOTE: map post-filter function.
|
@@ -867,7 +692,7 @@ def str2template(
|
|
867
692
|
if not isinstance(getter, str):
|
868
693
|
getter: str = str(getter)
|
869
694
|
|
870
|
-
value: str = value.replace(found.
|
695
|
+
value: str = value.replace(found.full, getter, 1)
|
871
696
|
|
872
697
|
return search_env_replace(value)
|
873
698
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: ddeutil-workflow
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.16
|
4
4
|
Summary: Lightweight workflow orchestration with less dependencies
|
5
5
|
Author-email: ddeutils <korawich.anu@gmail.com>
|
6
6
|
License: MIT
|
@@ -22,8 +22,8 @@ Classifier: Programming Language :: Python :: 3.13
|
|
22
22
|
Requires-Python: >=3.9.13
|
23
23
|
Description-Content-Type: text/markdown
|
24
24
|
License-File: LICENSE
|
25
|
-
Requires-Dist: ddeutil >=0.4.
|
26
|
-
Requires-Dist: ddeutil-io >=0.
|
25
|
+
Requires-Dist: ddeutil >=0.4.3
|
26
|
+
Requires-Dist: ddeutil-io[yaml] >=0.2.3
|
27
27
|
Requires-Dist: python-dotenv ==1.0.1
|
28
28
|
Requires-Dist: typer <1.0.0,==0.12.5
|
29
29
|
Requires-Dist: schedule <2.0.0,==1.2.2
|
@@ -179,29 +179,31 @@ The main configuration that use to dynamic changing with your propose of this
|
|
179
179
|
application. If any configuration values do not set yet, it will use default value
|
180
180
|
and do not raise any error to you.
|
181
181
|
|
182
|
-
| Environment
|
183
|
-
|
184
|
-
| `WORKFLOW_ROOT_PATH`
|
185
|
-
| `WORKFLOW_CORE_REGISTRY`
|
186
|
-
| `WORKFLOW_CORE_REGISTRY_FILTER`
|
187
|
-
| `WORKFLOW_CORE_PATH_CONF`
|
188
|
-
| `WORKFLOW_CORE_TIMEZONE`
|
189
|
-
| `WORKFLOW_CORE_STAGE_DEFAULT_ID`
|
190
|
-
| `WORKFLOW_CORE_STAGE_RAISE_ERROR`
|
191
|
-
| `
|
192
|
-
| `
|
193
|
-
| `
|
194
|
-
| `
|
195
|
-
| `
|
196
|
-
| `
|
197
|
-
| `
|
182
|
+
| Environment | Component | Default | Description | Remark |
|
183
|
+
|:----------------------------------------|-----------|----------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
|
184
|
+
| `WORKFLOW_ROOT_PATH` | Core | . | The root path of the workflow application | |
|
185
|
+
| `WORKFLOW_CORE_REGISTRY` | Core | src.ddeutil.workflow,tests.utils | List of importable string for the hook stage | |
|
186
|
+
| `WORKFLOW_CORE_REGISTRY_FILTER` | Core | ddeutil.workflow.utils | List of importable string for the filter template | |
|
187
|
+
| `WORKFLOW_CORE_PATH_CONF` | Core | conf | The config path that keep all template `.yaml` files | |
|
188
|
+
| `WORKFLOW_CORE_TIMEZONE` | Core | Asia/Bangkok | A Timezone string value that will pass to `ZoneInfo` object | |
|
189
|
+
| `WORKFLOW_CORE_STAGE_DEFAULT_ID` | Core | true | A flag that enable default stage ID that use for catch an execution output | |
|
190
|
+
| `WORKFLOW_CORE_STAGE_RAISE_ERROR` | Core | false | A flag that all stage raise StageException from stage execution | |
|
191
|
+
| `WORKFLOW_CORE_JOB_DEFAULT_ID` | Core | false | A flag that enable default job ID that use for catch an execution output. The ID that use will be sequence number. | |
|
192
|
+
| `WORKFLOW_CORE_MAX_NUM_POKING` | Core | 4 | | |
|
193
|
+
| `WORKFLOW_CORE_MAX_JOB_PARALLEL` | Core | 2 | The maximum job number that able to run parallel in workflow executor | |
|
194
|
+
| `WORKFLOW_CORE_WORKFLOW_ID_SIMPLE_MODE` | Core | true | | |
|
195
|
+
| `WORKFLOW_LOG_DEBUG_MODE` | Log | true | A flag that enable logging with debug level mode | |
|
196
|
+
| `WORKFLOW_LOG_ENABLE_WRITE` | Log | true | A flag that enable logging object saving log to its destination | |
|
197
|
+
| `WORKFLOW_APP_MAX_PROCESS` | Schedule | 2 | The maximum process worker number that run in scheduler app module | |
|
198
|
+
| `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` | Schedule | 100 | A schedule per process that run parallel | |
|
199
|
+
| `WORKFLOW_APP_STOP_BOUNDARY_DELTA` | Schedule | '{"minutes": 5, "seconds": 20}' | A time delta value that use to stop scheduler app in json string format | |
|
198
200
|
|
199
201
|
**API Application**:
|
200
202
|
|
201
|
-
| Environment
|
202
|
-
|
203
|
-
| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW`
|
204
|
-
| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE`
|
203
|
+
| Environment | Component | Default | Description | Remark |
|
204
|
+
|:--------------------------------------|-----------|---------|-----------------------------------------------------------------------------------|--------|
|
205
|
+
| `WORKFLOW_API_ENABLE_ROUTE_WORKFLOW` | API | true | A flag that enable workflow route to manage execute manually and workflow logging | |
|
206
|
+
| `WORKFLOW_API_ENABLE_ROUTE_SCHEDULE` | API | true | A flag that enable run scheduler | |
|
205
207
|
|
206
208
|
## :rocket: Deployment
|
207
209
|
|
@@ -224,3 +226,17 @@ like crontab job but via Python API.
|
|
224
226
|
> [!NOTE]
|
225
227
|
> If this package already deploy, it able to use
|
226
228
|
> `uvicorn ddeutil.workflow.api:app --host 127.0.0.1 --port 80 --workers 4`
|
229
|
+
|
230
|
+
### Docker Container
|
231
|
+
|
232
|
+
Create Docker image;
|
233
|
+
|
234
|
+
```shell
|
235
|
+
$ docker build -t ddeutil-workflow:latest -f .container/Dockerfile .
|
236
|
+
```
|
237
|
+
|
238
|
+
Run the above Docker image;
|
239
|
+
|
240
|
+
```shell
|
241
|
+
$ docker run -i ddeutil-workflow:latest
|
242
|
+
```
|