ddeutil-workflow 0.0.68__py3-none-any.whl → 0.0.70__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +14 -12
- ddeutil/workflow/api/log_conf.py +16 -29
- ddeutil/workflow/api/routes/logs.py +1 -1
- ddeutil/workflow/api/routes/workflows.py +3 -3
- ddeutil/workflow/audits.py +374 -0
- ddeutil/workflow/cli.py +70 -6
- ddeutil/workflow/conf.py +4 -51
- ddeutil/workflow/errors.py +7 -1
- ddeutil/workflow/event.py +2 -2
- ddeutil/workflow/job.py +9 -3
- ddeutil/workflow/result.py +10 -1
- ddeutil/workflow/reusables.py +1 -1
- ddeutil/workflow/{logs.py → traces.py} +224 -409
- ddeutil/workflow/utils.py +19 -11
- ddeutil/workflow/workflow.py +226 -18
- {ddeutil_workflow-0.0.68.dist-info → ddeutil_workflow-0.0.70.dist-info}/METADATA +29 -27
- ddeutil_workflow-0.0.70.dist-info/RECORD +30 -0
- ddeutil_workflow-0.0.68.dist-info/RECORD +0 -29
- {ddeutil_workflow-0.0.68.dist-info → ddeutil_workflow-0.0.70.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.68.dist-info → ddeutil_workflow-0.0.70.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.68.dist-info → ddeutil_workflow-0.0.70.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.68.dist-info → ddeutil_workflow-0.0.70.dist-info}/top_level.txt +0 -0
ddeutil/workflow/conf.py
CHANGED
@@ -7,12 +7,11 @@ from __future__ import annotations
|
|
7
7
|
|
8
8
|
import copy
|
9
9
|
import os
|
10
|
-
from abc import ABC, abstractmethod
|
11
10
|
from collections.abc import Iterator
|
12
11
|
from functools import cached_property
|
13
12
|
from inspect import isclass
|
14
13
|
from pathlib import Path
|
15
|
-
from typing import Final, Optional,
|
14
|
+
from typing import Final, Optional, TypeVar, Union
|
16
15
|
from zoneinfo import ZoneInfo
|
17
16
|
|
18
17
|
from ddeutil.core import str2bool
|
@@ -145,18 +144,6 @@ class Config: # pragma: no cov
|
|
145
144
|
def stage_default_id(self) -> bool:
|
146
145
|
return str2bool(env("CORE_STAGE_DEFAULT_ID", "false"))
|
147
146
|
|
148
|
-
@property
|
149
|
-
def max_cron_per_workflow(self) -> int:
|
150
|
-
"""The maximum on value that store in workflow model.
|
151
|
-
|
152
|
-
:rtype: int
|
153
|
-
"""
|
154
|
-
return int(env("CORE_MAX_CRON_PER_WORKFLOW", "5"))
|
155
|
-
|
156
|
-
@property
|
157
|
-
def max_queue_complete_hist(self) -> int:
|
158
|
-
return int(env("CORE_MAX_QUEUE_COMPLETE_HIST", "16"))
|
159
|
-
|
160
147
|
|
161
148
|
class APIConfig:
|
162
149
|
"""API Config object."""
|
@@ -170,23 +157,7 @@ class APIConfig:
|
|
170
157
|
return env("API_PREFIX_PATH", f"/api/v{self.version}")
|
171
158
|
|
172
159
|
|
173
|
-
class
|
174
|
-
"""Base Load object is the abstraction object for any Load object that
|
175
|
-
should to inherit from this base class.
|
176
|
-
"""
|
177
|
-
|
178
|
-
@classmethod
|
179
|
-
@abstractmethod
|
180
|
-
def find(cls, name: str, *args, **kwargs) -> DictData: ...
|
181
|
-
|
182
|
-
@classmethod
|
183
|
-
@abstractmethod
|
184
|
-
def finds(
|
185
|
-
cls, obj: object, *args, **kwargs
|
186
|
-
) -> Iterator[tuple[str, DictData]]: ...
|
187
|
-
|
188
|
-
|
189
|
-
class FileLoad(BaseLoad):
|
160
|
+
class YamlParser:
|
190
161
|
"""Base Load object that use to search config data by given some identity
|
191
162
|
value like name of `Workflow` or `Crontab` templates.
|
192
163
|
|
@@ -356,13 +327,13 @@ class FileLoad(BaseLoad):
|
|
356
327
|
*,
|
357
328
|
ignore_filename: Optional[str] = None,
|
358
329
|
) -> bool:
|
359
|
-
"""Check this file was ignored.
|
330
|
+
"""Check this file was ignored from the `.confignore` format.
|
360
331
|
|
361
332
|
:param file: (Path) A file path that want to check.
|
362
333
|
:param path: (Path) A config path that want to read the config
|
363
334
|
ignore file.
|
364
335
|
:param ignore_filename: (str) An ignore filename. Default is
|
365
|
-
|
336
|
+
``.confignore`` filename.
|
366
337
|
|
367
338
|
:rtype: bool
|
368
339
|
"""
|
@@ -428,24 +399,6 @@ def dynamic(
|
|
428
399
|
return extra
|
429
400
|
|
430
401
|
|
431
|
-
class Loader(Protocol): # pragma: no cov
|
432
|
-
type: str
|
433
|
-
path: Path
|
434
|
-
data: DictData
|
435
|
-
extras: DictData
|
436
|
-
externals: DictData
|
437
|
-
|
438
|
-
def __init__(self, *args, **kwargs) -> None: ...
|
439
|
-
|
440
|
-
@classmethod
|
441
|
-
def find(cls, name: str, *args, **kwargs) -> DictData: ...
|
442
|
-
|
443
|
-
@classmethod
|
444
|
-
def finds(
|
445
|
-
cls, obj: object, *args, **kwargs
|
446
|
-
) -> Iterator[tuple[str, DictData]]: ...
|
447
|
-
|
448
|
-
|
449
402
|
def pass_env(value: T) -> T: # pragma: no cov
|
450
403
|
"""Passing environment variable to an input value.
|
451
404
|
|
ddeutil/workflow/errors.py
CHANGED
@@ -38,8 +38,14 @@ def to_dict(exception: Exception, **kwargs) -> ErrorData: # pragma: no cov
|
|
38
38
|
|
39
39
|
|
40
40
|
class BaseError(Exception):
|
41
|
-
"""Base Workflow exception class will implement the
|
41
|
+
"""Base Workflow exception class will implement the ``refs`` argument for
|
42
42
|
making an error context to the result context.
|
43
|
+
|
44
|
+
Attributes:
|
45
|
+
refs: (:obj:str, optional)
|
46
|
+
context: (:obj:DictData)
|
47
|
+
params: (:obj:DictData)
|
48
|
+
|
43
49
|
"""
|
44
50
|
|
45
51
|
def __init__(
|
ddeutil/workflow/event.py
CHANGED
@@ -23,7 +23,7 @@ from typing_extensions import Self
|
|
23
23
|
|
24
24
|
from .__cron import WEEKDAYS, CronJob, CronJobYear, CronRunner, Options
|
25
25
|
from .__types import DictData, DictStr
|
26
|
-
from .conf import
|
26
|
+
from .conf import YamlParser
|
27
27
|
|
28
28
|
Interval = Literal["daily", "weekly", "monthly"]
|
29
29
|
|
@@ -139,7 +139,7 @@ class Crontab(BaseModel):
|
|
139
139
|
:rtype: Self
|
140
140
|
"""
|
141
141
|
extras: DictData = extras or {}
|
142
|
-
loader:
|
142
|
+
loader: YamlParser = YamlParser(name, extras=extras)
|
143
143
|
|
144
144
|
# NOTE: Validate the config type match with current connection model
|
145
145
|
if loader.type != cls.__name__:
|
ddeutil/workflow/job.py
CHANGED
@@ -402,13 +402,19 @@ class Job(BaseModel):
|
|
402
402
|
"""
|
403
403
|
# VALIDATE: Validate stage id should not duplicate.
|
404
404
|
rs: list[str] = []
|
405
|
+
rs_raise: list[str] = []
|
405
406
|
for stage in value:
|
406
407
|
name: str = stage.iden
|
407
408
|
if name in rs:
|
408
|
-
|
409
|
-
|
410
|
-
)
|
409
|
+
rs_raise.append(name)
|
410
|
+
continue
|
411
411
|
rs.append(name)
|
412
|
+
|
413
|
+
if rs_raise:
|
414
|
+
raise ValueError(
|
415
|
+
f"Stage name, {', '.join(repr(s) for s in rs_raise)}, should "
|
416
|
+
f"not be duplicate."
|
417
|
+
)
|
412
418
|
return value
|
413
419
|
|
414
420
|
@model_validator(mode="after")
|
ddeutil/workflow/result.py
CHANGED
@@ -13,6 +13,7 @@ from dataclasses import field
|
|
13
13
|
from datetime import datetime
|
14
14
|
from enum import Enum
|
15
15
|
from typing import Optional, Union
|
16
|
+
from zoneinfo import ZoneInfo
|
16
17
|
|
17
18
|
from pydantic import ConfigDict
|
18
19
|
from pydantic.dataclasses import dataclass
|
@@ -30,12 +31,20 @@ from . import (
|
|
30
31
|
WorkflowError,
|
31
32
|
)
|
32
33
|
from .__types import DictData
|
34
|
+
from .audits import TraceModel, get_trace
|
33
35
|
from .conf import dynamic
|
34
36
|
from .errors import ResultError
|
35
|
-
from .logs import TraceModel, get_dt_tznow, get_trace
|
36
37
|
from .utils import default_gen_id, gen_id, get_dt_now
|
37
38
|
|
38
39
|
|
40
|
+
def get_dt_tznow(tz: Optional[ZoneInfo] = None) -> datetime: # pragma: no cov
|
41
|
+
"""Return the current datetime object that passing the config timezone.
|
42
|
+
|
43
|
+
:rtype: datetime
|
44
|
+
"""
|
45
|
+
return get_dt_now(tz=dynamic("tz", f=tz))
|
46
|
+
|
47
|
+
|
39
48
|
class Status(str, Enum):
|
40
49
|
"""Status Int Enum object that use for tracking execution status to the
|
41
50
|
Result dataclass object.
|
ddeutil/workflow/reusables.py
CHANGED
@@ -44,7 +44,7 @@ from .errors import UtilError
|
|
44
44
|
T = TypeVar("T")
|
45
45
|
P = ParamSpec("P")
|
46
46
|
|
47
|
-
# NOTE: Adjust logging level of the
|
47
|
+
# NOTE: Adjust logging level of the ``asyncio`` to INFO level.
|
48
48
|
logging.getLogger("asyncio").setLevel(logging.INFO)
|
49
49
|
|
50
50
|
|