ddeutil-workflow 0.0.13__py3-none-any.whl → 0.0.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +4 -1
- ddeutil/workflow/__types.py +24 -8
- ddeutil/workflow/api.py +2 -2
- ddeutil/workflow/conf.py +41 -0
- ddeutil/workflow/cron.py +19 -12
- ddeutil/workflow/job.py +189 -153
- ddeutil/workflow/log.py +28 -14
- ddeutil/workflow/scheduler.py +233 -112
- ddeutil/workflow/stage.py +66 -33
- ddeutil/workflow/utils.py +106 -40
- {ddeutil_workflow-0.0.13.dist-info → ddeutil_workflow-0.0.14.dist-info}/METADATA +6 -4
- ddeutil_workflow-0.0.14.dist-info/RECORD +22 -0
- {ddeutil_workflow-0.0.13.dist-info → ddeutil_workflow-0.0.14.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.13.dist-info/RECORD +0 -21
- {ddeutil_workflow-0.0.13.dist-info → ddeutil_workflow-0.0.14.dist-info}/LICENSE +0 -0
- {ddeutil_workflow-0.0.13.dist-info → ddeutil_workflow-0.0.14.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.13.dist-info → ddeutil_workflow-0.0.14.dist-info}/top_level.txt +0 -0
ddeutil/workflow/log.py
CHANGED
@@ -12,7 +12,7 @@ from abc import ABC, abstractmethod
|
|
12
12
|
from datetime import datetime
|
13
13
|
from functools import lru_cache
|
14
14
|
from pathlib import Path
|
15
|
-
from typing import Optional, Union
|
15
|
+
from typing import ClassVar, Optional, Union
|
16
16
|
|
17
17
|
from ddeutil.core import str2bool
|
18
18
|
from pydantic import BaseModel, Field
|
@@ -20,12 +20,15 @@ from pydantic.functional_validators import model_validator
|
|
20
20
|
from typing_extensions import Self
|
21
21
|
|
22
22
|
from .__types import DictData
|
23
|
-
from .utils import
|
23
|
+
from .utils import load_config
|
24
24
|
|
25
25
|
|
26
26
|
@lru_cache
|
27
27
|
def get_logger(name: str):
|
28
|
-
"""Return logger with an input module name.
|
28
|
+
"""Return logger object with an input module name.
|
29
|
+
|
30
|
+
:param name: A module name that want to log.
|
31
|
+
"""
|
29
32
|
logger = logging.getLogger(name)
|
30
33
|
formatter = logging.Formatter(
|
31
34
|
fmt=(
|
@@ -45,7 +48,10 @@ def get_logger(name: str):
|
|
45
48
|
|
46
49
|
|
47
50
|
class BaseLog(BaseModel, ABC):
|
48
|
-
"""Base Log Pydantic Model abstraction that implement
|
51
|
+
"""Base Log Pydantic Model with abstraction class property that implement
|
52
|
+
only model fields. This model should to use with inherit to logging
|
53
|
+
sub-class like file, sqlite, etc.
|
54
|
+
"""
|
49
55
|
|
50
56
|
name: str = Field(description="A workflow name.")
|
51
57
|
on: str = Field(description="A cronjob string of this piepline schedule.")
|
@@ -61,7 +67,11 @@ class BaseLog(BaseModel, ABC):
|
|
61
67
|
update: datetime = Field(default_factory=datetime.now)
|
62
68
|
|
63
69
|
@model_validator(mode="after")
|
64
|
-
def __model_action(self):
|
70
|
+
def __model_action(self) -> Self:
|
71
|
+
"""Do before the Log action with WORKFLOW_LOG_ENABLE_WRITE env variable.
|
72
|
+
|
73
|
+
:rtype: Self
|
74
|
+
"""
|
65
75
|
if str2bool(os.getenv("WORKFLOW_LOG_ENABLE_WRITE", "false")):
|
66
76
|
self.do_before()
|
67
77
|
return self
|
@@ -71,7 +81,7 @@ class BaseLog(BaseModel, ABC):
|
|
71
81
|
|
72
82
|
@abstractmethod
|
73
83
|
def save(self, excluded: list[str] | None) -> None:
|
74
|
-
"""Save logging"""
|
84
|
+
"""Save this model logging to target logging store."""
|
75
85
|
raise NotImplementedError("Log should implement ``save`` method.")
|
76
86
|
|
77
87
|
|
@@ -81,13 +91,19 @@ class FileLog(BaseLog):
|
|
81
91
|
``self.save`` method for file.
|
82
92
|
"""
|
83
93
|
|
94
|
+
filename: ClassVar[str] = (
|
95
|
+
"./logs/workflow={name}/release={release:%Y%m%d%H%M%S}"
|
96
|
+
)
|
97
|
+
|
84
98
|
def do_before(self) -> None:
|
85
99
|
"""Create directory of release before saving log file."""
|
86
100
|
self.pointer().mkdir(parents=True, exist_ok=True)
|
87
101
|
|
88
102
|
@classmethod
|
89
103
|
def find_logs(cls, name: str):
|
90
|
-
pointer: Path =
|
104
|
+
pointer: Path = (
|
105
|
+
load_config().engine.paths.root / f"./logs/workflow={name}"
|
106
|
+
)
|
91
107
|
for file in pointer.glob("./release=*/*.log"):
|
92
108
|
with file.open(mode="r", encoding="utf-8") as f:
|
93
109
|
yield json.load(f)
|
@@ -96,7 +112,7 @@ class FileLog(BaseLog):
|
|
96
112
|
def find_log(cls, name: str, release: datetime | None = None):
|
97
113
|
if release is not None:
|
98
114
|
pointer: Path = (
|
99
|
-
|
115
|
+
load_config().engine.paths.root
|
100
116
|
/ f"./logs/workflow={name}/release={release:%Y%m%d%H%M%S}"
|
101
117
|
)
|
102
118
|
if not pointer.exists():
|
@@ -129,9 +145,8 @@ class FileLog(BaseLog):
|
|
129
145
|
return False
|
130
146
|
|
131
147
|
# NOTE: create pointer path that use the same logic of pointer method.
|
132
|
-
pointer: Path = (
|
133
|
-
|
134
|
-
/ f"./logs/workflow={name}/release={release:%Y%m%d%H%M%S}"
|
148
|
+
pointer: Path = load_config().engine.paths.root / cls.filename.format(
|
149
|
+
name=name, release=release
|
135
150
|
)
|
136
151
|
|
137
152
|
if not queue:
|
@@ -143,9 +158,8 @@ class FileLog(BaseLog):
|
|
143
158
|
|
144
159
|
:rtype: Path
|
145
160
|
"""
|
146
|
-
return (
|
147
|
-
|
148
|
-
/ f"./logs/workflow={self.name}/release={self.release:%Y%m%d%H%M%S}"
|
161
|
+
return load_config().engine.paths.root / self.filename.format(
|
162
|
+
name=self.name, release=self.release
|
149
163
|
)
|
150
164
|
|
151
165
|
def save(self, excluded: list[str] | None) -> Self:
|