ddeutil-workflow 0.0.50__py3-none-any.whl → 0.0.52__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +7 -56
- ddeutil/workflow/conf.py +2 -2
- ddeutil/workflow/cron.py +46 -20
- ddeutil/workflow/job.py +181 -100
- ddeutil/workflow/logs.py +22 -18
- ddeutil/workflow/params.py +56 -16
- ddeutil/workflow/reusables.py +4 -2
- ddeutil/workflow/scheduler.py +5 -1
- ddeutil/workflow/stages.py +320 -154
- ddeutil/workflow/workflow.py +83 -74
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.52.dist-info}/METADATA +69 -13
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.52.dist-info}/RECORD +16 -16
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.52.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.52.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.52.dist-info}/top_level.txt +0 -0
ddeutil/workflow/logs.py
CHANGED
@@ -224,18 +224,30 @@ class BaseTrace(ABC): # pragma: no cov
|
|
224
224
|
"Adjust make message method for this trace object before using."
|
225
225
|
)
|
226
226
|
|
227
|
-
def
|
228
|
-
|
229
|
-
|
227
|
+
def __logging(
|
228
|
+
self, message: str, mode: str, *, is_err: bool = False
|
229
|
+
) -> None:
|
230
|
+
"""Write trace log with append mode and logging this message with any
|
231
|
+
logging level.
|
230
232
|
|
231
233
|
:param message: (str) A message that want to log.
|
232
234
|
"""
|
233
235
|
msg: str = self.make_message(message)
|
234
236
|
|
235
|
-
if
|
236
|
-
self.
|
237
|
+
if mode != "debug" or (
|
238
|
+
mode == "debug" and dynamic("debug", extras=self.extras)
|
239
|
+
):
|
240
|
+
self.writer(msg, is_err=is_err)
|
241
|
+
|
242
|
+
getattr(logger, mode)(msg, stacklevel=3)
|
243
|
+
|
244
|
+
def debug(self, message: str):
|
245
|
+
"""Write trace log with append mode and logging this message with the
|
246
|
+
DEBUG level.
|
237
247
|
|
238
|
-
|
248
|
+
:param message: (str) A message that want to log.
|
249
|
+
"""
|
250
|
+
self.__logging(message, mode="debug")
|
239
251
|
|
240
252
|
def info(self, message: str) -> None:
|
241
253
|
"""Write trace log with append mode and logging this message with the
|
@@ -243,9 +255,7 @@ class BaseTrace(ABC): # pragma: no cov
|
|
243
255
|
|
244
256
|
:param message: (str) A message that want to log.
|
245
257
|
"""
|
246
|
-
|
247
|
-
self.writer(msg)
|
248
|
-
logger.info(msg, stacklevel=2)
|
258
|
+
self.__logging(message, mode="info")
|
249
259
|
|
250
260
|
def warning(self, message: str) -> None:
|
251
261
|
"""Write trace log with append mode and logging this message with the
|
@@ -253,9 +263,7 @@ class BaseTrace(ABC): # pragma: no cov
|
|
253
263
|
|
254
264
|
:param message: (str) A message that want to log.
|
255
265
|
"""
|
256
|
-
|
257
|
-
self.writer(msg)
|
258
|
-
logger.warning(msg, stacklevel=2)
|
266
|
+
self.__logging(message, mode="warning")
|
259
267
|
|
260
268
|
def error(self, message: str) -> None:
|
261
269
|
"""Write trace log with append mode and logging this message with the
|
@@ -263,9 +271,7 @@ class BaseTrace(ABC): # pragma: no cov
|
|
263
271
|
|
264
272
|
:param message: (str) A message that want to log.
|
265
273
|
"""
|
266
|
-
|
267
|
-
self.writer(msg, is_err=True)
|
268
|
-
logger.error(msg, stacklevel=2)
|
274
|
+
self.__logging(message, mode="error", is_err=True)
|
269
275
|
|
270
276
|
def exception(self, message: str) -> None:
|
271
277
|
"""Write trace log with append mode and logging this message with the
|
@@ -273,9 +279,7 @@ class BaseTrace(ABC): # pragma: no cov
|
|
273
279
|
|
274
280
|
:param message: (str) A message that want to log.
|
275
281
|
"""
|
276
|
-
|
277
|
-
self.writer(msg, is_err=True)
|
278
|
-
logger.exception(msg, stacklevel=2)
|
282
|
+
self.__logging(message, mode="exception", is_err=True)
|
279
283
|
|
280
284
|
async def adebug(self, message: str) -> None: # pragma: no cov
|
281
285
|
"""Async write trace log with append mode and logging this message with
|
ddeutil/workflow/params.py
CHANGED
@@ -12,29 +12,17 @@ passing an input value to target execution method.
|
|
12
12
|
"""
|
13
13
|
from __future__ import annotations
|
14
14
|
|
15
|
-
import decimal
|
16
15
|
from abc import ABC, abstractmethod
|
17
16
|
from datetime import date, datetime
|
17
|
+
from decimal import Decimal, InvalidOperation
|
18
18
|
from typing import Annotated, Any, Literal, Optional, TypeVar, Union
|
19
19
|
|
20
20
|
from ddeutil.core import str2dict, str2list
|
21
21
|
from pydantic import BaseModel, Field
|
22
22
|
|
23
|
-
from .__types import TupleStr
|
24
23
|
from .exceptions import ParamValueException
|
25
24
|
from .utils import get_d_now, get_dt_now
|
26
25
|
|
27
|
-
__all__: TupleStr = (
|
28
|
-
"ChoiceParam",
|
29
|
-
"DatetimeParam",
|
30
|
-
"DateParam",
|
31
|
-
"IntParam",
|
32
|
-
"Param",
|
33
|
-
"StrParam",
|
34
|
-
"ArrayParam",
|
35
|
-
"MapParam",
|
36
|
-
)
|
37
|
-
|
38
26
|
T = TypeVar("T")
|
39
27
|
|
40
28
|
|
@@ -201,11 +189,60 @@ class IntParam(DefaultParam):
|
|
201
189
|
return value
|
202
190
|
|
203
191
|
|
204
|
-
#
|
192
|
+
class FloatParam(DefaultParam): # pragma: no cov
|
193
|
+
type: Literal["float"] = "float"
|
194
|
+
precision: int = 6
|
195
|
+
|
196
|
+
def rounding(self, value: float) -> float:
|
197
|
+
round_str: str = f"{{0:.{self.precision}f}}"
|
198
|
+
return float(round_str.format(round(value, self.precision)))
|
199
|
+
|
200
|
+
def receive(self, value: Optional[Union[float, int, str]] = None) -> float:
|
201
|
+
|
202
|
+
if value in None:
|
203
|
+
return self.default
|
204
|
+
|
205
|
+
if isinstance(value, float):
|
206
|
+
return self.rounding(value)
|
207
|
+
elif isinstance(value, int):
|
208
|
+
return self.rounding(float(value))
|
209
|
+
elif not isinstance(value, str):
|
210
|
+
raise TypeError(
|
211
|
+
"Received value type does not math with str, float, or int."
|
212
|
+
)
|
213
|
+
|
214
|
+
try:
|
215
|
+
return self.rounding(float(value))
|
216
|
+
except Exception:
|
217
|
+
raise
|
218
|
+
|
219
|
+
|
205
220
|
class DecimalParam(DefaultParam): # pragma: no cov
|
221
|
+
"""Decimal parameter."""
|
222
|
+
|
206
223
|
type: Literal["decimal"] = "decimal"
|
224
|
+
precision: int = 6
|
225
|
+
|
226
|
+
def rounding(self, value: Decimal) -> Decimal:
|
227
|
+
return value.quantize(Decimal(10) ** -self.precision)
|
228
|
+
|
229
|
+
def receive(self, value: float | Decimal | None = None) -> Decimal:
|
207
230
|
|
208
|
-
|
231
|
+
if isinstance(value, float):
|
232
|
+
return self.rounding(Decimal(value))
|
233
|
+
elif isinstance(value, Decimal):
|
234
|
+
return self.rounding(value)
|
235
|
+
elif not isinstance(value, str):
|
236
|
+
raise TypeError(
|
237
|
+
"Received value type does not math with str, float, or decimal."
|
238
|
+
)
|
239
|
+
|
240
|
+
try:
|
241
|
+
return self.rounding(Decimal(value))
|
242
|
+
except InvalidOperation as e:
|
243
|
+
raise ValueError(
|
244
|
+
"String that want to convert to decimal type does not valid."
|
245
|
+
) from e
|
209
246
|
|
210
247
|
|
211
248
|
class ChoiceParam(BaseParam):
|
@@ -223,7 +260,8 @@ class ChoiceParam(BaseParam):
|
|
223
260
|
:rtype: str
|
224
261
|
"""
|
225
262
|
# NOTE:
|
226
|
-
# Return the first value in options if it does not pass any input
|
263
|
+
# Return the first value in options if it does not pass any input
|
264
|
+
# value.
|
227
265
|
if value is None:
|
228
266
|
return self.options[0]
|
229
267
|
if value not in self.options:
|
@@ -314,6 +352,8 @@ Param = Annotated[
|
|
314
352
|
ChoiceParam,
|
315
353
|
DatetimeParam,
|
316
354
|
DateParam,
|
355
|
+
FloatParam,
|
356
|
+
DecimalParam,
|
317
357
|
IntParam,
|
318
358
|
StrParam,
|
319
359
|
],
|
ddeutil/workflow/reusables.py
CHANGED
@@ -400,12 +400,14 @@ DecoratorTagFunc = Callable[[Callable[[...], Any]], ReturnTagFunc]
|
|
400
400
|
|
401
401
|
|
402
402
|
def tag(
|
403
|
-
name: str
|
403
|
+
name: Optional[str] = None,
|
404
|
+
alias: Optional[str] = None,
|
404
405
|
) -> DecoratorTagFunc: # pragma: no cov
|
405
406
|
"""Tag decorator function that set function attributes, ``tag`` and ``name``
|
406
407
|
for making registries variable.
|
407
408
|
|
408
409
|
:param: name: (str) A tag name for make different use-case of a function.
|
410
|
+
It will use 'latest' if this tag name does not set.
|
409
411
|
:param: alias: (str) A alias function name that keeping in registries.
|
410
412
|
If this value does not supply, it will use original function name
|
411
413
|
from `__name__` argument.
|
@@ -414,7 +416,7 @@ def tag(
|
|
414
416
|
"""
|
415
417
|
|
416
418
|
def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
|
417
|
-
func.tag = name
|
419
|
+
func.tag = name or "latest"
|
418
420
|
func.name = alias or func.__name__.replace("_", "-")
|
419
421
|
|
420
422
|
@wraps(func)
|
ddeutil/workflow/scheduler.py
CHANGED
@@ -781,6 +781,7 @@ def schedule_control(
|
|
781
781
|
def schedule_runner(
|
782
782
|
stop: datetime | None = None,
|
783
783
|
*,
|
784
|
+
max_process: int | None = None,
|
784
785
|
extras: DictData | None = None,
|
785
786
|
excluded: list[str] | None = None,
|
786
787
|
) -> Result: # pragma: no cov
|
@@ -790,6 +791,7 @@ def schedule_runner(
|
|
790
791
|
path by `WORKFLOW_APP_MAX_SCHEDULE_PER_PROCESS` value.
|
791
792
|
|
792
793
|
:param stop: A stop datetime object that force stop running scheduler.
|
794
|
+
:param max_process: (int) The maximum process that want to run this func.
|
793
795
|
:param extras: An extra parameter that want to override core config.
|
794
796
|
:param excluded: A list of schedule name that want to exclude from finding.
|
795
797
|
|
@@ -813,7 +815,9 @@ def schedule_runner(
|
|
813
815
|
context: DictData = {"schedules": [], "threads": []}
|
814
816
|
|
815
817
|
with ProcessPoolExecutor(
|
816
|
-
max_workers=dynamic(
|
818
|
+
max_workers=dynamic(
|
819
|
+
"max_schedule_process", f=max_process, extras=extras
|
820
|
+
),
|
817
821
|
) as executor:
|
818
822
|
|
819
823
|
futures: list[Future] = [
|