ddeutil-workflow 0.0.57__py3-none-any.whl → 0.0.59__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__cron.py +3 -3
- ddeutil/workflow/conf.py +7 -5
- ddeutil/workflow/event.py +12 -12
- ddeutil/workflow/exceptions.py +3 -3
- ddeutil/workflow/job.py +44 -36
- ddeutil/workflow/logs.py +123 -82
- ddeutil/workflow/params.py +13 -5
- ddeutil/workflow/result.py +18 -18
- ddeutil/workflow/reusables.py +9 -9
- ddeutil/workflow/scheduler.py +17 -15
- ddeutil/workflow/stages.py +149 -102
- ddeutil/workflow/utils.py +6 -6
- ddeutil/workflow/workflow.py +48 -48
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.59.dist-info}/METADATA +6 -3
- ddeutil_workflow-0.0.59.dist-info/RECORD +31 -0
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.59.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.57.dist-info/RECORD +0 -31
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.59.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.59.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.59.dist-info}/top_level.txt +0 -0
ddeutil/workflow/reusables.py
CHANGED
@@ -7,6 +7,7 @@
|
|
7
7
|
"""Reusables module that keep any templating functions."""
|
8
8
|
from __future__ import annotations
|
9
9
|
|
10
|
+
import copy
|
10
11
|
import inspect
|
11
12
|
import logging
|
12
13
|
from ast import Call, Constant, Expr, Module, Name, parse
|
@@ -35,7 +36,7 @@ logger = logging.getLogger("ddeutil.workflow")
|
|
35
36
|
logging.getLogger("asyncio").setLevel(logging.INFO)
|
36
37
|
|
37
38
|
|
38
|
-
FILTERS: dict[str,
|
39
|
+
FILTERS: dict[str, Callable] = { # pragma: no cov
|
39
40
|
"abs": abs,
|
40
41
|
"str": str,
|
41
42
|
"int": int,
|
@@ -258,9 +259,9 @@ def str2template(
|
|
258
259
|
value: str,
|
259
260
|
params: DictData,
|
260
261
|
*,
|
261
|
-
filters: dict[str, FilterRegistry]
|
262
|
+
filters: Optional[dict[str, FilterRegistry]] = None,
|
262
263
|
registers: Optional[list[str]] = None,
|
263
|
-
) -> str:
|
264
|
+
) -> Optional[str]:
|
264
265
|
"""(Sub-function) Pass param to template string that can search by
|
265
266
|
``RE_CALLER`` regular expression.
|
266
267
|
|
@@ -326,7 +327,7 @@ def str2template(
|
|
326
327
|
def param2template(
|
327
328
|
value: T,
|
328
329
|
params: DictData,
|
329
|
-
filters: dict[str, FilterRegistry]
|
330
|
+
filters: Optional[dict[str, FilterRegistry]] = None,
|
330
331
|
*,
|
331
332
|
extras: Optional[DictData] = None,
|
332
333
|
) -> T:
|
@@ -381,7 +382,7 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
|
|
381
382
|
|
382
383
|
|
383
384
|
@custom_filter("coalesce") # pragma: no cov
|
384
|
-
def coalesce(value: T
|
385
|
+
def coalesce(value: Optional[T], default: Any) -> T:
|
385
386
|
"""Coalesce with default value if the main value is None."""
|
386
387
|
return default if value is None else value
|
387
388
|
|
@@ -450,11 +451,10 @@ def make_registry(
|
|
450
451
|
:rtype: dict[str, Registry]
|
451
452
|
"""
|
452
453
|
rs: dict[str, Registry] = {}
|
453
|
-
regis_calls: list[str] =
|
454
|
-
"registry_caller", f=registries
|
455
|
-
)
|
454
|
+
regis_calls: list[str] = copy.deepcopy(
|
455
|
+
dynamic("registry_caller", f=registries)
|
456
|
+
)
|
456
457
|
regis_calls.extend(["ddeutil.vendors"])
|
457
|
-
|
458
458
|
for module in regis_calls:
|
459
459
|
# NOTE: try to sequential import task functions
|
460
460
|
try:
|
ddeutil/workflow/scheduler.py
CHANGED
@@ -57,7 +57,7 @@ except ImportError: # pragma: no cov
|
|
57
57
|
from .__cron import CronRunner
|
58
58
|
from .__types import DictData, TupleStr
|
59
59
|
from .conf import FileLoad, Loader, dynamic
|
60
|
-
from .event import
|
60
|
+
from .event import Crontab
|
61
61
|
from .exceptions import ScheduleException, WorkflowException
|
62
62
|
from .logs import Audit, get_audit
|
63
63
|
from .result import SUCCESS, Result
|
@@ -103,9 +103,9 @@ class ScheduleWorkflow(BaseModel):
|
|
103
103
|
description="An alias name of workflow that use for schedule model.",
|
104
104
|
)
|
105
105
|
name: str = Field(description="A workflow name.")
|
106
|
-
on: list[
|
106
|
+
on: list[Crontab] = Field(
|
107
107
|
default_factory=list,
|
108
|
-
description="An override the list of
|
108
|
+
description="An override the list of Crontab object values.",
|
109
109
|
)
|
110
110
|
values: DictData = Field(
|
111
111
|
default_factory=dict,
|
@@ -158,15 +158,17 @@ class ScheduleWorkflow(BaseModel):
|
|
158
158
|
return data
|
159
159
|
|
160
160
|
@field_validator("on", mode="after")
|
161
|
-
def __on_no_dup__(
|
161
|
+
def __on_no_dup__(
|
162
|
+
cls, value: list[Crontab], info: ValidationInfo
|
163
|
+
) -> list[Crontab]:
|
162
164
|
"""Validate the on fields should not contain duplicate values and if it
|
163
165
|
contains every minute value, it should have only one on value.
|
164
166
|
|
165
|
-
:param value: (list[
|
167
|
+
:param value: (list[Crontab]) A list of `Crontab` object.
|
166
168
|
:param info: (ValidationInfo) An validation info object for getting an
|
167
169
|
extra parameter.
|
168
170
|
|
169
|
-
:rtype: list[
|
171
|
+
:rtype: list[Crontab]
|
170
172
|
"""
|
171
173
|
set_ons: set[str] = {str(on.cronjob) for on in value}
|
172
174
|
if len(set_ons) != len(value):
|
@@ -209,7 +211,7 @@ class ScheduleWorkflow(BaseModel):
|
|
209
211
|
|
210
212
|
# IMPORTANT: Create the default 'on' value if it does not pass the `on`
|
211
213
|
# field to the Schedule object.
|
212
|
-
ons: list[
|
214
|
+
ons: list[Crontab] = self.on or wf.on.copy()
|
213
215
|
workflow_tasks: list[WorkflowTask] = []
|
214
216
|
for on in ons:
|
215
217
|
|
@@ -333,9 +335,9 @@ class Schedule(BaseModel):
|
|
333
335
|
def pending(
|
334
336
|
self,
|
335
337
|
*,
|
336
|
-
stop: datetime
|
338
|
+
stop: Optional[datetime] = None,
|
337
339
|
audit: type[Audit] | None = None,
|
338
|
-
parent_run_id: str
|
340
|
+
parent_run_id: Optional[str] = None,
|
339
341
|
) -> Result: # pragma: no cov
|
340
342
|
"""Pending this schedule tasks with the schedule package.
|
341
343
|
|
@@ -382,7 +384,7 @@ DecoratorCancelJob = Callable[[ReturnResultOrCancel], ReturnResultOrCancel]
|
|
382
384
|
|
383
385
|
def catch_exceptions(
|
384
386
|
cancel_on_failure: bool = False,
|
385
|
-
parent_run_id: str
|
387
|
+
parent_run_id: Optional[str] = None,
|
386
388
|
) -> DecoratorCancelJob:
|
387
389
|
"""Catch exception error from scheduler job that running with schedule
|
388
390
|
package and return CancelJob if this function raise an error.
|
@@ -438,7 +440,7 @@ def schedule_task(
|
|
438
440
|
threads: ReleaseThreads,
|
439
441
|
audit: type[Audit],
|
440
442
|
*,
|
441
|
-
parent_run_id: str
|
443
|
+
parent_run_id: Optional[str] = None,
|
442
444
|
extras: Optional[DictData] = None,
|
443
445
|
) -> ResultOrCancel:
|
444
446
|
"""Schedule task function that generate thread of workflow task release
|
@@ -556,7 +558,7 @@ def schedule_task(
|
|
556
558
|
|
557
559
|
def monitor(
|
558
560
|
threads: ReleaseThreads,
|
559
|
-
parent_run_id: str
|
561
|
+
parent_run_id: Optional[str] = None,
|
560
562
|
) -> None: # pragma: no cov
|
561
563
|
"""Monitoring function that running every five minute for track long-running
|
562
564
|
thread instance from the schedule_control function that run every minute.
|
@@ -683,11 +685,11 @@ def scheduler_pending(
|
|
683
685
|
|
684
686
|
def schedule_control(
|
685
687
|
schedules: list[str],
|
686
|
-
stop: datetime
|
688
|
+
stop: Optional[datetime] = None,
|
687
689
|
*,
|
688
690
|
extras: DictData | None = None,
|
689
691
|
audit: type[Audit] | None = None,
|
690
|
-
parent_run_id: str
|
692
|
+
parent_run_id: Optional[str] = None,
|
691
693
|
) -> Result: # pragma: no cov
|
692
694
|
"""Scheduler control function that run the chuck of schedules every minute
|
693
695
|
and this function release monitoring thread for tracking undead thread in
|
@@ -742,7 +744,7 @@ def schedule_control(
|
|
742
744
|
|
743
745
|
|
744
746
|
def schedule_runner(
|
745
|
-
stop: datetime
|
747
|
+
stop: Optional[datetime] = None,
|
746
748
|
*,
|
747
749
|
max_process: int | None = None,
|
748
750
|
extras: DictData | None = None,
|