ddeutil-workflow 0.0.57__py3-none-any.whl → 0.0.58__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/conf.py +1 -1
- ddeutil/workflow/event.py +10 -10
- ddeutil/workflow/job.py +9 -2
- ddeutil/workflow/logs.py +46 -32
- ddeutil/workflow/params.py +4 -0
- ddeutil/workflow/scheduler.py +9 -7
- ddeutil/workflow/stages.py +81 -34
- ddeutil/workflow/workflow.py +17 -17
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.58.dist-info}/METADATA +1 -1
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.58.dist-info}/RECORD +15 -15
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.58.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.58.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.58.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.57.dist-info → ddeutil_workflow-0.0.58.dist-info}/top_level.txt +0 -0
ddeutil/workflow/__about__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__: str = "0.0.
|
1
|
+
__version__: str = "0.0.58"
|
ddeutil/workflow/conf.py
CHANGED
@@ -218,7 +218,7 @@ class BaseLoad(ABC): # pragma: no cov
|
|
218
218
|
|
219
219
|
class FileLoad(BaseLoad):
|
220
220
|
"""Base Load object that use to search config data by given some identity
|
221
|
-
value like name of `Workflow` or `
|
221
|
+
value like name of `Workflow` or `Crontab` templates.
|
222
222
|
|
223
223
|
:param name: (str) A name of key of config data that read with YAML
|
224
224
|
Environment object.
|
ddeutil/workflow/event.py
CHANGED
@@ -3,8 +3,8 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
-
"""Event module that store all event object. Now, it has only `
|
7
|
-
model these are schedule with crontab event.
|
6
|
+
"""Event module that store all event object. Now, it has only `Crontab` and
|
7
|
+
`CrontabYear` model these are schedule with crontab event.
|
8
8
|
"""
|
9
9
|
from __future__ import annotations
|
10
10
|
|
@@ -63,9 +63,9 @@ def interval2crontab(
|
|
63
63
|
return f"{h} {m} {'1' if interval == 'monthly' else '*'} * {d}"
|
64
64
|
|
65
65
|
|
66
|
-
class
|
67
|
-
"""
|
68
|
-
and generate CronRunner object from this crontab value.
|
66
|
+
class Crontab(BaseModel):
|
67
|
+
"""Cron event model (Warped the CronJob object by Pydantic model) to keep
|
68
|
+
crontab value and generate CronRunner object from this crontab value.
|
69
69
|
|
70
70
|
Methods:
|
71
71
|
- generate: is the main use-case of this schedule object.
|
@@ -128,7 +128,7 @@ class On(BaseModel):
|
|
128
128
|
extras: DictData | None = None,
|
129
129
|
) -> Self:
|
130
130
|
"""Constructor from the name of config loader that will use loader
|
131
|
-
object for getting the `
|
131
|
+
object for getting the `Crontab` data.
|
132
132
|
|
133
133
|
:param name: (str) A name of config that will get from loader.
|
134
134
|
:param extras: (DictData) An extra parameter that use to override core
|
@@ -172,7 +172,7 @@ class On(BaseModel):
|
|
172
172
|
def __prepare_values(cls, data: Any) -> Any:
|
173
173
|
"""Extract tz key from value and change name to timezone key.
|
174
174
|
|
175
|
-
:param data: (DictData) A data that want to pass for create an
|
175
|
+
:param data: (DictData) A data that want to pass for create an Crontab
|
176
176
|
model.
|
177
177
|
|
178
178
|
:rtype: DictData
|
@@ -265,9 +265,9 @@ class On(BaseModel):
|
|
265
265
|
return runner
|
266
266
|
|
267
267
|
|
268
|
-
class
|
269
|
-
"""
|
270
|
-
some data schedule tools like AWS Glue.
|
268
|
+
class CrontabYear(Crontab):
|
269
|
+
"""Cron event with enhance Year Pydantic model for limit year matrix that
|
270
|
+
use by some data schedule tools like AWS Glue.
|
271
271
|
"""
|
272
272
|
|
273
273
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
ddeutil/workflow/job.py
CHANGED
@@ -839,7 +839,7 @@ def local_execute(
|
|
839
839
|
ls: str = "Fail-Fast" if fail_fast_flag else "All-Completed"
|
840
840
|
workers: int = job.strategy.max_parallel
|
841
841
|
result.trace.info(
|
842
|
-
f"[JOB]: Execute {ls}: {job.id} with {workers} "
|
842
|
+
f"[JOB]: Execute {ls}: {job.id!r} with {workers} "
|
843
843
|
f"worker{'s' if workers > 1 else ''}."
|
844
844
|
)
|
845
845
|
|
@@ -886,7 +886,14 @@ def local_execute(
|
|
886
886
|
future.cancel()
|
887
887
|
time.sleep(0.075)
|
888
888
|
|
889
|
-
nd: str =
|
889
|
+
nd: str = (
|
890
|
+
(
|
891
|
+
f", {len(not_done)} strateg"
|
892
|
+
f"{'ies' if len(not_done) > 1 else 'y'} not run!!!"
|
893
|
+
)
|
894
|
+
if not_done
|
895
|
+
else ""
|
896
|
+
)
|
890
897
|
result.trace.debug(f"[JOB]: ... Job was set Fail-Fast{nd}")
|
891
898
|
done: list[Future] = as_completed(futures)
|
892
899
|
|
ddeutil/workflow/logs.py
CHANGED
@@ -70,18 +70,28 @@ def get_dt_tznow() -> datetime: # pragma: no cov
|
|
70
70
|
return get_dt_now(tz=config.tz)
|
71
71
|
|
72
72
|
|
73
|
+
PREFIX_LOGS: dict[str, dict] = {
|
74
|
+
"CALLER": {"emoji": ""},
|
75
|
+
"STAGE": {"emoji": ""},
|
76
|
+
"JOB": {"emoji": ""},
|
77
|
+
"WORKFLOW": {"emoji": "🏃"},
|
78
|
+
"RELEASE": {"emoji": ""},
|
79
|
+
"POKE": {"emoji": ""},
|
80
|
+
} # pragma: no cov
|
81
|
+
|
82
|
+
|
73
83
|
class TraceMeta(BaseModel): # pragma: no cov
|
74
84
|
"""Trace Metadata model for making the current metadata of this CPU, Memory
|
75
85
|
process, and thread data.
|
76
86
|
"""
|
77
87
|
|
78
|
-
mode: Literal["stdout", "stderr"]
|
79
|
-
datetime: str
|
80
|
-
process: int
|
81
|
-
thread: int
|
82
|
-
message: str
|
83
|
-
filename: str
|
84
|
-
lineno: int
|
88
|
+
mode: Literal["stdout", "stderr"] = Field(description="A meta mode.")
|
89
|
+
datetime: str = Field(description="A datetime in string format.")
|
90
|
+
process: int = Field(description="A process ID.")
|
91
|
+
thread: int = Field(description="A thread ID.")
|
92
|
+
message: str = Field(description="A message log.")
|
93
|
+
filename: str = Field(description="A filename of this log.")
|
94
|
+
lineno: int = Field(description="A line number of this log.")
|
85
95
|
|
86
96
|
@classmethod
|
87
97
|
def make(
|
@@ -142,11 +152,9 @@ class TraceData(BaseModel): # pragma: no cov
|
|
142
152
|
"""
|
143
153
|
data: DictStr = {"stdout": "", "stderr": "", "meta": []}
|
144
154
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
if (file / "stderr.txt").exists():
|
149
|
-
data["stderr"] = (file / "stderr.txt").read_text(encoding="utf-8")
|
155
|
+
for mode in ("stdout", "stderr"):
|
156
|
+
if (file / f"{mode}.txt").exists():
|
157
|
+
data[mode] = (file / f"{mode}.txt").read_text(encoding="utf-8")
|
150
158
|
|
151
159
|
if (file / "metadata.json").exists():
|
152
160
|
data["meta"] = [
|
@@ -288,16 +296,30 @@ class BaseTrace(ABC): # pragma: no cov
|
|
288
296
|
"""
|
289
297
|
self.__logging(message, mode="exception", is_err=True)
|
290
298
|
|
299
|
+
async def __alogging(
|
300
|
+
self, message: str, mode: str, *, is_err: bool = False
|
301
|
+
) -> None:
|
302
|
+
"""Write trace log with append mode and logging this message with any
|
303
|
+
logging level.
|
304
|
+
|
305
|
+
:param message: (str) A message that want to log.
|
306
|
+
"""
|
307
|
+
msg: str = prepare_newline(self.make_message(message))
|
308
|
+
|
309
|
+
if mode != "debug" or (
|
310
|
+
mode == "debug" and dynamic("debug", extras=self.extras)
|
311
|
+
):
|
312
|
+
await self.awriter(msg, is_err=is_err)
|
313
|
+
|
314
|
+
getattr(logger, mode)(msg, stacklevel=3)
|
315
|
+
|
291
316
|
async def adebug(self, message: str) -> None: # pragma: no cov
|
292
317
|
"""Async write trace log with append mode and logging this message with
|
293
318
|
the DEBUG level.
|
294
319
|
|
295
320
|
:param message: (str) A message that want to log.
|
296
321
|
"""
|
297
|
-
|
298
|
-
if dynamic("debug", extras=self.extras):
|
299
|
-
await self.awriter(msg)
|
300
|
-
logger.info(msg, stacklevel=2)
|
322
|
+
await self.__alogging(message, mode="debug")
|
301
323
|
|
302
324
|
async def ainfo(self, message: str) -> None: # pragma: no cov
|
303
325
|
"""Async write trace log with append mode and logging this message with
|
@@ -305,9 +327,7 @@ class BaseTrace(ABC): # pragma: no cov
|
|
305
327
|
|
306
328
|
:param message: (str) A message that want to log.
|
307
329
|
"""
|
308
|
-
|
309
|
-
await self.awriter(msg)
|
310
|
-
logger.info(msg, stacklevel=2)
|
330
|
+
await self.__alogging(message, mode="info")
|
311
331
|
|
312
332
|
async def awarning(self, message: str) -> None: # pragma: no cov
|
313
333
|
"""Async write trace log with append mode and logging this message with
|
@@ -315,9 +335,7 @@ class BaseTrace(ABC): # pragma: no cov
|
|
315
335
|
|
316
336
|
:param message: (str) A message that want to log.
|
317
337
|
"""
|
318
|
-
|
319
|
-
await self.awriter(msg)
|
320
|
-
logger.warning(msg, stacklevel=2)
|
338
|
+
await self.__alogging(message, mode="warning")
|
321
339
|
|
322
340
|
async def aerror(self, message: str) -> None: # pragma: no cov
|
323
341
|
"""Async write trace log with append mode and logging this message with
|
@@ -325,9 +343,7 @@ class BaseTrace(ABC): # pragma: no cov
|
|
325
343
|
|
326
344
|
:param message: (str) A message that want to log.
|
327
345
|
"""
|
328
|
-
|
329
|
-
await self.awriter(msg, is_err=True)
|
330
|
-
logger.error(msg, stacklevel=2)
|
346
|
+
await self.__alogging(message, mode="error", is_err=True)
|
331
347
|
|
332
348
|
async def aexception(self, message: str) -> None: # pragma: no cov
|
333
349
|
"""Async write trace log with append mode and logging this message with
|
@@ -335,9 +351,7 @@ class BaseTrace(ABC): # pragma: no cov
|
|
335
351
|
|
336
352
|
:param message: (str) A message that want to log.
|
337
353
|
"""
|
338
|
-
|
339
|
-
await self.awriter(msg, is_err=True)
|
340
|
-
logger.exception(msg, stacklevel=2)
|
354
|
+
await self.__alogging(message, mode="exception", is_err=True)
|
341
355
|
|
342
356
|
|
343
357
|
class FileTrace(BaseTrace): # pragma: no cov
|
@@ -351,7 +365,7 @@ class FileTrace(BaseTrace): # pragma: no cov
|
|
351
365
|
) -> Iterator[TraceData]: # pragma: no cov
|
352
366
|
"""Find trace logs.
|
353
367
|
|
354
|
-
:param path: (Path)
|
368
|
+
:param path: (Path) A trace path that want to find.
|
355
369
|
:param extras: An extra parameter that want to override core config.
|
356
370
|
"""
|
357
371
|
for file in sorted(
|
@@ -364,16 +378,16 @@ class FileTrace(BaseTrace): # pragma: no cov
|
|
364
378
|
def find_trace_with_id(
|
365
379
|
cls,
|
366
380
|
run_id: str,
|
367
|
-
force_raise: bool = True,
|
368
381
|
*,
|
382
|
+
force_raise: bool = True,
|
369
383
|
path: Path | None = None,
|
370
384
|
extras: Optional[DictData] = None,
|
371
385
|
) -> TraceData:
|
372
386
|
"""Find trace log with an input specific run ID.
|
373
387
|
|
374
388
|
:param run_id: A running ID of trace log.
|
375
|
-
:param force_raise:
|
376
|
-
:param path:
|
389
|
+
:param force_raise: (bool)
|
390
|
+
:param path: (Path)
|
377
391
|
:param extras: An extra parameter that want to override core config.
|
378
392
|
"""
|
379
393
|
base_path: Path = path or dynamic("trace_path", extras=extras)
|
ddeutil/workflow/params.py
CHANGED
@@ -190,10 +190,13 @@ class IntParam(DefaultParam):
|
|
190
190
|
|
191
191
|
|
192
192
|
class FloatParam(DefaultParam): # pragma: no cov
|
193
|
+
"""Float parameter."""
|
194
|
+
|
193
195
|
type: Literal["float"] = "float"
|
194
196
|
precision: int = 6
|
195
197
|
|
196
198
|
def rounding(self, value: float) -> float:
|
199
|
+
"""Rounding float value with the specific precision field."""
|
197
200
|
round_str: str = f"{{0:.{self.precision}f}}"
|
198
201
|
return float(round_str.format(round(value, self.precision)))
|
199
202
|
|
@@ -224,6 +227,7 @@ class DecimalParam(DefaultParam): # pragma: no cov
|
|
224
227
|
precision: int = 6
|
225
228
|
|
226
229
|
def rounding(self, value: Decimal) -> Decimal:
|
230
|
+
"""Rounding float value with the specific precision field."""
|
227
231
|
return value.quantize(Decimal(10) ** -self.precision)
|
228
232
|
|
229
233
|
def receive(self, value: float | Decimal | None = None) -> Decimal:
|
ddeutil/workflow/scheduler.py
CHANGED
@@ -57,7 +57,7 @@ except ImportError: # pragma: no cov
|
|
57
57
|
from .__cron import CronRunner
|
58
58
|
from .__types import DictData, TupleStr
|
59
59
|
from .conf import FileLoad, Loader, dynamic
|
60
|
-
from .event import
|
60
|
+
from .event import Crontab
|
61
61
|
from .exceptions import ScheduleException, WorkflowException
|
62
62
|
from .logs import Audit, get_audit
|
63
63
|
from .result import SUCCESS, Result
|
@@ -103,9 +103,9 @@ class ScheduleWorkflow(BaseModel):
|
|
103
103
|
description="An alias name of workflow that use for schedule model.",
|
104
104
|
)
|
105
105
|
name: str = Field(description="A workflow name.")
|
106
|
-
on: list[
|
106
|
+
on: list[Crontab] = Field(
|
107
107
|
default_factory=list,
|
108
|
-
description="An override the list of
|
108
|
+
description="An override the list of Crontab object values.",
|
109
109
|
)
|
110
110
|
values: DictData = Field(
|
111
111
|
default_factory=dict,
|
@@ -158,15 +158,17 @@ class ScheduleWorkflow(BaseModel):
|
|
158
158
|
return data
|
159
159
|
|
160
160
|
@field_validator("on", mode="after")
|
161
|
-
def __on_no_dup__(
|
161
|
+
def __on_no_dup__(
|
162
|
+
cls, value: list[Crontab], info: ValidationInfo
|
163
|
+
) -> list[Crontab]:
|
162
164
|
"""Validate the on fields should not contain duplicate values and if it
|
163
165
|
contains every minute value, it should have only one on value.
|
164
166
|
|
165
|
-
:param value: (list[
|
167
|
+
:param value: (list[Crontab]) A list of `Crontab` object.
|
166
168
|
:param info: (ValidationInfo) An validation info object for getting an
|
167
169
|
extra parameter.
|
168
170
|
|
169
|
-
:rtype: list[
|
171
|
+
:rtype: list[Crontab]
|
170
172
|
"""
|
171
173
|
set_ons: set[str] = {str(on.cronjob) for on in value}
|
172
174
|
if len(set_ons) != len(value):
|
@@ -209,7 +211,7 @@ class ScheduleWorkflow(BaseModel):
|
|
209
211
|
|
210
212
|
# IMPORTANT: Create the default 'on' value if it does not pass the `on`
|
211
213
|
# field to the Schedule object.
|
212
|
-
ons: list[
|
214
|
+
ons: list[Crontab] = self.on or wf.on.copy()
|
213
215
|
workflow_tasks: list[WorkflowTask] = []
|
214
216
|
for on in ons:
|
215
217
|
|
ddeutil/workflow/stages.py
CHANGED
@@ -507,13 +507,13 @@ class EmptyStage(BaseAsyncStage):
|
|
507
507
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
508
508
|
extras=self.extras,
|
509
509
|
)
|
510
|
-
|
511
|
-
|
512
|
-
message: str = "..."
|
513
|
-
else:
|
514
|
-
message: str = param2template(
|
510
|
+
message: str = (
|
511
|
+
param2template(
|
515
512
|
dedent(self.echo.strip("\n")), params, extras=self.extras
|
516
513
|
)
|
514
|
+
if self.echo
|
515
|
+
else "..."
|
516
|
+
)
|
517
517
|
|
518
518
|
result.trace.info(
|
519
519
|
f"[STAGE]: Execute Empty-Stage: {self.name!r}: ( {message} )"
|
@@ -546,12 +546,13 @@ class EmptyStage(BaseAsyncStage):
|
|
546
546
|
extras=self.extras,
|
547
547
|
)
|
548
548
|
|
549
|
-
|
550
|
-
|
551
|
-
else:
|
552
|
-
message: str = param2template(
|
549
|
+
message: str = (
|
550
|
+
param2template(
|
553
551
|
dedent(self.echo.strip("\n")), params, extras=self.extras
|
554
552
|
)
|
553
|
+
if self.echo
|
554
|
+
else "..."
|
555
|
+
)
|
555
556
|
|
556
557
|
result.trace.info(f"[STAGE]: Empty-Stage: {self.name!r}: ( {message} )")
|
557
558
|
if self.sleep > 0:
|
@@ -1303,12 +1304,43 @@ class TriggerStage(BaseStage):
|
|
1303
1304
|
else "."
|
1304
1305
|
)
|
1305
1306
|
raise StageException(
|
1306
|
-
f"Trigger workflow return
|
1307
|
+
f"Trigger workflow return `FAILED` status{err_msg}"
|
1307
1308
|
)
|
1308
1309
|
return rs
|
1309
1310
|
|
1310
1311
|
|
1311
|
-
class
|
1312
|
+
class BaseNestedStage(BaseStage):
|
1313
|
+
"""Base Nested Stage model. This model is use for checking the child stage
|
1314
|
+
is the nested stage or not.
|
1315
|
+
"""
|
1316
|
+
|
1317
|
+
@abstractmethod
|
1318
|
+
def execute(
|
1319
|
+
self,
|
1320
|
+
params: DictData,
|
1321
|
+
*,
|
1322
|
+
result: Result | None = None,
|
1323
|
+
event: Event | None = None,
|
1324
|
+
) -> Result:
|
1325
|
+
"""Execute abstraction method that action something by sub-model class.
|
1326
|
+
This is important method that make this class is able to be the nested
|
1327
|
+
stage.
|
1328
|
+
|
1329
|
+
:param params: (DictData) A parameter data that want to use in this
|
1330
|
+
execution.
|
1331
|
+
:param result: (Result) A result object for keeping context and status
|
1332
|
+
data.
|
1333
|
+
:param event: (Event) An event manager that use to track parent execute
|
1334
|
+
was not force stopped.
|
1335
|
+
|
1336
|
+
:rtype: Result
|
1337
|
+
"""
|
1338
|
+
raise NotImplementedError(
|
1339
|
+
"Nested-Stage should implement `execute` method."
|
1340
|
+
)
|
1341
|
+
|
1342
|
+
|
1343
|
+
class ParallelStage(BaseNestedStage):
|
1312
1344
|
"""Parallel stage executor that execute branch stages with multithreading.
|
1313
1345
|
This stage let you set the fix branches for running child stage inside it on
|
1314
1346
|
multithread pool.
|
@@ -1510,19 +1542,14 @@ class ParallelStage(BaseStage):
|
|
1510
1542
|
future.result()
|
1511
1543
|
except StageException as e:
|
1512
1544
|
status = FAILED
|
1513
|
-
result.trace.error(
|
1514
|
-
f"[STAGE]: Error Handler:||{e.__class__.__name__}:||{e}"
|
1515
|
-
)
|
1516
1545
|
if "errors" in context:
|
1517
1546
|
context["errors"][e.refs] = e.to_dict()
|
1518
1547
|
else:
|
1519
1548
|
context["errors"] = e.to_dict(with_refs=True)
|
1520
|
-
except CancelledError:
|
1521
|
-
pass
|
1522
1549
|
return result.catch(status=status, context=context)
|
1523
1550
|
|
1524
1551
|
|
1525
|
-
class ForEachStage(
|
1552
|
+
class ForEachStage(BaseNestedStage):
|
1526
1553
|
"""For-Each stage executor that execute all stages with each item in the
|
1527
1554
|
foreach list.
|
1528
1555
|
|
@@ -1563,9 +1590,17 @@ class ForEachStage(BaseStage):
|
|
1563
1590
|
"will be sequential mode if this value equal 1."
|
1564
1591
|
),
|
1565
1592
|
)
|
1593
|
+
use_index_as_key: bool = Field(
|
1594
|
+
default=False,
|
1595
|
+
description=(
|
1596
|
+
"A flag for using the loop index as a key instead item value. "
|
1597
|
+
"This flag allow to skip checking duplicate item step."
|
1598
|
+
),
|
1599
|
+
)
|
1566
1600
|
|
1567
1601
|
def execute_item(
|
1568
1602
|
self,
|
1603
|
+
index: int,
|
1569
1604
|
item: StrOrInt,
|
1570
1605
|
params: DictData,
|
1571
1606
|
result: Result,
|
@@ -1575,6 +1610,7 @@ class ForEachStage(BaseStage):
|
|
1575
1610
|
"""Execute all nested stage that set on this stage with specific foreach
|
1576
1611
|
item parameter.
|
1577
1612
|
|
1613
|
+
:param index: (int) An index value of foreach loop.
|
1578
1614
|
:param item: (str | int) An item that want to execution.
|
1579
1615
|
:param params: (DictData) A parameter data.
|
1580
1616
|
:param result: (Result) A Result instance for return context and status.
|
@@ -1588,8 +1624,9 @@ class ForEachStage(BaseStage):
|
|
1588
1624
|
:rtype: Result
|
1589
1625
|
"""
|
1590
1626
|
result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
|
1627
|
+
key: StrOrInt = index if self.use_index_as_key else item
|
1591
1628
|
context: DictData = copy.deepcopy(params)
|
1592
|
-
context.update({"item": item})
|
1629
|
+
context.update({"item": item, "loop": index})
|
1593
1630
|
output: DictData = {"item": item, "stages": {}}
|
1594
1631
|
for stage in self.stages:
|
1595
1632
|
|
@@ -1608,14 +1645,14 @@ class ForEachStage(BaseStage):
|
|
1608
1645
|
result.catch(
|
1609
1646
|
status=CANCEL,
|
1610
1647
|
foreach={
|
1611
|
-
|
1648
|
+
key: {
|
1612
1649
|
"item": item,
|
1613
1650
|
"stages": filter_func(output.pop("stages", {})),
|
1614
1651
|
"errors": StageException(error_msg).to_dict(),
|
1615
1652
|
}
|
1616
1653
|
},
|
1617
1654
|
)
|
1618
|
-
raise StageException(error_msg, refs=
|
1655
|
+
raise StageException(error_msg, refs=key)
|
1619
1656
|
|
1620
1657
|
try:
|
1621
1658
|
rs: Result = stage.handler_execute(
|
@@ -1631,14 +1668,14 @@ class ForEachStage(BaseStage):
|
|
1631
1668
|
result.catch(
|
1632
1669
|
status=FAILED,
|
1633
1670
|
foreach={
|
1634
|
-
|
1671
|
+
key: {
|
1635
1672
|
"item": item,
|
1636
1673
|
"stages": filter_func(output.pop("stages", {})),
|
1637
1674
|
"errors": e.to_dict(),
|
1638
1675
|
},
|
1639
1676
|
},
|
1640
1677
|
)
|
1641
|
-
raise StageException(str(e), refs=
|
1678
|
+
raise StageException(str(e), refs=key) from e
|
1642
1679
|
|
1643
1680
|
if rs.status == FAILED:
|
1644
1681
|
error_msg: str = (
|
@@ -1649,19 +1686,19 @@ class ForEachStage(BaseStage):
|
|
1649
1686
|
result.catch(
|
1650
1687
|
status=FAILED,
|
1651
1688
|
foreach={
|
1652
|
-
|
1689
|
+
key: {
|
1653
1690
|
"item": item,
|
1654
1691
|
"stages": filter_func(output.pop("stages", {})),
|
1655
1692
|
"errors": StageException(error_msg).to_dict(),
|
1656
1693
|
},
|
1657
1694
|
},
|
1658
1695
|
)
|
1659
|
-
raise StageException(error_msg, refs=
|
1696
|
+
raise StageException(error_msg, refs=key)
|
1660
1697
|
|
1661
1698
|
return result.catch(
|
1662
1699
|
status=SUCCESS,
|
1663
1700
|
foreach={
|
1664
|
-
|
1701
|
+
key: {
|
1665
1702
|
"item": item,
|
1666
1703
|
"stages": filter_func(output.pop("stages", {})),
|
1667
1704
|
},
|
@@ -1700,6 +1737,11 @@ class ForEachStage(BaseStage):
|
|
1700
1737
|
# [VALIDATE]: Type of the foreach should be `list` type.
|
1701
1738
|
if not isinstance(foreach, list):
|
1702
1739
|
raise TypeError(f"Does not support foreach: {foreach!r}")
|
1740
|
+
elif len(set(foreach)) != len(foreach) and not self.use_index_as_key:
|
1741
|
+
raise ValueError(
|
1742
|
+
"Foreach item should not duplicate. If this stage must to pass "
|
1743
|
+
"duplicate item, it should set `use_index_as_key: true`."
|
1744
|
+
)
|
1703
1745
|
|
1704
1746
|
result.trace.info(f"[STAGE]: Execute Foreach-Stage: {foreach!r}.")
|
1705
1747
|
result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
|
@@ -1721,12 +1763,13 @@ class ForEachStage(BaseStage):
|
|
1721
1763
|
futures: list[Future] = [
|
1722
1764
|
executor.submit(
|
1723
1765
|
self.execute_item,
|
1766
|
+
index=i,
|
1724
1767
|
item=item,
|
1725
1768
|
params=params,
|
1726
1769
|
result=result,
|
1727
1770
|
event=event,
|
1728
1771
|
)
|
1729
|
-
for item in foreach
|
1772
|
+
for i, item in enumerate(foreach, start=0)
|
1730
1773
|
]
|
1731
1774
|
context: DictData = {}
|
1732
1775
|
status: Status = SUCCESS
|
@@ -1741,7 +1784,14 @@ class ForEachStage(BaseStage):
|
|
1741
1784
|
future.cancel()
|
1742
1785
|
time.sleep(0.075)
|
1743
1786
|
|
1744
|
-
nd: str =
|
1787
|
+
nd: str = (
|
1788
|
+
(
|
1789
|
+
f", {len(not_done)} item"
|
1790
|
+
f"{'s' if len(not_done) > 1 else ''} not run!!!"
|
1791
|
+
)
|
1792
|
+
if not_done
|
1793
|
+
else ""
|
1794
|
+
)
|
1745
1795
|
result.trace.debug(
|
1746
1796
|
f"[STAGE]: ... Foreach-Stage set failed event{nd}"
|
1747
1797
|
)
|
@@ -1752,9 +1802,6 @@ class ForEachStage(BaseStage):
|
|
1752
1802
|
future.result()
|
1753
1803
|
except StageException as e:
|
1754
1804
|
status = FAILED
|
1755
|
-
result.trace.error(
|
1756
|
-
f"[STAGE]: Error Handler:||{e.__class__.__name__}:||{e}"
|
1757
|
-
)
|
1758
1805
|
if "errors" in context:
|
1759
1806
|
context["errors"][e.refs] = e.to_dict()
|
1760
1807
|
else:
|
@@ -1764,7 +1811,7 @@ class ForEachStage(BaseStage):
|
|
1764
1811
|
return result.catch(status=status, context=context)
|
1765
1812
|
|
1766
1813
|
|
1767
|
-
class UntilStage(
|
1814
|
+
class UntilStage(BaseNestedStage):
|
1768
1815
|
"""Until stage executor that will run stages in each loop until it valid
|
1769
1816
|
with stop loop condition.
|
1770
1817
|
|
@@ -2015,7 +2062,7 @@ class Match(BaseModel):
|
|
2015
2062
|
)
|
2016
2063
|
|
2017
2064
|
|
2018
|
-
class CaseStage(
|
2065
|
+
class CaseStage(BaseNestedStage):
|
2019
2066
|
"""Case stage executor that execute all stages if the condition was matched.
|
2020
2067
|
|
2021
2068
|
Data Validate:
|
@@ -2259,7 +2306,7 @@ class RaiseStage(BaseAsyncStage):
|
|
2259
2306
|
extras=self.extras,
|
2260
2307
|
)
|
2261
2308
|
message: str = param2template(self.message, params, extras=self.extras)
|
2262
|
-
result.trace.info(f"[STAGE]: Execute Raise-Stage: {message
|
2309
|
+
result.trace.info(f"[STAGE]: Execute Raise-Stage: ( {message} )")
|
2263
2310
|
raise StageException(message)
|
2264
2311
|
|
2265
2312
|
async def axecute(
|
@@ -2286,7 +2333,7 @@ class RaiseStage(BaseAsyncStage):
|
|
2286
2333
|
extras=self.extras,
|
2287
2334
|
)
|
2288
2335
|
message: str = param2template(self.message, params, extras=self.extras)
|
2289
|
-
await result.trace.ainfo(f"[STAGE]: Execute Raise-Stage: {message
|
2336
|
+
await result.trace.ainfo(f"[STAGE]: Execute Raise-Stage: ( {message} )")
|
2290
2337
|
raise StageException(message)
|
2291
2338
|
|
2292
2339
|
|
ddeutil/workflow/workflow.py
CHANGED
@@ -41,7 +41,7 @@ from typing_extensions import Self
|
|
41
41
|
from .__cron import CronRunner
|
42
42
|
from .__types import DictData, TupleStr
|
43
43
|
from .conf import FileLoad, Loader, dynamic
|
44
|
-
from .event import
|
44
|
+
from .event import Crontab
|
45
45
|
from .exceptions import WorkflowException
|
46
46
|
from .job import Job
|
47
47
|
from .logs import Audit, get_audit
|
@@ -314,7 +314,7 @@ class ReleaseQueue:
|
|
314
314
|
|
315
315
|
|
316
316
|
class Workflow(BaseModel):
|
317
|
-
"""Workflow model that use to keep the `Job` and `
|
317
|
+
"""Workflow model that use to keep the `Job` and `Crontab` models.
|
318
318
|
|
319
319
|
This is the main future of this project because it uses to be workflow
|
320
320
|
data for running everywhere that you want or using it to scheduler task in
|
@@ -338,9 +338,9 @@ class Workflow(BaseModel):
|
|
338
338
|
default_factory=dict,
|
339
339
|
description="A parameters that need to use on this workflow.",
|
340
340
|
)
|
341
|
-
on: list[
|
341
|
+
on: list[Crontab] = Field(
|
342
342
|
default_factory=list,
|
343
|
-
description="A list of
|
343
|
+
description="A list of Crontab instance for this workflow schedule.",
|
344
344
|
)
|
345
345
|
jobs: dict[str, Job] = Field(
|
346
346
|
default_factory=dict,
|
@@ -447,9 +447,9 @@ class Workflow(BaseModel):
|
|
447
447
|
@field_validator("on", mode="after")
|
448
448
|
def __on_no_dup_and_reach_limit__(
|
449
449
|
cls,
|
450
|
-
value: list[
|
450
|
+
value: list[Crontab],
|
451
451
|
info: ValidationInfo,
|
452
|
-
) -> list[
|
452
|
+
) -> list[Crontab]:
|
453
453
|
"""Validate the on fields should not contain duplicate values and if it
|
454
454
|
contains the every minute value more than one value, it will remove to
|
455
455
|
only one value.
|
@@ -458,7 +458,7 @@ class Workflow(BaseModel):
|
|
458
458
|
|
459
459
|
:param value: A list of on object.
|
460
460
|
|
461
|
-
:rtype: list[
|
461
|
+
:rtype: list[Crontab]
|
462
462
|
"""
|
463
463
|
set_ons: set[str] = {str(on.cronjob) for on in value}
|
464
464
|
if len(set_ons) != len(value):
|
@@ -1022,7 +1022,14 @@ class Workflow(BaseModel):
|
|
1022
1022
|
extras=self.extras,
|
1023
1023
|
)
|
1024
1024
|
context: DictData = self.parameterize(params)
|
1025
|
-
|
1025
|
+
event: Event = event or Event()
|
1026
|
+
max_job_parallel: int = dynamic(
|
1027
|
+
"max_job_parallel", f=max_job_parallel, extras=self.extras
|
1028
|
+
)
|
1029
|
+
result.trace.info(
|
1030
|
+
f"[WORKFLOW]: Execute: {self.name!r} ("
|
1031
|
+
f"{'parallel' if max_job_parallel > 1 else 'sequential'} jobs)"
|
1032
|
+
)
|
1026
1033
|
if not self.jobs:
|
1027
1034
|
result.trace.warning(f"[WORKFLOW]: {self.name!r} does not set jobs")
|
1028
1035
|
return result.catch(status=SUCCESS, context=context)
|
@@ -1035,16 +1042,9 @@ class Workflow(BaseModel):
|
|
1035
1042
|
timeout: int = dynamic(
|
1036
1043
|
"max_job_exec_timeout", f=timeout, extras=self.extras
|
1037
1044
|
)
|
1038
|
-
|
1039
|
-
result.trace.debug(
|
1040
|
-
f"[WORKFLOW]: ... Run {self.name!r} with non-threading."
|
1041
|
-
)
|
1042
|
-
max_job_parallel: int = dynamic(
|
1043
|
-
"max_job_parallel", f=max_job_parallel, extras=self.extras
|
1044
|
-
)
|
1045
|
+
|
1045
1046
|
with ThreadPoolExecutor(
|
1046
|
-
max_workers=max_job_parallel,
|
1047
|
-
thread_name_prefix="wf_exec_non_threading_",
|
1047
|
+
max_workers=max_job_parallel, thread_name_prefix="wf_exec_"
|
1048
1048
|
) as executor:
|
1049
1049
|
futures: list[Future] = []
|
1050
1050
|
|
@@ -1,20 +1,20 @@
|
|
1
|
-
ddeutil/workflow/__about__.py,sha256=
|
1
|
+
ddeutil/workflow/__about__.py,sha256=aH1VtwOIsTYSvZ--4Gjq5U7sAIC_RxyBRrJoMOYdUMs,28
|
2
2
|
ddeutil/workflow/__cron.py,sha256=yLWN_1MtcN5Uc3Dinq5lpsjW1_0HmIM5tEm-o_q0Spw,28527
|
3
3
|
ddeutil/workflow/__init__.py,sha256=NXEhjzKFdIGa-jtIq9HXChLCjSXNPd8VJ8ltggxbBO8,1371
|
4
4
|
ddeutil/workflow/__main__.py,sha256=x-sYedl4T8p6054aySk-EQX6vhytvPR0HvaBNYxMzp0,364
|
5
5
|
ddeutil/workflow/__types.py,sha256=7xXy6ynpT6Do6U5A-XYSVuinE2g-4wlZGGJ1NACK1BE,4343
|
6
|
-
ddeutil/workflow/conf.py,sha256=
|
7
|
-
ddeutil/workflow/event.py,sha256=
|
6
|
+
ddeutil/workflow/conf.py,sha256=x3ZCvz_NCpfqMV2DQtMMdYN4pMN9s6AEX9uFsIpiqz0,14827
|
7
|
+
ddeutil/workflow/event.py,sha256=VAXUwkuKwaH2gpqc2g0qTE1EhO0dAi46b-RSEtBYvnc,10397
|
8
8
|
ddeutil/workflow/exceptions.py,sha256=HNXkZLaoWa6ejYG1NdwlUAyZiJWbsjjOJ9DjIPaM-aw,2343
|
9
|
-
ddeutil/workflow/job.py,sha256=
|
10
|
-
ddeutil/workflow/logs.py,sha256=
|
11
|
-
ddeutil/workflow/params.py,sha256=
|
9
|
+
ddeutil/workflow/job.py,sha256=FYfnJnSKVLtyVzM0VrcMRXOk_m_YH4vPCvfmzvaOiZ8,35241
|
10
|
+
ddeutil/workflow/logs.py,sha256=Jkcj42-GdK5kTY0w2y8PTCPyofRjfG5EVSirWICrjv4,27618
|
11
|
+
ddeutil/workflow/params.py,sha256=QCI5u2gCzi9vR8_emjyJaVevTrr81ofhFK_vPHfPf2k,11560
|
12
12
|
ddeutil/workflow/result.py,sha256=yEV_IXtiC8x-4zx6DKal5swebjtOWdKakv-WuhNyiNQ,5891
|
13
13
|
ddeutil/workflow/reusables.py,sha256=iXcS7Gg-71qVX4ln0ILTDx03cTtUnj_rNoXHTVdVrxc,17636
|
14
|
-
ddeutil/workflow/scheduler.py,sha256=
|
15
|
-
ddeutil/workflow/stages.py,sha256=
|
14
|
+
ddeutil/workflow/scheduler.py,sha256=8btWD5dDgTHyx92MJvFWbN79dDTAvVuaLzjm4c_HQvo,27239
|
15
|
+
ddeutil/workflow/stages.py,sha256=83pP3kLjTKIV66XsMm8rvEnF4ZNknxQTk80MePb_e5U,92032
|
16
16
|
ddeutil/workflow/utils.py,sha256=wrL9nAVPOFWEvgniELAHbB_NGVX5QeL9DkzHEE35LE8,8766
|
17
|
-
ddeutil/workflow/workflow.py,sha256=
|
17
|
+
ddeutil/workflow/workflow.py,sha256=bJ-CCv4U8EOg73qQKVjdAQ0xU82OShUCJGdSgfa8dRs,44785
|
18
18
|
ddeutil/workflow/api/__init__.py,sha256=kY30dL8HPY8tY_GBmm7y_3OdoXzB1-EA2a96PLU0AQw,5278
|
19
19
|
ddeutil/workflow/api/logs.py,sha256=NMTnOnsBrDB5129329xF2myLdrb-z9k1MQrmrP7qXJw,1818
|
20
20
|
ddeutil/workflow/api/utils.py,sha256=uTtUFVLpiYYahXvCVx8sueRQ03K2Xw1id_gW3IMmX1U,5295
|
@@ -23,9 +23,9 @@ ddeutil/workflow/api/routes/job.py,sha256=8X5VLDJH6PumyNIY6JGRNBsf2gWN0eG9DzxRPS
|
|
23
23
|
ddeutil/workflow/api/routes/logs.py,sha256=U6vOni3wd-ZTOwd3yVdSOpgyRmNdcgfngU5KlLM3Cww,5383
|
24
24
|
ddeutil/workflow/api/routes/schedules.py,sha256=14RnaJKEGMSJtncI1H_QQVZNBe_jDS40PPRO6qFc3i0,4805
|
25
25
|
ddeutil/workflow/api/routes/workflows.py,sha256=GJu5PiXEylswrXylEImpncySjeU9chrvrtjhiMCw2RQ,4529
|
26
|
-
ddeutil_workflow-0.0.
|
27
|
-
ddeutil_workflow-0.0.
|
28
|
-
ddeutil_workflow-0.0.
|
29
|
-
ddeutil_workflow-0.0.
|
30
|
-
ddeutil_workflow-0.0.
|
31
|
-
ddeutil_workflow-0.0.
|
26
|
+
ddeutil_workflow-0.0.58.dist-info/licenses/LICENSE,sha256=nGFZ1QEhhhWeMHf9n99_fdt4vQaXS29xWKxt-OcLywk,1085
|
27
|
+
ddeutil_workflow-0.0.58.dist-info/METADATA,sha256=1jRIZa2yDn_5BojKYl-Fvpbkb5rUSHwPnZl5JlDmSqA,19228
|
28
|
+
ddeutil_workflow-0.0.58.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
|
29
|
+
ddeutil_workflow-0.0.58.dist-info/entry_points.txt,sha256=qDTpPSauL0ciO6T4iSVt8bJeYrVEkkoEEw_RlGx6Kgk,63
|
30
|
+
ddeutil_workflow-0.0.58.dist-info/top_level.txt,sha256=m9M6XeSWDwt_yMsmH6gcOjHZVK5O0-vgtNBuncHjzW4,8
|
31
|
+
ddeutil_workflow-0.0.58.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|