ddeutil-workflow 0.0.64__py3-none-any.whl → 0.0.66__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +1 -1
- ddeutil/workflow/__main__.py +1 -27
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/cli.py +66 -0
- ddeutil/workflow/conf.py +1 -9
- ddeutil/workflow/{exceptions.py → errors.py} +46 -11
- ddeutil/workflow/job.py +247 -120
- ddeutil/workflow/logs.py +1 -1
- ddeutil/workflow/params.py +11 -11
- ddeutil/workflow/result.py +84 -10
- ddeutil/workflow/reusables.py +15 -17
- ddeutil/workflow/stages.py +685 -450
- ddeutil/workflow/utils.py +33 -0
- ddeutil/workflow/workflow.py +177 -664
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.66.dist-info}/METADATA +15 -13
- ddeutil_workflow-0.0.66.dist-info/RECORD +29 -0
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.66.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.64.dist-info/RECORD +0 -28
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.66.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.66.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.66.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -3,9 +3,9 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
-
"""Stages module include all stage model that
|
7
|
-
of this workflow engine. The stage handle the minimize task that run
|
8
|
-
thread (same thread at its job owner) that mean it is the lowest executor that
|
6
|
+
"""Stages module include all stage model that implemented to be the minimum execution
|
7
|
+
layer of this workflow core engine. The stage handle the minimize task that run
|
8
|
+
in a thread (same thread at its job owner) that mean it is the lowest executor that
|
9
9
|
you can track logs.
|
10
10
|
|
11
11
|
The output of stage execution only return SUCCESS or CANCEL status because
|
@@ -17,9 +17,11 @@ the stage execution method.
|
|
17
17
|
|
18
18
|
Execution --> Ok ┬--( handler )--> Result with `SUCCESS` or `CANCEL`
|
19
19
|
|
|
20
|
-
|
20
|
+
├--( handler )--> Result with `FAILED` (Set `raise_error` flag)
|
21
|
+
|
|
22
|
+
╰--( handler )---> Result with `SKIP`
|
21
23
|
|
22
|
-
--> Error ---( handler )--> Raise
|
24
|
+
--> Error ---( handler )--> Raise StageError(...)
|
23
25
|
|
24
26
|
On the context I/O that pass to a stage object at execute process. The
|
25
27
|
execute method receives a `params={"params": {...}}` value for passing template
|
@@ -55,14 +57,26 @@ from textwrap import dedent
|
|
55
57
|
from threading import Event
|
56
58
|
from typing import Annotated, Any, Optional, TypeVar, Union, get_type_hints
|
57
59
|
|
60
|
+
from ddeutil.core import str2list
|
58
61
|
from pydantic import BaseModel, Field, ValidationError
|
59
|
-
from pydantic.functional_validators import model_validator
|
62
|
+
from pydantic.functional_validators import field_validator, model_validator
|
60
63
|
from typing_extensions import Self
|
61
64
|
|
65
|
+
from . import StageCancelError, StageRetryError
|
62
66
|
from .__types import DictData, DictStr, StrOrInt, StrOrNone, TupleStr
|
63
67
|
from .conf import dynamic, pass_env
|
64
|
-
from .
|
65
|
-
from .result import
|
68
|
+
from .errors import StageError, StageSkipError, to_dict
|
69
|
+
from .result import (
|
70
|
+
CANCEL,
|
71
|
+
FAILED,
|
72
|
+
SKIP,
|
73
|
+
SUCCESS,
|
74
|
+
WAIT,
|
75
|
+
Result,
|
76
|
+
Status,
|
77
|
+
get_status_from_error,
|
78
|
+
validate_statuses,
|
79
|
+
)
|
66
80
|
from .reusables import (
|
67
81
|
TagFunc,
|
68
82
|
create_model_from_caller,
|
@@ -76,6 +90,7 @@ from .utils import (
|
|
76
90
|
filter_func,
|
77
91
|
gen_id,
|
78
92
|
make_exec,
|
93
|
+
to_train,
|
79
94
|
)
|
80
95
|
|
81
96
|
T = TypeVar("T")
|
@@ -106,6 +121,12 @@ class BaseStage(BaseModel, ABC):
|
|
106
121
|
name: str = Field(
|
107
122
|
description="A stage name that want to logging when start execution.",
|
108
123
|
)
|
124
|
+
desc: StrOrNone = Field(
|
125
|
+
default=None,
|
126
|
+
description=(
|
127
|
+
"A stage description that use to logging when start execution."
|
128
|
+
),
|
129
|
+
)
|
109
130
|
condition: StrOrNone = Field(
|
110
131
|
default=None,
|
111
132
|
description=(
|
@@ -124,6 +145,14 @@ class BaseStage(BaseModel, ABC):
|
|
124
145
|
"""
|
125
146
|
return self.id or self.name
|
126
147
|
|
148
|
+
@field_validator("desc", mode="after")
|
149
|
+
def ___prepare_desc__(cls, value: str) -> str:
|
150
|
+
"""Prepare description string that was created on a template.
|
151
|
+
|
152
|
+
:rtype: str
|
153
|
+
"""
|
154
|
+
return dedent(value.lstrip("\n"))
|
155
|
+
|
127
156
|
@model_validator(mode="after")
|
128
157
|
def __prepare_running_id(self) -> Self:
|
129
158
|
"""Prepare stage running ID that use default value of field and this
|
@@ -135,14 +164,12 @@ class BaseStage(BaseModel, ABC):
|
|
135
164
|
|
136
165
|
:rtype: Self
|
137
166
|
"""
|
138
|
-
|
139
167
|
# VALIDATE: Validate stage id and name should not dynamic with params
|
140
168
|
# template. (allow only matrix)
|
141
169
|
if not_in_template(self.id) or not_in_template(self.name):
|
142
170
|
raise ValueError(
|
143
171
|
"Stage name and ID should only template with 'matrix.'"
|
144
172
|
)
|
145
|
-
|
146
173
|
return self
|
147
174
|
|
148
175
|
@abstractmethod
|
@@ -175,47 +202,41 @@ class BaseStage(BaseModel, ABC):
|
|
175
202
|
parent_run_id: StrOrNone = None,
|
176
203
|
result: Optional[Result] = None,
|
177
204
|
event: Optional[Event] = None,
|
178
|
-
raise_error: Optional[bool] = None,
|
179
205
|
) -> Union[Result, DictData]:
|
180
206
|
"""Handler stage execution result from the stage `execute` method.
|
181
207
|
|
182
|
-
This
|
183
|
-
|
184
|
-
|
185
|
-
`raise_error` parameter to True.
|
208
|
+
This handler strategy will catch and mapping message to the result
|
209
|
+
context data before returning. All possible status that will return from
|
210
|
+
this method be:
|
186
211
|
|
187
|
-
|
212
|
+
Handler --> Ok --> Result
|
188
213
|
|-status: SUCCESS
|
189
214
|
╰-context:
|
190
215
|
╰-outputs: ...
|
191
216
|
|
192
217
|
--> Ok --> Result
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
218
|
+
╰-status: CANCEL
|
219
|
+
|
220
|
+
--> Ok --> Result
|
221
|
+
╰-status: SKIP
|
197
222
|
|
198
|
-
--> Ok --> Result
|
223
|
+
--> Ok --> Result
|
199
224
|
|-status: FAILED
|
200
225
|
╰-errors:
|
201
226
|
|-name: ...
|
202
227
|
╰-message: ...
|
203
228
|
|
204
|
-
--> Error --> Raise StageException(...)
|
205
|
-
|
206
229
|
On the last step, it will set the running ID on a return result
|
207
230
|
object from the current stage ID before release the final result.
|
208
231
|
|
209
232
|
:param params: (DictData) A parameter data.
|
210
|
-
:param run_id: (str) A running stage ID.
|
211
|
-
:param parent_run_id: (str) A parent running ID.
|
233
|
+
:param run_id: (str) A running stage ID. (Default is None)
|
234
|
+
:param parent_run_id: (str) A parent running ID. (Default is None)
|
212
235
|
:param result: (Result) A result object for keeping context and status
|
213
236
|
data before execution.
|
237
|
+
(Default is None)
|
214
238
|
:param event: (Event) An event manager that pass to the stage execution.
|
215
|
-
|
216
|
-
|
217
|
-
:raise StageException: If the raise_error was set and the execution
|
218
|
-
raise any error.
|
239
|
+
(Default is None)
|
219
240
|
|
220
241
|
:rtype: Result
|
221
242
|
"""
|
@@ -227,21 +248,71 @@ class BaseStage(BaseModel, ABC):
|
|
227
248
|
extras=self.extras,
|
228
249
|
)
|
229
250
|
try:
|
230
|
-
|
251
|
+
result.trace.info(
|
252
|
+
f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
|
253
|
+
f"{self.name!r}."
|
254
|
+
)
|
255
|
+
if self.desc:
|
256
|
+
result.trace.debug(f"[STAGE]: Description:||{self.desc}||")
|
257
|
+
|
258
|
+
if self.is_skipped(params):
|
259
|
+
raise StageSkipError(
|
260
|
+
f"Skip because condition {self.condition} was valid."
|
261
|
+
)
|
262
|
+
# NOTE: Start call wrapped execution method that will use custom
|
263
|
+
# execution before the real execution from inherit stage model.
|
264
|
+
result_caught: Result = self.__execute(
|
265
|
+
params, result=result, event=event
|
266
|
+
)
|
267
|
+
if result_caught.status == WAIT:
|
268
|
+
raise StageError(
|
269
|
+
"Status from execution should not return waiting status."
|
270
|
+
)
|
271
|
+
return result_caught
|
272
|
+
|
273
|
+
# NOTE: Catch this error in this line because the execution can raise
|
274
|
+
# this exception class at other location.
|
275
|
+
except (
|
276
|
+
StageSkipError,
|
277
|
+
StageCancelError,
|
278
|
+
StageError,
|
279
|
+
) as e: # pragma: no cov
|
280
|
+
result.trace.info(
|
281
|
+
f"[STAGE]: Handler:||{e.__class__.__name__}: {e}||"
|
282
|
+
f"{traceback.format_exc()}"
|
283
|
+
)
|
284
|
+
return result.catch(
|
285
|
+
status=get_status_from_error(e),
|
286
|
+
context=(
|
287
|
+
None
|
288
|
+
if isinstance(e, StageSkipError)
|
289
|
+
else {"errors": e.to_dict()}
|
290
|
+
),
|
291
|
+
)
|
231
292
|
except Exception as e:
|
232
|
-
e_name: str = e.__class__.__name__
|
233
293
|
result.trace.error(
|
234
|
-
f"[STAGE]: Error Handler:||{
|
294
|
+
f"[STAGE]: Error Handler:||{e.__class__.__name__}: {e}||"
|
235
295
|
f"{traceback.format_exc()}"
|
236
296
|
)
|
237
|
-
if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
|
238
|
-
if isinstance(e, StageException):
|
239
|
-
raise
|
240
|
-
raise StageException(
|
241
|
-
f"{self.__class__.__name__}: {e_name}: {e}"
|
242
|
-
) from e
|
243
297
|
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
244
298
|
|
299
|
+
def __execute(
|
300
|
+
self, params: DictData, result: Result, event: Optional[Event]
|
301
|
+
) -> Result:
|
302
|
+
"""Wrapped the execute method before returning to handler execution.
|
303
|
+
|
304
|
+
:param params: (DictData) A parameter data that want to use in this
|
305
|
+
execution.
|
306
|
+
:param result: (Result) A result object for keeping context and status
|
307
|
+
data.
|
308
|
+
:param event: (Event) An event manager that use to track parent execute
|
309
|
+
was not force stopped.
|
310
|
+
|
311
|
+
:rtype: Result
|
312
|
+
"""
|
313
|
+
result.catch(status=WAIT)
|
314
|
+
return self.execute(params, result=result, event=event)
|
315
|
+
|
245
316
|
def set_outputs(self, output: DictData, to: DictData) -> DictData:
|
246
317
|
"""Set an outputs from execution result context to the received context
|
247
318
|
with a `to` input parameter. The result context from stage execution
|
@@ -289,20 +360,15 @@ class BaseStage(BaseModel, ABC):
|
|
289
360
|
):
|
290
361
|
return to
|
291
362
|
|
292
|
-
|
363
|
+
_id: str = self.gen_id(params=to)
|
364
|
+
output: DictData = copy.deepcopy(output)
|
293
365
|
errors: DictData = (
|
294
|
-
{"errors": output.pop("errors"
|
366
|
+
{"errors": output.pop("errors")} if "errors" in output else {}
|
295
367
|
)
|
296
|
-
|
297
|
-
{"
|
298
|
-
if "skipped" in output
|
299
|
-
else {}
|
368
|
+
status: dict[str, Status] = (
|
369
|
+
{"status": output.pop("status")} if "status" in output else {}
|
300
370
|
)
|
301
|
-
to["stages"][
|
302
|
-
"outputs": copy.deepcopy(output),
|
303
|
-
**skipping,
|
304
|
-
**errors,
|
305
|
-
}
|
371
|
+
to["stages"][_id] = {"outputs": output} | errors | status
|
306
372
|
return to
|
307
373
|
|
308
374
|
def get_outputs(self, output: DictData) -> DictData:
|
@@ -331,14 +397,15 @@ class BaseStage(BaseModel, ABC):
|
|
331
397
|
:param params: (DictData) A parameters that want to pass to condition
|
332
398
|
template.
|
333
399
|
|
334
|
-
:raise
|
400
|
+
:raise StageError: When it has any error raise from the eval
|
335
401
|
condition statement.
|
336
|
-
:raise
|
402
|
+
:raise StageError: When return type of the eval condition statement
|
337
403
|
does not return with boolean type.
|
338
404
|
|
339
405
|
:rtype: bool
|
340
406
|
"""
|
341
|
-
|
407
|
+
# NOTE: Support for condition value is empty string.
|
408
|
+
if not self.condition:
|
342
409
|
return False
|
343
410
|
|
344
411
|
try:
|
@@ -354,13 +421,13 @@ class BaseStage(BaseModel, ABC):
|
|
354
421
|
raise TypeError("Return type of condition does not be boolean")
|
355
422
|
return not rs
|
356
423
|
except Exception as e:
|
357
|
-
raise
|
424
|
+
raise StageError(f"{e.__class__.__name__}: {e}") from e
|
358
425
|
|
359
426
|
def gen_id(self, params: DictData) -> str:
|
360
427
|
"""Generate stage ID that dynamic use stage's name if it ID does not
|
361
428
|
set.
|
362
429
|
|
363
|
-
:param params: A parameter data.
|
430
|
+
:param params: (DictData) A parameter or context data.
|
364
431
|
|
365
432
|
:rtype: str
|
366
433
|
"""
|
@@ -372,8 +439,16 @@ class BaseStage(BaseModel, ABC):
|
|
372
439
|
)
|
373
440
|
)
|
374
441
|
|
442
|
+
@property
|
443
|
+
def is_nested(self) -> bool:
|
444
|
+
"""Return true if this stage is nested stage.
|
445
|
+
|
446
|
+
:rtype: bool
|
447
|
+
"""
|
448
|
+
return False
|
449
|
+
|
375
450
|
|
376
|
-
class BaseAsyncStage(BaseStage):
|
451
|
+
class BaseAsyncStage(BaseStage, ABC):
|
377
452
|
"""Base Async Stage model to make any stage model allow async execution for
|
378
453
|
optimize CPU and Memory on the current node. If you want to implement any
|
379
454
|
custom async stage, you can inherit this class and implement
|
@@ -383,18 +458,6 @@ class BaseAsyncStage(BaseStage):
|
|
383
458
|
model.
|
384
459
|
"""
|
385
460
|
|
386
|
-
@abstractmethod
|
387
|
-
def execute(
|
388
|
-
self,
|
389
|
-
params: DictData,
|
390
|
-
*,
|
391
|
-
result: Optional[Result] = None,
|
392
|
-
event: Optional[Event] = None,
|
393
|
-
) -> Result:
|
394
|
-
raise NotImplementedError(
|
395
|
-
"Async Stage should implement `execute` method."
|
396
|
-
)
|
397
|
-
|
398
461
|
@abstractmethod
|
399
462
|
async def axecute(
|
400
463
|
self,
|
@@ -427,7 +490,6 @@ class BaseAsyncStage(BaseStage):
|
|
427
490
|
parent_run_id: StrOrNone = None,
|
428
491
|
result: Optional[Result] = None,
|
429
492
|
event: Optional[Event] = None,
|
430
|
-
raise_error: Optional[bool] = None,
|
431
493
|
) -> Result:
|
432
494
|
"""Async Handler stage execution result from the stage `execute` method.
|
433
495
|
|
@@ -437,7 +499,6 @@ class BaseAsyncStage(BaseStage):
|
|
437
499
|
:param result: (Result) A Result instance for return context and status.
|
438
500
|
:param event: (Event) An Event manager instance that use to cancel this
|
439
501
|
execution if it forces stopped by parent execution.
|
440
|
-
:param raise_error: (bool) A flag that all this method raise error
|
441
502
|
|
442
503
|
:rtype: Result
|
443
504
|
"""
|
@@ -448,22 +509,142 @@ class BaseAsyncStage(BaseStage):
|
|
448
509
|
id_logic=self.iden,
|
449
510
|
extras=self.extras,
|
450
511
|
)
|
451
|
-
|
452
512
|
try:
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
513
|
+
await result.trace.ainfo(
|
514
|
+
f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
|
515
|
+
f"{self.name!r}."
|
516
|
+
)
|
517
|
+
if self.desc:
|
518
|
+
await result.trace.adebug(
|
519
|
+
f"[STAGE]: Description:||{self.desc}||"
|
520
|
+
)
|
521
|
+
|
522
|
+
if self.is_skipped(params=params):
|
523
|
+
raise StageSkipError(
|
524
|
+
f"Skip because condition {self.condition} was valid."
|
525
|
+
)
|
464
526
|
|
527
|
+
# NOTE: Start call wrapped execution method that will use custom
|
528
|
+
# execution before the real execution from inherit stage model.
|
529
|
+
result_caught: Result = await self.__axecute(
|
530
|
+
params, result=result, event=event
|
531
|
+
)
|
532
|
+
if result_caught.status == WAIT:
|
533
|
+
raise StageError(
|
534
|
+
"Status from execution should not return waiting status."
|
535
|
+
)
|
536
|
+
return result_caught
|
537
|
+
|
538
|
+
# NOTE: Catch this error in this line because the execution can raise
|
539
|
+
# this exception class at other location.
|
540
|
+
except (
|
541
|
+
StageSkipError,
|
542
|
+
StageCancelError,
|
543
|
+
StageError,
|
544
|
+
) as e: # pragma: no cov
|
545
|
+
await result.trace.ainfo(
|
546
|
+
f"[STAGE]: Skip Handler:||{e.__class__.__name__}: {e}||"
|
547
|
+
f"{traceback.format_exc()}"
|
548
|
+
)
|
549
|
+
return result.catch(
|
550
|
+
status=get_status_from_error(e),
|
551
|
+
context=(
|
552
|
+
{"errors": e.to_dict()}
|
553
|
+
if isinstance(e, StageError)
|
554
|
+
else None
|
555
|
+
),
|
556
|
+
)
|
557
|
+
except Exception as e:
|
558
|
+
await result.trace.aerror(
|
559
|
+
f"[STAGE]: Error Handler:||{e.__class__.__name__}: {e}||"
|
560
|
+
f"{traceback.format_exc()}"
|
561
|
+
)
|
465
562
|
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
466
563
|
|
564
|
+
async def __axecute(
|
565
|
+
self, params: DictData, result: Result, event: Optional[Event]
|
566
|
+
) -> Result:
|
567
|
+
"""Wrapped the axecute method before returning to handler axecute.
|
568
|
+
|
569
|
+
:param params: (DictData) A parameter data that want to use in this
|
570
|
+
execution.
|
571
|
+
:param result: (Result) A result object for keeping context and status
|
572
|
+
data.
|
573
|
+
:param event: (Event) An event manager that use to track parent execute
|
574
|
+
was not force stopped.
|
575
|
+
|
576
|
+
:rtype: Result
|
577
|
+
"""
|
578
|
+
result.catch(status=WAIT)
|
579
|
+
return await self.axecute(params, result=result, event=event)
|
580
|
+
|
581
|
+
|
582
|
+
class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
583
|
+
"""Base Retry Stage model that will execute again when it raises with the
|
584
|
+
`StageRetryError`.
|
585
|
+
"""
|
586
|
+
|
587
|
+
retry: int = Field(
|
588
|
+
default=0,
|
589
|
+
ge=0,
|
590
|
+
lt=20,
|
591
|
+
description="Retry number if stage execution get the error.",
|
592
|
+
)
|
593
|
+
|
594
|
+
def __execute(
|
595
|
+
self,
|
596
|
+
params: DictData,
|
597
|
+
result: Result,
|
598
|
+
event: Optional[Event],
|
599
|
+
) -> Result:
|
600
|
+
"""Wrapped the execute method with retry strategy before returning to
|
601
|
+
handler execute.
|
602
|
+
|
603
|
+
:param params: (DictData) A parameter data that want to use in this
|
604
|
+
execution.
|
605
|
+
:param result: (Result) A result object for keeping context and status
|
606
|
+
data.
|
607
|
+
:param event: (Event) An event manager that use to track parent execute
|
608
|
+
was not force stopped.
|
609
|
+
|
610
|
+
:rtype: Result
|
611
|
+
"""
|
612
|
+
current_retry: int = 0
|
613
|
+
with current_retry < (self.retry + 1):
|
614
|
+
try:
|
615
|
+
result.catch(status=WAIT, context={"retry": current_retry})
|
616
|
+
return self.execute(params, result=result, event=event)
|
617
|
+
except StageRetryError:
|
618
|
+
current_retry += 1
|
619
|
+
raise StageError(f"Reach the maximum of retry number: {self.retry}.")
|
620
|
+
|
621
|
+
async def __axecute(
|
622
|
+
self,
|
623
|
+
params: DictData,
|
624
|
+
result: Result,
|
625
|
+
event: Optional[Event],
|
626
|
+
) -> Result:
|
627
|
+
"""Wrapped the axecute method with retry strategy before returning to
|
628
|
+
handler axecute.
|
629
|
+
|
630
|
+
:param params: (DictData) A parameter data that want to use in this
|
631
|
+
execution.
|
632
|
+
:param result: (Result) A result object for keeping context and status
|
633
|
+
data.
|
634
|
+
:param event: (Event) An event manager that use to track parent execute
|
635
|
+
was not force stopped.
|
636
|
+
|
637
|
+
:rtype: Result
|
638
|
+
"""
|
639
|
+
current_retry: int = 0
|
640
|
+
with current_retry < (self.retry + 1):
|
641
|
+
try:
|
642
|
+
result.catch(status=WAIT, context={"retry": current_retry})
|
643
|
+
return await self.axecute(params, result=result, event=event)
|
644
|
+
except StageRetryError:
|
645
|
+
current_retry += 1
|
646
|
+
raise StageError(f"Reach the maximum of retry number: {self.retry}.")
|
647
|
+
|
467
648
|
|
468
649
|
class EmptyStage(BaseAsyncStage):
|
469
650
|
"""Empty stage executor that do nothing and log the `message` field to
|
@@ -527,12 +708,15 @@ class EmptyStage(BaseAsyncStage):
|
|
527
708
|
else "..."
|
528
709
|
)
|
529
710
|
|
530
|
-
|
531
|
-
|
532
|
-
|
711
|
+
if event and event.is_set():
|
712
|
+
raise StageCancelError(
|
713
|
+
"Execution was canceled from the event before start parallel."
|
714
|
+
)
|
715
|
+
|
716
|
+
result.trace.info(f"[STAGE]: Message: ( {message} )")
|
533
717
|
if self.sleep > 0:
|
534
718
|
if self.sleep > 5:
|
535
|
-
result.trace.info(f"[STAGE]: ...
|
719
|
+
result.trace.info(f"[STAGE]: Sleep ... ({self.sleep} sec)")
|
536
720
|
time.sleep(self.sleep)
|
537
721
|
return result.catch(status=SUCCESS)
|
538
722
|
|
@@ -566,11 +750,16 @@ class EmptyStage(BaseAsyncStage):
|
|
566
750
|
else "..."
|
567
751
|
)
|
568
752
|
|
569
|
-
|
753
|
+
if event and event.is_set():
|
754
|
+
raise StageCancelError(
|
755
|
+
"Execution was canceled from the event before start parallel."
|
756
|
+
)
|
757
|
+
|
758
|
+
result.trace.info(f"[STAGE]: Message: ( {message} )")
|
570
759
|
if self.sleep > 0:
|
571
760
|
if self.sleep > 5:
|
572
761
|
await result.trace.ainfo(
|
573
|
-
f"[STAGE]: ...
|
762
|
+
f"[STAGE]: Sleep ... ({self.sleep} sec)"
|
574
763
|
)
|
575
764
|
await asyncio.sleep(self.sleep)
|
576
765
|
return result.catch(status=SUCCESS)
|
@@ -703,19 +892,15 @@ class BashStage(BaseAsyncStage):
|
|
703
892
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
704
893
|
extras=self.extras,
|
705
894
|
)
|
706
|
-
|
707
|
-
result.trace.info(f"[STAGE]: Execute Shell-Stage: {self.name}")
|
708
|
-
|
709
895
|
bash: str = param2template(
|
710
896
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
711
897
|
)
|
712
|
-
|
713
898
|
with self.create_sh_file(
|
714
899
|
bash=bash,
|
715
900
|
env=param2template(self.env, params, extras=self.extras),
|
716
901
|
run_id=result.run_id,
|
717
902
|
) as sh:
|
718
|
-
result.trace.debug(f"[STAGE]:
|
903
|
+
result.trace.debug(f"[STAGE]: Create `{sh[1]}` file.")
|
719
904
|
rs: CompletedProcess = subprocess.run(
|
720
905
|
sh,
|
721
906
|
shell=False,
|
@@ -726,7 +911,7 @@ class BashStage(BaseAsyncStage):
|
|
726
911
|
)
|
727
912
|
if rs.returncode > 0:
|
728
913
|
e: str = rs.stderr.removesuffix("\n")
|
729
|
-
raise
|
914
|
+
raise StageError(
|
730
915
|
f"Subprocess: {e}\n---( statement )---\n```bash\n{bash}\n```"
|
731
916
|
)
|
732
917
|
return result.catch(
|
@@ -759,17 +944,15 @@ class BashStage(BaseAsyncStage):
|
|
759
944
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
760
945
|
extras=self.extras,
|
761
946
|
)
|
762
|
-
await result.trace.ainfo(f"[STAGE]: Execute Shell-Stage: {self.name}")
|
763
947
|
bash: str = param2template(
|
764
948
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
765
949
|
)
|
766
|
-
|
767
950
|
async with self.async_create_sh_file(
|
768
951
|
bash=bash,
|
769
952
|
env=param2template(self.env, params, extras=self.extras),
|
770
953
|
run_id=result.run_id,
|
771
954
|
) as sh:
|
772
|
-
await result.trace.adebug(f"[STAGE]:
|
955
|
+
await result.trace.adebug(f"[STAGE]: Create `{sh[1]}` file.")
|
773
956
|
rs: CompletedProcess = subprocess.run(
|
774
957
|
sh,
|
775
958
|
shell=False,
|
@@ -781,7 +964,7 @@ class BashStage(BaseAsyncStage):
|
|
781
964
|
|
782
965
|
if rs.returncode > 0:
|
783
966
|
e: str = rs.stderr.removesuffix("\n")
|
784
|
-
raise
|
967
|
+
raise StageError(
|
785
968
|
f"Subprocess: {e}\n---( statement )---\n```bash\n{bash}\n```"
|
786
969
|
)
|
787
970
|
return result.catch(
|
@@ -888,7 +1071,6 @@ class PyStage(BaseAsyncStage):
|
|
888
1071
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
889
1072
|
extras=self.extras,
|
890
1073
|
)
|
891
|
-
|
892
1074
|
lc: DictData = {}
|
893
1075
|
gb: DictData = (
|
894
1076
|
globals()
|
@@ -896,8 +1078,6 @@ class PyStage(BaseAsyncStage):
|
|
896
1078
|
| {"result": result}
|
897
1079
|
)
|
898
1080
|
|
899
|
-
result.trace.info(f"[STAGE]: Execute Py-Stage: {self.name}")
|
900
|
-
|
901
1081
|
# WARNING: The exec build-in function is very dangerous. So, it
|
902
1082
|
# should use the re module to validate exec-string before running.
|
903
1083
|
exec(
|
@@ -921,6 +1101,7 @@ class PyStage(BaseAsyncStage):
|
|
921
1101
|
and not ismodule(gb[k])
|
922
1102
|
and not isclass(gb[k])
|
923
1103
|
and not isfunction(gb[k])
|
1104
|
+
and k in params
|
924
1105
|
)
|
925
1106
|
},
|
926
1107
|
},
|
@@ -956,8 +1137,6 @@ class PyStage(BaseAsyncStage):
|
|
956
1137
|
| param2template(self.vars, params, extras=self.extras)
|
957
1138
|
| {"result": result}
|
958
1139
|
)
|
959
|
-
await result.trace.ainfo(f"[STAGE]: Execute Py-Stage: {self.name}")
|
960
|
-
|
961
1140
|
# WARNING: The exec build-in function is very dangerous. So, it
|
962
1141
|
# should use the re module to validate exec-string before running.
|
963
1142
|
exec(
|
@@ -978,6 +1157,7 @@ class PyStage(BaseAsyncStage):
|
|
978
1157
|
and not ismodule(gb[k])
|
979
1158
|
and not isclass(gb[k])
|
980
1159
|
and not isfunction(gb[k])
|
1160
|
+
and k in params
|
981
1161
|
)
|
982
1162
|
},
|
983
1163
|
},
|
@@ -1060,7 +1240,7 @@ class CallStage(BaseAsyncStage):
|
|
1060
1240
|
)()
|
1061
1241
|
|
1062
1242
|
result.trace.info(
|
1063
|
-
f"[STAGE]:
|
1243
|
+
f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'"
|
1064
1244
|
)
|
1065
1245
|
|
1066
1246
|
# VALIDATE: check input task caller parameters that exists before
|
@@ -1094,6 +1274,11 @@ class CallStage(BaseAsyncStage):
|
|
1094
1274
|
if "result" not in sig.parameters and not has_keyword:
|
1095
1275
|
args.pop("result")
|
1096
1276
|
|
1277
|
+
if event and event.is_set():
|
1278
|
+
raise StageCancelError(
|
1279
|
+
"Execution was canceled from the event before start parallel."
|
1280
|
+
)
|
1281
|
+
|
1097
1282
|
args = self.validate_model_args(call_func, args, result)
|
1098
1283
|
if inspect.iscoroutinefunction(call_func):
|
1099
1284
|
loop = asyncio.get_event_loop()
|
@@ -1148,7 +1333,7 @@ class CallStage(BaseAsyncStage):
|
|
1148
1333
|
)()
|
1149
1334
|
|
1150
1335
|
await result.trace.ainfo(
|
1151
|
-
f"[STAGE]:
|
1336
|
+
f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'"
|
1152
1337
|
)
|
1153
1338
|
|
1154
1339
|
# VALIDATE: check input task caller parameters that exists before
|
@@ -1182,7 +1367,7 @@ class CallStage(BaseAsyncStage):
|
|
1182
1367
|
if "result" not in sig.parameters and not has_keyword:
|
1183
1368
|
args.pop("result")
|
1184
1369
|
|
1185
|
-
args = self.validate_model_args(call_func, args, result)
|
1370
|
+
args: DictData = self.validate_model_args(call_func, args, result)
|
1186
1371
|
if inspect.iscoroutinefunction(call_func):
|
1187
1372
|
rs: DictOrModel = await call_func(
|
1188
1373
|
**param2template(args, params, extras=self.extras)
|
@@ -1212,20 +1397,20 @@ class CallStage(BaseAsyncStage):
|
|
1212
1397
|
) -> DictData:
|
1213
1398
|
"""Validate an input arguments before passing to the caller function.
|
1214
1399
|
|
1215
|
-
:param func: A tag function that want to get typing.
|
1216
|
-
:param args: An arguments before passing to this tag
|
1400
|
+
:param func: (TagFunc) A tag function that want to get typing.
|
1401
|
+
:param args: (DictData) An arguments before passing to this tag func.
|
1217
1402
|
:param result: (Result) A result object for keeping context and status
|
1218
1403
|
data.
|
1219
1404
|
|
1220
1405
|
:rtype: DictData
|
1221
1406
|
"""
|
1222
1407
|
try:
|
1223
|
-
model_instance = create_model_from_caller(
|
1224
|
-
|
1408
|
+
model_instance: BaseModel = create_model_from_caller(
|
1409
|
+
func
|
1410
|
+
).model_validate(args)
|
1411
|
+
override: DictData = dict(model_instance)
|
1225
1412
|
args.update(override)
|
1226
|
-
|
1227
1413
|
type_hints: dict[str, Any] = get_type_hints(func)
|
1228
|
-
|
1229
1414
|
for arg in type_hints:
|
1230
1415
|
|
1231
1416
|
if arg == "return":
|
@@ -1233,10 +1418,11 @@ class CallStage(BaseAsyncStage):
|
|
1233
1418
|
|
1234
1419
|
if arg.removeprefix("_") in args:
|
1235
1420
|
args[arg] = args.pop(arg.removeprefix("_"))
|
1421
|
+
continue
|
1236
1422
|
|
1237
1423
|
return args
|
1238
1424
|
except ValidationError as e:
|
1239
|
-
raise
|
1425
|
+
raise StageError(
|
1240
1426
|
"Validate argument from the caller function raise invalid type."
|
1241
1427
|
) from e
|
1242
1428
|
except TypeError as e:
|
@@ -1247,7 +1433,42 @@ class CallStage(BaseAsyncStage):
|
|
1247
1433
|
return args
|
1248
1434
|
|
1249
1435
|
|
1250
|
-
class
|
1436
|
+
class BaseNestedStage(BaseStage, ABC):
|
1437
|
+
"""Base Nested Stage model. This model is use for checking the child stage
|
1438
|
+
is the nested stage or not.
|
1439
|
+
"""
|
1440
|
+
|
1441
|
+
def set_outputs(self, output: DictData, to: DictData) -> DictData:
|
1442
|
+
"""Override the set outputs method that support for nested-stage."""
|
1443
|
+
return super().set_outputs(output, to=to)
|
1444
|
+
|
1445
|
+
def get_outputs(self, output: DictData) -> DictData:
|
1446
|
+
"""Override the get outputs method that support for nested-stage"""
|
1447
|
+
return super().get_outputs(output)
|
1448
|
+
|
1449
|
+
@property
|
1450
|
+
def is_nested(self) -> bool:
|
1451
|
+
"""Check if this stage is a nested stage or not.
|
1452
|
+
|
1453
|
+
:rtype: bool
|
1454
|
+
"""
|
1455
|
+
return True
|
1456
|
+
|
1457
|
+
@staticmethod
|
1458
|
+
def mark_errors(context: DictData, error: StageError) -> None:
|
1459
|
+
"""Make the errors context result with the refs value depends on the nested
|
1460
|
+
execute func.
|
1461
|
+
|
1462
|
+
:param context: (DictData) A context data.
|
1463
|
+
:param error: (StageError) A stage exception object.
|
1464
|
+
"""
|
1465
|
+
if "errors" in context:
|
1466
|
+
context["errors"][error.refs] = error.to_dict()
|
1467
|
+
else:
|
1468
|
+
context["errors"] = error.to_dict(with_refs=True)
|
1469
|
+
|
1470
|
+
|
1471
|
+
class TriggerStage(BaseNestedStage):
|
1251
1472
|
"""Trigger workflow executor stage that run an input trigger Workflow
|
1252
1473
|
execute method. This is the stage that allow you to create the reusable
|
1253
1474
|
Workflow template with dynamic parameters.
|
@@ -1279,13 +1500,14 @@ class TriggerStage(BaseStage):
|
|
1279
1500
|
event: Optional[Event] = None,
|
1280
1501
|
) -> Result:
|
1281
1502
|
"""Trigger another workflow execution. It will wait the trigger
|
1282
|
-
workflow running complete before catching its result
|
1503
|
+
workflow running complete before catching its result and raise error
|
1504
|
+
when the result status does not be SUCCESS.
|
1283
1505
|
|
1284
1506
|
:param params: (DictData) A parameter data.
|
1285
1507
|
:param result: (Result) A result object for keeping context and status
|
1286
|
-
data.
|
1508
|
+
data. (Default is None)
|
1287
1509
|
:param event: (Event) An event manager that use to track parent execute
|
1288
|
-
was not force stopped.
|
1510
|
+
was not force stopped. (Default is None)
|
1289
1511
|
|
1290
1512
|
:rtype: Result
|
1291
1513
|
"""
|
@@ -1297,57 +1519,28 @@ class TriggerStage(BaseStage):
|
|
1297
1519
|
)
|
1298
1520
|
|
1299
1521
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
1300
|
-
result
|
1301
|
-
rs: Result = Workflow.from_conf(
|
1522
|
+
result: Result = Workflow.from_conf(
|
1302
1523
|
name=pass_env(_trigger),
|
1303
|
-
extras=self.extras
|
1524
|
+
extras=self.extras,
|
1304
1525
|
).execute(
|
1526
|
+
# NOTE: Should not use the `pass_env` function on this params parameter.
|
1305
1527
|
params=param2template(self.params, params, extras=self.extras),
|
1306
1528
|
run_id=None,
|
1307
1529
|
parent_run_id=result.parent_run_id,
|
1308
1530
|
event=event,
|
1309
1531
|
)
|
1310
|
-
if
|
1532
|
+
if result.status == FAILED:
|
1311
1533
|
err_msg: StrOrNone = (
|
1312
1534
|
f" with:\n{msg}"
|
1313
|
-
if (msg :=
|
1535
|
+
if (msg := result.context.get("errors", {}).get("message"))
|
1314
1536
|
else "."
|
1315
1537
|
)
|
1316
|
-
raise
|
1317
|
-
|
1318
|
-
)
|
1319
|
-
|
1320
|
-
|
1321
|
-
|
1322
|
-
class BaseNestedStage(BaseStage):
|
1323
|
-
"""Base Nested Stage model. This model is use for checking the child stage
|
1324
|
-
is the nested stage or not.
|
1325
|
-
"""
|
1326
|
-
|
1327
|
-
@abstractmethod
|
1328
|
-
def execute(
|
1329
|
-
self,
|
1330
|
-
params: DictData,
|
1331
|
-
*,
|
1332
|
-
result: Optional[Result] = None,
|
1333
|
-
event: Optional[Event] = None,
|
1334
|
-
) -> Result:
|
1335
|
-
"""Execute abstraction method that action something by sub-model class.
|
1336
|
-
This is important method that make this class is able to be the nested
|
1337
|
-
stage.
|
1338
|
-
|
1339
|
-
:param params: (DictData) A parameter data that want to use in this
|
1340
|
-
execution.
|
1341
|
-
:param result: (Result) A result object for keeping context and status
|
1342
|
-
data.
|
1343
|
-
:param event: (Event) An event manager that use to track parent execute
|
1344
|
-
was not force stopped.
|
1345
|
-
|
1346
|
-
:rtype: Result
|
1347
|
-
"""
|
1348
|
-
raise NotImplementedError(
|
1349
|
-
"Nested-Stage should implement `execute` method."
|
1350
|
-
)
|
1538
|
+
raise StageError(f"Trigger workflow was failed{err_msg}")
|
1539
|
+
elif result.status == CANCEL:
|
1540
|
+
raise StageCancelError("Trigger workflow was cancel.")
|
1541
|
+
elif result.status == SKIP:
|
1542
|
+
raise StageSkipError("Trigger workflow was skipped.")
|
1543
|
+
return result
|
1351
1544
|
|
1352
1545
|
|
1353
1546
|
class ParallelStage(BaseNestedStage):
|
@@ -1368,10 +1561,14 @@ class ParallelStage(BaseNestedStage):
|
|
1368
1561
|
... "echo": "Start run with branch 1",
|
1369
1562
|
... "sleep": 3,
|
1370
1563
|
... },
|
1564
|
+
... {
|
1565
|
+
... "name": "Echo second stage",
|
1566
|
+
... "echo": "Start run with branch 1",
|
1567
|
+
... },
|
1371
1568
|
... ],
|
1372
1569
|
... "branch02": [
|
1373
1570
|
... {
|
1374
|
-
... "name": "Echo
|
1571
|
+
... "name": "Echo first stage",
|
1375
1572
|
... "echo": "Start run with branch 2",
|
1376
1573
|
... "sleep": 1,
|
1377
1574
|
... },
|
@@ -1401,94 +1598,119 @@ class ParallelStage(BaseNestedStage):
|
|
1401
1598
|
result: Result,
|
1402
1599
|
*,
|
1403
1600
|
event: Optional[Event] = None,
|
1404
|
-
) -> Result:
|
1405
|
-
"""Execute all stage
|
1601
|
+
) -> tuple[Status, Result]:
|
1602
|
+
"""Execute branch that will execute all nested-stage that was set in
|
1603
|
+
this stage with specific branch ID.
|
1406
1604
|
|
1407
1605
|
:param branch: (str) A branch ID.
|
1408
1606
|
:param params: (DictData) A parameter data.
|
1409
1607
|
:param result: (Result) A Result instance for return context and status.
|
1410
1608
|
:param event: (Event) An Event manager instance that use to cancel this
|
1411
1609
|
execution if it forces stopped by parent execution.
|
1610
|
+
(Default is None)
|
1412
1611
|
|
1413
|
-
:
|
1612
|
+
:raise StageCancelError: If event was set.
|
1613
|
+
:raise StageCancelError: If result from a nested-stage return canceled
|
1614
|
+
status.
|
1615
|
+
:raise StageError: If result from a nested-stage return failed status.
|
1616
|
+
|
1617
|
+
:rtype: tuple[Status, Result]
|
1414
1618
|
"""
|
1415
1619
|
result.trace.debug(f"[STAGE]: Execute Branch: {branch!r}")
|
1620
|
+
|
1621
|
+
# NOTE: Create nested-context
|
1416
1622
|
context: DictData = copy.deepcopy(params)
|
1417
1623
|
context.update({"branch": branch})
|
1418
|
-
|
1419
|
-
|
1624
|
+
nestet_context: DictData = {"branch": branch, "stages": {}}
|
1625
|
+
|
1626
|
+
total_stage: int = len(self.parallel[branch])
|
1627
|
+
skips: list[bool] = [False] * total_stage
|
1628
|
+
for i, stage in enumerate(self.parallel[branch], start=0):
|
1420
1629
|
|
1421
1630
|
if self.extras:
|
1422
1631
|
stage.extras = self.extras
|
1423
1632
|
|
1424
|
-
if stage.is_skipped(params=context):
|
1425
|
-
result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
|
1426
|
-
stage.set_outputs(output={"skipped": True}, to=output)
|
1427
|
-
continue
|
1428
|
-
|
1429
1633
|
if event and event.is_set():
|
1430
1634
|
error_msg: str = (
|
1431
|
-
"Branch
|
1432
|
-
"
|
1635
|
+
"Branch execution was canceled from the event before "
|
1636
|
+
"start branch execution."
|
1433
1637
|
)
|
1434
1638
|
result.catch(
|
1435
1639
|
status=CANCEL,
|
1436
1640
|
parallel={
|
1437
1641
|
branch: {
|
1642
|
+
"status": CANCEL,
|
1438
1643
|
"branch": branch,
|
1439
|
-
"stages": filter_func(
|
1440
|
-
|
1644
|
+
"stages": filter_func(
|
1645
|
+
nestet_context.pop("stages", {})
|
1646
|
+
),
|
1647
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
1441
1648
|
}
|
1442
1649
|
},
|
1443
1650
|
)
|
1444
|
-
raise
|
1651
|
+
raise StageCancelError(error_msg, refs=branch)
|
1445
1652
|
|
1446
|
-
|
1447
|
-
|
1448
|
-
|
1449
|
-
|
1450
|
-
|
1451
|
-
|
1452
|
-
|
1653
|
+
rs: Result = stage.handler_execute(
|
1654
|
+
params=context,
|
1655
|
+
run_id=result.run_id,
|
1656
|
+
parent_run_id=result.parent_run_id,
|
1657
|
+
event=event,
|
1658
|
+
)
|
1659
|
+
stage.set_outputs(rs.context, to=nestet_context)
|
1660
|
+
stage.set_outputs(stage.get_outputs(nestet_context), to=context)
|
1661
|
+
|
1662
|
+
if rs.status == SKIP:
|
1663
|
+
skips[i] = True
|
1664
|
+
continue
|
1665
|
+
|
1666
|
+
elif rs.status == FAILED: # pragma: no cov
|
1667
|
+
error_msg: str = (
|
1668
|
+
f"Branch execution was break because its nested-stage, "
|
1669
|
+
f"{stage.iden!r}, failed."
|
1453
1670
|
)
|
1454
|
-
stage.set_outputs(rs.context, to=output)
|
1455
|
-
stage.set_outputs(stage.get_outputs(output), to=context)
|
1456
|
-
except StageException as e:
|
1457
1671
|
result.catch(
|
1458
1672
|
status=FAILED,
|
1459
1673
|
parallel={
|
1460
1674
|
branch: {
|
1675
|
+
"status": FAILED,
|
1461
1676
|
"branch": branch,
|
1462
|
-
"stages": filter_func(
|
1463
|
-
|
1677
|
+
"stages": filter_func(
|
1678
|
+
nestet_context.pop("stages", {})
|
1679
|
+
),
|
1680
|
+
"errors": StageError(error_msg).to_dict(),
|
1464
1681
|
},
|
1465
1682
|
},
|
1466
1683
|
)
|
1467
|
-
raise
|
1684
|
+
raise StageError(error_msg, refs=branch)
|
1468
1685
|
|
1469
|
-
|
1686
|
+
elif rs.status == CANCEL:
|
1470
1687
|
error_msg: str = (
|
1471
|
-
|
1472
|
-
|
1688
|
+
"Branch execution was canceled from the event after "
|
1689
|
+
"end branch execution."
|
1473
1690
|
)
|
1474
1691
|
result.catch(
|
1475
|
-
status=
|
1692
|
+
status=CANCEL,
|
1476
1693
|
parallel={
|
1477
1694
|
branch: {
|
1695
|
+
"status": CANCEL,
|
1478
1696
|
"branch": branch,
|
1479
|
-
"stages": filter_func(
|
1480
|
-
|
1481
|
-
|
1697
|
+
"stages": filter_func(
|
1698
|
+
nestet_context.pop("stages", {})
|
1699
|
+
),
|
1700
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
1701
|
+
}
|
1482
1702
|
},
|
1483
1703
|
)
|
1484
|
-
raise
|
1704
|
+
raise StageCancelError(error_msg, refs=branch)
|
1485
1705
|
|
1486
|
-
|
1487
|
-
|
1706
|
+
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
1707
|
+
return status, result.catch(
|
1708
|
+
status=status,
|
1488
1709
|
parallel={
|
1489
1710
|
branch: {
|
1711
|
+
"status": status,
|
1490
1712
|
"branch": branch,
|
1491
|
-
"stages": filter_func(
|
1713
|
+
"stages": filter_func(nestet_context.pop("stages", {})),
|
1492
1714
|
},
|
1493
1715
|
},
|
1494
1716
|
)
|
@@ -1506,6 +1728,7 @@ class ParallelStage(BaseNestedStage):
|
|
1506
1728
|
:param result: (Result) A Result instance for return context and status.
|
1507
1729
|
:param event: (Event) An Event manager instance that use to cancel this
|
1508
1730
|
execution if it forces stopped by parent execution.
|
1731
|
+
(Default is None)
|
1509
1732
|
|
1510
1733
|
:rtype: Result
|
1511
1734
|
"""
|
@@ -1514,28 +1737,18 @@ class ParallelStage(BaseNestedStage):
|
|
1514
1737
|
extras=self.extras,
|
1515
1738
|
)
|
1516
1739
|
event: Event = event or Event()
|
1517
|
-
result.trace.info(
|
1518
|
-
|
1740
|
+
result.trace.info(f"[STAGE]: Parallel with {self.max_workers} workers.")
|
1741
|
+
result.catch(
|
1742
|
+
status=WAIT,
|
1743
|
+
context={"workers": self.max_workers, "parallel": {}},
|
1519
1744
|
)
|
1520
|
-
|
1745
|
+
len_parallel: int = len(self.parallel)
|
1521
1746
|
if event and event.is_set():
|
1522
|
-
|
1523
|
-
|
1524
|
-
context={
|
1525
|
-
"errors": StageException(
|
1526
|
-
"Stage was canceled from event that had set "
|
1527
|
-
"before stage parallel execution."
|
1528
|
-
).to_dict()
|
1529
|
-
},
|
1747
|
+
raise StageCancelError(
|
1748
|
+
"Execution was canceled from the event before start parallel."
|
1530
1749
|
)
|
1531
1750
|
|
1532
|
-
with ThreadPoolExecutor(
|
1533
|
-
max_workers=self.max_workers, thread_name_prefix="stage_parallel_"
|
1534
|
-
) as executor:
|
1535
|
-
|
1536
|
-
context: DictData = {}
|
1537
|
-
status: Status = SUCCESS
|
1538
|
-
|
1751
|
+
with ThreadPoolExecutor(self.max_workers, "stp") as executor:
|
1539
1752
|
futures: list[Future] = [
|
1540
1753
|
executor.submit(
|
1541
1754
|
self.execute_branch,
|
@@ -1546,17 +1759,18 @@ class ParallelStage(BaseNestedStage):
|
|
1546
1759
|
)
|
1547
1760
|
for branch in self.parallel
|
1548
1761
|
]
|
1549
|
-
|
1550
|
-
|
1762
|
+
context: DictData = {}
|
1763
|
+
statuses: list[Status] = [WAIT] * len_parallel
|
1764
|
+
for i, future in enumerate(as_completed(futures), start=0):
|
1551
1765
|
try:
|
1552
|
-
future.result()
|
1553
|
-
except
|
1554
|
-
|
1555
|
-
|
1556
|
-
|
1557
|
-
|
1558
|
-
|
1559
|
-
|
1766
|
+
statuses[i], _ = future.result()
|
1767
|
+
except StageError as e:
|
1768
|
+
statuses[i] = get_status_from_error(e)
|
1769
|
+
self.mark_errors(context, e)
|
1770
|
+
return result.catch(
|
1771
|
+
status=validate_statuses(statuses),
|
1772
|
+
context=context,
|
1773
|
+
)
|
1560
1774
|
|
1561
1775
|
|
1562
1776
|
class ForEachStage(BaseNestedStage):
|
@@ -1616,9 +1830,12 @@ class ForEachStage(BaseNestedStage):
|
|
1616
1830
|
result: Result,
|
1617
1831
|
*,
|
1618
1832
|
event: Optional[Event] = None,
|
1619
|
-
) -> Result:
|
1620
|
-
"""Execute all nested
|
1621
|
-
item
|
1833
|
+
) -> tuple[Status, Result]:
|
1834
|
+
"""Execute item that will execute all nested-stage that was set in this
|
1835
|
+
stage with specific foreach item.
|
1836
|
+
|
1837
|
+
This method will create the nested-context from an input context
|
1838
|
+
data and use it instead the context data.
|
1622
1839
|
|
1623
1840
|
:param index: (int) An index value of foreach loop.
|
1624
1841
|
:param item: (str | int) An item that want to execution.
|
@@ -1626,91 +1843,114 @@ class ForEachStage(BaseNestedStage):
|
|
1626
1843
|
:param result: (Result) A Result instance for return context and status.
|
1627
1844
|
:param event: (Event) An Event manager instance that use to cancel this
|
1628
1845
|
execution if it forces stopped by parent execution.
|
1846
|
+
(Default is None)
|
1629
1847
|
|
1630
|
-
|
1631
|
-
|
1632
|
-
:raise StageException: If the result from execution has `FAILED` status.
|
1848
|
+
This method should raise error when it wants to stop the foreach
|
1849
|
+
loop such as cancel event or getting the failed status.
|
1633
1850
|
|
1634
|
-
:
|
1851
|
+
:raise StageCancelError: If event was set.
|
1852
|
+
:raise StageError: If the stage execution raise any Exception error.
|
1853
|
+
:raise StageError: If the result from execution has `FAILED` status.
|
1854
|
+
|
1855
|
+
:rtype: tuple[Status, Result]
|
1635
1856
|
"""
|
1636
1857
|
result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
|
1637
1858
|
key: StrOrInt = index if self.use_index_as_key else item
|
1859
|
+
|
1860
|
+
# NOTE: Create nested-context data from the passing context.
|
1638
1861
|
context: DictData = copy.deepcopy(params)
|
1639
1862
|
context.update({"item": item, "loop": index})
|
1640
|
-
|
1641
|
-
|
1863
|
+
nestet_context: DictData = {"item": item, "stages": {}}
|
1864
|
+
|
1865
|
+
total_stage: int = len(self.stages)
|
1866
|
+
skips: list[bool] = [False] * total_stage
|
1867
|
+
for i, stage in enumerate(self.stages, start=0):
|
1642
1868
|
|
1643
1869
|
if self.extras:
|
1644
1870
|
stage.extras = self.extras
|
1645
1871
|
|
1646
|
-
if stage.is_skipped(params=context):
|
1647
|
-
result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
|
1648
|
-
stage.set_outputs(output={"skipped": True}, to=output)
|
1649
|
-
continue
|
1650
|
-
|
1651
1872
|
if event and event.is_set():
|
1652
1873
|
error_msg: str = (
|
1653
|
-
"Item
|
1874
|
+
"Item execution was canceled from the event before start "
|
1875
|
+
"item execution."
|
1654
1876
|
)
|
1655
1877
|
result.catch(
|
1656
1878
|
status=CANCEL,
|
1657
1879
|
foreach={
|
1658
1880
|
key: {
|
1881
|
+
"status": CANCEL,
|
1659
1882
|
"item": item,
|
1660
|
-
"stages": filter_func(
|
1661
|
-
|
1883
|
+
"stages": filter_func(
|
1884
|
+
nestet_context.pop("stages", {})
|
1885
|
+
),
|
1886
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
1662
1887
|
}
|
1663
1888
|
},
|
1664
1889
|
)
|
1665
|
-
raise
|
1890
|
+
raise StageCancelError(error_msg, refs=key)
|
1666
1891
|
|
1667
|
-
|
1668
|
-
|
1669
|
-
|
1670
|
-
|
1671
|
-
|
1672
|
-
|
1673
|
-
|
1892
|
+
rs: Result = stage.handler_execute(
|
1893
|
+
params=context,
|
1894
|
+
run_id=result.run_id,
|
1895
|
+
parent_run_id=result.parent_run_id,
|
1896
|
+
event=event,
|
1897
|
+
)
|
1898
|
+
stage.set_outputs(rs.context, to=nestet_context)
|
1899
|
+
stage.set_outputs(stage.get_outputs(nestet_context), to=context)
|
1900
|
+
|
1901
|
+
if rs.status == SKIP:
|
1902
|
+
skips[i] = True
|
1903
|
+
continue
|
1904
|
+
|
1905
|
+
elif rs.status == FAILED: # pragma: no cov
|
1906
|
+
error_msg: str = (
|
1907
|
+
f"Item execution was break because its nested-stage, "
|
1908
|
+
f"{stage.iden!r}, failed."
|
1674
1909
|
)
|
1675
|
-
|
1676
|
-
stage.set_outputs(stage.get_outputs(output), to=context)
|
1677
|
-
except StageException as e:
|
1910
|
+
result.trace.warning(f"[STAGE]: {error_msg}")
|
1678
1911
|
result.catch(
|
1679
1912
|
status=FAILED,
|
1680
1913
|
foreach={
|
1681
1914
|
key: {
|
1915
|
+
"status": FAILED,
|
1682
1916
|
"item": item,
|
1683
|
-
"stages": filter_func(
|
1684
|
-
|
1917
|
+
"stages": filter_func(
|
1918
|
+
nestet_context.pop("stages", {})
|
1919
|
+
),
|
1920
|
+
"errors": StageError(error_msg).to_dict(),
|
1685
1921
|
},
|
1686
1922
|
},
|
1687
1923
|
)
|
1688
|
-
raise
|
1924
|
+
raise StageError(error_msg, refs=key)
|
1689
1925
|
|
1690
|
-
|
1926
|
+
elif rs.status == CANCEL:
|
1691
1927
|
error_msg: str = (
|
1692
|
-
|
1693
|
-
|
1928
|
+
"Item execution was canceled from the event after "
|
1929
|
+
"end item execution."
|
1694
1930
|
)
|
1695
|
-
result.trace.warning(f"[STAGE]: {error_msg}")
|
1696
1931
|
result.catch(
|
1697
|
-
status=
|
1932
|
+
status=CANCEL,
|
1698
1933
|
foreach={
|
1699
1934
|
key: {
|
1935
|
+
"status": CANCEL,
|
1700
1936
|
"item": item,
|
1701
|
-
"stages": filter_func(
|
1702
|
-
|
1703
|
-
|
1937
|
+
"stages": filter_func(
|
1938
|
+
nestet_context.pop("stages", {})
|
1939
|
+
),
|
1940
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
1941
|
+
}
|
1704
1942
|
},
|
1705
1943
|
)
|
1706
|
-
raise
|
1944
|
+
raise StageCancelError(error_msg, refs=key)
|
1707
1945
|
|
1708
|
-
|
1709
|
-
|
1946
|
+
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
1947
|
+
return status, result.catch(
|
1948
|
+
status=status,
|
1710
1949
|
foreach={
|
1711
1950
|
key: {
|
1951
|
+
"status": status,
|
1712
1952
|
"item": item,
|
1713
|
-
"stages": filter_func(
|
1953
|
+
"stages": filter_func(nestet_context.pop("stages", {})),
|
1714
1954
|
},
|
1715
1955
|
},
|
1716
1956
|
)
|
@@ -1724,6 +1964,10 @@ class ForEachStage(BaseNestedStage):
|
|
1724
1964
|
) -> Result:
|
1725
1965
|
"""Execute the stages that pass each item form the foreach field.
|
1726
1966
|
|
1967
|
+
This stage will use fail-fast strategy if it was set concurrency
|
1968
|
+
value more than 1. It will cancel all nested-stage execution when it has
|
1969
|
+
any item loop raise failed or canceled error.
|
1970
|
+
|
1727
1971
|
:param params: (DictData) A parameter data.
|
1728
1972
|
:param result: (Result) A Result instance for return context and status.
|
1729
1973
|
:param event: (Event) An Event manager instance that use to cancel this
|
@@ -1738,38 +1982,42 @@ class ForEachStage(BaseNestedStage):
|
|
1738
1982
|
extras=self.extras,
|
1739
1983
|
)
|
1740
1984
|
event: Event = event or Event()
|
1741
|
-
foreach: Union[list[str], list[int]] = (
|
1985
|
+
foreach: Union[list[str], list[int]] = pass_env(
|
1742
1986
|
param2template(self.foreach, params, extras=self.extras)
|
1743
|
-
if isinstance(self.foreach, str)
|
1744
|
-
else self.foreach
|
1745
1987
|
)
|
1746
1988
|
|
1989
|
+
# [NOTE]: Force convert str to list.
|
1990
|
+
if isinstance(foreach, str):
|
1991
|
+
try:
|
1992
|
+
foreach: list[Any] = str2list(foreach)
|
1993
|
+
except ValueError as e:
|
1994
|
+
raise TypeError(
|
1995
|
+
f"Does not support string foreach: {foreach!r} that can "
|
1996
|
+
f"not convert to list."
|
1997
|
+
) from e
|
1998
|
+
|
1747
1999
|
# [VALIDATE]: Type of the foreach should be `list` type.
|
1748
|
-
|
1749
|
-
raise TypeError(
|
2000
|
+
elif not isinstance(foreach, list):
|
2001
|
+
raise TypeError(
|
2002
|
+
f"Does not support foreach: {foreach!r} ({type(foreach)})"
|
2003
|
+
)
|
2004
|
+
# [Validate]: Value in the foreach item should not be duplicate when the
|
2005
|
+
# `use_index_as_key` field did not set.
|
1750
2006
|
elif len(set(foreach)) != len(foreach) and not self.use_index_as_key:
|
1751
2007
|
raise ValueError(
|
1752
2008
|
"Foreach item should not duplicate. If this stage must to pass "
|
1753
2009
|
"duplicate item, it should set `use_index_as_key: true`."
|
1754
2010
|
)
|
1755
2011
|
|
1756
|
-
result.trace.info(f"[STAGE]:
|
2012
|
+
result.trace.info(f"[STAGE]: Foreach: {foreach!r}.")
|
1757
2013
|
result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
|
2014
|
+
len_foreach: int = len(foreach)
|
1758
2015
|
if event and event.is_set():
|
1759
|
-
|
1760
|
-
|
1761
|
-
context={
|
1762
|
-
"errors": StageException(
|
1763
|
-
"Stage was canceled from event that had set "
|
1764
|
-
"before stage foreach execution."
|
1765
|
-
).to_dict()
|
1766
|
-
},
|
2016
|
+
raise StageCancelError(
|
2017
|
+
"Execution was canceled from the event before start foreach."
|
1767
2018
|
)
|
1768
2019
|
|
1769
|
-
with ThreadPoolExecutor(
|
1770
|
-
max_workers=self.concurrent, thread_name_prefix="stage_foreach_"
|
1771
|
-
) as executor:
|
1772
|
-
|
2020
|
+
with ThreadPoolExecutor(self.concurrent, "stf") as executor:
|
1773
2021
|
futures: list[Future] = [
|
1774
2022
|
executor.submit(
|
1775
2023
|
self.execute_item,
|
@@ -1781,19 +2029,21 @@ class ForEachStage(BaseNestedStage):
|
|
1781
2029
|
)
|
1782
2030
|
for i, item in enumerate(foreach, start=0)
|
1783
2031
|
]
|
2032
|
+
|
1784
2033
|
context: DictData = {}
|
1785
|
-
|
2034
|
+
statuses: list[Status] = [WAIT] * len_foreach
|
2035
|
+
fail_fast: bool = False
|
1786
2036
|
|
1787
2037
|
done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
|
1788
2038
|
if len(list(done)) != len(futures):
|
1789
2039
|
result.trace.warning(
|
1790
|
-
"[STAGE]: Set event for stop pending for-each stage."
|
2040
|
+
"[STAGE]: Set the event for stop pending for-each stage."
|
1791
2041
|
)
|
1792
2042
|
event.set()
|
1793
2043
|
for future in not_done:
|
1794
2044
|
future.cancel()
|
1795
|
-
time.sleep(0.075)
|
1796
2045
|
|
2046
|
+
time.sleep(0.025)
|
1797
2047
|
nd: str = (
|
1798
2048
|
(
|
1799
2049
|
f", {len(not_done)} item"
|
@@ -1806,18 +2056,24 @@ class ForEachStage(BaseNestedStage):
|
|
1806
2056
|
f"[STAGE]: ... Foreach-Stage set failed event{nd}"
|
1807
2057
|
)
|
1808
2058
|
done: Iterator[Future] = as_completed(futures)
|
2059
|
+
fail_fast = True
|
1809
2060
|
|
1810
|
-
for future in done:
|
2061
|
+
for i, future in enumerate(done, start=0):
|
1811
2062
|
try:
|
1812
|
-
future.result()
|
1813
|
-
except
|
1814
|
-
|
1815
|
-
|
1816
|
-
context["errors"][e.refs] = e.to_dict()
|
1817
|
-
else:
|
1818
|
-
context["errors"] = e.to_dict(with_refs=True)
|
2063
|
+
statuses[i], _ = future.result()
|
2064
|
+
except StageError as e:
|
2065
|
+
statuses[i] = get_status_from_error(e)
|
2066
|
+
self.mark_errors(context, e)
|
1819
2067
|
except CancelledError:
|
1820
2068
|
pass
|
2069
|
+
|
2070
|
+
status: Status = validate_statuses(statuses)
|
2071
|
+
|
2072
|
+
# NOTE: Prepare status because it does not cancel from parent event but
|
2073
|
+
# cancel from failed item execution.
|
2074
|
+
if fail_fast and status == CANCEL:
|
2075
|
+
status = FAILED
|
2076
|
+
|
1821
2077
|
return result.catch(status=status, context=context)
|
1822
2078
|
|
1823
2079
|
|
@@ -1876,8 +2132,9 @@ class UntilStage(BaseNestedStage):
|
|
1876
2132
|
params: DictData,
|
1877
2133
|
result: Result,
|
1878
2134
|
event: Optional[Event] = None,
|
1879
|
-
) -> tuple[Result, T]:
|
1880
|
-
"""Execute all stage
|
2135
|
+
) -> tuple[Status, Result, T]:
|
2136
|
+
"""Execute loop that will execute all nested-stage that was set in this
|
2137
|
+
stage with specific loop and item.
|
1881
2138
|
|
1882
2139
|
:param item: (T) An item that want to execution.
|
1883
2140
|
:param loop: (int) A number of loop.
|
@@ -1886,98 +2143,115 @@ class UntilStage(BaseNestedStage):
|
|
1886
2143
|
:param event: (Event) An Event manager instance that use to cancel this
|
1887
2144
|
execution if it forces stopped by parent execution.
|
1888
2145
|
|
1889
|
-
:rtype: tuple[Result, T]
|
2146
|
+
:rtype: tuple[Status, Result, T]
|
1890
2147
|
:return: Return a pair of Result and changed item.
|
1891
2148
|
"""
|
1892
|
-
result.trace.debug(f"[STAGE]:
|
2149
|
+
result.trace.debug(f"[STAGE]: Execute Loop: {loop} (Item {item!r})")
|
2150
|
+
|
2151
|
+
# NOTE: Create nested-context
|
1893
2152
|
context: DictData = copy.deepcopy(params)
|
1894
|
-
context.update({"item": item})
|
1895
|
-
|
1896
|
-
|
1897
|
-
|
2153
|
+
context.update({"item": item, "loop": loop})
|
2154
|
+
nestet_context: DictData = {"loop": loop, "item": item, "stages": {}}
|
2155
|
+
|
2156
|
+
next_item: Optional[T] = None
|
2157
|
+
total_stage: int = len(self.stages)
|
2158
|
+
skips: list[bool] = [False] * total_stage
|
2159
|
+
for i, stage in enumerate(self.stages, start=0):
|
1898
2160
|
|
1899
2161
|
if self.extras:
|
1900
2162
|
stage.extras = self.extras
|
1901
2163
|
|
1902
|
-
if stage.is_skipped(params=context):
|
1903
|
-
result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
|
1904
|
-
stage.set_outputs(output={"skipped": True}, to=output)
|
1905
|
-
continue
|
1906
|
-
|
1907
2164
|
if event and event.is_set():
|
1908
2165
|
error_msg: str = (
|
1909
|
-
"Loop
|
1910
|
-
"
|
2166
|
+
"Loop execution was canceled from the event before start "
|
2167
|
+
"loop execution."
|
1911
2168
|
)
|
1912
|
-
|
1913
|
-
|
1914
|
-
|
1915
|
-
|
1916
|
-
|
1917
|
-
|
1918
|
-
|
1919
|
-
|
1920
|
-
"
|
1921
|
-
|
1922
|
-
|
1923
|
-
|
1924
|
-
|
2169
|
+
result.catch(
|
2170
|
+
status=CANCEL,
|
2171
|
+
until={
|
2172
|
+
loop: {
|
2173
|
+
"status": CANCEL,
|
2174
|
+
"loop": loop,
|
2175
|
+
"item": item,
|
2176
|
+
"stages": filter_func(
|
2177
|
+
nestet_context.pop("stages", {})
|
2178
|
+
),
|
2179
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
2180
|
+
}
|
2181
|
+
},
|
1925
2182
|
)
|
2183
|
+
raise StageCancelError(error_msg, refs=loop)
|
1926
2184
|
|
1927
|
-
|
1928
|
-
|
1929
|
-
|
1930
|
-
|
1931
|
-
|
1932
|
-
|
1933
|
-
|
1934
|
-
|
1935
|
-
|
2185
|
+
rs: Result = stage.handler_execute(
|
2186
|
+
params=context,
|
2187
|
+
run_id=result.run_id,
|
2188
|
+
parent_run_id=result.parent_run_id,
|
2189
|
+
event=event,
|
2190
|
+
)
|
2191
|
+
stage.set_outputs(rs.context, to=nestet_context)
|
2192
|
+
|
2193
|
+
if "item" in (_output := stage.get_outputs(nestet_context)):
|
2194
|
+
next_item = _output["item"]
|
1936
2195
|
|
1937
|
-
|
1938
|
-
next_item = _output["item"]
|
2196
|
+
stage.set_outputs(_output, to=context)
|
1939
2197
|
|
1940
|
-
|
1941
|
-
|
2198
|
+
if rs.status == SKIP:
|
2199
|
+
skips[i] = True
|
2200
|
+
continue
|
2201
|
+
|
2202
|
+
elif rs.status == FAILED:
|
2203
|
+
error_msg: str = (
|
2204
|
+
f"Loop execution was break because its nested-stage, "
|
2205
|
+
f"{stage.iden!r}, failed."
|
2206
|
+
)
|
1942
2207
|
result.catch(
|
1943
2208
|
status=FAILED,
|
1944
2209
|
until={
|
1945
2210
|
loop: {
|
2211
|
+
"status": FAILED,
|
1946
2212
|
"loop": loop,
|
1947
2213
|
"item": item,
|
1948
|
-
"stages": filter_func(
|
1949
|
-
|
2214
|
+
"stages": filter_func(
|
2215
|
+
nestet_context.pop("stages", {})
|
2216
|
+
),
|
2217
|
+
"errors": StageError(error_msg).to_dict(),
|
1950
2218
|
}
|
1951
2219
|
},
|
1952
2220
|
)
|
1953
|
-
raise
|
2221
|
+
raise StageError(error_msg, refs=loop)
|
1954
2222
|
|
1955
|
-
|
2223
|
+
elif rs.status == CANCEL:
|
1956
2224
|
error_msg: str = (
|
1957
|
-
|
1958
|
-
|
2225
|
+
"Loop execution was canceled from the event after "
|
2226
|
+
"end loop execution."
|
1959
2227
|
)
|
1960
2228
|
result.catch(
|
1961
|
-
status=
|
2229
|
+
status=CANCEL,
|
1962
2230
|
until={
|
1963
2231
|
loop: {
|
2232
|
+
"status": CANCEL,
|
1964
2233
|
"loop": loop,
|
1965
2234
|
"item": item,
|
1966
|
-
"stages": filter_func(
|
1967
|
-
|
2235
|
+
"stages": filter_func(
|
2236
|
+
nestet_context.pop("stages", {})
|
2237
|
+
),
|
2238
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
1968
2239
|
}
|
1969
2240
|
},
|
1970
2241
|
)
|
1971
|
-
raise
|
2242
|
+
raise StageCancelError(error_msg, refs=loop)
|
1972
2243
|
|
2244
|
+
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
1973
2245
|
return (
|
2246
|
+
status,
|
1974
2247
|
result.catch(
|
1975
|
-
status=
|
2248
|
+
status=status,
|
1976
2249
|
until={
|
1977
2250
|
loop: {
|
2251
|
+
"status": status,
|
1978
2252
|
"loop": loop,
|
1979
2253
|
"item": item,
|
1980
|
-
"stages": filter_func(
|
2254
|
+
"stages": filter_func(nestet_context.pop("stages", {})),
|
1981
2255
|
}
|
1982
2256
|
},
|
1983
2257
|
),
|
@@ -1991,12 +2265,14 @@ class UntilStage(BaseNestedStage):
|
|
1991
2265
|
result: Optional[Result] = None,
|
1992
2266
|
event: Optional[Event] = None,
|
1993
2267
|
) -> Result:
|
1994
|
-
"""Execute until loop with checking until condition
|
2268
|
+
"""Execute until loop with checking the until condition before release
|
2269
|
+
the next loop.
|
1995
2270
|
|
1996
2271
|
:param params: (DictData) A parameter data.
|
1997
2272
|
:param result: (Result) A Result instance for return context and status.
|
1998
2273
|
:param event: (Event) An Event manager instance that use to cancel this
|
1999
2274
|
execution if it forces stopped by parent execution.
|
2275
|
+
(Default is None)
|
2000
2276
|
|
2001
2277
|
:rtype: Result
|
2002
2278
|
"""
|
@@ -2004,29 +2280,24 @@ class UntilStage(BaseNestedStage):
|
|
2004
2280
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
2005
2281
|
extras=self.extras,
|
2006
2282
|
)
|
2007
|
-
|
2008
|
-
result.trace.info(f"[STAGE]:
|
2009
|
-
item: Union[str, int, bool] =
|
2010
|
-
self.item, params, extras=self.extras
|
2283
|
+
event: Event = event or Event()
|
2284
|
+
result.trace.info(f"[STAGE]: Until: {self.until!r}")
|
2285
|
+
item: Union[str, int, bool] = pass_env(
|
2286
|
+
param2template(self.item, params, extras=self.extras)
|
2011
2287
|
)
|
2012
2288
|
loop: int = 1
|
2013
|
-
|
2289
|
+
until_rs: bool = True
|
2014
2290
|
exceed_loop: bool = False
|
2015
2291
|
result.catch(status=WAIT, context={"until": {}})
|
2016
|
-
|
2292
|
+
statuses: list[Status] = []
|
2293
|
+
while until_rs and not (exceed_loop := (loop > self.max_loop)):
|
2017
2294
|
|
2018
2295
|
if event and event.is_set():
|
2019
|
-
|
2020
|
-
|
2021
|
-
context={
|
2022
|
-
"errors": StageException(
|
2023
|
-
"Stage was canceled from event that had set "
|
2024
|
-
"before stage loop execution."
|
2025
|
-
).to_dict()
|
2026
|
-
},
|
2296
|
+
raise StageCancelError(
|
2297
|
+
"Execution was canceled from the event before start loop."
|
2027
2298
|
)
|
2028
2299
|
|
2029
|
-
result, item = self.execute_loop(
|
2300
|
+
status, result, item = self.execute_loop(
|
2030
2301
|
item=item,
|
2031
2302
|
loop=loop,
|
2032
2303
|
params=params,
|
@@ -2036,34 +2307,39 @@ class UntilStage(BaseNestedStage):
|
|
2036
2307
|
|
2037
2308
|
loop += 1
|
2038
2309
|
if item is None:
|
2310
|
+
item: int = loop
|
2039
2311
|
result.trace.warning(
|
2040
|
-
f"[STAGE]:
|
2041
|
-
f"default."
|
2312
|
+
f"[STAGE]: Return loop not set the item. It uses loop: "
|
2313
|
+
f"{loop} by default."
|
2042
2314
|
)
|
2043
|
-
item: int = loop
|
2044
2315
|
|
2045
2316
|
next_track: bool = eval(
|
2046
|
-
|
2047
|
-
|
2048
|
-
|
2049
|
-
|
2317
|
+
pass_env(
|
2318
|
+
param2template(
|
2319
|
+
self.until,
|
2320
|
+
params | {"item": item, "loop": loop},
|
2321
|
+
extras=self.extras,
|
2322
|
+
),
|
2050
2323
|
),
|
2051
2324
|
globals() | params | {"item": item},
|
2052
2325
|
{},
|
2053
2326
|
)
|
2054
2327
|
if not isinstance(next_track, bool):
|
2055
|
-
raise
|
2328
|
+
raise TypeError(
|
2056
2329
|
"Return type of until condition not be `boolean`, getting"
|
2057
2330
|
f": {next_track!r}"
|
2058
2331
|
)
|
2059
|
-
|
2060
|
-
|
2332
|
+
until_rs: bool = not next_track
|
2333
|
+
statuses.append(status)
|
2334
|
+
delay(0.005)
|
2061
2335
|
|
2062
2336
|
if exceed_loop:
|
2063
|
-
|
2064
|
-
f"
|
2337
|
+
error_msg: str = (
|
2338
|
+
f"Loop was exceed the maximum {self.max_loop} "
|
2339
|
+
f"loop{'s' if self.max_loop > 1 else ''}."
|
2065
2340
|
)
|
2066
|
-
|
2341
|
+
raise StageError(error_msg)
|
2342
|
+
return result.catch(status=validate_statuses(statuses))
|
2067
2343
|
|
2068
2344
|
|
2069
2345
|
class Match(BaseModel):
|
@@ -2147,11 +2423,6 @@ class CaseStage(BaseNestedStage):
|
|
2147
2423
|
if self.extras:
|
2148
2424
|
stage.extras = self.extras
|
2149
2425
|
|
2150
|
-
if stage.is_skipped(params=context):
|
2151
|
-
result.trace.info(f"[STAGE]: ... Skip stage: {stage.iden!r}")
|
2152
|
-
stage.set_outputs(output={"skipped": True}, to=output)
|
2153
|
-
continue
|
2154
|
-
|
2155
2426
|
if event and event.is_set():
|
2156
2427
|
error_msg: str = (
|
2157
2428
|
"Case-Stage was canceled from event that had set before "
|
@@ -2162,29 +2433,18 @@ class CaseStage(BaseNestedStage):
|
|
2162
2433
|
context={
|
2163
2434
|
"case": case,
|
2164
2435
|
"stages": filter_func(output.pop("stages", {})),
|
2165
|
-
"errors":
|
2436
|
+
"errors": StageError(error_msg).to_dict(),
|
2166
2437
|
},
|
2167
2438
|
)
|
2168
2439
|
|
2169
|
-
|
2170
|
-
|
2171
|
-
|
2172
|
-
|
2173
|
-
|
2174
|
-
|
2175
|
-
|
2176
|
-
|
2177
|
-
stage.set_outputs(rs.context, to=output)
|
2178
|
-
stage.set_outputs(stage.get_outputs(output), to=context)
|
2179
|
-
except StageException as e:
|
2180
|
-
return result.catch(
|
2181
|
-
status=FAILED,
|
2182
|
-
context={
|
2183
|
-
"case": case,
|
2184
|
-
"stages": filter_func(output.pop("stages", {})),
|
2185
|
-
"errors": e.to_dict(),
|
2186
|
-
},
|
2187
|
-
)
|
2440
|
+
rs: Result = stage.handler_execute(
|
2441
|
+
params=context,
|
2442
|
+
run_id=result.run_id,
|
2443
|
+
parent_run_id=result.parent_run_id,
|
2444
|
+
event=event,
|
2445
|
+
)
|
2446
|
+
stage.set_outputs(rs.context, to=output)
|
2447
|
+
stage.set_outputs(stage.get_outputs(output), to=context)
|
2188
2448
|
|
2189
2449
|
if rs.status == FAILED:
|
2190
2450
|
error_msg: str = (
|
@@ -2196,7 +2456,7 @@ class CaseStage(BaseNestedStage):
|
|
2196
2456
|
context={
|
2197
2457
|
"case": case,
|
2198
2458
|
"stages": filter_func(output.pop("stages", {})),
|
2199
|
-
"errors":
|
2459
|
+
"errors": StageError(error_msg).to_dict(),
|
2200
2460
|
},
|
2201
2461
|
)
|
2202
2462
|
return result.catch(
|
@@ -2230,7 +2490,7 @@ class CaseStage(BaseNestedStage):
|
|
2230
2490
|
|
2231
2491
|
_case: StrOrNone = param2template(self.case, params, extras=self.extras)
|
2232
2492
|
|
2233
|
-
result.trace.info(f"[STAGE]:
|
2493
|
+
result.trace.info(f"[STAGE]: Case: {_case!r}.")
|
2234
2494
|
_else: Optional[Match] = None
|
2235
2495
|
stages: Optional[list[Stage]] = None
|
2236
2496
|
for match in self.match:
|
@@ -2239,39 +2499,28 @@ class CaseStage(BaseNestedStage):
|
|
2239
2499
|
continue
|
2240
2500
|
|
2241
2501
|
_condition: str = param2template(c, params, extras=self.extras)
|
2242
|
-
if stages is None and _case == _condition:
|
2502
|
+
if stages is None and pass_env(_case) == pass_env(_condition):
|
2243
2503
|
stages: list[Stage] = match.stages
|
2244
2504
|
|
2245
2505
|
if stages is None:
|
2246
2506
|
if _else is None:
|
2247
2507
|
if not self.skip_not_match:
|
2248
|
-
raise
|
2508
|
+
raise StageError(
|
2249
2509
|
"This stage does not set else for support not match "
|
2250
2510
|
"any case."
|
2251
2511
|
)
|
2252
|
-
|
2253
|
-
"
|
2254
|
-
|
2255
|
-
error_msg: str = (
|
2256
|
-
"Case-Stage was canceled because it does not match any "
|
2257
|
-
"case and else condition does not set too."
|
2258
|
-
)
|
2259
|
-
return result.catch(
|
2260
|
-
status=CANCEL,
|
2261
|
-
context={"errors": StageException(error_msg).to_dict()},
|
2512
|
+
raise StageSkipError(
|
2513
|
+
"Execution was skipped because it does not match any "
|
2514
|
+
"case and the else condition does not set too."
|
2262
2515
|
)
|
2516
|
+
|
2263
2517
|
_case: str = "_"
|
2264
2518
|
stages: list[Stage] = _else.stages
|
2265
2519
|
|
2266
2520
|
if event and event.is_set():
|
2267
|
-
|
2268
|
-
|
2269
|
-
|
2270
|
-
"errors": StageException(
|
2271
|
-
"Stage was canceled from event that had set before "
|
2272
|
-
"case-stage execution."
|
2273
|
-
).to_dict()
|
2274
|
-
},
|
2521
|
+
raise StageCancelError(
|
2522
|
+
"Execution was canceled from the event before start "
|
2523
|
+
"case execution."
|
2275
2524
|
)
|
2276
2525
|
|
2277
2526
|
return self.execute_case(
|
@@ -2280,7 +2529,7 @@ class CaseStage(BaseNestedStage):
|
|
2280
2529
|
|
2281
2530
|
|
2282
2531
|
class RaiseStage(BaseAsyncStage):
|
2283
|
-
"""Raise error stage executor that raise `
|
2532
|
+
"""Raise error stage executor that raise `StageError` that use a message
|
2284
2533
|
field for making error message before raise.
|
2285
2534
|
|
2286
2535
|
Data Validate:
|
@@ -2293,7 +2542,7 @@ class RaiseStage(BaseAsyncStage):
|
|
2293
2542
|
|
2294
2543
|
message: str = Field(
|
2295
2544
|
description=(
|
2296
|
-
"An error message that want to raise with `
|
2545
|
+
"An error message that want to raise with `StageError` class"
|
2297
2546
|
),
|
2298
2547
|
alias="raise",
|
2299
2548
|
)
|
@@ -2305,7 +2554,7 @@ class RaiseStage(BaseAsyncStage):
|
|
2305
2554
|
result: Optional[Result] = None,
|
2306
2555
|
event: Optional[Event] = None,
|
2307
2556
|
) -> Result:
|
2308
|
-
"""Raise the
|
2557
|
+
"""Raise the StageError object with the message field execution.
|
2309
2558
|
|
2310
2559
|
:param params: (DictData) A parameter data.
|
2311
2560
|
:param result: (Result) A Result instance for return context and status.
|
@@ -2317,8 +2566,8 @@ class RaiseStage(BaseAsyncStage):
|
|
2317
2566
|
extras=self.extras,
|
2318
2567
|
)
|
2319
2568
|
message: str = param2template(self.message, params, extras=self.extras)
|
2320
|
-
result.trace.info(f"[STAGE]:
|
2321
|
-
raise
|
2569
|
+
result.trace.info(f"[STAGE]: Message: ( {message} )")
|
2570
|
+
raise StageError(message)
|
2322
2571
|
|
2323
2572
|
async def axecute(
|
2324
2573
|
self,
|
@@ -2345,7 +2594,7 @@ class RaiseStage(BaseAsyncStage):
|
|
2345
2594
|
)
|
2346
2595
|
message: str = param2template(self.message, params, extras=self.extras)
|
2347
2596
|
await result.trace.ainfo(f"[STAGE]: Execute Raise-Stage: ( {message} )")
|
2348
|
-
raise
|
2597
|
+
raise StageError(message)
|
2349
2598
|
|
2350
2599
|
|
2351
2600
|
class DockerStage(BaseStage): # pragma: no cov
|
@@ -2439,7 +2688,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2439
2688
|
)
|
2440
2689
|
return result.catch(
|
2441
2690
|
status=CANCEL,
|
2442
|
-
context={"errors":
|
2691
|
+
context={"errors": StageError(error_msg).to_dict()},
|
2443
2692
|
)
|
2444
2693
|
|
2445
2694
|
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
|
@@ -2509,9 +2758,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2509
2758
|
extras=self.extras,
|
2510
2759
|
)
|
2511
2760
|
|
2512
|
-
result.trace.info(
|
2513
|
-
f"[STAGE]: Execute Docker-Stage: {self.image}:{self.tag}"
|
2514
|
-
)
|
2761
|
+
result.trace.info(f"[STAGE]: Docker: {self.image}:{self.tag}")
|
2515
2762
|
raise NotImplementedError("Docker Stage does not implement yet.")
|
2516
2763
|
|
2517
2764
|
|
@@ -2610,8 +2857,6 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2610
2857
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
2611
2858
|
extras=self.extras,
|
2612
2859
|
)
|
2613
|
-
|
2614
|
-
result.trace.info(f"[STAGE]: Execute VirtualPy-Stage: {self.name}")
|
2615
2860
|
run: str = param2template(dedent(self.run), params, extras=self.extras)
|
2616
2861
|
with self.create_py_file(
|
2617
2862
|
py=run,
|
@@ -2619,19 +2864,9 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2619
2864
|
deps=param2template(self.deps, params, extras=self.extras),
|
2620
2865
|
run_id=result.run_id,
|
2621
2866
|
) as py:
|
2622
|
-
result.trace.debug(f"[STAGE]:
|
2623
|
-
try:
|
2624
|
-
import uv
|
2625
|
-
|
2626
|
-
_ = uv
|
2627
|
-
except ImportError:
|
2628
|
-
raise ImportError(
|
2629
|
-
"The VirtualPyStage need you to install `uv` before"
|
2630
|
-
"execution."
|
2631
|
-
) from None
|
2632
|
-
|
2867
|
+
result.trace.debug(f"[STAGE]: Create `{py}` file.")
|
2633
2868
|
rs: CompletedProcess = subprocess.run(
|
2634
|
-
["uv", "run", py, "--no-cache"],
|
2869
|
+
["python", "-m", "uv", "run", py, "--no-cache"],
|
2635
2870
|
# ["uv", "run", "--python", "3.9", py],
|
2636
2871
|
shell=False,
|
2637
2872
|
capture_output=True,
|
@@ -2645,7 +2880,7 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2645
2880
|
if "\\x00" in rs.stderr
|
2646
2881
|
else rs.stderr
|
2647
2882
|
).removesuffix("\n")
|
2648
|
-
raise
|
2883
|
+
raise StageError(
|
2649
2884
|
f"Subprocess: {e}\nRunning Statement:\n---\n"
|
2650
2885
|
f"```python\n{run}\n```"
|
2651
2886
|
)
|