ddeutil-workflow 0.0.64__py3-none-any.whl → 0.0.65__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +1 -1
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/conf.py +0 -4
- ddeutil/workflow/{exceptions.py → errors.py} +49 -11
- ddeutil/workflow/job.py +249 -118
- ddeutil/workflow/params.py +11 -11
- ddeutil/workflow/result.py +86 -10
- ddeutil/workflow/reusables.py +15 -17
- ddeutil/workflow/stages.py +676 -450
- ddeutil/workflow/utils.py +33 -0
- ddeutil/workflow/workflow.py +163 -664
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.65.dist-info}/METADATA +14 -12
- ddeutil_workflow-0.0.65.dist-info/RECORD +28 -0
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.65.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.64.dist-info/RECORD +0 -28
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.65.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.65.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.65.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -3,9 +3,9 @@
|
|
3
3
|
# Licensed under the MIT License. See LICENSE in the project root for
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
|
-
"""Stages module include all stage model that
|
7
|
-
of this workflow engine. The stage handle the minimize task that run
|
8
|
-
thread (same thread at its job owner) that mean it is the lowest executor that
|
6
|
+
"""Stages module include all stage model that implemented to be the minimum execution
|
7
|
+
layer of this workflow core engine. The stage handle the minimize task that run
|
8
|
+
in a thread (same thread at its job owner) that mean it is the lowest executor that
|
9
9
|
you can track logs.
|
10
10
|
|
11
11
|
The output of stage execution only return SUCCESS or CANCEL status because
|
@@ -17,9 +17,11 @@ the stage execution method.
|
|
17
17
|
|
18
18
|
Execution --> Ok ┬--( handler )--> Result with `SUCCESS` or `CANCEL`
|
19
19
|
|
|
20
|
-
|
20
|
+
├--( handler )--> Result with `FAILED` (Set `raise_error` flag)
|
21
|
+
|
|
22
|
+
╰--( handler )---> Result with `SKIP`
|
21
23
|
|
22
|
-
--> Error ---( handler )--> Raise
|
24
|
+
--> Error ---( handler )--> Raise StageError(...)
|
23
25
|
|
24
26
|
On the context I/O that pass to a stage object at execute process. The
|
25
27
|
execute method receives a `params={"params": {...}}` value for passing template
|
@@ -55,14 +57,26 @@ from textwrap import dedent
|
|
55
57
|
from threading import Event
|
56
58
|
from typing import Annotated, Any, Optional, TypeVar, Union, get_type_hints
|
57
59
|
|
60
|
+
from ddeutil.core import str2list
|
58
61
|
from pydantic import BaseModel, Field, ValidationError
|
59
|
-
from pydantic.functional_validators import model_validator
|
62
|
+
from pydantic.functional_validators import field_validator, model_validator
|
60
63
|
from typing_extensions import Self
|
61
64
|
|
65
|
+
from . import StageCancelError, StageRetryError
|
62
66
|
from .__types import DictData, DictStr, StrOrInt, StrOrNone, TupleStr
|
63
67
|
from .conf import dynamic, pass_env
|
64
|
-
from .
|
65
|
-
from .result import
|
68
|
+
from .errors import StageError, StageSkipError, to_dict
|
69
|
+
from .result import (
|
70
|
+
CANCEL,
|
71
|
+
FAILED,
|
72
|
+
SKIP,
|
73
|
+
SUCCESS,
|
74
|
+
WAIT,
|
75
|
+
Result,
|
76
|
+
Status,
|
77
|
+
get_status_from_error,
|
78
|
+
validate_statuses,
|
79
|
+
)
|
66
80
|
from .reusables import (
|
67
81
|
TagFunc,
|
68
82
|
create_model_from_caller,
|
@@ -76,6 +90,7 @@ from .utils import (
|
|
76
90
|
filter_func,
|
77
91
|
gen_id,
|
78
92
|
make_exec,
|
93
|
+
to_train,
|
79
94
|
)
|
80
95
|
|
81
96
|
T = TypeVar("T")
|
@@ -106,6 +121,12 @@ class BaseStage(BaseModel, ABC):
|
|
106
121
|
name: str = Field(
|
107
122
|
description="A stage name that want to logging when start execution.",
|
108
123
|
)
|
124
|
+
desc: StrOrNone = Field(
|
125
|
+
default=None,
|
126
|
+
description=(
|
127
|
+
"A stage description that use to logging when start execution."
|
128
|
+
),
|
129
|
+
)
|
109
130
|
condition: StrOrNone = Field(
|
110
131
|
default=None,
|
111
132
|
description=(
|
@@ -124,6 +145,14 @@ class BaseStage(BaseModel, ABC):
|
|
124
145
|
"""
|
125
146
|
return self.id or self.name
|
126
147
|
|
148
|
+
@field_validator("desc", mode="after")
|
149
|
+
def ___prepare_desc__(cls, value: str) -> str:
|
150
|
+
"""Prepare description string that was created on a template.
|
151
|
+
|
152
|
+
:rtype: str
|
153
|
+
"""
|
154
|
+
return dedent(value.lstrip("\n"))
|
155
|
+
|
127
156
|
@model_validator(mode="after")
|
128
157
|
def __prepare_running_id(self) -> Self:
|
129
158
|
"""Prepare stage running ID that use default value of field and this
|
@@ -135,14 +164,12 @@ class BaseStage(BaseModel, ABC):
|
|
135
164
|
|
136
165
|
:rtype: Self
|
137
166
|
"""
|
138
|
-
|
139
167
|
# VALIDATE: Validate stage id and name should not dynamic with params
|
140
168
|
# template. (allow only matrix)
|
141
169
|
if not_in_template(self.id) or not_in_template(self.name):
|
142
170
|
raise ValueError(
|
143
171
|
"Stage name and ID should only template with 'matrix.'"
|
144
172
|
)
|
145
|
-
|
146
173
|
return self
|
147
174
|
|
148
175
|
@abstractmethod
|
@@ -175,47 +202,41 @@ class BaseStage(BaseModel, ABC):
|
|
175
202
|
parent_run_id: StrOrNone = None,
|
176
203
|
result: Optional[Result] = None,
|
177
204
|
event: Optional[Event] = None,
|
178
|
-
raise_error: Optional[bool] = None,
|
179
205
|
) -> Union[Result, DictData]:
|
180
206
|
"""Handler stage execution result from the stage `execute` method.
|
181
207
|
|
182
|
-
This
|
183
|
-
|
184
|
-
|
185
|
-
`raise_error` parameter to True.
|
208
|
+
This handler strategy will catch and mapping message to the result
|
209
|
+
context data before returning. All possible status that will return from
|
210
|
+
this method be:
|
186
211
|
|
187
|
-
|
212
|
+
Handler --> Ok --> Result
|
188
213
|
|-status: SUCCESS
|
189
214
|
╰-context:
|
190
215
|
╰-outputs: ...
|
191
216
|
|
192
217
|
--> Ok --> Result
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
218
|
+
╰-status: CANCEL
|
219
|
+
|
220
|
+
--> Ok --> Result
|
221
|
+
╰-status: SKIP
|
197
222
|
|
198
|
-
--> Ok --> Result
|
223
|
+
--> Ok --> Result
|
199
224
|
|-status: FAILED
|
200
225
|
╰-errors:
|
201
226
|
|-name: ...
|
202
227
|
╰-message: ...
|
203
228
|
|
204
|
-
--> Error --> Raise StageException(...)
|
205
|
-
|
206
229
|
On the last step, it will set the running ID on a return result
|
207
230
|
object from the current stage ID before release the final result.
|
208
231
|
|
209
232
|
:param params: (DictData) A parameter data.
|
210
|
-
:param run_id: (str) A running stage ID.
|
211
|
-
:param parent_run_id: (str) A parent running ID.
|
233
|
+
:param run_id: (str) A running stage ID. (Default is None)
|
234
|
+
:param parent_run_id: (str) A parent running ID. (Default is None)
|
212
235
|
:param result: (Result) A result object for keeping context and status
|
213
236
|
data before execution.
|
237
|
+
(Default is None)
|
214
238
|
:param event: (Event) An event manager that pass to the stage execution.
|
215
|
-
|
216
|
-
|
217
|
-
:raise StageException: If the raise_error was set and the execution
|
218
|
-
raise any error.
|
239
|
+
(Default is None)
|
219
240
|
|
220
241
|
:rtype: Result
|
221
242
|
"""
|
@@ -227,21 +248,71 @@ class BaseStage(BaseModel, ABC):
|
|
227
248
|
extras=self.extras,
|
228
249
|
)
|
229
250
|
try:
|
230
|
-
|
251
|
+
result.trace.info(
|
252
|
+
f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
|
253
|
+
f"{self.name!r}."
|
254
|
+
)
|
255
|
+
if self.desc:
|
256
|
+
result.trace.debug(f"[STAGE]: Description:||{self.desc}||")
|
257
|
+
|
258
|
+
if self.is_skipped(params):
|
259
|
+
raise StageSkipError(
|
260
|
+
f"Skip because condition {self.condition} was valid."
|
261
|
+
)
|
262
|
+
# NOTE: Start call wrapped execution method that will use custom
|
263
|
+
# execution before the real execution from inherit stage model.
|
264
|
+
result_caught: Result = self.__execute(
|
265
|
+
params, result=result, event=event
|
266
|
+
)
|
267
|
+
if result_caught.status == WAIT:
|
268
|
+
raise StageError(
|
269
|
+
"Status from execution should not return waiting status."
|
270
|
+
)
|
271
|
+
return result_caught
|
272
|
+
|
273
|
+
# NOTE: Catch this error in this line because the execution can raise
|
274
|
+
# this exception class at other location.
|
275
|
+
except (
|
276
|
+
StageSkipError,
|
277
|
+
StageCancelError,
|
278
|
+
StageError,
|
279
|
+
) as e: # pragma: no cov
|
280
|
+
result.trace.info(
|
281
|
+
f"[STAGE]: Handler:||{e.__class__.__name__}: {e}||"
|
282
|
+
f"{traceback.format_exc()}"
|
283
|
+
)
|
284
|
+
return result.catch(
|
285
|
+
status=get_status_from_error(e),
|
286
|
+
context=(
|
287
|
+
None
|
288
|
+
if isinstance(e, StageSkipError)
|
289
|
+
else {"errors": e.to_dict()}
|
290
|
+
),
|
291
|
+
)
|
231
292
|
except Exception as e:
|
232
|
-
e_name: str = e.__class__.__name__
|
233
293
|
result.trace.error(
|
234
|
-
f"[STAGE]: Error Handler:||{
|
294
|
+
f"[STAGE]: Error Handler:||{e.__class__.__name__}: {e}||"
|
235
295
|
f"{traceback.format_exc()}"
|
236
296
|
)
|
237
|
-
if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
|
238
|
-
if isinstance(e, StageException):
|
239
|
-
raise
|
240
|
-
raise StageException(
|
241
|
-
f"{self.__class__.__name__}: {e_name}: {e}"
|
242
|
-
) from e
|
243
297
|
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
244
298
|
|
299
|
+
def __execute(
|
300
|
+
self, params: DictData, result: Result, event: Optional[Event]
|
301
|
+
) -> Result:
|
302
|
+
"""Wrapped the execute method before returning to handler execution.
|
303
|
+
|
304
|
+
:param params: (DictData) A parameter data that want to use in this
|
305
|
+
execution.
|
306
|
+
:param result: (Result) A result object for keeping context and status
|
307
|
+
data.
|
308
|
+
:param event: (Event) An event manager that use to track parent execute
|
309
|
+
was not force stopped.
|
310
|
+
|
311
|
+
:rtype: Result
|
312
|
+
"""
|
313
|
+
result.catch(status=WAIT)
|
314
|
+
return self.execute(params, result=result, event=event)
|
315
|
+
|
245
316
|
def set_outputs(self, output: DictData, to: DictData) -> DictData:
|
246
317
|
"""Set an outputs from execution result context to the received context
|
247
318
|
with a `to` input parameter. The result context from stage execution
|
@@ -289,20 +360,15 @@ class BaseStage(BaseModel, ABC):
|
|
289
360
|
):
|
290
361
|
return to
|
291
362
|
|
292
|
-
|
363
|
+
_id: str = self.gen_id(params=to)
|
364
|
+
output: DictData = copy.deepcopy(output)
|
293
365
|
errors: DictData = (
|
294
|
-
{"errors": output.pop("errors"
|
366
|
+
{"errors": output.pop("errors")} if "errors" in output else {}
|
295
367
|
)
|
296
|
-
|
297
|
-
{"
|
298
|
-
if "skipped" in output
|
299
|
-
else {}
|
368
|
+
status: dict[str, Status] = (
|
369
|
+
{"status": output.pop("status")} if "status" in output else {}
|
300
370
|
)
|
301
|
-
to["stages"][
|
302
|
-
"outputs": copy.deepcopy(output),
|
303
|
-
**skipping,
|
304
|
-
**errors,
|
305
|
-
}
|
371
|
+
to["stages"][_id] = {"outputs": output} | errors | status
|
306
372
|
return to
|
307
373
|
|
308
374
|
def get_outputs(self, output: DictData) -> DictData:
|
@@ -331,14 +397,15 @@ class BaseStage(BaseModel, ABC):
|
|
331
397
|
:param params: (DictData) A parameters that want to pass to condition
|
332
398
|
template.
|
333
399
|
|
334
|
-
:raise
|
400
|
+
:raise StageError: When it has any error raise from the eval
|
335
401
|
condition statement.
|
336
|
-
:raise
|
402
|
+
:raise StageError: When return type of the eval condition statement
|
337
403
|
does not return with boolean type.
|
338
404
|
|
339
405
|
:rtype: bool
|
340
406
|
"""
|
341
|
-
|
407
|
+
# NOTE: Support for condition value is empty string.
|
408
|
+
if not self.condition:
|
342
409
|
return False
|
343
410
|
|
344
411
|
try:
|
@@ -354,13 +421,13 @@ class BaseStage(BaseModel, ABC):
|
|
354
421
|
raise TypeError("Return type of condition does not be boolean")
|
355
422
|
return not rs
|
356
423
|
except Exception as e:
|
357
|
-
raise
|
424
|
+
raise StageError(f"{e.__class__.__name__}: {e}") from e
|
358
425
|
|
359
426
|
def gen_id(self, params: DictData) -> str:
|
360
427
|
"""Generate stage ID that dynamic use stage's name if it ID does not
|
361
428
|
set.
|
362
429
|
|
363
|
-
:param params: A parameter data.
|
430
|
+
:param params: (DictData) A parameter or context data.
|
364
431
|
|
365
432
|
:rtype: str
|
366
433
|
"""
|
@@ -372,8 +439,16 @@ class BaseStage(BaseModel, ABC):
|
|
372
439
|
)
|
373
440
|
)
|
374
441
|
|
442
|
+
@property
|
443
|
+
def is_nested(self) -> bool:
|
444
|
+
"""Return true if this stage is nested stage.
|
445
|
+
|
446
|
+
:rtype: bool
|
447
|
+
"""
|
448
|
+
return False
|
449
|
+
|
375
450
|
|
376
|
-
class BaseAsyncStage(BaseStage):
|
451
|
+
class BaseAsyncStage(BaseStage, ABC):
|
377
452
|
"""Base Async Stage model to make any stage model allow async execution for
|
378
453
|
optimize CPU and Memory on the current node. If you want to implement any
|
379
454
|
custom async stage, you can inherit this class and implement
|
@@ -383,18 +458,6 @@ class BaseAsyncStage(BaseStage):
|
|
383
458
|
model.
|
384
459
|
"""
|
385
460
|
|
386
|
-
@abstractmethod
|
387
|
-
def execute(
|
388
|
-
self,
|
389
|
-
params: DictData,
|
390
|
-
*,
|
391
|
-
result: Optional[Result] = None,
|
392
|
-
event: Optional[Event] = None,
|
393
|
-
) -> Result:
|
394
|
-
raise NotImplementedError(
|
395
|
-
"Async Stage should implement `execute` method."
|
396
|
-
)
|
397
|
-
|
398
461
|
@abstractmethod
|
399
462
|
async def axecute(
|
400
463
|
self,
|
@@ -427,7 +490,6 @@ class BaseAsyncStage(BaseStage):
|
|
427
490
|
parent_run_id: StrOrNone = None,
|
428
491
|
result: Optional[Result] = None,
|
429
492
|
event: Optional[Event] = None,
|
430
|
-
raise_error: Optional[bool] = None,
|
431
493
|
) -> Result:
|
432
494
|
"""Async Handler stage execution result from the stage `execute` method.
|
433
495
|
|
@@ -437,7 +499,6 @@ class BaseAsyncStage(BaseStage):
|
|
437
499
|
:param result: (Result) A Result instance for return context and status.
|
438
500
|
:param event: (Event) An Event manager instance that use to cancel this
|
439
501
|
execution if it forces stopped by parent execution.
|
440
|
-
:param raise_error: (bool) A flag that all this method raise error
|
441
502
|
|
442
503
|
:rtype: Result
|
443
504
|
"""
|
@@ -448,22 +509,142 @@ class BaseAsyncStage(BaseStage):
|
|
448
509
|
id_logic=self.iden,
|
449
510
|
extras=self.extras,
|
450
511
|
)
|
451
|
-
|
452
512
|
try:
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
513
|
+
await result.trace.ainfo(
|
514
|
+
f"[STAGE]: Handler {to_train(self.__class__.__name__)}: "
|
515
|
+
f"{self.name!r}."
|
516
|
+
)
|
517
|
+
if self.desc:
|
518
|
+
await result.trace.adebug(
|
519
|
+
f"[STAGE]: Description:||{self.desc}||"
|
520
|
+
)
|
521
|
+
|
522
|
+
if self.is_skipped(params=params):
|
523
|
+
raise StageSkipError(
|
524
|
+
f"Skip because condition {self.condition} was valid."
|
525
|
+
)
|
464
526
|
|
527
|
+
# NOTE: Start call wrapped execution method that will use custom
|
528
|
+
# execution before the real execution from inherit stage model.
|
529
|
+
result_caught: Result = await self.__axecute(
|
530
|
+
params, result=result, event=event
|
531
|
+
)
|
532
|
+
if result_caught.status == WAIT:
|
533
|
+
raise StageError(
|
534
|
+
"Status from execution should not return waiting status."
|
535
|
+
)
|
536
|
+
return result_caught
|
537
|
+
|
538
|
+
# NOTE: Catch this error in this line because the execution can raise
|
539
|
+
# this exception class at other location.
|
540
|
+
except (
|
541
|
+
StageSkipError,
|
542
|
+
StageCancelError,
|
543
|
+
StageError,
|
544
|
+
) as e: # pragma: no cov
|
545
|
+
await result.trace.ainfo(
|
546
|
+
f"[STAGE]: Skip Handler:||{e.__class__.__name__}: {e}||"
|
547
|
+
f"{traceback.format_exc()}"
|
548
|
+
)
|
549
|
+
return result.catch(
|
550
|
+
status=get_status_from_error(e),
|
551
|
+
context=(
|
552
|
+
{"errors": e.to_dict()}
|
553
|
+
if isinstance(e, StageError)
|
554
|
+
else None
|
555
|
+
),
|
556
|
+
)
|
557
|
+
except Exception as e:
|
558
|
+
await result.trace.aerror(
|
559
|
+
f"[STAGE]: Error Handler:||{e.__class__.__name__}: {e}||"
|
560
|
+
f"{traceback.format_exc()}"
|
561
|
+
)
|
465
562
|
return result.catch(status=FAILED, context={"errors": to_dict(e)})
|
466
563
|
|
564
|
+
async def __axecute(
|
565
|
+
self, params: DictData, result: Result, event: Optional[Event]
|
566
|
+
) -> Result:
|
567
|
+
"""Wrapped the axecute method before returning to handler axecute.
|
568
|
+
|
569
|
+
:param params: (DictData) A parameter data that want to use in this
|
570
|
+
execution.
|
571
|
+
:param result: (Result) A result object for keeping context and status
|
572
|
+
data.
|
573
|
+
:param event: (Event) An event manager that use to track parent execute
|
574
|
+
was not force stopped.
|
575
|
+
|
576
|
+
:rtype: Result
|
577
|
+
"""
|
578
|
+
result.catch(status=WAIT)
|
579
|
+
return await self.axecute(params, result=result, event=event)
|
580
|
+
|
581
|
+
|
582
|
+
class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
583
|
+
"""Base Retry Stage model that will execute again when it raises with the
|
584
|
+
`StageRetryError`.
|
585
|
+
"""
|
586
|
+
|
587
|
+
retry: int = Field(
|
588
|
+
default=0,
|
589
|
+
ge=0,
|
590
|
+
lt=20,
|
591
|
+
description="Retry number if stage execution get the error.",
|
592
|
+
)
|
593
|
+
|
594
|
+
def __execute(
|
595
|
+
self,
|
596
|
+
params: DictData,
|
597
|
+
result: Result,
|
598
|
+
event: Optional[Event],
|
599
|
+
) -> Result:
|
600
|
+
"""Wrapped the execute method with retry strategy before returning to
|
601
|
+
handler execute.
|
602
|
+
|
603
|
+
:param params: (DictData) A parameter data that want to use in this
|
604
|
+
execution.
|
605
|
+
:param result: (Result) A result object for keeping context and status
|
606
|
+
data.
|
607
|
+
:param event: (Event) An event manager that use to track parent execute
|
608
|
+
was not force stopped.
|
609
|
+
|
610
|
+
:rtype: Result
|
611
|
+
"""
|
612
|
+
current_retry: int = 0
|
613
|
+
with current_retry < (self.retry + 1):
|
614
|
+
try:
|
615
|
+
result.catch(status=WAIT, context={"retry": current_retry})
|
616
|
+
return self.execute(params, result=result, event=event)
|
617
|
+
except StageRetryError:
|
618
|
+
current_retry += 1
|
619
|
+
raise StageError(f"Reach the maximum of retry number: {self.retry}.")
|
620
|
+
|
621
|
+
async def __axecute(
|
622
|
+
self,
|
623
|
+
params: DictData,
|
624
|
+
result: Result,
|
625
|
+
event: Optional[Event],
|
626
|
+
) -> Result:
|
627
|
+
"""Wrapped the axecute method with retry strategy before returning to
|
628
|
+
handler axecute.
|
629
|
+
|
630
|
+
:param params: (DictData) A parameter data that want to use in this
|
631
|
+
execution.
|
632
|
+
:param result: (Result) A result object for keeping context and status
|
633
|
+
data.
|
634
|
+
:param event: (Event) An event manager that use to track parent execute
|
635
|
+
was not force stopped.
|
636
|
+
|
637
|
+
:rtype: Result
|
638
|
+
"""
|
639
|
+
current_retry: int = 0
|
640
|
+
with current_retry < (self.retry + 1):
|
641
|
+
try:
|
642
|
+
result.catch(status=WAIT, context={"retry": current_retry})
|
643
|
+
return await self.axecute(params, result=result, event=event)
|
644
|
+
except StageRetryError:
|
645
|
+
current_retry += 1
|
646
|
+
raise StageError(f"Reach the maximum of retry number: {self.retry}.")
|
647
|
+
|
467
648
|
|
468
649
|
class EmptyStage(BaseAsyncStage):
|
469
650
|
"""Empty stage executor that do nothing and log the `message` field to
|
@@ -527,12 +708,15 @@ class EmptyStage(BaseAsyncStage):
|
|
527
708
|
else "..."
|
528
709
|
)
|
529
710
|
|
530
|
-
|
531
|
-
|
532
|
-
|
711
|
+
if event and event.is_set():
|
712
|
+
raise StageCancelError(
|
713
|
+
"Execution was canceled from the event before start parallel."
|
714
|
+
)
|
715
|
+
|
716
|
+
result.trace.info(f"[STAGE]: Message: ( {message} )")
|
533
717
|
if self.sleep > 0:
|
534
718
|
if self.sleep > 5:
|
535
|
-
result.trace.info(f"[STAGE]: ...
|
719
|
+
result.trace.info(f"[STAGE]: Sleep ... ({self.sleep} sec)")
|
536
720
|
time.sleep(self.sleep)
|
537
721
|
return result.catch(status=SUCCESS)
|
538
722
|
|
@@ -566,11 +750,16 @@ class EmptyStage(BaseAsyncStage):
|
|
566
750
|
else "..."
|
567
751
|
)
|
568
752
|
|
569
|
-
|
753
|
+
if event and event.is_set():
|
754
|
+
raise StageCancelError(
|
755
|
+
"Execution was canceled from the event before start parallel."
|
756
|
+
)
|
757
|
+
|
758
|
+
result.trace.info(f"[STAGE]: Message: ( {message} )")
|
570
759
|
if self.sleep > 0:
|
571
760
|
if self.sleep > 5:
|
572
761
|
await result.trace.ainfo(
|
573
|
-
f"[STAGE]: ...
|
762
|
+
f"[STAGE]: Sleep ... ({self.sleep} sec)"
|
574
763
|
)
|
575
764
|
await asyncio.sleep(self.sleep)
|
576
765
|
return result.catch(status=SUCCESS)
|
@@ -703,19 +892,15 @@ class BashStage(BaseAsyncStage):
|
|
703
892
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
704
893
|
extras=self.extras,
|
705
894
|
)
|
706
|
-
|
707
|
-
result.trace.info(f"[STAGE]: Execute Shell-Stage: {self.name}")
|
708
|
-
|
709
895
|
bash: str = param2template(
|
710
896
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
711
897
|
)
|
712
|
-
|
713
898
|
with self.create_sh_file(
|
714
899
|
bash=bash,
|
715
900
|
env=param2template(self.env, params, extras=self.extras),
|
716
901
|
run_id=result.run_id,
|
717
902
|
) as sh:
|
718
|
-
result.trace.debug(f"[STAGE]:
|
903
|
+
result.trace.debug(f"[STAGE]: Create `{sh[1]}` file.")
|
719
904
|
rs: CompletedProcess = subprocess.run(
|
720
905
|
sh,
|
721
906
|
shell=False,
|
@@ -726,7 +911,7 @@ class BashStage(BaseAsyncStage):
|
|
726
911
|
)
|
727
912
|
if rs.returncode > 0:
|
728
913
|
e: str = rs.stderr.removesuffix("\n")
|
729
|
-
raise
|
914
|
+
raise StageError(
|
730
915
|
f"Subprocess: {e}\n---( statement )---\n```bash\n{bash}\n```"
|
731
916
|
)
|
732
917
|
return result.catch(
|
@@ -759,17 +944,15 @@ class BashStage(BaseAsyncStage):
|
|
759
944
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
760
945
|
extras=self.extras,
|
761
946
|
)
|
762
|
-
await result.trace.ainfo(f"[STAGE]: Execute Shell-Stage: {self.name}")
|
763
947
|
bash: str = param2template(
|
764
948
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
765
949
|
)
|
766
|
-
|
767
950
|
async with self.async_create_sh_file(
|
768
951
|
bash=bash,
|
769
952
|
env=param2template(self.env, params, extras=self.extras),
|
770
953
|
run_id=result.run_id,
|
771
954
|
) as sh:
|
772
|
-
await result.trace.adebug(f"[STAGE]:
|
955
|
+
await result.trace.adebug(f"[STAGE]: Create `{sh[1]}` file.")
|
773
956
|
rs: CompletedProcess = subprocess.run(
|
774
957
|
sh,
|
775
958
|
shell=False,
|
@@ -781,7 +964,7 @@ class BashStage(BaseAsyncStage):
|
|
781
964
|
|
782
965
|
if rs.returncode > 0:
|
783
966
|
e: str = rs.stderr.removesuffix("\n")
|
784
|
-
raise
|
967
|
+
raise StageError(
|
785
968
|
f"Subprocess: {e}\n---( statement )---\n```bash\n{bash}\n```"
|
786
969
|
)
|
787
970
|
return result.catch(
|
@@ -888,7 +1071,6 @@ class PyStage(BaseAsyncStage):
|
|
888
1071
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
889
1072
|
extras=self.extras,
|
890
1073
|
)
|
891
|
-
|
892
1074
|
lc: DictData = {}
|
893
1075
|
gb: DictData = (
|
894
1076
|
globals()
|
@@ -896,8 +1078,6 @@ class PyStage(BaseAsyncStage):
|
|
896
1078
|
| {"result": result}
|
897
1079
|
)
|
898
1080
|
|
899
|
-
result.trace.info(f"[STAGE]: Execute Py-Stage: {self.name}")
|
900
|
-
|
901
1081
|
# WARNING: The exec build-in function is very dangerous. So, it
|
902
1082
|
# should use the re module to validate exec-string before running.
|
903
1083
|
exec(
|
@@ -921,6 +1101,7 @@ class PyStage(BaseAsyncStage):
|
|
921
1101
|
and not ismodule(gb[k])
|
922
1102
|
and not isclass(gb[k])
|
923
1103
|
and not isfunction(gb[k])
|
1104
|
+
and k in params
|
924
1105
|
)
|
925
1106
|
},
|
926
1107
|
},
|
@@ -956,8 +1137,6 @@ class PyStage(BaseAsyncStage):
|
|
956
1137
|
| param2template(self.vars, params, extras=self.extras)
|
957
1138
|
| {"result": result}
|
958
1139
|
)
|
959
|
-
await result.trace.ainfo(f"[STAGE]: Execute Py-Stage: {self.name}")
|
960
|
-
|
961
1140
|
# WARNING: The exec build-in function is very dangerous. So, it
|
962
1141
|
# should use the re module to validate exec-string before running.
|
963
1142
|
exec(
|
@@ -978,6 +1157,7 @@ class PyStage(BaseAsyncStage):
|
|
978
1157
|
and not ismodule(gb[k])
|
979
1158
|
and not isclass(gb[k])
|
980
1159
|
and not isfunction(gb[k])
|
1160
|
+
and k in params
|
981
1161
|
)
|
982
1162
|
},
|
983
1163
|
},
|
@@ -1060,7 +1240,7 @@ class CallStage(BaseAsyncStage):
|
|
1060
1240
|
)()
|
1061
1241
|
|
1062
1242
|
result.trace.info(
|
1063
|
-
f"[STAGE]:
|
1243
|
+
f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'"
|
1064
1244
|
)
|
1065
1245
|
|
1066
1246
|
# VALIDATE: check input task caller parameters that exists before
|
@@ -1094,6 +1274,11 @@ class CallStage(BaseAsyncStage):
|
|
1094
1274
|
if "result" not in sig.parameters and not has_keyword:
|
1095
1275
|
args.pop("result")
|
1096
1276
|
|
1277
|
+
if event and event.is_set():
|
1278
|
+
raise StageCancelError(
|
1279
|
+
"Execution was canceled from the event before start parallel."
|
1280
|
+
)
|
1281
|
+
|
1097
1282
|
args = self.validate_model_args(call_func, args, result)
|
1098
1283
|
if inspect.iscoroutinefunction(call_func):
|
1099
1284
|
loop = asyncio.get_event_loop()
|
@@ -1148,7 +1333,7 @@ class CallStage(BaseAsyncStage):
|
|
1148
1333
|
)()
|
1149
1334
|
|
1150
1335
|
await result.trace.ainfo(
|
1151
|
-
f"[STAGE]:
|
1336
|
+
f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'"
|
1152
1337
|
)
|
1153
1338
|
|
1154
1339
|
# VALIDATE: check input task caller parameters that exists before
|
@@ -1182,7 +1367,7 @@ class CallStage(BaseAsyncStage):
|
|
1182
1367
|
if "result" not in sig.parameters and not has_keyword:
|
1183
1368
|
args.pop("result")
|
1184
1369
|
|
1185
|
-
args = self.validate_model_args(call_func, args, result)
|
1370
|
+
args: DictData = self.validate_model_args(call_func, args, result)
|
1186
1371
|
if inspect.iscoroutinefunction(call_func):
|
1187
1372
|
rs: DictOrModel = await call_func(
|
1188
1373
|
**param2template(args, params, extras=self.extras)
|
@@ -1212,20 +1397,20 @@ class CallStage(BaseAsyncStage):
|
|
1212
1397
|
) -> DictData:
|
1213
1398
|
"""Validate an input arguments before passing to the caller function.
|
1214
1399
|
|
1215
|
-
:param func: A tag function that want to get typing.
|
1216
|
-
:param args: An arguments before passing to this tag
|
1400
|
+
:param func: (TagFunc) A tag function that want to get typing.
|
1401
|
+
:param args: (DictData) An arguments before passing to this tag func.
|
1217
1402
|
:param result: (Result) A result object for keeping context and status
|
1218
1403
|
data.
|
1219
1404
|
|
1220
1405
|
:rtype: DictData
|
1221
1406
|
"""
|
1222
1407
|
try:
|
1223
|
-
model_instance = create_model_from_caller(
|
1224
|
-
|
1408
|
+
model_instance: BaseModel = create_model_from_caller(
|
1409
|
+
func
|
1410
|
+
).model_validate(args)
|
1411
|
+
override: DictData = dict(model_instance)
|
1225
1412
|
args.update(override)
|
1226
|
-
|
1227
1413
|
type_hints: dict[str, Any] = get_type_hints(func)
|
1228
|
-
|
1229
1414
|
for arg in type_hints:
|
1230
1415
|
|
1231
1416
|
if arg == "return":
|
@@ -1233,10 +1418,11 @@ class CallStage(BaseAsyncStage):
|
|
1233
1418
|
|
1234
1419
|
if arg.removeprefix("_") in args:
|
1235
1420
|
args[arg] = args.pop(arg.removeprefix("_"))
|
1421
|
+
continue
|
1236
1422
|
|
1237
1423
|
return args
|
1238
1424
|
except ValidationError as e:
|
1239
|
-
raise
|
1425
|
+
raise StageError(
|
1240
1426
|
"Validate argument from the caller function raise invalid type."
|
1241
1427
|
) from e
|
1242
1428
|
except TypeError as e:
|
@@ -1247,7 +1433,42 @@ class CallStage(BaseAsyncStage):
|
|
1247
1433
|
return args
|
1248
1434
|
|
1249
1435
|
|
1250
|
-
class
|
1436
|
+
class BaseNestedStage(BaseStage, ABC):
|
1437
|
+
"""Base Nested Stage model. This model is use for checking the child stage
|
1438
|
+
is the nested stage or not.
|
1439
|
+
"""
|
1440
|
+
|
1441
|
+
def set_outputs(self, output: DictData, to: DictData) -> DictData:
|
1442
|
+
"""Override the set outputs method that support for nested-stage."""
|
1443
|
+
return super().set_outputs(output, to=to)
|
1444
|
+
|
1445
|
+
def get_outputs(self, output: DictData) -> DictData:
|
1446
|
+
"""Override the get outputs method that support for nested-stage"""
|
1447
|
+
return super().get_outputs(output)
|
1448
|
+
|
1449
|
+
@property
|
1450
|
+
def is_nested(self) -> bool:
|
1451
|
+
"""Check if this stage is a nested stage or not.
|
1452
|
+
|
1453
|
+
:rtype: bool
|
1454
|
+
"""
|
1455
|
+
return True
|
1456
|
+
|
1457
|
+
@staticmethod
|
1458
|
+
def mark_errors(context: DictData, error: StageError) -> None:
|
1459
|
+
"""Make the errors context result with the refs value depends on the nested
|
1460
|
+
execute func.
|
1461
|
+
|
1462
|
+
:param context: (DictData) A context data.
|
1463
|
+
:param error: (StageError) A stage exception object.
|
1464
|
+
"""
|
1465
|
+
if "errors" in context:
|
1466
|
+
context["errors"][error.refs] = error.to_dict()
|
1467
|
+
else:
|
1468
|
+
context["errors"] = error.to_dict(with_refs=True)
|
1469
|
+
|
1470
|
+
|
1471
|
+
class TriggerStage(BaseNestedStage):
|
1251
1472
|
"""Trigger workflow executor stage that run an input trigger Workflow
|
1252
1473
|
execute method. This is the stage that allow you to create the reusable
|
1253
1474
|
Workflow template with dynamic parameters.
|
@@ -1279,13 +1500,14 @@ class TriggerStage(BaseStage):
|
|
1279
1500
|
event: Optional[Event] = None,
|
1280
1501
|
) -> Result:
|
1281
1502
|
"""Trigger another workflow execution. It will wait the trigger
|
1282
|
-
workflow running complete before catching its result
|
1503
|
+
workflow running complete before catching its result and raise error
|
1504
|
+
when the result status does not be SUCCESS.
|
1283
1505
|
|
1284
1506
|
:param params: (DictData) A parameter data.
|
1285
1507
|
:param result: (Result) A result object for keeping context and status
|
1286
|
-
data.
|
1508
|
+
data. (Default is None)
|
1287
1509
|
:param event: (Event) An event manager that use to track parent execute
|
1288
|
-
was not force stopped.
|
1510
|
+
was not force stopped. (Default is None)
|
1289
1511
|
|
1290
1512
|
:rtype: Result
|
1291
1513
|
"""
|
@@ -1297,57 +1519,28 @@ class TriggerStage(BaseStage):
|
|
1297
1519
|
)
|
1298
1520
|
|
1299
1521
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
1300
|
-
result
|
1301
|
-
rs: Result = Workflow.from_conf(
|
1522
|
+
result: Result = Workflow.from_conf(
|
1302
1523
|
name=pass_env(_trigger),
|
1303
|
-
extras=self.extras
|
1524
|
+
extras=self.extras,
|
1304
1525
|
).execute(
|
1526
|
+
# NOTE: Should not use the `pass_env` function on this params parameter.
|
1305
1527
|
params=param2template(self.params, params, extras=self.extras),
|
1306
1528
|
run_id=None,
|
1307
1529
|
parent_run_id=result.parent_run_id,
|
1308
1530
|
event=event,
|
1309
1531
|
)
|
1310
|
-
if
|
1532
|
+
if result.status == FAILED:
|
1311
1533
|
err_msg: StrOrNone = (
|
1312
1534
|
f" with:\n{msg}"
|
1313
|
-
if (msg :=
|
1535
|
+
if (msg := result.context.get("errors", {}).get("message"))
|
1314
1536
|
else "."
|
1315
1537
|
)
|
1316
|
-
raise
|
1317
|
-
|
1318
|
-
)
|
1319
|
-
|
1320
|
-
|
1321
|
-
|
1322
|
-
class BaseNestedStage(BaseStage):
|
1323
|
-
"""Base Nested Stage model. This model is use for checking the child stage
|
1324
|
-
is the nested stage or not.
|
1325
|
-
"""
|
1326
|
-
|
1327
|
-
@abstractmethod
|
1328
|
-
def execute(
|
1329
|
-
self,
|
1330
|
-
params: DictData,
|
1331
|
-
*,
|
1332
|
-
result: Optional[Result] = None,
|
1333
|
-
event: Optional[Event] = None,
|
1334
|
-
) -> Result:
|
1335
|
-
"""Execute abstraction method that action something by sub-model class.
|
1336
|
-
This is important method that make this class is able to be the nested
|
1337
|
-
stage.
|
1338
|
-
|
1339
|
-
:param params: (DictData) A parameter data that want to use in this
|
1340
|
-
execution.
|
1341
|
-
:param result: (Result) A result object for keeping context and status
|
1342
|
-
data.
|
1343
|
-
:param event: (Event) An event manager that use to track parent execute
|
1344
|
-
was not force stopped.
|
1345
|
-
|
1346
|
-
:rtype: Result
|
1347
|
-
"""
|
1348
|
-
raise NotImplementedError(
|
1349
|
-
"Nested-Stage should implement `execute` method."
|
1350
|
-
)
|
1538
|
+
raise StageError(f"Trigger workflow was failed{err_msg}")
|
1539
|
+
elif result.status == CANCEL:
|
1540
|
+
raise StageCancelError("Trigger workflow was cancel.")
|
1541
|
+
elif result.status == SKIP:
|
1542
|
+
raise StageSkipError("Trigger workflow was skipped.")
|
1543
|
+
return result
|
1351
1544
|
|
1352
1545
|
|
1353
1546
|
class ParallelStage(BaseNestedStage):
|
@@ -1368,10 +1561,14 @@ class ParallelStage(BaseNestedStage):
|
|
1368
1561
|
... "echo": "Start run with branch 1",
|
1369
1562
|
... "sleep": 3,
|
1370
1563
|
... },
|
1564
|
+
... {
|
1565
|
+
... "name": "Echo second stage",
|
1566
|
+
... "echo": "Start run with branch 1",
|
1567
|
+
... },
|
1371
1568
|
... ],
|
1372
1569
|
... "branch02": [
|
1373
1570
|
... {
|
1374
|
-
... "name": "Echo
|
1571
|
+
... "name": "Echo first stage",
|
1375
1572
|
... "echo": "Start run with branch 2",
|
1376
1573
|
... "sleep": 1,
|
1377
1574
|
... },
|
@@ -1401,94 +1598,116 @@ class ParallelStage(BaseNestedStage):
|
|
1401
1598
|
result: Result,
|
1402
1599
|
*,
|
1403
1600
|
event: Optional[Event] = None,
|
1404
|
-
) -> Result:
|
1405
|
-
"""Execute all stage
|
1601
|
+
) -> tuple[Status, Result]:
|
1602
|
+
"""Execute branch that will execute all nested-stage that was set in
|
1603
|
+
this stage with specific branch ID.
|
1406
1604
|
|
1407
1605
|
:param branch: (str) A branch ID.
|
1408
1606
|
:param params: (DictData) A parameter data.
|
1409
1607
|
:param result: (Result) A Result instance for return context and status.
|
1410
1608
|
:param event: (Event) An Event manager instance that use to cancel this
|
1411
1609
|
execution if it forces stopped by parent execution.
|
1610
|
+
(Default is None)
|
1412
1611
|
|
1413
|
-
:
|
1612
|
+
:raise StageCancelError: If event was set.
|
1613
|
+
|
1614
|
+
:rtype: tuple[Status, Result]
|
1414
1615
|
"""
|
1415
1616
|
result.trace.debug(f"[STAGE]: Execute Branch: {branch!r}")
|
1617
|
+
|
1618
|
+
# NOTE: Create nested-context
|
1416
1619
|
context: DictData = copy.deepcopy(params)
|
1417
1620
|
context.update({"branch": branch})
|
1418
|
-
|
1419
|
-
|
1621
|
+
nestet_context: DictData = {"branch": branch, "stages": {}}
|
1622
|
+
|
1623
|
+
total_stage: int = len(self.parallel[branch])
|
1624
|
+
skips: list[bool] = [False] * total_stage
|
1625
|
+
for i, stage in enumerate(self.parallel[branch], start=0):
|
1420
1626
|
|
1421
1627
|
if self.extras:
|
1422
1628
|
stage.extras = self.extras
|
1423
1629
|
|
1424
|
-
if stage.is_skipped(params=context):
|
1425
|
-
result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
|
1426
|
-
stage.set_outputs(output={"skipped": True}, to=output)
|
1427
|
-
continue
|
1428
|
-
|
1429
1630
|
if event and event.is_set():
|
1430
1631
|
error_msg: str = (
|
1431
|
-
"Branch
|
1432
|
-
"
|
1632
|
+
"Branch execution was canceled from the event before "
|
1633
|
+
"start branch execution."
|
1433
1634
|
)
|
1434
1635
|
result.catch(
|
1435
1636
|
status=CANCEL,
|
1436
1637
|
parallel={
|
1437
1638
|
branch: {
|
1639
|
+
"status": CANCEL,
|
1438
1640
|
"branch": branch,
|
1439
|
-
"stages": filter_func(
|
1440
|
-
|
1641
|
+
"stages": filter_func(
|
1642
|
+
nestet_context.pop("stages", {})
|
1643
|
+
),
|
1644
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
1441
1645
|
}
|
1442
1646
|
},
|
1443
1647
|
)
|
1444
|
-
raise
|
1648
|
+
raise StageCancelError(error_msg, refs=branch)
|
1445
1649
|
|
1446
|
-
|
1447
|
-
|
1448
|
-
|
1449
|
-
|
1450
|
-
|
1451
|
-
|
1452
|
-
|
1650
|
+
rs: Result = stage.handler_execute(
|
1651
|
+
params=context,
|
1652
|
+
run_id=result.run_id,
|
1653
|
+
parent_run_id=result.parent_run_id,
|
1654
|
+
event=event,
|
1655
|
+
)
|
1656
|
+
stage.set_outputs(rs.context, to=nestet_context)
|
1657
|
+
stage.set_outputs(stage.get_outputs(nestet_context), to=context)
|
1658
|
+
|
1659
|
+
if rs.status == SKIP:
|
1660
|
+
skips[i] = True
|
1661
|
+
continue
|
1662
|
+
|
1663
|
+
elif rs.status == FAILED: # pragma: no cov
|
1664
|
+
error_msg: str = (
|
1665
|
+
f"Branch execution was break because its nested-stage, "
|
1666
|
+
f"{stage.iden!r}, failed."
|
1453
1667
|
)
|
1454
|
-
stage.set_outputs(rs.context, to=output)
|
1455
|
-
stage.set_outputs(stage.get_outputs(output), to=context)
|
1456
|
-
except StageException as e:
|
1457
1668
|
result.catch(
|
1458
1669
|
status=FAILED,
|
1459
1670
|
parallel={
|
1460
1671
|
branch: {
|
1672
|
+
"status": FAILED,
|
1461
1673
|
"branch": branch,
|
1462
|
-
"stages": filter_func(
|
1463
|
-
|
1674
|
+
"stages": filter_func(
|
1675
|
+
nestet_context.pop("stages", {})
|
1676
|
+
),
|
1677
|
+
"errors": StageError(error_msg).to_dict(),
|
1464
1678
|
},
|
1465
1679
|
},
|
1466
1680
|
)
|
1467
|
-
raise
|
1681
|
+
raise StageError(error_msg, refs=branch)
|
1468
1682
|
|
1469
|
-
|
1683
|
+
elif rs.status == CANCEL:
|
1470
1684
|
error_msg: str = (
|
1471
|
-
|
1472
|
-
|
1685
|
+
"Branch execution was canceled from the event after "
|
1686
|
+
"end branch execution."
|
1473
1687
|
)
|
1474
1688
|
result.catch(
|
1475
|
-
status=
|
1689
|
+
status=CANCEL,
|
1476
1690
|
parallel={
|
1477
1691
|
branch: {
|
1692
|
+
"status": CANCEL,
|
1478
1693
|
"branch": branch,
|
1479
|
-
"stages": filter_func(
|
1480
|
-
|
1481
|
-
|
1694
|
+
"stages": filter_func(
|
1695
|
+
nestet_context.pop("stages", {})
|
1696
|
+
),
|
1697
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
1698
|
+
}
|
1482
1699
|
},
|
1483
1700
|
)
|
1484
|
-
raise
|
1701
|
+
raise StageCancelError(error_msg, refs=branch)
|
1485
1702
|
|
1486
|
-
|
1487
|
-
|
1703
|
+
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
1704
|
+
return status, result.catch(
|
1705
|
+
status=status,
|
1488
1706
|
parallel={
|
1489
1707
|
branch: {
|
1708
|
+
"status": status,
|
1490
1709
|
"branch": branch,
|
1491
|
-
"stages": filter_func(
|
1710
|
+
"stages": filter_func(nestet_context.pop("stages", {})),
|
1492
1711
|
},
|
1493
1712
|
},
|
1494
1713
|
)
|
@@ -1506,6 +1725,7 @@ class ParallelStage(BaseNestedStage):
|
|
1506
1725
|
:param result: (Result) A Result instance for return context and status.
|
1507
1726
|
:param event: (Event) An Event manager instance that use to cancel this
|
1508
1727
|
execution if it forces stopped by parent execution.
|
1728
|
+
(Default is None)
|
1509
1729
|
|
1510
1730
|
:rtype: Result
|
1511
1731
|
"""
|
@@ -1514,28 +1734,18 @@ class ParallelStage(BaseNestedStage):
|
|
1514
1734
|
extras=self.extras,
|
1515
1735
|
)
|
1516
1736
|
event: Event = event or Event()
|
1517
|
-
result.trace.info(
|
1518
|
-
|
1737
|
+
result.trace.info(f"[STAGE]: Parallel with {self.max_workers} workers.")
|
1738
|
+
result.catch(
|
1739
|
+
status=WAIT,
|
1740
|
+
context={"workers": self.max_workers, "parallel": {}},
|
1519
1741
|
)
|
1520
|
-
|
1742
|
+
len_parallel: int = len(self.parallel)
|
1521
1743
|
if event and event.is_set():
|
1522
|
-
|
1523
|
-
|
1524
|
-
context={
|
1525
|
-
"errors": StageException(
|
1526
|
-
"Stage was canceled from event that had set "
|
1527
|
-
"before stage parallel execution."
|
1528
|
-
).to_dict()
|
1529
|
-
},
|
1744
|
+
raise StageCancelError(
|
1745
|
+
"Execution was canceled from the event before start parallel."
|
1530
1746
|
)
|
1531
1747
|
|
1532
|
-
with ThreadPoolExecutor(
|
1533
|
-
max_workers=self.max_workers, thread_name_prefix="stage_parallel_"
|
1534
|
-
) as executor:
|
1535
|
-
|
1536
|
-
context: DictData = {}
|
1537
|
-
status: Status = SUCCESS
|
1538
|
-
|
1748
|
+
with ThreadPoolExecutor(self.max_workers, "stp") as executor:
|
1539
1749
|
futures: list[Future] = [
|
1540
1750
|
executor.submit(
|
1541
1751
|
self.execute_branch,
|
@@ -1546,17 +1756,18 @@ class ParallelStage(BaseNestedStage):
|
|
1546
1756
|
)
|
1547
1757
|
for branch in self.parallel
|
1548
1758
|
]
|
1549
|
-
|
1550
|
-
|
1759
|
+
context: DictData = {}
|
1760
|
+
statuses: list[Status] = [WAIT] * len_parallel
|
1761
|
+
for i, future in enumerate(as_completed(futures), start=0):
|
1551
1762
|
try:
|
1552
|
-
future.result()
|
1553
|
-
except
|
1554
|
-
|
1555
|
-
|
1556
|
-
|
1557
|
-
|
1558
|
-
|
1559
|
-
|
1763
|
+
statuses[i], _ = future.result()
|
1764
|
+
except StageError as e:
|
1765
|
+
statuses[i] = get_status_from_error(e)
|
1766
|
+
self.mark_errors(context, e)
|
1767
|
+
return result.catch(
|
1768
|
+
status=validate_statuses(statuses),
|
1769
|
+
context=context,
|
1770
|
+
)
|
1560
1771
|
|
1561
1772
|
|
1562
1773
|
class ForEachStage(BaseNestedStage):
|
@@ -1616,9 +1827,12 @@ class ForEachStage(BaseNestedStage):
|
|
1616
1827
|
result: Result,
|
1617
1828
|
*,
|
1618
1829
|
event: Optional[Event] = None,
|
1619
|
-
) -> Result:
|
1620
|
-
"""Execute all nested
|
1621
|
-
item
|
1830
|
+
) -> tuple[Status, Result]:
|
1831
|
+
"""Execute item that will execute all nested-stage that was set in this
|
1832
|
+
stage with specific foreach item.
|
1833
|
+
|
1834
|
+
This method will create the nested-context from an input context
|
1835
|
+
data and use it instead the context data.
|
1622
1836
|
|
1623
1837
|
:param index: (int) An index value of foreach loop.
|
1624
1838
|
:param item: (str | int) An item that want to execution.
|
@@ -1626,91 +1840,112 @@ class ForEachStage(BaseNestedStage):
|
|
1626
1840
|
:param result: (Result) A Result instance for return context and status.
|
1627
1841
|
:param event: (Event) An Event manager instance that use to cancel this
|
1628
1842
|
execution if it forces stopped by parent execution.
|
1843
|
+
(Default is None)
|
1629
1844
|
|
1630
|
-
|
1631
|
-
|
1632
|
-
:raise StageException: If the result from execution has `FAILED` status.
|
1845
|
+
This method should raise error when it wants to stop the foreach
|
1846
|
+
loop such as cancel event or getting the failed status.
|
1633
1847
|
|
1634
|
-
:
|
1848
|
+
:raise StageCancelError: If event was set.
|
1849
|
+
:raise StageError: If the stage execution raise any Exception error.
|
1850
|
+
:raise StageError: If the result from execution has `FAILED` status.
|
1851
|
+
|
1852
|
+
:rtype: tuple[Status, Result]
|
1635
1853
|
"""
|
1636
1854
|
result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
|
1637
1855
|
key: StrOrInt = index if self.use_index_as_key else item
|
1856
|
+
|
1638
1857
|
context: DictData = copy.deepcopy(params)
|
1639
1858
|
context.update({"item": item, "loop": index})
|
1640
|
-
|
1641
|
-
|
1859
|
+
nestet_context: DictData = {"item": item, "stages": {}}
|
1860
|
+
total_stage: int = len(self.stages)
|
1861
|
+
skips: list[bool] = [False] * total_stage
|
1862
|
+
for i, stage in enumerate(self.stages, start=0):
|
1642
1863
|
|
1643
1864
|
if self.extras:
|
1644
1865
|
stage.extras = self.extras
|
1645
1866
|
|
1646
|
-
if stage.is_skipped(params=context):
|
1647
|
-
result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
|
1648
|
-
stage.set_outputs(output={"skipped": True}, to=output)
|
1649
|
-
continue
|
1650
|
-
|
1651
1867
|
if event and event.is_set():
|
1652
1868
|
error_msg: str = (
|
1653
|
-
"Item
|
1869
|
+
"Item execution was canceled from the event before start "
|
1870
|
+
"item execution."
|
1654
1871
|
)
|
1655
1872
|
result.catch(
|
1656
1873
|
status=CANCEL,
|
1657
1874
|
foreach={
|
1658
1875
|
key: {
|
1876
|
+
"status": CANCEL,
|
1659
1877
|
"item": item,
|
1660
|
-
"stages": filter_func(
|
1661
|
-
|
1878
|
+
"stages": filter_func(
|
1879
|
+
nestet_context.pop("stages", {})
|
1880
|
+
),
|
1881
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
1662
1882
|
}
|
1663
1883
|
},
|
1664
1884
|
)
|
1665
|
-
raise
|
1885
|
+
raise StageCancelError(error_msg, refs=key)
|
1666
1886
|
|
1667
|
-
|
1668
|
-
|
1669
|
-
|
1670
|
-
|
1671
|
-
|
1672
|
-
|
1673
|
-
|
1887
|
+
rs: Result = stage.handler_execute(
|
1888
|
+
params=context,
|
1889
|
+
run_id=result.run_id,
|
1890
|
+
parent_run_id=result.parent_run_id,
|
1891
|
+
event=event,
|
1892
|
+
)
|
1893
|
+
stage.set_outputs(rs.context, to=nestet_context)
|
1894
|
+
stage.set_outputs(stage.get_outputs(nestet_context), to=context)
|
1895
|
+
|
1896
|
+
if rs.status == SKIP:
|
1897
|
+
skips[i] = True
|
1898
|
+
continue
|
1899
|
+
|
1900
|
+
elif rs.status == FAILED: # pragma: no cov
|
1901
|
+
error_msg: str = (
|
1902
|
+
f"Item execution was break because its nested-stage, "
|
1903
|
+
f"{stage.iden!r}, failed."
|
1674
1904
|
)
|
1675
|
-
|
1676
|
-
stage.set_outputs(stage.get_outputs(output), to=context)
|
1677
|
-
except StageException as e:
|
1905
|
+
result.trace.warning(f"[STAGE]: {error_msg}")
|
1678
1906
|
result.catch(
|
1679
1907
|
status=FAILED,
|
1680
1908
|
foreach={
|
1681
1909
|
key: {
|
1910
|
+
"status": FAILED,
|
1682
1911
|
"item": item,
|
1683
|
-
"stages": filter_func(
|
1684
|
-
|
1912
|
+
"stages": filter_func(
|
1913
|
+
nestet_context.pop("stages", {})
|
1914
|
+
),
|
1915
|
+
"errors": StageError(error_msg).to_dict(),
|
1685
1916
|
},
|
1686
1917
|
},
|
1687
1918
|
)
|
1688
|
-
raise
|
1919
|
+
raise StageError(error_msg, refs=key)
|
1689
1920
|
|
1690
|
-
|
1921
|
+
elif rs.status == CANCEL:
|
1691
1922
|
error_msg: str = (
|
1692
|
-
|
1693
|
-
|
1923
|
+
"Item execution was canceled from the event after "
|
1924
|
+
"end item execution."
|
1694
1925
|
)
|
1695
|
-
result.trace.warning(f"[STAGE]: {error_msg}")
|
1696
1926
|
result.catch(
|
1697
|
-
status=
|
1927
|
+
status=CANCEL,
|
1698
1928
|
foreach={
|
1699
1929
|
key: {
|
1930
|
+
"status": CANCEL,
|
1700
1931
|
"item": item,
|
1701
|
-
"stages": filter_func(
|
1702
|
-
|
1703
|
-
|
1932
|
+
"stages": filter_func(
|
1933
|
+
nestet_context.pop("stages", {})
|
1934
|
+
),
|
1935
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
1936
|
+
}
|
1704
1937
|
},
|
1705
1938
|
)
|
1706
|
-
raise
|
1939
|
+
raise StageCancelError(error_msg, refs=key)
|
1707
1940
|
|
1708
|
-
|
1709
|
-
|
1941
|
+
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
1942
|
+
return status, result.catch(
|
1943
|
+
status=status,
|
1710
1944
|
foreach={
|
1711
1945
|
key: {
|
1946
|
+
"status": status,
|
1712
1947
|
"item": item,
|
1713
|
-
"stages": filter_func(
|
1948
|
+
"stages": filter_func(nestet_context.pop("stages", {})),
|
1714
1949
|
},
|
1715
1950
|
},
|
1716
1951
|
)
|
@@ -1738,38 +1973,42 @@ class ForEachStage(BaseNestedStage):
|
|
1738
1973
|
extras=self.extras,
|
1739
1974
|
)
|
1740
1975
|
event: Event = event or Event()
|
1741
|
-
foreach: Union[list[str], list[int]] = (
|
1976
|
+
foreach: Union[list[str], list[int]] = pass_env(
|
1742
1977
|
param2template(self.foreach, params, extras=self.extras)
|
1743
|
-
if isinstance(self.foreach, str)
|
1744
|
-
else self.foreach
|
1745
1978
|
)
|
1746
1979
|
|
1980
|
+
# [NOTE]: Force convert str to list.
|
1981
|
+
if isinstance(foreach, str):
|
1982
|
+
try:
|
1983
|
+
foreach: list[Any] = str2list(foreach)
|
1984
|
+
except ValueError as e:
|
1985
|
+
raise TypeError(
|
1986
|
+
f"Does not support string foreach: {foreach!r} that can "
|
1987
|
+
f"not convert to list."
|
1988
|
+
) from e
|
1989
|
+
|
1747
1990
|
# [VALIDATE]: Type of the foreach should be `list` type.
|
1748
|
-
|
1749
|
-
raise TypeError(
|
1991
|
+
elif not isinstance(foreach, list):
|
1992
|
+
raise TypeError(
|
1993
|
+
f"Does not support foreach: {foreach!r} ({type(foreach)})"
|
1994
|
+
)
|
1995
|
+
# [Validate]: Value in the foreach item should not be duplicate when the
|
1996
|
+
# `use_index_as_key` field did not set.
|
1750
1997
|
elif len(set(foreach)) != len(foreach) and not self.use_index_as_key:
|
1751
1998
|
raise ValueError(
|
1752
1999
|
"Foreach item should not duplicate. If this stage must to pass "
|
1753
2000
|
"duplicate item, it should set `use_index_as_key: true`."
|
1754
2001
|
)
|
1755
2002
|
|
1756
|
-
result.trace.info(f"[STAGE]:
|
2003
|
+
result.trace.info(f"[STAGE]: Foreach: {foreach!r}.")
|
1757
2004
|
result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
|
2005
|
+
len_foreach: int = len(foreach)
|
1758
2006
|
if event and event.is_set():
|
1759
|
-
|
1760
|
-
|
1761
|
-
context={
|
1762
|
-
"errors": StageException(
|
1763
|
-
"Stage was canceled from event that had set "
|
1764
|
-
"before stage foreach execution."
|
1765
|
-
).to_dict()
|
1766
|
-
},
|
2007
|
+
raise StageCancelError(
|
2008
|
+
"Execution was canceled from the event before start foreach."
|
1767
2009
|
)
|
1768
2010
|
|
1769
|
-
with ThreadPoolExecutor(
|
1770
|
-
max_workers=self.concurrent, thread_name_prefix="stage_foreach_"
|
1771
|
-
) as executor:
|
1772
|
-
|
2011
|
+
with ThreadPoolExecutor(self.concurrent, "stf") as executor:
|
1773
2012
|
futures: list[Future] = [
|
1774
2013
|
executor.submit(
|
1775
2014
|
self.execute_item,
|
@@ -1781,19 +2020,21 @@ class ForEachStage(BaseNestedStage):
|
|
1781
2020
|
)
|
1782
2021
|
for i, item in enumerate(foreach, start=0)
|
1783
2022
|
]
|
2023
|
+
|
1784
2024
|
context: DictData = {}
|
1785
|
-
|
2025
|
+
statuses: list[Status] = [WAIT] * len_foreach
|
2026
|
+
fail_fast: bool = False
|
1786
2027
|
|
1787
2028
|
done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
|
1788
2029
|
if len(list(done)) != len(futures):
|
1789
2030
|
result.trace.warning(
|
1790
|
-
"[STAGE]: Set event for stop pending for-each stage."
|
2031
|
+
"[STAGE]: Set the event for stop pending for-each stage."
|
1791
2032
|
)
|
1792
2033
|
event.set()
|
1793
2034
|
for future in not_done:
|
1794
2035
|
future.cancel()
|
1795
|
-
time.sleep(0.075)
|
1796
2036
|
|
2037
|
+
time.sleep(0.025)
|
1797
2038
|
nd: str = (
|
1798
2039
|
(
|
1799
2040
|
f", {len(not_done)} item"
|
@@ -1806,18 +2047,24 @@ class ForEachStage(BaseNestedStage):
|
|
1806
2047
|
f"[STAGE]: ... Foreach-Stage set failed event{nd}"
|
1807
2048
|
)
|
1808
2049
|
done: Iterator[Future] = as_completed(futures)
|
2050
|
+
fail_fast = True
|
1809
2051
|
|
1810
|
-
for future in done:
|
2052
|
+
for i, future in enumerate(done, start=0):
|
1811
2053
|
try:
|
1812
|
-
future.result()
|
1813
|
-
except
|
1814
|
-
|
1815
|
-
|
1816
|
-
context["errors"][e.refs] = e.to_dict()
|
1817
|
-
else:
|
1818
|
-
context["errors"] = e.to_dict(with_refs=True)
|
2054
|
+
statuses[i], _ = future.result()
|
2055
|
+
except StageError as e:
|
2056
|
+
statuses[i] = get_status_from_error(e)
|
2057
|
+
self.mark_errors(context, e)
|
1819
2058
|
except CancelledError:
|
1820
2059
|
pass
|
2060
|
+
|
2061
|
+
status: Status = validate_statuses(statuses)
|
2062
|
+
|
2063
|
+
# NOTE: Prepare status because it does not cancel from parent event but
|
2064
|
+
# cancel from failed item execution.
|
2065
|
+
if fail_fast and status == CANCEL:
|
2066
|
+
status = FAILED
|
2067
|
+
|
1821
2068
|
return result.catch(status=status, context=context)
|
1822
2069
|
|
1823
2070
|
|
@@ -1876,8 +2123,9 @@ class UntilStage(BaseNestedStage):
|
|
1876
2123
|
params: DictData,
|
1877
2124
|
result: Result,
|
1878
2125
|
event: Optional[Event] = None,
|
1879
|
-
) -> tuple[Result, T]:
|
1880
|
-
"""Execute all stage
|
2126
|
+
) -> tuple[Status, Result, T]:
|
2127
|
+
"""Execute loop that will execute all nested-stage that was set in this
|
2128
|
+
stage with specific loop and item.
|
1881
2129
|
|
1882
2130
|
:param item: (T) An item that want to execution.
|
1883
2131
|
:param loop: (int) A number of loop.
|
@@ -1886,98 +2134,115 @@ class UntilStage(BaseNestedStage):
|
|
1886
2134
|
:param event: (Event) An Event manager instance that use to cancel this
|
1887
2135
|
execution if it forces stopped by parent execution.
|
1888
2136
|
|
1889
|
-
:rtype: tuple[Result, T]
|
2137
|
+
:rtype: tuple[Status, Result, T]
|
1890
2138
|
:return: Return a pair of Result and changed item.
|
1891
2139
|
"""
|
1892
|
-
result.trace.debug(f"[STAGE]:
|
2140
|
+
result.trace.debug(f"[STAGE]: Execute Loop: {loop} (Item {item!r})")
|
2141
|
+
|
2142
|
+
# NOTE: Create nested-context
|
1893
2143
|
context: DictData = copy.deepcopy(params)
|
1894
|
-
context.update({"item": item})
|
1895
|
-
|
1896
|
-
|
1897
|
-
|
2144
|
+
context.update({"item": item, "loop": loop})
|
2145
|
+
nestet_context: DictData = {"loop": loop, "item": item, "stages": {}}
|
2146
|
+
|
2147
|
+
next_item: Optional[T] = None
|
2148
|
+
total_stage: int = len(self.stages)
|
2149
|
+
skips: list[bool] = [False] * total_stage
|
2150
|
+
for i, stage in enumerate(self.stages, start=0):
|
1898
2151
|
|
1899
2152
|
if self.extras:
|
1900
2153
|
stage.extras = self.extras
|
1901
2154
|
|
1902
|
-
if stage.is_skipped(params=context):
|
1903
|
-
result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
|
1904
|
-
stage.set_outputs(output={"skipped": True}, to=output)
|
1905
|
-
continue
|
1906
|
-
|
1907
2155
|
if event and event.is_set():
|
1908
2156
|
error_msg: str = (
|
1909
|
-
"Loop
|
1910
|
-
"
|
2157
|
+
"Loop execution was canceled from the event before start "
|
2158
|
+
"loop execution."
|
1911
2159
|
)
|
1912
|
-
|
1913
|
-
|
1914
|
-
|
1915
|
-
|
1916
|
-
|
1917
|
-
|
1918
|
-
|
1919
|
-
|
1920
|
-
"
|
1921
|
-
|
1922
|
-
|
1923
|
-
|
1924
|
-
|
2160
|
+
result.catch(
|
2161
|
+
status=CANCEL,
|
2162
|
+
until={
|
2163
|
+
loop: {
|
2164
|
+
"status": CANCEL,
|
2165
|
+
"loop": loop,
|
2166
|
+
"item": item,
|
2167
|
+
"stages": filter_func(
|
2168
|
+
nestet_context.pop("stages", {})
|
2169
|
+
),
|
2170
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
2171
|
+
}
|
2172
|
+
},
|
1925
2173
|
)
|
2174
|
+
raise StageCancelError(error_msg, refs=loop)
|
1926
2175
|
|
1927
|
-
|
1928
|
-
|
1929
|
-
|
1930
|
-
|
1931
|
-
|
1932
|
-
|
1933
|
-
|
1934
|
-
|
1935
|
-
|
2176
|
+
rs: Result = stage.handler_execute(
|
2177
|
+
params=context,
|
2178
|
+
run_id=result.run_id,
|
2179
|
+
parent_run_id=result.parent_run_id,
|
2180
|
+
event=event,
|
2181
|
+
)
|
2182
|
+
stage.set_outputs(rs.context, to=nestet_context)
|
2183
|
+
|
2184
|
+
if "item" in (_output := stage.get_outputs(nestet_context)):
|
2185
|
+
next_item = _output["item"]
|
1936
2186
|
|
1937
|
-
|
1938
|
-
next_item = _output["item"]
|
2187
|
+
stage.set_outputs(_output, to=context)
|
1939
2188
|
|
1940
|
-
|
1941
|
-
|
2189
|
+
if rs.status == SKIP:
|
2190
|
+
skips[i] = True
|
2191
|
+
continue
|
2192
|
+
|
2193
|
+
elif rs.status == FAILED:
|
2194
|
+
error_msg: str = (
|
2195
|
+
f"Loop execution was break because its nested-stage, "
|
2196
|
+
f"{stage.iden!r}, failed."
|
2197
|
+
)
|
1942
2198
|
result.catch(
|
1943
2199
|
status=FAILED,
|
1944
2200
|
until={
|
1945
2201
|
loop: {
|
2202
|
+
"status": FAILED,
|
1946
2203
|
"loop": loop,
|
1947
2204
|
"item": item,
|
1948
|
-
"stages": filter_func(
|
1949
|
-
|
2205
|
+
"stages": filter_func(
|
2206
|
+
nestet_context.pop("stages", {})
|
2207
|
+
),
|
2208
|
+
"errors": StageError(error_msg).to_dict(),
|
1950
2209
|
}
|
1951
2210
|
},
|
1952
2211
|
)
|
1953
|
-
raise
|
2212
|
+
raise StageError(error_msg, refs=loop)
|
1954
2213
|
|
1955
|
-
|
2214
|
+
elif rs.status == CANCEL:
|
1956
2215
|
error_msg: str = (
|
1957
|
-
|
1958
|
-
|
2216
|
+
"Loop execution was canceled from the event after "
|
2217
|
+
"end loop execution."
|
1959
2218
|
)
|
1960
2219
|
result.catch(
|
1961
|
-
status=
|
2220
|
+
status=CANCEL,
|
1962
2221
|
until={
|
1963
2222
|
loop: {
|
2223
|
+
"status": CANCEL,
|
1964
2224
|
"loop": loop,
|
1965
2225
|
"item": item,
|
1966
|
-
"stages": filter_func(
|
1967
|
-
|
2226
|
+
"stages": filter_func(
|
2227
|
+
nestet_context.pop("stages", {})
|
2228
|
+
),
|
2229
|
+
"errors": StageCancelError(error_msg).to_dict(),
|
1968
2230
|
}
|
1969
2231
|
},
|
1970
2232
|
)
|
1971
|
-
raise
|
2233
|
+
raise StageCancelError(error_msg, refs=loop)
|
1972
2234
|
|
2235
|
+
status: Status = SKIP if sum(skips) == total_stage else SUCCESS
|
1973
2236
|
return (
|
2237
|
+
status,
|
1974
2238
|
result.catch(
|
1975
|
-
status=
|
2239
|
+
status=status,
|
1976
2240
|
until={
|
1977
2241
|
loop: {
|
2242
|
+
"status": status,
|
1978
2243
|
"loop": loop,
|
1979
2244
|
"item": item,
|
1980
|
-
"stages": filter_func(
|
2245
|
+
"stages": filter_func(nestet_context.pop("stages", {})),
|
1981
2246
|
}
|
1982
2247
|
},
|
1983
2248
|
),
|
@@ -1991,12 +2256,14 @@ class UntilStage(BaseNestedStage):
|
|
1991
2256
|
result: Optional[Result] = None,
|
1992
2257
|
event: Optional[Event] = None,
|
1993
2258
|
) -> Result:
|
1994
|
-
"""Execute until loop with checking until condition
|
2259
|
+
"""Execute until loop with checking the until condition before release
|
2260
|
+
the next loop.
|
1995
2261
|
|
1996
2262
|
:param params: (DictData) A parameter data.
|
1997
2263
|
:param result: (Result) A Result instance for return context and status.
|
1998
2264
|
:param event: (Event) An Event manager instance that use to cancel this
|
1999
2265
|
execution if it forces stopped by parent execution.
|
2266
|
+
(Default is None)
|
2000
2267
|
|
2001
2268
|
:rtype: Result
|
2002
2269
|
"""
|
@@ -2004,29 +2271,24 @@ class UntilStage(BaseNestedStage):
|
|
2004
2271
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
2005
2272
|
extras=self.extras,
|
2006
2273
|
)
|
2007
|
-
|
2008
|
-
result.trace.info(f"[STAGE]:
|
2009
|
-
item: Union[str, int, bool] =
|
2010
|
-
self.item, params, extras=self.extras
|
2274
|
+
event: Event = event or Event()
|
2275
|
+
result.trace.info(f"[STAGE]: Until: {self.until!r}")
|
2276
|
+
item: Union[str, int, bool] = pass_env(
|
2277
|
+
param2template(self.item, params, extras=self.extras)
|
2011
2278
|
)
|
2012
2279
|
loop: int = 1
|
2013
|
-
|
2280
|
+
until_rs: bool = True
|
2014
2281
|
exceed_loop: bool = False
|
2015
2282
|
result.catch(status=WAIT, context={"until": {}})
|
2016
|
-
|
2283
|
+
statuses: list[Status] = []
|
2284
|
+
while until_rs and not (exceed_loop := (loop > self.max_loop)):
|
2017
2285
|
|
2018
2286
|
if event and event.is_set():
|
2019
|
-
|
2020
|
-
|
2021
|
-
context={
|
2022
|
-
"errors": StageException(
|
2023
|
-
"Stage was canceled from event that had set "
|
2024
|
-
"before stage loop execution."
|
2025
|
-
).to_dict()
|
2026
|
-
},
|
2287
|
+
raise StageCancelError(
|
2288
|
+
"Execution was canceled from the event before start loop."
|
2027
2289
|
)
|
2028
2290
|
|
2029
|
-
result, item = self.execute_loop(
|
2291
|
+
status, result, item = self.execute_loop(
|
2030
2292
|
item=item,
|
2031
2293
|
loop=loop,
|
2032
2294
|
params=params,
|
@@ -2036,34 +2298,39 @@ class UntilStage(BaseNestedStage):
|
|
2036
2298
|
|
2037
2299
|
loop += 1
|
2038
2300
|
if item is None:
|
2301
|
+
item: int = loop
|
2039
2302
|
result.trace.warning(
|
2040
|
-
f"[STAGE]:
|
2041
|
-
f"default."
|
2303
|
+
f"[STAGE]: Return loop not set the item. It uses loop: "
|
2304
|
+
f"{loop} by default."
|
2042
2305
|
)
|
2043
|
-
item: int = loop
|
2044
2306
|
|
2045
2307
|
next_track: bool = eval(
|
2046
|
-
|
2047
|
-
|
2048
|
-
|
2049
|
-
|
2308
|
+
pass_env(
|
2309
|
+
param2template(
|
2310
|
+
self.until,
|
2311
|
+
params | {"item": item, "loop": loop},
|
2312
|
+
extras=self.extras,
|
2313
|
+
),
|
2050
2314
|
),
|
2051
2315
|
globals() | params | {"item": item},
|
2052
2316
|
{},
|
2053
2317
|
)
|
2054
2318
|
if not isinstance(next_track, bool):
|
2055
|
-
raise
|
2319
|
+
raise TypeError(
|
2056
2320
|
"Return type of until condition not be `boolean`, getting"
|
2057
2321
|
f": {next_track!r}"
|
2058
2322
|
)
|
2059
|
-
|
2060
|
-
|
2323
|
+
until_rs: bool = not next_track
|
2324
|
+
statuses.append(status)
|
2325
|
+
delay(0.005)
|
2061
2326
|
|
2062
2327
|
if exceed_loop:
|
2063
|
-
|
2064
|
-
f"
|
2328
|
+
error_msg: str = (
|
2329
|
+
f"Loop was exceed the maximum {self.max_loop} "
|
2330
|
+
f"loop{'s' if self.max_loop > 1 else ''}."
|
2065
2331
|
)
|
2066
|
-
|
2332
|
+
raise StageError(error_msg)
|
2333
|
+
return result.catch(status=validate_statuses(statuses))
|
2067
2334
|
|
2068
2335
|
|
2069
2336
|
class Match(BaseModel):
|
@@ -2147,11 +2414,6 @@ class CaseStage(BaseNestedStage):
|
|
2147
2414
|
if self.extras:
|
2148
2415
|
stage.extras = self.extras
|
2149
2416
|
|
2150
|
-
if stage.is_skipped(params=context):
|
2151
|
-
result.trace.info(f"[STAGE]: ... Skip stage: {stage.iden!r}")
|
2152
|
-
stage.set_outputs(output={"skipped": True}, to=output)
|
2153
|
-
continue
|
2154
|
-
|
2155
2417
|
if event and event.is_set():
|
2156
2418
|
error_msg: str = (
|
2157
2419
|
"Case-Stage was canceled from event that had set before "
|
@@ -2162,29 +2424,18 @@ class CaseStage(BaseNestedStage):
|
|
2162
2424
|
context={
|
2163
2425
|
"case": case,
|
2164
2426
|
"stages": filter_func(output.pop("stages", {})),
|
2165
|
-
"errors":
|
2427
|
+
"errors": StageError(error_msg).to_dict(),
|
2166
2428
|
},
|
2167
2429
|
)
|
2168
2430
|
|
2169
|
-
|
2170
|
-
|
2171
|
-
|
2172
|
-
|
2173
|
-
|
2174
|
-
|
2175
|
-
|
2176
|
-
|
2177
|
-
stage.set_outputs(rs.context, to=output)
|
2178
|
-
stage.set_outputs(stage.get_outputs(output), to=context)
|
2179
|
-
except StageException as e:
|
2180
|
-
return result.catch(
|
2181
|
-
status=FAILED,
|
2182
|
-
context={
|
2183
|
-
"case": case,
|
2184
|
-
"stages": filter_func(output.pop("stages", {})),
|
2185
|
-
"errors": e.to_dict(),
|
2186
|
-
},
|
2187
|
-
)
|
2431
|
+
rs: Result = stage.handler_execute(
|
2432
|
+
params=context,
|
2433
|
+
run_id=result.run_id,
|
2434
|
+
parent_run_id=result.parent_run_id,
|
2435
|
+
event=event,
|
2436
|
+
)
|
2437
|
+
stage.set_outputs(rs.context, to=output)
|
2438
|
+
stage.set_outputs(stage.get_outputs(output), to=context)
|
2188
2439
|
|
2189
2440
|
if rs.status == FAILED:
|
2190
2441
|
error_msg: str = (
|
@@ -2196,7 +2447,7 @@ class CaseStage(BaseNestedStage):
|
|
2196
2447
|
context={
|
2197
2448
|
"case": case,
|
2198
2449
|
"stages": filter_func(output.pop("stages", {})),
|
2199
|
-
"errors":
|
2450
|
+
"errors": StageError(error_msg).to_dict(),
|
2200
2451
|
},
|
2201
2452
|
)
|
2202
2453
|
return result.catch(
|
@@ -2230,7 +2481,7 @@ class CaseStage(BaseNestedStage):
|
|
2230
2481
|
|
2231
2482
|
_case: StrOrNone = param2template(self.case, params, extras=self.extras)
|
2232
2483
|
|
2233
|
-
result.trace.info(f"[STAGE]:
|
2484
|
+
result.trace.info(f"[STAGE]: Case: {_case!r}.")
|
2234
2485
|
_else: Optional[Match] = None
|
2235
2486
|
stages: Optional[list[Stage]] = None
|
2236
2487
|
for match in self.match:
|
@@ -2239,39 +2490,28 @@ class CaseStage(BaseNestedStage):
|
|
2239
2490
|
continue
|
2240
2491
|
|
2241
2492
|
_condition: str = param2template(c, params, extras=self.extras)
|
2242
|
-
if stages is None and _case == _condition:
|
2493
|
+
if stages is None and pass_env(_case) == pass_env(_condition):
|
2243
2494
|
stages: list[Stage] = match.stages
|
2244
2495
|
|
2245
2496
|
if stages is None:
|
2246
2497
|
if _else is None:
|
2247
2498
|
if not self.skip_not_match:
|
2248
|
-
raise
|
2499
|
+
raise StageError(
|
2249
2500
|
"This stage does not set else for support not match "
|
2250
2501
|
"any case."
|
2251
2502
|
)
|
2252
|
-
|
2253
|
-
"
|
2254
|
-
|
2255
|
-
error_msg: str = (
|
2256
|
-
"Case-Stage was canceled because it does not match any "
|
2257
|
-
"case and else condition does not set too."
|
2258
|
-
)
|
2259
|
-
return result.catch(
|
2260
|
-
status=CANCEL,
|
2261
|
-
context={"errors": StageException(error_msg).to_dict()},
|
2503
|
+
raise StageSkipError(
|
2504
|
+
"Execution was skipped because it does not match any "
|
2505
|
+
"case and the else condition does not set too."
|
2262
2506
|
)
|
2507
|
+
|
2263
2508
|
_case: str = "_"
|
2264
2509
|
stages: list[Stage] = _else.stages
|
2265
2510
|
|
2266
2511
|
if event and event.is_set():
|
2267
|
-
|
2268
|
-
|
2269
|
-
|
2270
|
-
"errors": StageException(
|
2271
|
-
"Stage was canceled from event that had set before "
|
2272
|
-
"case-stage execution."
|
2273
|
-
).to_dict()
|
2274
|
-
},
|
2512
|
+
raise StageCancelError(
|
2513
|
+
"Execution was canceled from the event before start "
|
2514
|
+
"case execution."
|
2275
2515
|
)
|
2276
2516
|
|
2277
2517
|
return self.execute_case(
|
@@ -2280,7 +2520,7 @@ class CaseStage(BaseNestedStage):
|
|
2280
2520
|
|
2281
2521
|
|
2282
2522
|
class RaiseStage(BaseAsyncStage):
|
2283
|
-
"""Raise error stage executor that raise `
|
2523
|
+
"""Raise error stage executor that raise `StageError` that use a message
|
2284
2524
|
field for making error message before raise.
|
2285
2525
|
|
2286
2526
|
Data Validate:
|
@@ -2293,7 +2533,7 @@ class RaiseStage(BaseAsyncStage):
|
|
2293
2533
|
|
2294
2534
|
message: str = Field(
|
2295
2535
|
description=(
|
2296
|
-
"An error message that want to raise with `
|
2536
|
+
"An error message that want to raise with `StageError` class"
|
2297
2537
|
),
|
2298
2538
|
alias="raise",
|
2299
2539
|
)
|
@@ -2305,7 +2545,7 @@ class RaiseStage(BaseAsyncStage):
|
|
2305
2545
|
result: Optional[Result] = None,
|
2306
2546
|
event: Optional[Event] = None,
|
2307
2547
|
) -> Result:
|
2308
|
-
"""Raise the
|
2548
|
+
"""Raise the StageError object with the message field execution.
|
2309
2549
|
|
2310
2550
|
:param params: (DictData) A parameter data.
|
2311
2551
|
:param result: (Result) A Result instance for return context and status.
|
@@ -2317,8 +2557,8 @@ class RaiseStage(BaseAsyncStage):
|
|
2317
2557
|
extras=self.extras,
|
2318
2558
|
)
|
2319
2559
|
message: str = param2template(self.message, params, extras=self.extras)
|
2320
|
-
result.trace.info(f"[STAGE]:
|
2321
|
-
raise
|
2560
|
+
result.trace.info(f"[STAGE]: Message: ( {message} )")
|
2561
|
+
raise StageError(message)
|
2322
2562
|
|
2323
2563
|
async def axecute(
|
2324
2564
|
self,
|
@@ -2345,7 +2585,7 @@ class RaiseStage(BaseAsyncStage):
|
|
2345
2585
|
)
|
2346
2586
|
message: str = param2template(self.message, params, extras=self.extras)
|
2347
2587
|
await result.trace.ainfo(f"[STAGE]: Execute Raise-Stage: ( {message} )")
|
2348
|
-
raise
|
2588
|
+
raise StageError(message)
|
2349
2589
|
|
2350
2590
|
|
2351
2591
|
class DockerStage(BaseStage): # pragma: no cov
|
@@ -2439,7 +2679,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2439
2679
|
)
|
2440
2680
|
return result.catch(
|
2441
2681
|
status=CANCEL,
|
2442
|
-
context={"errors":
|
2682
|
+
context={"errors": StageError(error_msg).to_dict()},
|
2443
2683
|
)
|
2444
2684
|
|
2445
2685
|
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
|
@@ -2509,9 +2749,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
2509
2749
|
extras=self.extras,
|
2510
2750
|
)
|
2511
2751
|
|
2512
|
-
result.trace.info(
|
2513
|
-
f"[STAGE]: Execute Docker-Stage: {self.image}:{self.tag}"
|
2514
|
-
)
|
2752
|
+
result.trace.info(f"[STAGE]: Docker: {self.image}:{self.tag}")
|
2515
2753
|
raise NotImplementedError("Docker Stage does not implement yet.")
|
2516
2754
|
|
2517
2755
|
|
@@ -2610,8 +2848,6 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2610
2848
|
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
2611
2849
|
extras=self.extras,
|
2612
2850
|
)
|
2613
|
-
|
2614
|
-
result.trace.info(f"[STAGE]: Execute VirtualPy-Stage: {self.name}")
|
2615
2851
|
run: str = param2template(dedent(self.run), params, extras=self.extras)
|
2616
2852
|
with self.create_py_file(
|
2617
2853
|
py=run,
|
@@ -2619,19 +2855,9 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2619
2855
|
deps=param2template(self.deps, params, extras=self.extras),
|
2620
2856
|
run_id=result.run_id,
|
2621
2857
|
) as py:
|
2622
|
-
result.trace.debug(f"[STAGE]:
|
2623
|
-
try:
|
2624
|
-
import uv
|
2625
|
-
|
2626
|
-
_ = uv
|
2627
|
-
except ImportError:
|
2628
|
-
raise ImportError(
|
2629
|
-
"The VirtualPyStage need you to install `uv` before"
|
2630
|
-
"execution."
|
2631
|
-
) from None
|
2632
|
-
|
2858
|
+
result.trace.debug(f"[STAGE]: Create `{py}` file.")
|
2633
2859
|
rs: CompletedProcess = subprocess.run(
|
2634
|
-
["uv", "run", py, "--no-cache"],
|
2860
|
+
["python", "-m", "uv", "run", py, "--no-cache"],
|
2635
2861
|
# ["uv", "run", "--python", "3.9", py],
|
2636
2862
|
shell=False,
|
2637
2863
|
capture_output=True,
|
@@ -2645,7 +2871,7 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
2645
2871
|
if "\\x00" in rs.stderr
|
2646
2872
|
else rs.stderr
|
2647
2873
|
).removesuffix("\n")
|
2648
|
-
raise
|
2874
|
+
raise StageError(
|
2649
2875
|
f"Subprocess: {e}\nRunning Statement:\n---\n"
|
2650
2876
|
f"```python\n{run}\n```"
|
2651
2877
|
)
|