ddeutil-workflow 0.0.84__py3-none-any.whl → 0.0.86__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +4 -4
- ddeutil/workflow/api/routes/job.py +3 -2
- ddeutil/workflow/audits.py +8 -6
- ddeutil/workflow/conf.py +9 -20
- ddeutil/workflow/errors.py +34 -19
- ddeutil/workflow/job.py +303 -159
- ddeutil/workflow/plugins/providers/az.py +2 -2
- ddeutil/workflow/result.py +46 -55
- ddeutil/workflow/stages.py +540 -458
- ddeutil/workflow/traces.py +259 -261
- ddeutil/workflow/workflow.py +304 -361
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.86.dist-info}/METADATA +13 -16
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.86.dist-info}/RECORD +18 -18
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.86.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.86.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.86.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.84.dist-info → ddeutil_workflow-0.0.86.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -122,6 +122,7 @@ from .utils import (
|
|
122
122
|
extract_id,
|
123
123
|
filter_func,
|
124
124
|
gen_id,
|
125
|
+
get_dt_now,
|
125
126
|
make_exec,
|
126
127
|
to_train,
|
127
128
|
)
|
@@ -133,17 +134,17 @@ DictOrModel = Union[DictData, BaseModel]
|
|
133
134
|
class BaseStage(BaseModel, ABC):
|
134
135
|
"""Abstract base class for all stage implementations.
|
135
136
|
|
136
|
-
|
137
|
-
It defines the common interface and metadata fields that all stages
|
138
|
-
implement, ensuring consistent behavior across different stage types.
|
137
|
+
BaseStage provides the foundation for all stage types in the workflow
|
138
|
+
system. It defines the common interface and metadata fields that all stages
|
139
|
+
must implement, ensuring consistent behavior across different stage types.
|
139
140
|
|
140
141
|
This abstract class handles core stage functionality including:
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
142
|
+
- Stage identification and naming
|
143
|
+
- Conditional execution logic
|
144
|
+
- Output management and templating
|
145
|
+
- Execution lifecycle management
|
145
146
|
|
146
|
-
|
147
|
+
Custom stages should inherit from this class and implement the abstract
|
147
148
|
`process()` method to define their specific execution behavior.
|
148
149
|
|
149
150
|
Attributes:
|
@@ -162,7 +163,6 @@ class BaseStage(BaseModel, ABC):
|
|
162
163
|
...
|
163
164
|
... def process(self, params: DictData, **kwargs) -> Result:
|
164
165
|
... return Result(status=SUCCESS)
|
165
|
-
```
|
166
166
|
"""
|
167
167
|
|
168
168
|
action_stage: ClassVar[bool] = False
|
@@ -190,7 +190,7 @@ class BaseStage(BaseModel, ABC):
|
|
190
190
|
default=None,
|
191
191
|
description=(
|
192
192
|
"A stage condition statement to allow stage executable. This field "
|
193
|
-
"
|
193
|
+
"alias with `if` key."
|
194
194
|
),
|
195
195
|
alias="if",
|
196
196
|
)
|
@@ -240,7 +240,8 @@ class BaseStage(BaseModel, ABC):
|
|
240
240
|
|
241
241
|
Args:
|
242
242
|
value (Any): An any value.
|
243
|
-
params (DictData):
|
243
|
+
params (DictData): A parameter data that want to use in this
|
244
|
+
execution.
|
244
245
|
|
245
246
|
Returns:
|
246
247
|
Any: A templated value.
|
@@ -260,14 +261,22 @@ class BaseStage(BaseModel, ABC):
|
|
260
261
|
"""Process abstraction method that action something by sub-model class.
|
261
262
|
This is important method that make this class is able to be the stage.
|
262
263
|
|
264
|
+
For process method, it designs to break process with any status by
|
265
|
+
raise it with a specific exception class.
|
266
|
+
|
267
|
+
- StageError -> FAILED
|
268
|
+
- StageSkipError -> SKIP
|
269
|
+
- StageCancelError -> CANCEL
|
270
|
+
|
263
271
|
Args:
|
264
272
|
params (DictData): A parameter data that want to use in this
|
265
273
|
execution.
|
266
274
|
run_id (str): A running stage ID.
|
267
|
-
context (DictData): A context data
|
268
|
-
|
269
|
-
|
270
|
-
|
275
|
+
context (DictData): A context data that was passed from handler
|
276
|
+
method.
|
277
|
+
parent_run_id (str, default None): A parent running ID.
|
278
|
+
event (Event, default None): An event manager that use to track
|
279
|
+
parent process was not force stopped.
|
271
280
|
|
272
281
|
Returns:
|
273
282
|
Result: The execution result with status and context data.
|
@@ -308,10 +317,10 @@ class BaseStage(BaseModel, ABC):
|
|
308
317
|
object from the current stage ID before release the final result.
|
309
318
|
|
310
319
|
Args:
|
311
|
-
params: A parameter data.
|
312
|
-
run_id: A running
|
313
|
-
event: An event manager that pass to the stage
|
314
|
-
|
320
|
+
params (DictData): A parameter data.
|
321
|
+
run_id (str, default None): A running ID.
|
322
|
+
event (Event, default None): An event manager that pass to the stage
|
323
|
+
execution.
|
315
324
|
|
316
325
|
Returns:
|
317
326
|
Result: The execution result with updated status and context.
|
@@ -320,13 +329,16 @@ class BaseStage(BaseModel, ABC):
|
|
320
329
|
parent_run_id, run_id = extract_id(
|
321
330
|
self.iden, run_id=run_id, extras=self.extras
|
322
331
|
)
|
323
|
-
context: DictData = {
|
332
|
+
context: DictData = {
|
333
|
+
"status": WAIT,
|
334
|
+
"info": {"exec_start": get_dt_now()},
|
335
|
+
}
|
324
336
|
trace: Trace = get_trace(
|
325
337
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
326
338
|
)
|
327
339
|
try:
|
328
340
|
_id: str = (
|
329
|
-
f" with ID: {
|
341
|
+
f" with ID: {self.pass_template(self.id, params=params)!r}"
|
330
342
|
if self.id
|
331
343
|
else ""
|
332
344
|
)
|
@@ -347,82 +359,73 @@ class BaseStage(BaseModel, ABC):
|
|
347
359
|
|
348
360
|
# NOTE: Start call wrapped execution method that will use custom
|
349
361
|
# execution before the real execution from inherit stage model.
|
350
|
-
|
362
|
+
result: Result = self._execute(
|
351
363
|
params,
|
352
|
-
run_id=run_id,
|
353
364
|
context=context,
|
354
|
-
|
365
|
+
trace=trace,
|
355
366
|
event=event,
|
356
367
|
)
|
357
|
-
if
|
368
|
+
if result.status == WAIT: # pragma: no cov
|
358
369
|
raise StageError(
|
359
370
|
"Status from execution should not return waiting status."
|
360
371
|
)
|
361
|
-
return
|
362
|
-
{"execution_time": time.monotonic() - ts}
|
363
|
-
)
|
372
|
+
return result
|
364
373
|
|
365
374
|
# NOTE: Catch this error in this line because the execution can raise
|
366
375
|
# this exception class at other location.
|
367
|
-
except
|
368
|
-
|
369
|
-
StageNestedSkipError,
|
370
|
-
StageNestedError,
|
371
|
-
StageError,
|
372
|
-
) as e: # pragma: no cov
|
376
|
+
except StageError as e: # pragma: no cov
|
377
|
+
updated: Optional[DictData] = {"errors": e.to_dict()}
|
373
378
|
if isinstance(e, StageNestedError):
|
374
|
-
trace.
|
379
|
+
trace.error(f"[STAGE]: ⚠️ Nested: {e}")
|
375
380
|
elif isinstance(e, (StageSkipError, StageNestedSkipError)):
|
376
|
-
trace.
|
381
|
+
trace.error(f"[STAGE]: ⏭️ Skip: {e}", module="stage")
|
382
|
+
updated = None
|
383
|
+
elif e.allow_traceback:
|
384
|
+
trace.error(
|
385
|
+
f"[STAGE]: 📢 Stage Failed:||🚨 {traceback.format_exc()}||"
|
386
|
+
)
|
377
387
|
else:
|
378
|
-
trace.
|
379
|
-
f"[STAGE]: Stage Failed
|
388
|
+
trace.error(
|
389
|
+
f"[STAGE]: 🤫 Stage Failed with disable traceback:||{e}"
|
380
390
|
)
|
381
391
|
st: Status = get_status_from_error(e)
|
382
|
-
return Result(
|
383
|
-
run_id=run_id,
|
384
|
-
parent_run_id=parent_run_id,
|
392
|
+
return Result.from_trace(trace).catch(
|
385
393
|
status=st,
|
386
|
-
context=catch(
|
387
|
-
context,
|
388
|
-
status=st,
|
389
|
-
updated=(
|
390
|
-
None
|
391
|
-
if isinstance(e, (StageSkipError, StageNestedSkipError))
|
392
|
-
else {"errors": e.to_dict()}
|
393
|
-
),
|
394
|
-
),
|
395
|
-
info={"execution_time": time.monotonic() - ts},
|
396
|
-
extras=self.extras,
|
394
|
+
context=catch(context, status=st, updated=updated),
|
397
395
|
)
|
398
396
|
except Exception as e:
|
399
397
|
trace.error(
|
400
|
-
f"
|
398
|
+
f"💥 Error Failed:||🚨 {traceback.format_exc()}||",
|
399
|
+
module="stage",
|
401
400
|
)
|
402
|
-
return Result(
|
403
|
-
run_id=run_id,
|
404
|
-
parent_run_id=parent_run_id,
|
401
|
+
return Result.from_trace(trace).catch(
|
405
402
|
status=FAILED,
|
406
403
|
context=catch(
|
407
404
|
context, status=FAILED, updated={"errors": to_dict(e)}
|
408
405
|
),
|
409
|
-
info={"execution_time": time.monotonic() - ts},
|
410
|
-
extras=self.extras,
|
411
406
|
)
|
407
|
+
finally:
|
408
|
+
context["info"].update(
|
409
|
+
{
|
410
|
+
"exec_end": get_dt_now(),
|
411
|
+
"exec_latency": round(time.monotonic() - ts, 6),
|
412
|
+
}
|
413
|
+
)
|
414
|
+
trace.debug("End Handler stage execution.", module="stage")
|
412
415
|
|
413
416
|
def _execute(
|
414
417
|
self,
|
415
418
|
params: DictData,
|
416
|
-
run_id: str,
|
417
419
|
context: DictData,
|
418
|
-
|
420
|
+
trace: Trace,
|
419
421
|
event: Optional[Event] = None,
|
420
422
|
) -> Result:
|
421
423
|
"""Wrapped the process method before returning to handler execution.
|
422
424
|
|
423
425
|
Args:
|
424
|
-
params: A parameter data that want to use in this
|
425
|
-
|
426
|
+
params: A parameter data that want to use in this execution.
|
427
|
+
context:
|
428
|
+
trace (Trace):
|
426
429
|
event: An event manager that use to track parent process
|
427
430
|
was not force stopped.
|
428
431
|
|
@@ -432,9 +435,9 @@ class BaseStage(BaseModel, ABC):
|
|
432
435
|
catch(context, status=WAIT)
|
433
436
|
return self.process(
|
434
437
|
params,
|
435
|
-
run_id=run_id,
|
438
|
+
run_id=trace.run_id,
|
436
439
|
context=context,
|
437
|
-
parent_run_id=parent_run_id,
|
440
|
+
parent_run_id=trace.parent_run_id,
|
438
441
|
event=event,
|
439
442
|
)
|
440
443
|
|
@@ -451,7 +454,7 @@ class BaseStage(BaseModel, ABC):
|
|
451
454
|
For example of setting output method, If you receive process output
|
452
455
|
and want to set on the `to` like;
|
453
456
|
|
454
|
-
... (i) output: {'foo': 'bar', '
|
457
|
+
... (i) output: {'foo': 'bar', 'status': SUCCESS, 'info': {}}
|
455
458
|
... (ii) to: {'stages': {}}
|
456
459
|
|
457
460
|
The received context in the `to` argument will be;
|
@@ -460,7 +463,8 @@ class BaseStage(BaseModel, ABC):
|
|
460
463
|
'stages': {
|
461
464
|
'<stage-id>': {
|
462
465
|
'outputs': {'foo': 'bar'},
|
463
|
-
'
|
466
|
+
'status': SUCCESS,
|
467
|
+
'info': {},
|
464
468
|
}
|
465
469
|
}
|
466
470
|
}
|
@@ -501,8 +505,13 @@ class BaseStage(BaseModel, ABC):
|
|
501
505
|
status: dict[str, Status] = (
|
502
506
|
{"status": output.pop("status")} if "status" in output else {}
|
503
507
|
)
|
508
|
+
info: DictData = (
|
509
|
+
{"info": output.pop("info")} if "info" in output else {}
|
510
|
+
)
|
504
511
|
kwargs: DictData = kwargs or {}
|
505
|
-
to["stages"][_id] =
|
512
|
+
to["stages"][_id] = (
|
513
|
+
{"outputs": output} | errors | status | info | kwargs
|
514
|
+
)
|
506
515
|
return to
|
507
516
|
|
508
517
|
def get_outputs(self, output: DictData) -> DictData:
|
@@ -530,15 +539,18 @@ class BaseStage(BaseModel, ABC):
|
|
530
539
|
"""Return true if condition of this stage do not correct. This process
|
531
540
|
use build-in eval function to execute the if-condition.
|
532
541
|
|
533
|
-
:
|
534
|
-
|
542
|
+
Args:
|
543
|
+
params (DictData): A parameters that want to pass to condition
|
544
|
+
template.
|
535
545
|
|
536
|
-
:
|
537
|
-
|
538
|
-
|
539
|
-
|
546
|
+
Raises:
|
547
|
+
StageError: When it has any error raise from the eval
|
548
|
+
condition statement.
|
549
|
+
StageError: When return type of the eval condition statement
|
550
|
+
does not return with boolean type.
|
540
551
|
|
541
|
-
:
|
552
|
+
Returns:
|
553
|
+
bool: True if the condition is valid with the current parameters.
|
542
554
|
"""
|
543
555
|
# NOTE: Support for condition value is empty string.
|
544
556
|
if not self.condition:
|
@@ -566,9 +578,11 @@ class BaseStage(BaseModel, ABC):
|
|
566
578
|
"""Generate stage ID that dynamic use stage's name if it ID does not
|
567
579
|
set.
|
568
580
|
|
569
|
-
:
|
581
|
+
Args:
|
582
|
+
params (DictData): A parameter or context data.
|
570
583
|
|
571
|
-
:
|
584
|
+
Returns:
|
585
|
+
str: An ID that already generated from id or name fields.
|
572
586
|
"""
|
573
587
|
return (
|
574
588
|
param2template(self.id, params=params, extras=self.extras)
|
@@ -643,7 +657,7 @@ class BaseStage(BaseModel, ABC):
|
|
643
657
|
*,
|
644
658
|
parent_run_id: Optional[str] = None,
|
645
659
|
event: Optional[Event] = None,
|
646
|
-
) ->
|
660
|
+
) -> Result:
|
647
661
|
"""Pre-process method that will use to run with dry-run mode, and it
|
648
662
|
should be used replace of process method when workflow release set with
|
649
663
|
DRYRUN mode.
|
@@ -689,6 +703,8 @@ class BaseStage(BaseModel, ABC):
|
|
689
703
|
"""Convert the current Stage model to the EmptyStage model for dry-run
|
690
704
|
mode if the `action_stage` class attribute has set.
|
691
705
|
|
706
|
+
Some use-case for this method is use for deactivate.
|
707
|
+
|
692
708
|
Args:
|
693
709
|
sleep (int, default 0.35): An adjustment sleep time.
|
694
710
|
message (str, default None): A message that want to override default
|
@@ -774,15 +790,19 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
774
790
|
Result: The execution result with status and context data.
|
775
791
|
"""
|
776
792
|
ts: float = time.monotonic()
|
777
|
-
parent_run_id
|
778
|
-
|
779
|
-
|
793
|
+
parent_run_id, run_id = extract_id(
|
794
|
+
self.iden, run_id=run_id, extras=self.extras
|
795
|
+
)
|
796
|
+
context: DictData = {
|
797
|
+
"status": WAIT,
|
798
|
+
"info": {"exec_start": get_dt_now()},
|
799
|
+
}
|
780
800
|
trace: Trace = get_trace(
|
781
801
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
782
802
|
)
|
783
803
|
try:
|
784
804
|
_id: str = (
|
785
|
-
f" with ID: {
|
805
|
+
f" with ID: {self.pass_template(self.id, params=params)!r}"
|
786
806
|
if self.id
|
787
807
|
else ""
|
788
808
|
)
|
@@ -803,66 +823,60 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
803
823
|
|
804
824
|
# NOTE: Start call wrapped execution method that will use custom
|
805
825
|
# execution before the real execution from inherit stage model.
|
806
|
-
|
826
|
+
result: Result = await self._axecute(
|
807
827
|
params,
|
808
828
|
run_id=run_id,
|
809
829
|
context=context,
|
810
830
|
parent_run_id=parent_run_id,
|
811
831
|
event=event,
|
812
832
|
)
|
813
|
-
if
|
833
|
+
if result.status == WAIT: # pragma: no cov
|
814
834
|
raise StageError(
|
815
835
|
"Status from execution should not return waiting status."
|
816
836
|
)
|
817
|
-
return
|
837
|
+
return result
|
818
838
|
|
819
839
|
# NOTE: Catch this error in this line because the execution can raise
|
820
840
|
# this exception class at other location.
|
821
|
-
except
|
822
|
-
|
823
|
-
StageNestedSkipError,
|
824
|
-
StageNestedError,
|
825
|
-
StageError,
|
826
|
-
) as e: # pragma: no cov
|
841
|
+
except StageError as e: # pragma: no cov
|
842
|
+
updated: Optional[DictData] = {"errors": e.to_dict()}
|
827
843
|
if isinstance(e, StageNestedError):
|
828
|
-
await trace.
|
844
|
+
await trace.aerror(f"[STAGE]: ⚠️ Nested: {e}")
|
829
845
|
elif isinstance(e, (StageSkipError, StageNestedSkipError)):
|
830
|
-
await trace.
|
846
|
+
await trace.aerror(f"[STAGE]: ⏭️ Skip: {e}")
|
847
|
+
updated = None
|
848
|
+
elif e.allow_traceback:
|
849
|
+
await trace.aerror(
|
850
|
+
f"[STAGE]: 📢 Stage Failed:||🚨 {traceback.format_exc()}||"
|
851
|
+
)
|
831
852
|
else:
|
832
|
-
await trace.
|
833
|
-
f"[STAGE]: Stage Failed
|
853
|
+
await trace.aerror(
|
854
|
+
f"[STAGE]: 🤫 Stage Failed with disable traceback:||{e}"
|
834
855
|
)
|
835
856
|
st: Status = get_status_from_error(e)
|
836
|
-
return Result(
|
837
|
-
run_id=run_id,
|
838
|
-
parent_run_id=parent_run_id,
|
857
|
+
return Result.from_trace(trace).catch(
|
839
858
|
status=st,
|
840
|
-
context=catch(
|
841
|
-
context,
|
842
|
-
status=st,
|
843
|
-
updated=(
|
844
|
-
None
|
845
|
-
if isinstance(e, (StageSkipError, StageNestedSkipError))
|
846
|
-
else {"errors": e.to_dict()}
|
847
|
-
),
|
848
|
-
),
|
849
|
-
info={"execution_time": time.monotonic() - ts},
|
850
|
-
extras=self.extras,
|
859
|
+
context=catch(context, status=st, updated=updated),
|
851
860
|
)
|
852
861
|
except Exception as e:
|
853
862
|
await trace.aerror(
|
854
|
-
f"
|
863
|
+
f"💥 Error Failed:||🚨 {traceback.format_exc()}||",
|
864
|
+
module="stage",
|
855
865
|
)
|
856
|
-
return Result(
|
857
|
-
run_id=run_id,
|
858
|
-
parent_run_id=parent_run_id,
|
866
|
+
return Result.from_trace(trace).catch(
|
859
867
|
status=FAILED,
|
860
868
|
context=catch(
|
861
869
|
context, status=FAILED, updated={"errors": to_dict(e)}
|
862
870
|
),
|
863
|
-
info={"execution_time": time.monotonic() - ts},
|
864
|
-
extras=self.extras,
|
865
871
|
)
|
872
|
+
finally:
|
873
|
+
context["info"].update(
|
874
|
+
{
|
875
|
+
"exec_end": get_dt_now(),
|
876
|
+
"exec_latency": time.monotonic() - ts,
|
877
|
+
}
|
878
|
+
)
|
879
|
+
trace.debug("[STAGE]: End Handler stage process.")
|
866
880
|
|
867
881
|
async def _axecute(
|
868
882
|
self,
|
@@ -898,6 +912,7 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
898
912
|
`StageRetryError`.
|
899
913
|
"""
|
900
914
|
|
915
|
+
action_stage: ClassVar[bool] = True
|
901
916
|
retry: int = Field(
|
902
917
|
default=0,
|
903
918
|
ge=0,
|
@@ -911,9 +926,8 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
911
926
|
def _execute(
|
912
927
|
self,
|
913
928
|
params: DictData,
|
914
|
-
run_id: str,
|
915
929
|
context: DictData,
|
916
|
-
|
930
|
+
trace: Trace,
|
917
931
|
event: Optional[Event] = None,
|
918
932
|
) -> Result:
|
919
933
|
"""Wrapped the execute method with retry strategy before returning to
|
@@ -931,9 +945,6 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
931
945
|
current_retry: int = 0
|
932
946
|
exception: Exception
|
933
947
|
catch(context, status=WAIT)
|
934
|
-
trace: Trace = get_trace(
|
935
|
-
run_id, parent_run_id=parent_run_id, extras=self.extras
|
936
|
-
)
|
937
948
|
# NOTE: First execution for not pass to retry step if it passes.
|
938
949
|
try:
|
939
950
|
if (
|
@@ -942,25 +953,33 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
942
953
|
):
|
943
954
|
return self.dryrun(
|
944
955
|
params | {"retry": current_retry},
|
945
|
-
run_id=run_id,
|
956
|
+
run_id=trace.run_id,
|
946
957
|
context=context,
|
947
|
-
parent_run_id=parent_run_id,
|
958
|
+
parent_run_id=trace.parent_run_id,
|
948
959
|
event=event,
|
949
960
|
)
|
950
961
|
return self.process(
|
951
962
|
params | {"retry": current_retry},
|
952
|
-
run_id=run_id,
|
963
|
+
run_id=trace.run_id,
|
953
964
|
context=context,
|
954
|
-
parent_run_id=parent_run_id,
|
965
|
+
parent_run_id=trace.parent_run_id,
|
955
966
|
event=event,
|
956
967
|
)
|
968
|
+
except (
|
969
|
+
StageNestedSkipError,
|
970
|
+
StageNestedCancelError,
|
971
|
+
StageSkipError,
|
972
|
+
StageCancelError,
|
973
|
+
):
|
974
|
+
trace.debug("[STAGE]: process raise skip or cancel error.")
|
975
|
+
raise
|
957
976
|
except Exception as e:
|
977
|
+
if self.retry == 0:
|
978
|
+
raise
|
979
|
+
|
958
980
|
current_retry += 1
|
959
981
|
exception = e
|
960
982
|
|
961
|
-
if self.retry == 0:
|
962
|
-
raise exception
|
963
|
-
|
964
983
|
trace.warning(
|
965
984
|
f"[STAGE]: Retry count: {current_retry} ... "
|
966
985
|
f"( {exception.__class__.__name__} )"
|
@@ -979,16 +998,16 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
979
998
|
):
|
980
999
|
return self.dryrun(
|
981
1000
|
params | {"retry": current_retry},
|
982
|
-
run_id=run_id,
|
1001
|
+
run_id=trace.run_id,
|
983
1002
|
context=context,
|
984
|
-
parent_run_id=parent_run_id,
|
1003
|
+
parent_run_id=trace.parent_run_id,
|
985
1004
|
event=event,
|
986
1005
|
)
|
987
1006
|
return self.process(
|
988
1007
|
params | {"retry": current_retry},
|
989
|
-
run_id=run_id,
|
1008
|
+
run_id=trace.run_id,
|
990
1009
|
context=context,
|
991
|
-
parent_run_id=parent_run_id,
|
1010
|
+
parent_run_id=trace.parent_run_id,
|
992
1011
|
event=event,
|
993
1012
|
)
|
994
1013
|
except (
|
@@ -1060,13 +1079,21 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
1060
1079
|
parent_run_id=parent_run_id,
|
1061
1080
|
event=event,
|
1062
1081
|
)
|
1082
|
+
except (
|
1083
|
+
StageNestedSkipError,
|
1084
|
+
StageNestedCancelError,
|
1085
|
+
StageSkipError,
|
1086
|
+
StageCancelError,
|
1087
|
+
):
|
1088
|
+
await trace.adebug("[STAGE]: process raise skip or cancel error.")
|
1089
|
+
raise
|
1063
1090
|
except Exception as e:
|
1091
|
+
if self.retry == 0:
|
1092
|
+
raise
|
1093
|
+
|
1064
1094
|
current_retry += 1
|
1065
1095
|
exception = e
|
1066
1096
|
|
1067
|
-
if self.retry == 0:
|
1068
|
-
raise exception
|
1069
|
-
|
1070
1097
|
await trace.awarning(
|
1071
1098
|
f"[STAGE]: Retry count: {current_retry} ... "
|
1072
1099
|
f"( {exception.__class__.__name__} )"
|
@@ -1136,24 +1163,16 @@ class EmptyStage(BaseAsyncStage):
|
|
1136
1163
|
for a specified duration, making it useful for workflow timing control
|
1137
1164
|
and debugging scenarios.
|
1138
1165
|
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
- name: "Debug Parameters"
|
1147
|
-
echo: "Processing file: ${{ params.filename }}"
|
1148
|
-
```
|
1149
|
-
|
1150
|
-
>>> stage = EmptyStage(
|
1151
|
-
... name="Status Update",
|
1152
|
-
... echo="Processing completed successfully",
|
1153
|
-
... sleep=1.0
|
1154
|
-
... )
|
1166
|
+
Examples:
|
1167
|
+
>>> stage = EmptyStage.model_validate({
|
1168
|
+
... "id": "empty-stage",
|
1169
|
+
... "name": "Status Update",
|
1170
|
+
... "echo": "Processing completed successfully",
|
1171
|
+
... "sleep": 1.0,
|
1172
|
+
... })
|
1155
1173
|
"""
|
1156
1174
|
|
1175
|
+
action_stage: ClassVar[bool] = True
|
1157
1176
|
echo: StrOrNone = Field(
|
1158
1177
|
default=None,
|
1159
1178
|
description=(
|
@@ -1187,13 +1206,17 @@ class EmptyStage(BaseAsyncStage):
|
|
1187
1206
|
without calling logging function.
|
1188
1207
|
|
1189
1208
|
Args:
|
1190
|
-
params: A parameter data that want to use in this
|
1209
|
+
params (DictData): A parameter data that want to use in this
|
1191
1210
|
execution.
|
1192
|
-
run_id: A running stage ID.
|
1193
|
-
context: A context data
|
1194
|
-
|
1195
|
-
|
1196
|
-
|
1211
|
+
run_id (str): A running stage ID.
|
1212
|
+
context (DictData): A context data that was passed from handler
|
1213
|
+
method.
|
1214
|
+
parent_run_id (str, default None): A parent running ID.
|
1215
|
+
event (Event, default None): An event manager that use to track
|
1216
|
+
parent process was not force stopped.
|
1217
|
+
|
1218
|
+
Raises:
|
1219
|
+
StageCancelError: If event was set before start process.
|
1197
1220
|
|
1198
1221
|
Returns:
|
1199
1222
|
Result: The execution result with status and context data.
|
@@ -1202,9 +1225,7 @@ class EmptyStage(BaseAsyncStage):
|
|
1202
1225
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1203
1226
|
)
|
1204
1227
|
message: str = (
|
1205
|
-
|
1206
|
-
dedent(self.echo.strip("\n")), params, extras=self.extras
|
1207
|
-
)
|
1228
|
+
self.pass_template(dedent(self.echo.strip("\n")), params=params)
|
1208
1229
|
if self.echo
|
1209
1230
|
else "..."
|
1210
1231
|
)
|
@@ -1217,12 +1238,8 @@ class EmptyStage(BaseAsyncStage):
|
|
1217
1238
|
if self.sleep > 5:
|
1218
1239
|
trace.info(f"[STAGE]: Sleep ... ({self.sleep} sec)")
|
1219
1240
|
time.sleep(self.sleep)
|
1220
|
-
return Result(
|
1221
|
-
|
1222
|
-
parent_run_id=parent_run_id,
|
1223
|
-
status=SUCCESS,
|
1224
|
-
context=catch(context=context, status=SUCCESS),
|
1225
|
-
extras=self.extras,
|
1241
|
+
return Result.from_trace(trace).catch(
|
1242
|
+
status=SUCCESS, context=catch(context=context, status=SUCCESS)
|
1226
1243
|
)
|
1227
1244
|
|
1228
1245
|
async def async_process(
|
@@ -1238,13 +1255,17 @@ class EmptyStage(BaseAsyncStage):
|
|
1238
1255
|
stdout.
|
1239
1256
|
|
1240
1257
|
Args:
|
1241
|
-
params: A parameter data that want to use in this
|
1258
|
+
params (DictData): A parameter data that want to use in this
|
1242
1259
|
execution.
|
1243
|
-
run_id: A running stage ID.
|
1244
|
-
context: A context data
|
1245
|
-
|
1246
|
-
|
1247
|
-
|
1260
|
+
run_id (str): A running stage ID.
|
1261
|
+
context (DictData): A context data that was passed from handler
|
1262
|
+
method.
|
1263
|
+
parent_run_id (str, default None): A parent running ID.
|
1264
|
+
event (Event, default None): An event manager that use to track
|
1265
|
+
parent process was not force stopped.
|
1266
|
+
|
1267
|
+
Raises:
|
1268
|
+
StageCancelError: If event was set before start process.
|
1248
1269
|
|
1249
1270
|
Returns:
|
1250
1271
|
Result: The execution result with status and context data.
|
@@ -1253,9 +1274,7 @@ class EmptyStage(BaseAsyncStage):
|
|
1253
1274
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
1254
1275
|
)
|
1255
1276
|
message: str = (
|
1256
|
-
|
1257
|
-
dedent(self.echo.strip("\n")), params, extras=self.extras
|
1258
|
-
)
|
1277
|
+
self.pass_template(dedent(self.echo.strip("\n")), params=params)
|
1259
1278
|
if self.echo
|
1260
1279
|
else "..."
|
1261
1280
|
)
|
@@ -1268,12 +1287,8 @@ class EmptyStage(BaseAsyncStage):
|
|
1268
1287
|
if self.sleep > 5:
|
1269
1288
|
await trace.ainfo(f"[STAGE]: Sleep ... ({self.sleep} sec)")
|
1270
1289
|
await asyncio.sleep(self.sleep)
|
1271
|
-
return Result(
|
1272
|
-
|
1273
|
-
parent_run_id=parent_run_id,
|
1274
|
-
status=SUCCESS,
|
1275
|
-
context=catch(context=context, status=SUCCESS),
|
1276
|
-
extras=self.extras,
|
1290
|
+
return Result.from_trace(trace).catch(
|
1291
|
+
status=SUCCESS, context=catch(context=context, status=SUCCESS)
|
1277
1292
|
)
|
1278
1293
|
|
1279
1294
|
|
@@ -1287,17 +1302,17 @@ class BashStage(BaseRetryStage):
|
|
1287
1302
|
statement. Thus, it will write the `.sh` file before start running bash
|
1288
1303
|
command for fix this issue.
|
1289
1304
|
|
1290
|
-
|
1291
|
-
>>> stage = {
|
1305
|
+
Examples:
|
1306
|
+
>>> stage = BaseStage.model_validate({
|
1307
|
+
... "id": "bash-stage",
|
1292
1308
|
... "name": "The Shell stage execution",
|
1293
1309
|
... "bash": 'echo "Hello $FOO"',
|
1294
1310
|
... "env": {
|
1295
1311
|
... "FOO": "BAR",
|
1296
1312
|
... },
|
1297
|
-
... }
|
1313
|
+
... })
|
1298
1314
|
"""
|
1299
1315
|
|
1300
|
-
action_stage: ClassVar[bool] = True
|
1301
1316
|
bash: str = Field(
|
1302
1317
|
description=(
|
1303
1318
|
"A bash statement that want to execute via Python subprocess."
|
@@ -1312,17 +1327,20 @@ class BashStage(BaseRetryStage):
|
|
1312
1327
|
)
|
1313
1328
|
|
1314
1329
|
@contextlib.asynccontextmanager
|
1315
|
-
async def
|
1330
|
+
async def async_make_sh_file(
|
1316
1331
|
self, bash: str, env: DictStr, run_id: StrOrNone = None
|
1317
1332
|
) -> AsyncIterator[TupleStr]:
|
1318
1333
|
"""Async create and write `.sh` file with the `aiofiles` package.
|
1319
1334
|
|
1320
|
-
:
|
1321
|
-
|
1322
|
-
|
1323
|
-
|
1335
|
+
Args:
|
1336
|
+
bash (str): A bash statement.
|
1337
|
+
env (DictStr): An environment variable that set before run bash.
|
1338
|
+
run_id (StrOrNone, default None): A running stage ID that use for
|
1339
|
+
writing `.sh` file instead generate by UUID4.
|
1324
1340
|
|
1325
|
-
:
|
1341
|
+
Returns:
|
1342
|
+
AsyncIterator[TupleStr]: Return context of prepared bash statement
|
1343
|
+
that want to execute.
|
1326
1344
|
"""
|
1327
1345
|
import aiofiles
|
1328
1346
|
|
@@ -1342,25 +1360,28 @@ class BashStage(BaseRetryStage):
|
|
1342
1360
|
# NOTE: Make this .sh file able to executable.
|
1343
1361
|
make_exec(f"./{f_name}")
|
1344
1362
|
|
1345
|
-
|
1346
|
-
|
1347
|
-
|
1348
|
-
|
1363
|
+
try:
|
1364
|
+
yield f_shebang, f_name
|
1365
|
+
finally:
|
1366
|
+
# Note: Remove .sh file that use to run bash.
|
1367
|
+
Path(f"./{f_name}").unlink()
|
1349
1368
|
|
1350
1369
|
@contextlib.contextmanager
|
1351
|
-
def
|
1370
|
+
def make_sh_file(
|
1352
1371
|
self, bash: str, env: DictStr, run_id: StrOrNone = None
|
1353
1372
|
) -> Iterator[TupleStr]:
|
1354
1373
|
"""Create and write the `.sh` file before giving this file name to
|
1355
1374
|
context. After that, it will auto delete this file automatic.
|
1356
1375
|
|
1357
|
-
:
|
1358
|
-
|
1359
|
-
|
1360
|
-
|
1376
|
+
Args:
|
1377
|
+
bash (str): A bash statement.
|
1378
|
+
env (DictStr): An environment variable that set before run bash.
|
1379
|
+
run_id (StrOrNone, default None): A running stage ID that use for
|
1380
|
+
writing `.sh` file instead generate by UUID4.
|
1361
1381
|
|
1362
|
-
:
|
1363
|
-
|
1382
|
+
Returns:
|
1383
|
+
Iterator[TupleStr]: Return context of prepared bash statement that
|
1384
|
+
want to execute.
|
1364
1385
|
"""
|
1365
1386
|
f_name: str = f"{run_id or uuid.uuid4()}.sh"
|
1366
1387
|
f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
|
@@ -1378,10 +1399,11 @@ class BashStage(BaseRetryStage):
|
|
1378
1399
|
# NOTE: Make this .sh file able to executable.
|
1379
1400
|
make_exec(f"./{f_name}")
|
1380
1401
|
|
1381
|
-
|
1382
|
-
|
1383
|
-
|
1384
|
-
|
1402
|
+
try:
|
1403
|
+
yield f_shebang, f_name
|
1404
|
+
finally:
|
1405
|
+
# Note: Remove .sh file that use to run bash.
|
1406
|
+
Path(f"./{f_name}").unlink()
|
1385
1407
|
|
1386
1408
|
@staticmethod
|
1387
1409
|
def prepare_std(value: str) -> Optional[str]:
|
@@ -1402,13 +1424,19 @@ class BashStage(BaseRetryStage):
|
|
1402
1424
|
`return_code`, `stdout`, and `stderr`.
|
1403
1425
|
|
1404
1426
|
Args:
|
1405
|
-
params: A parameter data that want to use in this
|
1427
|
+
params (DictData): A parameter data that want to use in this
|
1406
1428
|
execution.
|
1407
|
-
run_id: A running stage ID.
|
1408
|
-
context: A context data
|
1409
|
-
|
1410
|
-
|
1411
|
-
|
1429
|
+
run_id (str): A running stage ID.
|
1430
|
+
context (DictData): A context data that was passed from handler
|
1431
|
+
method.
|
1432
|
+
parent_run_id (str, default None): A parent running ID.
|
1433
|
+
event (Event, default None): An event manager that use to track
|
1434
|
+
parent process was not force stopped.
|
1435
|
+
|
1436
|
+
Raises:
|
1437
|
+
StageCancelError: If event was set before start process.
|
1438
|
+
StageError: If the return code form subprocess run function gather
|
1439
|
+
than 0.
|
1412
1440
|
|
1413
1441
|
Returns:
|
1414
1442
|
Result: The execution result with status and context data.
|
@@ -1419,12 +1447,16 @@ class BashStage(BaseRetryStage):
|
|
1419
1447
|
bash: str = param2template(
|
1420
1448
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
1421
1449
|
)
|
1422
|
-
with self.
|
1450
|
+
with self.make_sh_file(
|
1423
1451
|
bash=bash,
|
1424
1452
|
env=param2template(self.env, params, extras=self.extras),
|
1425
1453
|
run_id=run_id,
|
1426
1454
|
) as sh:
|
1427
|
-
|
1455
|
+
|
1456
|
+
if event and event.is_set():
|
1457
|
+
raise StageCancelError("Cancel before start bash process.")
|
1458
|
+
|
1459
|
+
trace.debug(f"[STAGE]: Create `{sh[1]}` file.", module="stage")
|
1428
1460
|
rs: CompletedProcess = subprocess.run(
|
1429
1461
|
sh,
|
1430
1462
|
shell=False,
|
@@ -1466,13 +1498,19 @@ class BashStage(BaseRetryStage):
|
|
1466
1498
|
stdout.
|
1467
1499
|
|
1468
1500
|
Args:
|
1469
|
-
params: A parameter data that want to use in this
|
1501
|
+
params (DictData): A parameter data that want to use in this
|
1470
1502
|
execution.
|
1471
|
-
run_id: A running stage ID.
|
1472
|
-
context: A context data
|
1473
|
-
|
1474
|
-
|
1475
|
-
|
1503
|
+
run_id (str): A running stage ID.
|
1504
|
+
context (DictData): A context data that was passed from handler
|
1505
|
+
method.
|
1506
|
+
parent_run_id (str, default None): A parent running ID.
|
1507
|
+
event (Event, default None): An event manager that use to track
|
1508
|
+
parent process was not force stopped.
|
1509
|
+
|
1510
|
+
Raises:
|
1511
|
+
StageCancelError: If event was set before start process.
|
1512
|
+
StageError: If the return code form subprocess run function gather
|
1513
|
+
than 0.
|
1476
1514
|
|
1477
1515
|
Returns:
|
1478
1516
|
Result: The execution result with status and context data.
|
@@ -1483,11 +1521,15 @@ class BashStage(BaseRetryStage):
|
|
1483
1521
|
bash: str = param2template(
|
1484
1522
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
1485
1523
|
)
|
1486
|
-
async with self.
|
1524
|
+
async with self.async_make_sh_file(
|
1487
1525
|
bash=bash,
|
1488
1526
|
env=param2template(self.env, params, extras=self.extras),
|
1489
1527
|
run_id=run_id,
|
1490
1528
|
) as sh:
|
1529
|
+
|
1530
|
+
if event and event.is_set():
|
1531
|
+
raise StageCancelError("Cancel before start bash process.")
|
1532
|
+
|
1491
1533
|
await trace.adebug(f"[STAGE]: Create `{sh[1]}` file.")
|
1492
1534
|
rs: CompletedProcess = subprocess.run(
|
1493
1535
|
sh,
|
@@ -1497,7 +1539,6 @@ class BashStage(BaseRetryStage):
|
|
1497
1539
|
text=True,
|
1498
1540
|
encoding="utf-8",
|
1499
1541
|
)
|
1500
|
-
|
1501
1542
|
if rs.returncode > 0:
|
1502
1543
|
e: str = rs.stderr.removesuffix("\n")
|
1503
1544
|
e_bash: str = bash.replace("\n", "\n\t")
|
@@ -1532,17 +1573,17 @@ class PyStage(BaseRetryStage):
|
|
1532
1573
|
module to validate exec-string before running or exclude the `os` package
|
1533
1574
|
from the current globals variable.
|
1534
1575
|
|
1535
|
-
|
1536
|
-
>>> stage = {
|
1576
|
+
Examples:
|
1577
|
+
>>> stage = PyStage.model_validate({
|
1578
|
+
... "id": "py-stage",
|
1537
1579
|
... "name": "Python stage execution",
|
1538
1580
|
... "run": 'print(f"Hello {VARIABLE}")',
|
1539
1581
|
... "vars": {
|
1540
1582
|
... "VARIABLE": "WORLD",
|
1541
1583
|
... },
|
1542
|
-
... }
|
1584
|
+
... })
|
1543
1585
|
"""
|
1544
1586
|
|
1545
|
-
action_stage: ClassVar[bool] = True
|
1546
1587
|
run: str = Field(
|
1547
1588
|
description="A Python string statement that want to run with `exec`.",
|
1548
1589
|
)
|
@@ -1557,11 +1598,13 @@ class PyStage(BaseRetryStage):
|
|
1557
1598
|
@staticmethod
|
1558
1599
|
def filter_locals(values: DictData) -> Iterator[str]:
|
1559
1600
|
"""Filter a locals mapping values that be module, class, or
|
1560
|
-
__annotations__
|
1601
|
+
`__annotations__`.
|
1561
1602
|
|
1562
|
-
:
|
1603
|
+
Args:
|
1604
|
+
values: (DictData) A locals values that want to filter.
|
1563
1605
|
|
1564
|
-
:
|
1606
|
+
Returns:
|
1607
|
+
Iterator[str]: Iter string value.
|
1565
1608
|
"""
|
1566
1609
|
for value in values:
|
1567
1610
|
|
@@ -1570,6 +1613,7 @@ class PyStage(BaseRetryStage):
|
|
1570
1613
|
or (value.startswith("__") and value.endswith("__"))
|
1571
1614
|
or ismodule(values[value])
|
1572
1615
|
or isclass(values[value])
|
1616
|
+
or value in ("trace",)
|
1573
1617
|
):
|
1574
1618
|
continue
|
1575
1619
|
|
@@ -1581,12 +1625,14 @@ class PyStage(BaseRetryStage):
|
|
1581
1625
|
"""Override set an outputs method for the Python execution process that
|
1582
1626
|
extract output from all the locals values.
|
1583
1627
|
|
1584
|
-
:
|
1585
|
-
output
|
1586
|
-
|
1587
|
-
|
1628
|
+
Args:
|
1629
|
+
output (DictData): An output data that want to extract to an
|
1630
|
+
output key.
|
1631
|
+
to (DictData): A context data that want to add output result.
|
1632
|
+
info (DictData):
|
1588
1633
|
|
1589
|
-
:
|
1634
|
+
Returns:
|
1635
|
+
DictData: A context data that have merged with the output data.
|
1590
1636
|
"""
|
1591
1637
|
output: DictData = output.copy()
|
1592
1638
|
lc: DictData = output.pop("locals", {})
|
@@ -1638,18 +1684,13 @@ class PyStage(BaseRetryStage):
|
|
1638
1684
|
}
|
1639
1685
|
)
|
1640
1686
|
|
1687
|
+
if event and event.is_set():
|
1688
|
+
raise StageCancelError("Cancel before start exec process.")
|
1689
|
+
|
1641
1690
|
# WARNING: The exec build-in function is very dangerous. So, it
|
1642
1691
|
# should use the re module to validate exec-string before running.
|
1643
|
-
exec(
|
1644
|
-
|
1645
|
-
param2template(dedent(self.run), params, extras=self.extras)
|
1646
|
-
),
|
1647
|
-
gb,
|
1648
|
-
lc,
|
1649
|
-
)
|
1650
|
-
return Result(
|
1651
|
-
run_id=run_id,
|
1652
|
-
parent_run_id=parent_run_id,
|
1692
|
+
exec(self.pass_template(dedent(self.run), params), gb, lc)
|
1693
|
+
return Result.from_trace(trace).catch(
|
1653
1694
|
status=SUCCESS,
|
1654
1695
|
context=catch(
|
1655
1696
|
context=context,
|
@@ -1670,7 +1711,6 @@ class PyStage(BaseRetryStage):
|
|
1670
1711
|
},
|
1671
1712
|
},
|
1672
1713
|
),
|
1673
|
-
extras=self.extras,
|
1674
1714
|
)
|
1675
1715
|
|
1676
1716
|
async def async_process(
|
@@ -1689,13 +1729,17 @@ class PyStage(BaseRetryStage):
|
|
1689
1729
|
- https://stackoverflow.com/questions/44859165/async-exec-in-python
|
1690
1730
|
|
1691
1731
|
Args:
|
1692
|
-
params: A parameter data that want to use in this
|
1732
|
+
params (DictData): A parameter data that want to use in this
|
1693
1733
|
execution.
|
1694
|
-
run_id: A running stage ID.
|
1695
|
-
context: A context data
|
1696
|
-
|
1697
|
-
|
1698
|
-
|
1734
|
+
run_id (str): A running stage ID.
|
1735
|
+
context (DictData): A context data that was passed from handler
|
1736
|
+
method.
|
1737
|
+
parent_run_id (str, default None): A parent running ID.
|
1738
|
+
event (Event, default None): An event manager that use to track
|
1739
|
+
parent process was not force stopped.
|
1740
|
+
|
1741
|
+
Raises:
|
1742
|
+
StageCancelError: If event was set before start process.
|
1699
1743
|
|
1700
1744
|
Returns:
|
1701
1745
|
Result: The execution result with status and context data.
|
@@ -1718,16 +1762,14 @@ class PyStage(BaseRetryStage):
|
|
1718
1762
|
)
|
1719
1763
|
}
|
1720
1764
|
)
|
1765
|
+
|
1766
|
+
if event and event.is_set():
|
1767
|
+
raise StageCancelError("Cancel before start exec process.")
|
1768
|
+
|
1721
1769
|
# WARNING: The exec build-in function is very dangerous. So, it
|
1722
1770
|
# should use the re module to validate exec-string before running.
|
1723
|
-
exec(
|
1724
|
-
|
1725
|
-
gb,
|
1726
|
-
lc,
|
1727
|
-
)
|
1728
|
-
return Result(
|
1729
|
-
run_id=run_id,
|
1730
|
-
parent_run_id=parent_run_id,
|
1771
|
+
exec(self.pass_template(dedent(self.run), params), gb, lc)
|
1772
|
+
return Result.from_trace(trace).catch(
|
1731
1773
|
status=SUCCESS,
|
1732
1774
|
context=catch(
|
1733
1775
|
context=context,
|
@@ -1748,7 +1790,6 @@ class PyStage(BaseRetryStage):
|
|
1748
1790
|
},
|
1749
1791
|
},
|
1750
1792
|
),
|
1751
|
-
extras=self.extras,
|
1752
1793
|
)
|
1753
1794
|
|
1754
1795
|
|
@@ -1773,15 +1814,15 @@ class CallStage(BaseRetryStage):
|
|
1773
1814
|
The caller registry to get a caller function should importable by the
|
1774
1815
|
current Python execution pointer.
|
1775
1816
|
|
1776
|
-
|
1777
|
-
>>> stage = {
|
1817
|
+
Examples:
|
1818
|
+
>>> stage = CallStage.model_validate({
|
1819
|
+
... "id": "call-stage",
|
1778
1820
|
... "name": "Task stage execution",
|
1779
1821
|
... "uses": "tasks/function-name@tag-name",
|
1780
1822
|
... "args": {"arg01": "BAR", "kwarg01": 10},
|
1781
|
-
... }
|
1823
|
+
... })
|
1782
1824
|
"""
|
1783
1825
|
|
1784
|
-
action_stage: ClassVar[bool] = True
|
1785
1826
|
uses: str = Field(
|
1786
1827
|
description=(
|
1787
1828
|
"A caller function with registry importer syntax that use to load "
|
@@ -1798,26 +1839,36 @@ class CallStage(BaseRetryStage):
|
|
1798
1839
|
)
|
1799
1840
|
|
1800
1841
|
@field_validator("args", mode="before")
|
1801
|
-
def __validate_args_key(cls,
|
1842
|
+
def __validate_args_key(cls, data: Any) -> Any:
|
1802
1843
|
"""Validate argument keys on the ``args`` field should not include the
|
1803
1844
|
special keys.
|
1804
1845
|
|
1805
|
-
:
|
1846
|
+
Args:
|
1847
|
+
data (Any): A data that want to check the special keys.
|
1806
1848
|
|
1807
|
-
:
|
1849
|
+
Returns:
|
1850
|
+
Any: An any data.
|
1808
1851
|
"""
|
1809
|
-
if isinstance(
|
1810
|
-
k in
|
1852
|
+
if isinstance(data, dict) and any(
|
1853
|
+
k in data for k in ("result", "extras")
|
1811
1854
|
):
|
1812
1855
|
raise ValueError(
|
1813
1856
|
"The argument on workflow template for the caller stage "
|
1814
1857
|
"should not pass `result` and `extras`. They are special "
|
1815
1858
|
"arguments."
|
1816
1859
|
)
|
1817
|
-
return
|
1860
|
+
return data
|
1818
1861
|
|
1819
1862
|
def get_caller(self, params: DictData) -> Callable[[], TagFunc]:
|
1820
|
-
"""Get the lazy TagFuc object from registry.
|
1863
|
+
"""Get the lazy TagFuc object from registry.
|
1864
|
+
|
1865
|
+
Args:
|
1866
|
+
params (DictData): A parameters.
|
1867
|
+
|
1868
|
+
Returns:
|
1869
|
+
Callable[[], TagFunc]: A lazy partial function that return the
|
1870
|
+
TagFunc object.
|
1871
|
+
"""
|
1821
1872
|
return extract_call(
|
1822
1873
|
param2template(self.uses, params, extras=self.extras),
|
1823
1874
|
registries=self.extras.get("registry_caller"),
|
@@ -1835,13 +1886,19 @@ class CallStage(BaseRetryStage):
|
|
1835
1886
|
"""Execute this caller function with its argument parameter.
|
1836
1887
|
|
1837
1888
|
Args:
|
1838
|
-
params: A parameter data that want to use in this
|
1889
|
+
params (DictData): A parameter data that want to use in this
|
1839
1890
|
execution.
|
1840
|
-
run_id: A running stage ID.
|
1841
|
-
context: A context data
|
1842
|
-
|
1843
|
-
|
1844
|
-
|
1891
|
+
run_id (str): A running stage ID.
|
1892
|
+
context (DictData): A context data that was passed from handler
|
1893
|
+
method.
|
1894
|
+
parent_run_id (str, default None): A parent running ID.
|
1895
|
+
event (Event, default None): An event manager that use to track
|
1896
|
+
parent process was not force stopped.
|
1897
|
+
|
1898
|
+
Raises:
|
1899
|
+
ValueError: If the necessary parameters do not exist in args field.
|
1900
|
+
TypeError: If the returning type of caller function does not match
|
1901
|
+
with dict type.
|
1845
1902
|
|
1846
1903
|
Returns:
|
1847
1904
|
Result: The execution result with status and context data.
|
@@ -1864,7 +1921,9 @@ class CallStage(BaseRetryStage):
|
|
1864
1921
|
),
|
1865
1922
|
"extras": self.extras,
|
1866
1923
|
} | self.pass_template(self.args, params)
|
1867
|
-
|
1924
|
+
|
1925
|
+
# NOTE: Catch the necessary parameters.
|
1926
|
+
sig: inspect.Signature = inspect.signature(call_func)
|
1868
1927
|
necessary_params: list[str] = []
|
1869
1928
|
has_keyword: bool = False
|
1870
1929
|
for k in sig.parameters:
|
@@ -1883,11 +1942,9 @@ class CallStage(BaseRetryStage):
|
|
1883
1942
|
(k.removeprefix("_") not in args and k not in args)
|
1884
1943
|
for k in necessary_params
|
1885
1944
|
):
|
1886
|
-
|
1887
|
-
necessary_params
|
1888
|
-
|
1889
|
-
if "extras" in necessary_params:
|
1890
|
-
necessary_params.remove("extras")
|
1945
|
+
for k in ("result", "extras"):
|
1946
|
+
if k in necessary_params:
|
1947
|
+
necessary_params.remove(k)
|
1891
1948
|
|
1892
1949
|
args.pop("result")
|
1893
1950
|
args.pop("extras")
|
@@ -1897,18 +1954,15 @@ class CallStage(BaseRetryStage):
|
|
1897
1954
|
)
|
1898
1955
|
|
1899
1956
|
if not has_keyword:
|
1900
|
-
|
1901
|
-
|
1957
|
+
for k in ("result", "extras"):
|
1958
|
+
if k not in sig.parameters:
|
1959
|
+
args.pop(k)
|
1902
1960
|
|
1903
|
-
|
1904
|
-
args.pop("extras")
|
1961
|
+
args: DictData = self.validate_model_args(call_func, args)
|
1905
1962
|
|
1906
1963
|
if event and event.is_set():
|
1907
1964
|
raise StageCancelError("Cancel before start call process.")
|
1908
1965
|
|
1909
|
-
args: DictData = self.validate_model_args(
|
1910
|
-
call_func, args, run_id, parent_run_id, extras=self.extras
|
1911
|
-
)
|
1912
1966
|
if inspect.iscoroutinefunction(call_func):
|
1913
1967
|
loop = asyncio.get_event_loop()
|
1914
1968
|
rs: DictData = loop.run_until_complete(
|
@@ -1962,6 +2016,11 @@ class CallStage(BaseRetryStage):
|
|
1962
2016
|
event: An event manager that use to track parent process
|
1963
2017
|
was not force stopped.
|
1964
2018
|
|
2019
|
+
Raises:
|
2020
|
+
ValueError: If the necessary parameters do not exist in args field.
|
2021
|
+
TypeError: If the returning type of caller function does not match
|
2022
|
+
with dict type.
|
2023
|
+
|
1965
2024
|
Returns:
|
1966
2025
|
Result: The execution result with status and context data.
|
1967
2026
|
"""
|
@@ -1985,7 +2044,9 @@ class CallStage(BaseRetryStage):
|
|
1985
2044
|
),
|
1986
2045
|
"extras": self.extras,
|
1987
2046
|
} | self.pass_template(self.args, params)
|
1988
|
-
|
2047
|
+
|
2048
|
+
# NOTE: Catch the necessary parameters.
|
2049
|
+
sig: inspect.Signature = inspect.signature(call_func)
|
1989
2050
|
necessary_params: list[str] = []
|
1990
2051
|
has_keyword: bool = False
|
1991
2052
|
for k in sig.parameters:
|
@@ -2003,11 +2064,9 @@ class CallStage(BaseRetryStage):
|
|
2003
2064
|
(k.removeprefix("_") not in args and k not in args)
|
2004
2065
|
for k in necessary_params
|
2005
2066
|
):
|
2006
|
-
|
2007
|
-
necessary_params
|
2008
|
-
|
2009
|
-
if "extras" in necessary_params:
|
2010
|
-
necessary_params.remove("extras")
|
2067
|
+
for k in ("result", "extras"):
|
2068
|
+
if k in necessary_params:
|
2069
|
+
necessary_params.remove(k)
|
2011
2070
|
|
2012
2071
|
args.pop("result")
|
2013
2072
|
args.pop("extras")
|
@@ -2017,18 +2076,15 @@ class CallStage(BaseRetryStage):
|
|
2017
2076
|
)
|
2018
2077
|
|
2019
2078
|
if not has_keyword:
|
2020
|
-
|
2021
|
-
|
2079
|
+
for k in ("result", "extras"):
|
2080
|
+
if k not in sig.parameters:
|
2081
|
+
args.pop(k)
|
2022
2082
|
|
2023
|
-
|
2024
|
-
args.pop("extras")
|
2083
|
+
args: DictData = self.validate_model_args(call_func, args)
|
2025
2084
|
|
2026
2085
|
if event and event.is_set():
|
2027
2086
|
raise StageCancelError("Cancel before start call process.")
|
2028
2087
|
|
2029
|
-
args: DictData = self.validate_model_args(
|
2030
|
-
call_func, args, run_id, parent_run_id, extras=self.extras
|
2031
|
-
)
|
2032
2088
|
if inspect.iscoroutinefunction(call_func):
|
2033
2089
|
rs: DictOrModel = await call_func(
|
2034
2090
|
**param2template(args, params, extras=self.extras)
|
@@ -2061,24 +2117,18 @@ class CallStage(BaseRetryStage):
|
|
2061
2117
|
)
|
2062
2118
|
|
2063
2119
|
@staticmethod
|
2064
|
-
def validate_model_args(
|
2065
|
-
func: TagFunc,
|
2066
|
-
args: DictData,
|
2067
|
-
run_id: str,
|
2068
|
-
parent_run_id: Optional[str] = None,
|
2069
|
-
extras: Optional[DictData] = None,
|
2070
|
-
) -> DictData:
|
2120
|
+
def validate_model_args(func: TagFunc, args: DictData) -> DictData:
|
2071
2121
|
"""Validate an input arguments before passing to the caller function.
|
2072
2122
|
|
2073
2123
|
Args:
|
2074
2124
|
func (TagFunc): A tag function object that want to get typing.
|
2075
2125
|
args (DictData): An arguments before passing to this tag func.
|
2076
|
-
|
2077
|
-
|
2078
|
-
|
2126
|
+
|
2127
|
+
Raises:
|
2128
|
+
StageError: If model validation was raised the ValidationError.
|
2079
2129
|
|
2080
2130
|
Returns:
|
2081
|
-
DictData: A prepared args
|
2131
|
+
DictData: A prepared args parameter that validate with model args.
|
2082
2132
|
"""
|
2083
2133
|
try:
|
2084
2134
|
override: DictData = dict(
|
@@ -2101,15 +2151,6 @@ class CallStage(BaseRetryStage):
|
|
2101
2151
|
raise StageError(
|
2102
2152
|
"Validate argument from the caller function raise invalid type."
|
2103
2153
|
) from e
|
2104
|
-
except TypeError as e:
|
2105
|
-
trace: Trace = get_trace(
|
2106
|
-
run_id, parent_run_id=parent_run_id, extras=extras
|
2107
|
-
)
|
2108
|
-
trace.warning(
|
2109
|
-
f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
|
2110
|
-
f"parsing model args process."
|
2111
|
-
)
|
2112
|
-
return args
|
2113
2154
|
|
2114
2155
|
def dryrun(
|
2115
2156
|
self,
|
@@ -2119,12 +2160,23 @@ class CallStage(BaseRetryStage):
|
|
2119
2160
|
*,
|
2120
2161
|
parent_run_id: Optional[str] = None,
|
2121
2162
|
event: Optional[Event] = None,
|
2122
|
-
) ->
|
2163
|
+
) -> Result: # pragma: no cov
|
2123
2164
|
"""Override the dryrun method for this CallStage.
|
2124
2165
|
|
2125
2166
|
Steps:
|
2126
2167
|
- Pre-hook caller function that exist.
|
2127
2168
|
- Show function parameters
|
2169
|
+
|
2170
|
+
Args:
|
2171
|
+
params (DictData): A parameter data that want to use in this
|
2172
|
+
execution.
|
2173
|
+
run_id (str): A running stage ID.
|
2174
|
+
context (DictData): A context data that was passed from handler
|
2175
|
+
method.
|
2176
|
+
parent_run_id (str, default None): A parent running ID.
|
2177
|
+
event (Event, default None): An event manager that use to track
|
2178
|
+
parent process was not force stopped.
|
2179
|
+
|
2128
2180
|
"""
|
2129
2181
|
trace: Trace = get_trace(
|
2130
2182
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
@@ -2142,7 +2194,9 @@ class CallStage(BaseRetryStage):
|
|
2142
2194
|
),
|
2143
2195
|
"extras": self.extras,
|
2144
2196
|
} | self.pass_template(self.args, params)
|
2145
|
-
|
2197
|
+
|
2198
|
+
# NOTE: Catch the necessary parameters.
|
2199
|
+
sig: inspect.Signature = inspect.signature(call_func)
|
2146
2200
|
trace.debug(f"[STAGE]: {sig.parameters}")
|
2147
2201
|
necessary_params: list[str] = []
|
2148
2202
|
has_keyword: bool = False
|
@@ -2164,24 +2218,22 @@ class CallStage(BaseRetryStage):
|
|
2164
2218
|
if p in func_typed
|
2165
2219
|
)
|
2166
2220
|
map_type_args: str = "||".join(f"\t{a}: {type(a)}" for a in args)
|
2167
|
-
if not has_keyword:
|
2168
|
-
if "result" not in sig.parameters:
|
2169
|
-
args.pop("result")
|
2170
2221
|
|
2171
|
-
|
2172
|
-
|
2222
|
+
if not has_keyword:
|
2223
|
+
for k in ("result", "extras"):
|
2224
|
+
if k not in sig.parameters:
|
2225
|
+
args.pop(k)
|
2173
2226
|
|
2174
|
-
trace.
|
2227
|
+
trace.info(
|
2175
2228
|
f"[STAGE]: Details"
|
2176
2229
|
f"||Necessary Params:"
|
2177
2230
|
f"||{map_type}"
|
2231
|
+
f"||Supported Keyword Params: {has_keyword}"
|
2178
2232
|
f"||Return Type: {func_typed['return']}"
|
2179
2233
|
f"||Argument Params:"
|
2180
2234
|
f"||{map_type_args}"
|
2181
2235
|
f"||"
|
2182
2236
|
)
|
2183
|
-
if has_keyword:
|
2184
|
-
trace.debug("[STAGE]: This caller function support keyword param.")
|
2185
2237
|
return Result(
|
2186
2238
|
run_id=run_id,
|
2187
2239
|
parent_run_id=parent_run_id,
|
@@ -2191,71 +2243,6 @@ class CallStage(BaseRetryStage):
|
|
2191
2243
|
)
|
2192
2244
|
|
2193
2245
|
|
2194
|
-
class BaseNestedStage(BaseAsyncStage, ABC):
|
2195
|
-
"""Base Nested Stage model. This model is use for checking the child stage
|
2196
|
-
is the nested stage or not.
|
2197
|
-
"""
|
2198
|
-
|
2199
|
-
def set_outputs(
|
2200
|
-
self, output: DictData, to: DictData, info: Optional[DictData] = None
|
2201
|
-
) -> DictData:
|
2202
|
-
"""Override the set outputs method that support for nested-stage."""
|
2203
|
-
return super().set_outputs(output, to=to)
|
2204
|
-
|
2205
|
-
def get_outputs(self, output: DictData) -> DictData:
|
2206
|
-
"""Override the get outputs method that support for nested-stage"""
|
2207
|
-
return super().get_outputs(output)
|
2208
|
-
|
2209
|
-
@property
|
2210
|
-
def is_nested(self) -> bool:
|
2211
|
-
"""Check if this stage is a nested stage or not.
|
2212
|
-
|
2213
|
-
:rtype: bool
|
2214
|
-
"""
|
2215
|
-
return True
|
2216
|
-
|
2217
|
-
@staticmethod
|
2218
|
-
def mark_errors(context: DictData, error: StageError) -> None:
|
2219
|
-
"""Make the errors context result with the refs value depends on the nested
|
2220
|
-
execute func.
|
2221
|
-
|
2222
|
-
Args:
|
2223
|
-
context: (DictData) A context data.
|
2224
|
-
error: (StageError) A stage exception object.
|
2225
|
-
"""
|
2226
|
-
if "errors" in context:
|
2227
|
-
context["errors"][error.refs] = error.to_dict()
|
2228
|
-
else:
|
2229
|
-
context["errors"] = error.to_dict(with_refs=True)
|
2230
|
-
|
2231
|
-
async def async_process(
|
2232
|
-
self,
|
2233
|
-
params: DictData,
|
2234
|
-
run_id: str,
|
2235
|
-
context: DictData,
|
2236
|
-
*,
|
2237
|
-
parent_run_id: Optional[str] = None,
|
2238
|
-
event: Optional[Event] = None,
|
2239
|
-
) -> Result:
|
2240
|
-
"""Async process for nested-stage do not implement yet.
|
2241
|
-
|
2242
|
-
Args:
|
2243
|
-
params: A parameter data that want to use in this
|
2244
|
-
execution.
|
2245
|
-
run_id: A running stage ID.
|
2246
|
-
context: A context data.
|
2247
|
-
parent_run_id: A parent running ID. (Default is None)
|
2248
|
-
event: An event manager that use to track parent process
|
2249
|
-
was not force stopped.
|
2250
|
-
|
2251
|
-
Returns:
|
2252
|
-
Result: The execution result with status and context data.
|
2253
|
-
"""
|
2254
|
-
raise NotImplementedError(
|
2255
|
-
"The nested-stage does not implement the `axecute` method yet."
|
2256
|
-
)
|
2257
|
-
|
2258
|
-
|
2259
2246
|
class TriggerStage(BaseRetryStage):
|
2260
2247
|
"""Trigger workflow executor stage that run an input trigger Workflow
|
2261
2248
|
execute method. This is the stage that allow you to create the reusable
|
@@ -2264,12 +2251,13 @@ class TriggerStage(BaseRetryStage):
|
|
2264
2251
|
This stage does not allow to pass the workflow model directly to the
|
2265
2252
|
trigger field. A trigger workflow name should exist on the config path only.
|
2266
2253
|
|
2267
|
-
|
2268
|
-
>>> stage = {
|
2254
|
+
Examples:
|
2255
|
+
>>> stage = TriggerStage.model_validate({
|
2256
|
+
... "id": "trigger-stage",
|
2269
2257
|
... "name": "Trigger workflow stage execution",
|
2270
2258
|
... "trigger": 'workflow-name-for-loader',
|
2271
2259
|
... "params": {"run-date": "2024-08-01", "source": "src"},
|
2272
|
-
... }
|
2260
|
+
... })
|
2273
2261
|
"""
|
2274
2262
|
|
2275
2263
|
trigger: str = Field(
|
@@ -2316,51 +2304,38 @@ class TriggerStage(BaseRetryStage):
|
|
2316
2304
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
2317
2305
|
if _trigger == self.extras.get("__sys_exec_break_circle", "NOTSET"):
|
2318
2306
|
raise StageError("Circle execute via trigger itself workflow name.")
|
2307
|
+
|
2319
2308
|
trace.info(f"[NESTED]: Load Workflow Config: {_trigger!r}")
|
2320
|
-
|
2309
|
+
workflow: Workflow = Workflow.from_conf(
|
2321
2310
|
name=pass_env(_trigger),
|
2322
2311
|
extras=self.extras,
|
2323
|
-
)
|
2324
|
-
|
2312
|
+
)
|
2313
|
+
|
2314
|
+
if event and event.is_set():
|
2315
|
+
raise StageCancelError("Cancel before start trigger process.")
|
2316
|
+
|
2317
|
+
# IMPORTANT: Should not use the `pass_env` function on this `params`
|
2318
|
+
# parameter.
|
2319
|
+
result: Result = workflow.execute(
|
2325
2320
|
params=param2template(self.params, params, extras=self.extras),
|
2326
2321
|
run_id=parent_run_id,
|
2327
2322
|
event=event,
|
2328
2323
|
)
|
2324
|
+
catch(context, status=result.status, updated=result.context)
|
2329
2325
|
if result.status == FAILED:
|
2330
2326
|
err_msg: str = (
|
2331
2327
|
f" with:\n{msg}"
|
2332
2328
|
if (msg := result.context.get("errors", {}).get("message"))
|
2333
2329
|
else "."
|
2334
2330
|
)
|
2335
|
-
|
2336
|
-
|
2337
|
-
|
2338
|
-
"status": FAILED,
|
2339
|
-
"errors": StageError(
|
2340
|
-
f"Trigger workflow was failed{err_msg}"
|
2341
|
-
).to_dict(),
|
2342
|
-
},
|
2331
|
+
raise StageError(
|
2332
|
+
f"Trigger workflow was failed{err_msg}",
|
2333
|
+
allow_traceback=False,
|
2343
2334
|
)
|
2344
2335
|
elif result.status == CANCEL:
|
2345
|
-
|
2346
|
-
status=CANCEL,
|
2347
|
-
context={
|
2348
|
-
"status": CANCEL,
|
2349
|
-
"errors": StageCancelError(
|
2350
|
-
"Trigger workflow was cancel."
|
2351
|
-
).to_dict(),
|
2352
|
-
},
|
2353
|
-
)
|
2336
|
+
raise StageCancelError("Trigger workflow was cancel.")
|
2354
2337
|
elif result.status == SKIP:
|
2355
|
-
|
2356
|
-
status=SKIP,
|
2357
|
-
context={
|
2358
|
-
"status": SKIP,
|
2359
|
-
"errors": StageSkipError(
|
2360
|
-
"Trigger workflow was skipped."
|
2361
|
-
).to_dict(),
|
2362
|
-
},
|
2363
|
-
)
|
2338
|
+
raise StageSkipError("Trigger workflow was skipped.")
|
2364
2339
|
return result
|
2365
2340
|
|
2366
2341
|
async def async_process(
|
@@ -2372,7 +2347,7 @@ class TriggerStage(BaseRetryStage):
|
|
2372
2347
|
parent_run_id: Optional[str] = None,
|
2373
2348
|
event: Optional[Event] = None,
|
2374
2349
|
) -> Result: # pragma: no cov
|
2375
|
-
"""Async process for
|
2350
|
+
"""Async process for trigger-stage do not implement yet.
|
2376
2351
|
|
2377
2352
|
Args:
|
2378
2353
|
params: A parameter data that want to use in this
|
@@ -2391,6 +2366,73 @@ class TriggerStage(BaseRetryStage):
|
|
2391
2366
|
)
|
2392
2367
|
|
2393
2368
|
|
2369
|
+
class BaseNestedStage(BaseAsyncStage, ABC):
|
2370
|
+
"""Base Nested Stage model. This model is use for checking the child stage
|
2371
|
+
is the nested stage or not.
|
2372
|
+
"""
|
2373
|
+
|
2374
|
+
def set_outputs(
|
2375
|
+
self, output: DictData, to: DictData, info: Optional[DictData] = None
|
2376
|
+
) -> DictData:
|
2377
|
+
"""Override the set outputs method that support for nested-stage."""
|
2378
|
+
return super().set_outputs(output, to=to)
|
2379
|
+
|
2380
|
+
def get_outputs(self, output: DictData) -> DictData:
|
2381
|
+
"""Override the get outputs method that support for nested-stage"""
|
2382
|
+
return super().get_outputs(output)
|
2383
|
+
|
2384
|
+
@property
|
2385
|
+
def is_nested(self) -> bool:
|
2386
|
+
"""Check if this stage is a nested stage or not.
|
2387
|
+
|
2388
|
+
Returns:
|
2389
|
+
bool: True only.
|
2390
|
+
"""
|
2391
|
+
return True
|
2392
|
+
|
2393
|
+
@staticmethod
|
2394
|
+
def mark_errors(context: DictData, error: StageError) -> None:
|
2395
|
+
"""Make the errors context result with the refs value depends on the nested
|
2396
|
+
execute func.
|
2397
|
+
|
2398
|
+
Args:
|
2399
|
+
context (DictData): A context data.
|
2400
|
+
error (StageError): A stage exception object.
|
2401
|
+
"""
|
2402
|
+
if "errors" in context:
|
2403
|
+
context["errors"][error.refs] = error.to_dict()
|
2404
|
+
else:
|
2405
|
+
context["errors"] = error.to_dict(with_refs=True)
|
2406
|
+
|
2407
|
+
async def async_process(
|
2408
|
+
self,
|
2409
|
+
params: DictData,
|
2410
|
+
run_id: str,
|
2411
|
+
context: DictData,
|
2412
|
+
*,
|
2413
|
+
parent_run_id: Optional[str] = None,
|
2414
|
+
event: Optional[Event] = None,
|
2415
|
+
) -> Result:
|
2416
|
+
"""Async process for nested-stage do not implement yet.
|
2417
|
+
|
2418
|
+
Args:
|
2419
|
+
params (DictData): A parameter data that want to use in this
|
2420
|
+
execution.
|
2421
|
+
run_id (str): A running stage ID.
|
2422
|
+
context (DictData): A context data that was passed from handler
|
2423
|
+
method.
|
2424
|
+
parent_run_id (str, default None): A parent running ID.
|
2425
|
+
event (Event, default None): An event manager that use to track
|
2426
|
+
parent process was not force stopped.
|
2427
|
+
|
2428
|
+
Returns:
|
2429
|
+
Result: The execution result with status and context data.
|
2430
|
+
"""
|
2431
|
+
raise NotImplementedError(
|
2432
|
+
"The nested-stage does not implement the `axecute` method yet."
|
2433
|
+
)
|
2434
|
+
|
2435
|
+
|
2394
2436
|
class ParallelContext(TypedDict):
|
2395
2437
|
branch: str
|
2396
2438
|
stages: NotRequired[dict[str, Any]]
|
@@ -2404,8 +2446,9 @@ class ParallelStage(BaseNestedStage):
|
|
2404
2446
|
This stage is not the low-level stage model because it runs multi-stages
|
2405
2447
|
in this stage execution.
|
2406
2448
|
|
2407
|
-
|
2408
|
-
>>> stage = {
|
2449
|
+
Examples:
|
2450
|
+
>>> stage = ParallelStage.model_validate({
|
2451
|
+
... "id": "parallel-stage",
|
2409
2452
|
... "name": "Parallel stage execution.",
|
2410
2453
|
... "parallel": {
|
2411
2454
|
... "branch01": [
|
@@ -2427,7 +2470,7 @@ class ParallelStage(BaseNestedStage):
|
|
2427
2470
|
... },
|
2428
2471
|
... ],
|
2429
2472
|
... }
|
2430
|
-
... }
|
2473
|
+
... })
|
2431
2474
|
"""
|
2432
2475
|
|
2433
2476
|
parallel: dict[str, list[Stage]] = Field(
|
@@ -2677,8 +2720,9 @@ class ForEachStage(BaseNestedStage):
|
|
2677
2720
|
This stage is not the low-level stage model because it runs
|
2678
2721
|
multi-stages in this stage execution.
|
2679
2722
|
|
2680
|
-
|
2681
|
-
>>> stage = {
|
2723
|
+
Examples:
|
2724
|
+
>>> stage = ForEachStage.model_validate({
|
2725
|
+
... "id": "foreach-stage",
|
2682
2726
|
... "name": "For-each stage execution",
|
2683
2727
|
... "foreach": [1, 2, 3]
|
2684
2728
|
... "stages": [
|
@@ -2687,7 +2731,7 @@ class ForEachStage(BaseNestedStage):
|
|
2687
2731
|
... "echo": "Start run with item ${{ item }}"
|
2688
2732
|
... },
|
2689
2733
|
... ],
|
2690
|
-
... }
|
2734
|
+
... })
|
2691
2735
|
"""
|
2692
2736
|
|
2693
2737
|
foreach: EachType = Field(
|
@@ -2862,15 +2906,18 @@ class ForEachStage(BaseNestedStage):
|
|
2862
2906
|
"""Validate foreach value that already passed to this model.
|
2863
2907
|
|
2864
2908
|
Args:
|
2865
|
-
value:
|
2909
|
+
value (Any): An any foreach value.
|
2866
2910
|
|
2867
2911
|
Raises:
|
2868
2912
|
TypeError: If value can not try-convert to list type.
|
2869
|
-
ValueError:
|
2913
|
+
ValueError: If the foreach value is dict type.
|
2914
|
+
ValueError: If the foreach value contain duplication item without
|
2915
|
+
enable using index as key flag.
|
2870
2916
|
|
2871
2917
|
Returns:
|
2872
2918
|
list[Any]: list of item.
|
2873
2919
|
"""
|
2920
|
+
# NOTE: Try to cast a foreach with string type to list of items.
|
2874
2921
|
if isinstance(value, str):
|
2875
2922
|
try:
|
2876
2923
|
value: list[Any] = str2list(value)
|
@@ -2879,6 +2926,7 @@ class ForEachStage(BaseNestedStage):
|
|
2879
2926
|
f"Does not support string foreach: {value!r} that can "
|
2880
2927
|
f"not convert to list."
|
2881
2928
|
) from e
|
2929
|
+
|
2882
2930
|
# [VALIDATE]: Type of the foreach should be `list` type.
|
2883
2931
|
elif isinstance(value, dict):
|
2884
2932
|
raise TypeError(
|
@@ -3000,8 +3048,9 @@ class UntilStage(BaseNestedStage):
|
|
3000
3048
|
This stage is not the low-level stage model because it runs
|
3001
3049
|
multi-stages in this stage execution.
|
3002
3050
|
|
3003
|
-
|
3004
|
-
>>> stage = {
|
3051
|
+
Examples:
|
3052
|
+
>>> stage = UntilStage.model_validate({
|
3053
|
+
... "id": "until-stage",
|
3005
3054
|
... "name": "Until stage execution",
|
3006
3055
|
... "item": 1,
|
3007
3056
|
... "until": "${{ item }} > 3"
|
@@ -3014,7 +3063,7 @@ class UntilStage(BaseNestedStage):
|
|
3014
3063
|
... )
|
3015
3064
|
... },
|
3016
3065
|
... ],
|
3017
|
-
... }
|
3066
|
+
... })
|
3018
3067
|
"""
|
3019
3068
|
|
3020
3069
|
item: Union[str, int, bool] = Field(
|
@@ -3285,6 +3334,8 @@ class Match(BaseModel):
|
|
3285
3334
|
|
3286
3335
|
|
3287
3336
|
class Else(BaseModel):
|
3337
|
+
"""Else model for the Case Stage."""
|
3338
|
+
|
3288
3339
|
other: list[Stage] = Field(
|
3289
3340
|
description="A list of stage that does not match any case.",
|
3290
3341
|
alias="else",
|
@@ -3294,8 +3345,9 @@ class Else(BaseModel):
|
|
3294
3345
|
class CaseStage(BaseNestedStage):
|
3295
3346
|
"""Case stage executor that execute all stages if the condition was matched.
|
3296
3347
|
|
3297
|
-
|
3298
|
-
>>> stage = {
|
3348
|
+
Examples:
|
3349
|
+
>>> stage = CaseStage.model_validate({
|
3350
|
+
... "id": "case-stage",
|
3299
3351
|
... "name": "If stage execution.",
|
3300
3352
|
... "case": "${{ param.test }}",
|
3301
3353
|
... "match": [
|
@@ -3318,9 +3370,10 @@ class CaseStage(BaseNestedStage):
|
|
3318
3370
|
... ],
|
3319
3371
|
... },
|
3320
3372
|
... ],
|
3321
|
-
... }
|
3373
|
+
... })
|
3322
3374
|
|
3323
|
-
>>> stage = {
|
3375
|
+
>>> stage = CaseStage.model_validate({
|
3376
|
+
... "id": "case-stage",
|
3324
3377
|
... "name": "If stage execution.",
|
3325
3378
|
... "case": "${{ param.test }}",
|
3326
3379
|
... "match": [
|
@@ -3342,7 +3395,7 @@ class CaseStage(BaseNestedStage):
|
|
3342
3395
|
... ],
|
3343
3396
|
... },
|
3344
3397
|
... ],
|
3345
|
-
... }
|
3398
|
+
... })
|
3346
3399
|
|
3347
3400
|
"""
|
3348
3401
|
|
@@ -3361,9 +3414,16 @@ class CaseStage(BaseNestedStage):
|
|
3361
3414
|
|
3362
3415
|
@field_validator("match", mode="after")
|
3363
3416
|
def __validate_match(
|
3364
|
-
cls,
|
3417
|
+
cls,
|
3418
|
+
match: list[Union[Match, Else]],
|
3365
3419
|
) -> list[Union[Match, Else]]:
|
3366
|
-
"""Validate the match field should contain only one Else model.
|
3420
|
+
"""Validate the match field should contain only one Else model.
|
3421
|
+
|
3422
|
+
Raises:
|
3423
|
+
ValueError: If match field contain Else more than 1 model.
|
3424
|
+
ValueError: If match field contain Match with '_' case (it represent
|
3425
|
+
the else case) more than 1 model.
|
3426
|
+
"""
|
3367
3427
|
c_else_case: int = 0
|
3368
3428
|
c_else_model: int = 0
|
3369
3429
|
for m in match:
|
@@ -3562,8 +3622,10 @@ class CaseStage(BaseNestedStage):
|
|
3562
3622
|
case: StrOrNone = param2template(self.case, params, extras=self.extras)
|
3563
3623
|
trace.info(f"[NESTED]: Get Case: {case!r}.")
|
3564
3624
|
case, stages = self.extract_stages_from_case(case, params=params)
|
3625
|
+
|
3565
3626
|
if event and event.is_set():
|
3566
3627
|
raise StageCancelError("Cancel before start case process.")
|
3628
|
+
|
3567
3629
|
status, context = self._process_nested(
|
3568
3630
|
case=case,
|
3569
3631
|
stages=stages,
|
@@ -3585,11 +3647,12 @@ class RaiseStage(BaseAsyncStage):
|
|
3585
3647
|
"""Raise error stage executor that raise `StageError` that use a message
|
3586
3648
|
field for making error message before raise.
|
3587
3649
|
|
3588
|
-
|
3589
|
-
>>> stage = {
|
3650
|
+
Examples:
|
3651
|
+
>>> stage = RaiseStage.model_validate({
|
3652
|
+
... "id": "raise-stage",
|
3590
3653
|
... "name": "Raise stage",
|
3591
3654
|
... "raise": "raise this stage",
|
3592
|
-
... }
|
3655
|
+
... })
|
3593
3656
|
|
3594
3657
|
"""
|
3595
3658
|
|
@@ -3890,7 +3953,7 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3890
3953
|
)
|
3891
3954
|
|
3892
3955
|
@contextlib.contextmanager
|
3893
|
-
def
|
3956
|
+
def make_py_file(
|
3894
3957
|
self,
|
3895
3958
|
py: str,
|
3896
3959
|
values: DictData,
|
@@ -3966,13 +4029,14 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3966
4029
|
- Execution python file with `uv run` via Python subprocess module.
|
3967
4030
|
|
3968
4031
|
Args:
|
3969
|
-
params: A parameter data that want to use in this
|
4032
|
+
params (DictData): A parameter data that want to use in this
|
3970
4033
|
execution.
|
3971
|
-
run_id: A running stage ID.
|
3972
|
-
context: A context data
|
3973
|
-
|
3974
|
-
|
3975
|
-
|
4034
|
+
run_id (str): A running stage ID.
|
4035
|
+
context (DictData): A context data that was passed from handler
|
4036
|
+
method.
|
4037
|
+
parent_run_id (str, default None): A parent running ID.
|
4038
|
+
event (Event, default None): An event manager that use to track
|
4039
|
+
parent process was not force stopped.
|
3976
4040
|
|
3977
4041
|
Returns:
|
3978
4042
|
Result: The execution result with status and context data.
|
@@ -3981,12 +4045,18 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3981
4045
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3982
4046
|
)
|
3983
4047
|
run: str = param2template(dedent(self.run), params, extras=self.extras)
|
3984
|
-
with self.
|
4048
|
+
with self.make_py_file(
|
3985
4049
|
py=run,
|
3986
4050
|
values=param2template(self.vars, params, extras=self.extras),
|
3987
4051
|
deps=param2template(self.deps, params, extras=self.extras),
|
3988
4052
|
run_id=run_id,
|
3989
4053
|
) as py:
|
4054
|
+
|
4055
|
+
if event and event.is_set():
|
4056
|
+
raise StageCancelError(
|
4057
|
+
"Cancel before start virtual python process."
|
4058
|
+
)
|
4059
|
+
|
3990
4060
|
trace.debug(f"[STAGE]: Create `{py}` file.")
|
3991
4061
|
rs: CompletedProcess = subprocess.run(
|
3992
4062
|
["python", "-m", "uv", "run", py, "--no-cache"],
|
@@ -4032,6 +4102,18 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
4032
4102
|
parent_run_id: Optional[str] = None,
|
4033
4103
|
event: Optional[Event] = None,
|
4034
4104
|
) -> Result:
|
4105
|
+
"""Async execution method for this Virtual Python stage.
|
4106
|
+
|
4107
|
+
Args:
|
4108
|
+
params (DictData): A parameter data that want to use in this
|
4109
|
+
execution.
|
4110
|
+
run_id (str): A running stage ID.
|
4111
|
+
context (DictData): A context data that was passed from handler
|
4112
|
+
method.
|
4113
|
+
parent_run_id (str, default None): A parent running ID.
|
4114
|
+
event (Event, default None): An event manager that use to track
|
4115
|
+
parent process was not force stopped.
|
4116
|
+
"""
|
4035
4117
|
raise NotImplementedError(
|
4036
4118
|
"Async process of Virtual Python stage does not implement yet."
|
4037
4119
|
)
|